From 76ba5a643f0e66a98be7c198330a85c33f4b83f0 Mon Sep 17 00:00:00 2001 From: liangzelang Date: Mon, 22 Jun 2020 11:01:13 +0800 Subject: [PATCH 001/254] fix bug of type cast --- mindspore/nn/optim/adam.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mindspore/nn/optim/adam.py b/mindspore/nn/optim/adam.py index 5a40d30d5a..ba56af6219 100755 --- a/mindspore/nn/optim/adam.py +++ b/mindspore/nn/optim/adam.py @@ -72,9 +72,9 @@ def _update_run_op(beta1, beta2, eps, lr, weight_decay_tensor, param, m, v, grad update_with_lr = op_mul(lr, update) next_param = param_fp32 - op_reshape(update_with_lr, op_shape(param_fp32)) - next_v = F.depend(next_v, F.assign(param, op_cast(next_param, mstype.float16))) - next_v = F.depend(next_v, F.assign(m, op_cast(next_m, mstype.float16))) - next_v = F.depend(next_v, F.assign(v, op_cast(next_v, mstype.float16))) + next_v = F.depend(next_v, F.assign(param, op_cast(next_param, F.dtype(param)))) + next_v = F.depend(next_v, F.assign(m, op_cast(next_m, F.dtype(m)))) + next_v = F.depend(next_v, F.assign(v, op_cast(next_v, F.dtype(v)))) return next_v From d3ada156737b03ba4722ee3e11e2798e7947e768 Mon Sep 17 00:00:00 2001 From: ougongchang Date: Mon, 22 Jun 2020 17:21:04 +0800 Subject: [PATCH 002/254] Change the attribute to children, becuase the attribute has beed changed in dataset --- mindspore/train/callback/_summary_collector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mindspore/train/callback/_summary_collector.py b/mindspore/train/callback/_summary_collector.py index e2e4a9cc2d..f86c1a2d32 100644 --- a/mindspore/train/callback/_summary_collector.py +++ b/mindspore/train/callback/_summary_collector.py @@ -677,7 +677,7 @@ class SummaryCollector(Callback): return output_dataset.dataset_dir if isinstance(output_dataset, dataset_files_set): return output_dataset.dataset_files[0] - return self._get_dataset_path(output_dataset.input[0]) + return self._get_dataset_path(output_dataset.children[0]) @staticmethod def _get_ckpt_file_path(cb_params): From 4c056855e037467f1e248c404b2c9a8c9f1e79a4 Mon Sep 17 00:00:00 2001 From: Shida He Date: Thu, 11 Jun 2020 10:42:57 -0400 Subject: [PATCH 003/254] Implementation for mindspore debugger --- CMakeLists.txt | 2 +- build.sh | 15 +- cmake/external_libs/absl.cmake | 14 + cmake/external_libs/c-ares.cmake | 12 + cmake/external_libs/grpc.cmake | 110 ++++ cmake/external_libs/zlib.cmake | 9 + cmake/mind_expression.cmake | 10 + cmake/options.cmake | 5 + mindspore/ccsrc/CMakeLists.txt | 20 + mindspore/ccsrc/debug/CMakeLists.txt | 9 + mindspore/ccsrc/debug/debug_services.cc | 194 +++++++ mindspore/ccsrc/debug/debug_services.h | 95 +++ .../ccsrc/debug/debugger/debug_graph.proto | 316 ++++++++++ .../ccsrc/debug/debugger/debug_grpc.proto | 81 +++ mindspore/ccsrc/debug/debugger/debugger.cc | 488 ++++++++++++++++ mindspore/ccsrc/debug/debugger/debugger.h | 159 +++++ mindspore/ccsrc/debug/debugger/grpc_client.cc | 124 ++++ mindspore/ccsrc/debug/debugger/grpc_client.h | 61 ++ .../ccsrc/debug/debugger/proto_exporter.cc | 542 ++++++++++++++++++ mindspore/ccsrc/debug/tensor_data.h | 75 +++ mindspore/ccsrc/debug/tensor_load.h | 69 +++ .../device/ascend/ascend_device_address.cc | 50 ++ .../device/ascend/ascend_device_address.h | 7 + .../device/ascend/ascend_kernel_runtime.cc | 86 +++ .../device/ascend/ascend_kernel_runtime.h | 1 + mindspore/ccsrc/device/kernel_runtime.cc | 8 + mindspore/ccsrc/device/kernel_runtime.h | 8 + .../ccsrc/kernel/cpu/debug_cpu_kernel.cc | 50 ++ mindspore/ccsrc/kernel/cpu/debug_cpu_kernel.h | 41 ++ mindspore/ccsrc/operator/ops.cc | 1 + mindspore/ccsrc/operator/ops.h | 1 + mindspore/ccsrc/operator/prim_debug.cc | 18 +- mindspore/ccsrc/parallel/node_check.cc | 1 + mindspore/ccsrc/parallel/ops_info/ops_utils.h | 1 + mindspore/ccsrc/pipeline/pipeline.cc | 5 +- .../ccsrc/pipeline/static_analysis/prim.cc | 2 + .../ccsrc/pipeline/static_analysis/prim.h | 3 + mindspore/ccsrc/session/ascend_session.cc | 36 ++ mindspore/ccsrc/session/ascend_session.h | 1 + mindspore/ccsrc/session/cpu_session.cc | 16 +- mindspore/ccsrc/session/session_basic.h | 20 +- mindspore/ccsrc/transform/convert.cc | 1 + mindspore/ccsrc/utils/context/ms_context.cc | 4 + mindspore/ccsrc/vm/backend.cc | 4 + mindspore/ccsrc/vm/backend.h | 6 + mindspore/ops/_grad/grad_debug_ops.py | 9 + mindspore/ops/operations/__init__.py | 3 +- mindspore/ops/operations/debug_ops.py | 28 +- tests/ut/cpp/CMakeLists.txt | 6 + 49 files changed, 2818 insertions(+), 9 deletions(-) create mode 100644 cmake/external_libs/absl.cmake create mode 100644 cmake/external_libs/c-ares.cmake create mode 100644 cmake/external_libs/grpc.cmake create mode 100644 cmake/external_libs/zlib.cmake create mode 100644 mindspore/ccsrc/debug/debug_services.cc create mode 100644 mindspore/ccsrc/debug/debug_services.h create mode 100644 mindspore/ccsrc/debug/debugger/debug_graph.proto create mode 100644 mindspore/ccsrc/debug/debugger/debug_grpc.proto create mode 100644 mindspore/ccsrc/debug/debugger/debugger.cc create mode 100644 mindspore/ccsrc/debug/debugger/debugger.h create mode 100644 mindspore/ccsrc/debug/debugger/grpc_client.cc create mode 100644 mindspore/ccsrc/debug/debugger/grpc_client.h create mode 100644 mindspore/ccsrc/debug/debugger/proto_exporter.cc create mode 100644 mindspore/ccsrc/debug/tensor_data.h create mode 100644 mindspore/ccsrc/debug/tensor_load.h create mode 100644 mindspore/ccsrc/kernel/cpu/debug_cpu_kernel.cc create mode 100644 mindspore/ccsrc/kernel/cpu/debug_cpu_kernel.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 37c3288f12..324eca867b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -12,7 +12,7 @@ if (NOT CMAKE_SYSTEM_NAME MATCHES "Windows") endif () if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin") - set(CMAKE_CXX_FLAGS_RELEASE "$ENV{CXXFLAGS} -O2 -Werror -Wno-return-std-move -Wno-unused-private-field -Wno-unused-lambda-capture -Wno-sign-compare -Wno-overloaded-virtual -Wno-unneeded-internal-declaration -Wno-unused-variable -Wno-pessimizing-move -Wno-inconsistent-missing-override -DHALF_ENABLE_CPP11_USER_LITERALS=0 -D_FORTIFY_SOURCE=2") + set(CMAKE_CXX_FLAGS_RELEASE "$ENV{CXXFLAGS} -O2 -Werror -Wno-return-std-move -Wno-unused-private-field -Wno-unused-lambda-capture -Wno-sign-compare -Wno-overloaded-virtual -Wno-unneeded-internal-declaration -Wno-unused-variable -Wno-pessimizing-move -Wno-inconsistent-missing-override -DHALF_ENABLE_CPP11_USER_LITERALS=0 -D_FORTIFY_SOURCE=2") else() set(CMAKE_CXX_FLAGS_RELEASE "$ENV{CXXFLAGS} -O2 -Wl,--allow-shlib-undefined -DHALF_ENABLE_CPP11_USER_LITERALS=0 -D_FORTIFY_SOURCE=2") endif() diff --git a/build.sh b/build.sh index 70718bf89b..608b723425 100755 --- a/build.sh +++ b/build.sh @@ -25,7 +25,7 @@ usage() echo "Usage:" echo "bash build.sh [-d] [-r] [-v] [-c on|off] [-t on|off] [-g on|off] [-h] [-b ge] [-m infer|train] \\" echo " [-a on|off] [-Q on|off] [-p on|off] [-i] [-L] [-R] [-D on|off] [-j[n]] [-e gpu|d|cpu] \\" - echo " [-P on|off] [-z [on|off]] [-M on|off] [-V 9.2|10.1] [-I] [-K]" + echo " [-P on|off] [-z [on|off]] [-M on|off] [-V 9.2|10.1] [-I] [-K] [-B on|off]" echo "" echo "Options:" echo " -d Debug mode" @@ -54,6 +54,7 @@ usage() echo " -I Compile predict, default off" echo " -K Compile with AKG, default off" echo " -s Enable serving module, default off" + echo " -B Enable debugger, default off" } # check value of input is 'on' or 'off' @@ -94,8 +95,10 @@ checkopts() PREDICT_PLATFORM="" ENABLE_AKG="on" ENABLE_SERVING="off" + ENABLE_DEBUGGER="off" + # Process the options - while getopts 'drvj:c:t:hsb:a:g:p:ie:m:I:LRP:Q:D:zM:V:K:s' opt + while getopts 'drvj:c:t:hsb:a:g:p:ie:m:I:LRP:Q:D:zM:V:K:sB:' opt do OPTARG=$(echo ${OPTARG} | tr '[A-Z]' '[a-z]') case "${opt}" in @@ -240,6 +243,11 @@ checkopts() ENABLE_SERVING="on" echo "enable serving" ;; + B) + check_on_off $OPTARG B + ENABLE_DEBUGGER="on" + echo "enable debugger" + ;; *) echo "Unknown option ${opt}!" usage @@ -322,6 +330,9 @@ build_mindspore() if [[ "X$ENABLE_SERVING" = "Xon" ]]; then CMAKE_ARGS="${CMAKE_ARGS} -DENABLE_SERVING=ON" fi + if [[ "X$ENABLE_DEBUGGER" = "Xon" ]]; then + CMAKE_ARGS="${CMAKE_ARGS} -DENABLE_DEBUGGER=ON" + fi echo "${CMAKE_ARGS}" if [[ "X$INC_BUILD" = "Xoff" ]]; then diff --git a/cmake/external_libs/absl.cmake b/cmake/external_libs/absl.cmake new file mode 100644 index 0000000000..6087b65128 --- /dev/null +++ b/cmake/external_libs/absl.cmake @@ -0,0 +1,14 @@ +mindspore_add_pkg(absl + VER 20200225.2 + LIBS absl_strings absl_throw_delegate absl_raw_logging_internal absl_int128 absl_bad_optional_access + URL https://github.com/abseil/abseil-cpp/archive/20200225.2.tar.gz + MD5 73f2b6e72f1599a9139170c29482ddc4 + CMAKE_OPTION -DCMAKE_BUILD_TYPE:STRING=Release -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=TRUE) + +include_directories(${absl_INC}) + +add_library(mindspore::absl_strings ALIAS absl::absl_strings) +add_library(mindspore::absl_throw_delegate ALIAS absl::absl_throw_delegate) +add_library(mindspore::absl_raw_logging_internal ALIAS absl::absl_raw_logging_internal) +add_library(mindspore::absl_int128 ALIAS absl::absl_int128) +add_library(mindspore::absl_bad_optional_access ALIAS absl::absl_bad_optional_access) diff --git a/cmake/external_libs/c-ares.cmake b/cmake/external_libs/c-ares.cmake new file mode 100644 index 0000000000..9bb547f2db --- /dev/null +++ b/cmake/external_libs/c-ares.cmake @@ -0,0 +1,12 @@ +mindspore_add_pkg(c-ares + VER 1.15.0 + LIBS cares + URL https://github.com/c-ares/c-ares/releases/download/cares-1_15_0/c-ares-1.15.0.tar.gz + MD5 d2391da274653f7643270623e822dff7 + CMAKE_OPTION -DCMAKE_BUILD_TYPE:STRING=Release + -DCARES_SHARED:BOOL=OFF + -DCARES_STATIC:BOOL=ON + -DCARES_STATIC_PIC:BOOL=ON) + +include_directories(${c-ares_INC}) +add_library(mindspore::cares ALIAS c-ares::cares) diff --git a/cmake/external_libs/grpc.cmake b/cmake/external_libs/grpc.cmake new file mode 100644 index 0000000000..7496cfd88e --- /dev/null +++ b/cmake/external_libs/grpc.cmake @@ -0,0 +1,110 @@ +set(grpc_USE_STATIC_LIBS ON) +if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin") + set(grpc_CXXFLAGS "-fstack-protector-all -Wno-uninitialized -Wno-unused-parameter -fPIC -fvisibility=hidden -D_FORTIFY_SOURCE=2 -O2") +elseif (${CMAKE_SYSTEM_NAME} MATCHES "Windows") + set(grpc_CXXFLAGS "-fstack-protector-all -Wno-maybe-uninitialized -Wno-unused-parameter -fPIC -fvisibility=hidden -D_FORTIFY_SOURCE=2 -O2") +else() + set(grpc_CXXFLAGS "-fstack-protector-all -Wno-maybe-uninitialized -Wno-unused-parameter -fPIC -fvisibility=hidden -D_FORTIFY_SOURCE=2 -D_GLIBCXX_USE_CXX11_ABI=0 -O2") +endif() + +set(grpc_LDFLAGS "-Wl,-z,relro,-z,now,-z,noexecstack") + + +if (EXISTS ${protobuf_ROOT}/lib64) + set(_FINDPACKAGE_PROTOBUF_CONFIG_DIR "${protobuf_ROOT}/lib64/cmake/protobuf") +else() + set(_FINDPACKAGE_PROTOBUF_CONFIG_DIR "${protobuf_ROOT}/lib/cmake/protobuf") +endif() +message("grpc using Protobuf_DIR : " ${_FINDPACKAGE_PROTOBUF_CONFIG_DIR}) + +if (EXISTS ${absl_ROOT}/lib64) + set(_FINDPACKAGE_ABSL_CONFIG_DIR "${absl_ROOT}/lib64/cmake/absl") +else() + set(_FINDPACKAGE_ABSL_CONFIG_DIR "${absl_ROOT}/lib/cmake/absl") +endif() +message("grpc using absl_DIR : " ${_FINDPACKAGE_ABSL_CONFIG_DIR}) + +set(_CMAKE_ARGS_OPENSSL_ROOT_DIR "") +if (OPENSSL_ROOT_DIR) + set(_CMAKE_ARGS_OPENSSL_ROOT_DIR "-DOPENSSL_ROOT_DIR:PATH=${OPENSSL_ROOT_DIR}") +endif() + +mindspore_add_pkg(grpc + VER 1.27.3 + LIBS grpc++ grpc gpr upb address_sorting + EXE grpc_cpp_plugin + URL https://github.com/grpc/grpc/archive/v1.27.3.tar.gz + MD5 0c6c3fc8682d4262dd0e5e6fabe1a7e2 + CMAKE_OPTION -DCMAKE_BUILD_TYPE:STRING=Release + -DgRPC_INSTALL:BOOL=ON + -DgRPC_BUILD_TESTS:BOOL=OFF + -DgRPC_PROTOBUF_PROVIDER:STRING=package + -DgRPC_PROTOBUF_PACKAGE_TYPE:STRING=CONFIG + -DProtobuf_DIR:PATH=${_FINDPACKAGE_PROTOBUF_CONFIG_DIR} + -DgRPC_ZLIB_PROVIDER:STRING=package + -DZLIB_ROOT:PATH=${zlib_ROOT} + -DgRPC_ABSL_PROVIDER:STRING=package + -Dabsl_DIR:PATH=${_FINDPACKAGE_ABSL_CONFIG_DIR} + -DgRPC_CARES_PROVIDER:STRING=package + -Dc-ares_DIR:PATH=${c-ares_ROOT}/lib/cmake/c-ares + -DgRPC_SSL_PROVIDER:STRING=package + ${_CMAKE_ARGS_OPENSSL_ROOT_DIR} + ) + +include_directories(${grpc_INC}) + +add_library(mindspore::grpc++ ALIAS grpc::grpc++) + +# link other grpc libs +target_link_libraries(grpc::grpc++ INTERFACE grpc::grpc grpc::gpr grpc::upb grpc::address_sorting) + +# link built dependencies +target_link_libraries(grpc::grpc++ INTERFACE mindspore::z) +target_link_libraries(grpc::grpc++ INTERFACE mindspore::cares) +target_link_libraries(grpc::grpc++ INTERFACE mindspore::absl_strings mindspore::absl_throw_delegate + mindspore::absl_raw_logging_internal mindspore::absl_int128 mindspore::absl_bad_optional_access) + +# link system openssl +find_package(OpenSSL REQUIRED) +target_link_libraries(grpc::grpc++ INTERFACE OpenSSL::SSL OpenSSL::Crypto) + + +function(ms_grpc_generate c_var h_var) + if(NOT ARGN) + message(SEND_ERROR "Error: ms_grpc_generate() called without any proto files") + return() + endif() + + set(${c_var}) + set(${h_var}) + + foreach(file ${ARGN}) + get_filename_component(abs_file ${file} ABSOLUTE) + get_filename_component(file_name ${file} NAME_WE) + get_filename_component(file_dir ${abs_file} PATH) + file(RELATIVE_PATH rel_path ${CMAKE_CURRENT_SOURCE_DIR} ${file_dir}) + + list(APPEND ${c_var} "${CMAKE_BINARY_DIR}/proto/${file_name}.pb.cc") + list(APPEND ${h_var} "${CMAKE_BINARY_DIR}/proto/${file_name}.pb.h") + list(APPEND ${c_var} "${CMAKE_BINARY_DIR}/proto/${file_name}.grpc.pb.cc") + list(APPEND ${h_var} "${CMAKE_BINARY_DIR}/proto/${file_name}.grpc.pb.h") + + add_custom_command( + OUTPUT "${CMAKE_BINARY_DIR}/proto/${file_name}.pb.cc" + "${CMAKE_BINARY_DIR}/proto/${file_name}.pb.h" + "${CMAKE_BINARY_DIR}/proto/${file_name}.grpc.pb.cc" + "${CMAKE_BINARY_DIR}/proto/${file_name}.grpc.pb.h" + WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} + COMMAND ${CMAKE_COMMAND} -E make_directory "${CMAKE_BINARY_DIR}/proto" + COMMAND protobuf::protoc --version + COMMAND protobuf::protoc -I${file_dir} --cpp_out=${CMAKE_BINARY_DIR}/proto + --grpc_out=${CMAKE_BINARY_DIR}/proto --plugin=protoc-gen-grpc=$ ${abs_file} + DEPENDS protobuf::protoc grpc::grpc_cpp_plugin ${abs_file} + COMMENT "Running C++ gRPC compiler on ${file}" VERBATIM) + endforeach() + + set_source_files_properties(${${c_var}} ${${h_var}} PROPERTIES GENERATED TRUE) + set(${c_var} ${${c_var}} PARENT_SCOPE) + set(${h_var} ${${h_var}} PARENT_SCOPE) + +endfunction() diff --git a/cmake/external_libs/zlib.cmake b/cmake/external_libs/zlib.cmake new file mode 100644 index 0000000000..06532ed8d7 --- /dev/null +++ b/cmake/external_libs/zlib.cmake @@ -0,0 +1,9 @@ +mindspore_add_pkg(zlib + VER 1.2.11 + LIBS z + URL https://github.com/madler/zlib/archive/v1.2.11.tar.gz + MD5 0095d2d2d1f3442ce1318336637b695f + CMAKE_OPTION -DCMAKE_BUILD_TYPE:STRING=Release) + +include_directories(${zlib_INC}) +add_library(mindspore::z ALIAS zlib::z) diff --git a/cmake/mind_expression.cmake b/cmake/mind_expression.cmake index 86337c1dd2..403316ac47 100644 --- a/cmake/mind_expression.cmake +++ b/cmake/mind_expression.cmake @@ -14,6 +14,16 @@ include(${CMAKE_SOURCE_DIR}/cmake/external_libs/eigen.cmake) include(${CMAKE_SOURCE_DIR}/cmake/external_libs/json.cmake) include(${CMAKE_SOURCE_DIR}/cmake/dependency_securec.cmake) include(${CMAKE_SOURCE_DIR}/cmake/external_libs/protobuf.cmake) + +if (ENABLE_DEBUGGER) + # build dependencies of gRPC + include(${CMAKE_SOURCE_DIR}/cmake/external_libs/absl.cmake) + include(${CMAKE_SOURCE_DIR}/cmake/external_libs/c-ares.cmake) + include(${CMAKE_SOURCE_DIR}/cmake/external_libs/zlib.cmake) + # build gRPC + include(${CMAKE_SOURCE_DIR}/cmake/external_libs/grpc.cmake) +endif() + include(${CMAKE_SOURCE_DIR}/cmake/external_libs/pybind11.cmake) MESSAGE("go to link flatbuffers") include(${CMAKE_SOURCE_DIR}/cmake/external_libs/flatbuffers.cmake) diff --git a/cmake/options.cmake b/cmake/options.cmake index 3e03ed3339..33e4b47ef3 100644 --- a/cmake/options.cmake +++ b/cmake/options.cmake @@ -17,6 +17,7 @@ option(ENABLE_DUMP_E2E "Enable dump e2e file, default on" OFF) option(ENABLE_DUMP_IR "Enable dump funciton graph ir, default on" ON) option(ENABLE_MPI "enable mpi" OFF) option(ENABLE_AKG "enable akg" OFF) +option(ENABLE_DEBUGGER "enable debugger" OFF) if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") if (WIN32) @@ -112,3 +113,7 @@ endif() if(ENABLE_DUMP_E2E) add_compile_definitions(ENABLE_DUMP_E2E) endif() + +if(ENABLE_DEBUGGER) + add_compile_definitions(ENABLE_DEBUGGER) +endif() diff --git a/mindspore/ccsrc/CMakeLists.txt b/mindspore/ccsrc/CMakeLists.txt index 80f82fd7ea..0dc68783e8 100644 --- a/mindspore/ccsrc/CMakeLists.txt +++ b/mindspore/ccsrc/CMakeLists.txt @@ -71,6 +71,17 @@ message("onnx proto path is :" ${ONNX_PROTO}) ms_protobuf_generate(ONNX_PROTO_SRCS ONNX_PROTO_HDRS ${ONNX_PROTO}) list(APPEND MINDSPORE_PROTO_LIST ${ONNX_PROTO_SRCS}) +if (ENABLE_DEBUGGER) + # debugger: compile proto files + include_directories("${CMAKE_BINARY_DIR}/debug/debugger") + file(GLOB_RECURSE DEBUGGER_PROTO_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "debug/debugger/debug_graph.proto") + ms_protobuf_generate(DEBUGGER_PROTO_SRCS DEBUGGER_PROTO_HDRS ${DEBUGGER_PROTO_LIST}) + file(GLOB_RECURSE DEBUGGER_GRPC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "debug/debugger/debug_grpc.proto") + ms_grpc_generate(DEBUGGER_GRPC_SRCS DEBUGGER_GRPC_HDRS ${DEBUGGER_GRPC_LIST}) + list(APPEND MINDSPORE_PROTO_LIST ${DEBUGGER_PROTO_SRCS}) + list(APPEND MINDSPORE_PROTO_LIST ${DEBUGGER_GRPC_SRCS}) +endif () + if (ENABLE_DUMP_PROTO) include_directories(${CMAKE_BINARY_DIR}) file(GLOB_RECURSE PROTO_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "utils/node_strategy.proto") @@ -125,6 +136,14 @@ endforeach () set_property(SOURCE ${SUB_OBJECTS_SRC} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_ME) add_library(mindspore STATIC ${SUB_OBJECTS_SRC}) + +target_link_libraries(proto_input mindspore::protobuf) + +if (ENABLE_DEBUGGER) + # debugger: link grpc + target_link_libraries(proto_input mindspore::grpc++) +endif() + target_link_libraries(mindspore proto_input) if (ENABLE_CPU AND ENABLE_MPI) target_link_libraries(mindspore securec mindspore::flatbuffers mindspore::ompi) @@ -217,6 +236,7 @@ if (USE_GLOG) endif () if (ENABLE_DUMP_PROTO) + message("add protobuf lib to c_expression") target_link_libraries(_c_expression PRIVATE mindspore::protobuf) endif () diff --git a/mindspore/ccsrc/debug/CMakeLists.txt b/mindspore/ccsrc/debug/CMakeLists.txt index 30b10a17fd..ba0c5e07ac 100644 --- a/mindspore/ccsrc/debug/CMakeLists.txt +++ b/mindspore/ccsrc/debug/CMakeLists.txt @@ -10,6 +10,15 @@ set(_DEBUG_SRC_LIST "${CMAKE_CURRENT_SOURCE_DIR}/trace.cc" ) +if (ENABLE_DEBUGGER) + list(APPEND _DEBUG_SRC_LIST + "${CMAKE_CURRENT_SOURCE_DIR}/debugger/debugger.cc" + "${CMAKE_CURRENT_SOURCE_DIR}/debugger/grpc_client.cc" + "${CMAKE_CURRENT_SOURCE_DIR}/debugger/proto_exporter.cc" + "${CMAKE_CURRENT_SOURCE_DIR}/debug_services.cc" + ) +endif (ENABLE_DEBUGGER) + if (ENABLE_DUMP_E2E) list(APPEND _DEBUG_SRC_LIST "${CMAKE_CURRENT_SOURCE_DIR}/e2e_dump.cc") endif (ENABLE_DUMP_E2E) diff --git a/mindspore/ccsrc/debug/debug_services.cc b/mindspore/ccsrc/debug/debug_services.cc new file mode 100644 index 0000000000..8d46e00f19 --- /dev/null +++ b/mindspore/ccsrc/debug/debug_services.cc @@ -0,0 +1,194 @@ +/** + * Copyright 2019-2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "debug/debug_services.h" +namespace mindspore { + +DebugServices::DebugServices() { + tensor_loader_ = new TensorLoader(); + uint32_t iter_num = -1; + tensor_loader_->set_iter_num(iter_num); +} + +DebugServices::DebugServices(const DebugServices &other) { + tensor_loader_ = other.tensor_loader_; + watchpoint_table = other.watchpoint_table; +} + +DebugServices &DebugServices::operator=(const DebugServices &other) { + if (this != &other) { + tensor_loader_ = other.tensor_loader_; + watchpoint_table = other.watchpoint_table; + } + return *this; +} + +DebugServices::~DebugServices() { delete tensor_loader_; } + +void DebugServices::add_watchpoint(unsigned int id, unsigned int watch_condition, + const std::vector> &check_node_list) { + std::lock_guard lg(lock_); + + watchpoint_t watchpoint_item; + + watchpoint_item.id = id; + + if (watch_condition == 0) { + watchpoint_item.conditions.nan.enabled = true; + } else if (watch_condition == 1) { + watchpoint_item.conditions.inf.enabled = true; + watchpoint_item.conditions.neg_inf.enabled = true; + } + + watchpoint_item.check_node_list = check_node_list; + + watchpoint_table[id] = watchpoint_item; +} + +void DebugServices::remove_watchpoint(unsigned int id) { + std::lock_guard lg(lock_); + watchpoint_table.erase(id); +} + +void DebugServices::check_watchpoints(std::vector *name, std::vector *slot, + std::vector *data_ptr, std::vector *data_size, + std::vector *condition, std::vector *wacthpoint_id) { + std::lock_guard lg(lock_); + + std::vector> tensor_list = tensor_loader_->GetTensor(); + + std::string current_tensor_name; + std::unordered_map watchpoints_to_check_table; + + for (std::size_t i = 0; i < tensor_list.size(); i++) { + current_tensor_name = tensor_list[i]->GetName(); + mindspore::tensor::TensorPtr tensor_ptr = tensor_list[i]->GetTensor(); + int tensor_data_type = tensor_ptr->data_type_c(); + + // check if we need to analyze this node and for which watchpoints we will check + // create a list of watchpoints to check + watchpoints_to_check_table.clear(); + for (auto w_table_item : watchpoint_table) { + // if the watchpoint is checking for a nan or inf and the current tensor is not of a float type, then + // don't check the watchpoint for this tensor + if (std::get<1>(w_table_item).conditions.inf.enabled || std::get<1>(w_table_item).conditions.neg_inf.enabled || + std::get<1>(w_table_item).conditions.nan.enabled) { + if (tensor_data_type != kNumberTypeFloat16 && tensor_data_type != kNumberTypeFloat && + tensor_data_type != kNumberTypeFloat32 && tensor_data_type != kNumberTypeFloat64) { + continue; + } + } + + auto check_node_list = std::get<1>(w_table_item).check_node_list; + + for (auto check_node : check_node_list) { + std::string w_name = std::get<0>(check_node); + bool w_type = std::get<1>(check_node); + + // check if the current node tensor name is included the watchpoint + std::string current_node_name = current_tensor_name.substr(0, current_tensor_name.find_first_of(":")); + if ((w_type == true && (current_tensor_name.find(w_name) != string::npos || w_name == "*")) || + (w_type == false && current_node_name == w_name)) { + watchpoints_to_check_table[w_table_item.second.id] = w_table_item.second; + break; + } + } + } + + // check if no watchpoints are valid for the current tensor + if (watchpoints_to_check_table.empty()) { + continue; + } + + // need to add support for float16 and float64, and other types when we support conditions beyond inf and nan + if (tensor_data_type != kNumberTypeFloat && tensor_data_type != kNumberTypeFloat32) { + continue; + } + + float *start_addr = reinterpret_cast(tensor_ptr->data_c(false)); + unsigned int num_elements = (tensor_ptr->data().nbytes()) / sizeof(float); + + std::unordered_map::iterator it_w_table_check; + std::vector hit_encountered; + + for (unsigned int index = 0; index < num_elements; index++) { + float x = start_addr[index]; + it_w_table_check = watchpoints_to_check_table.begin(); + + while (it_w_table_check != watchpoints_to_check_table.end()) { + if ((it_w_table_check->second.conditions.inf.enabled || it_w_table_check->second.conditions.neg_inf.enabled) && + isinf(x)) { + hit_encountered.push_back(it_w_table_check->second.id); + } else if (it_w_table_check->second.conditions.nan.enabled && isnan(x)) { + hit_encountered.push_back(it_w_table_check->second.id); + } + + ++it_w_table_check; + } + + if (hit_encountered.size()) { + for (auto it_hit_id = hit_encountered.begin(); it_hit_id != hit_encountered.end(); ++it_hit_id) { + std::string name_no_slot = current_tensor_name.substr(0, current_tensor_name.find_first_of(":")); + name->push_back(name_no_slot); + + slot->push_back(std::to_string(tensor_list[i]->GetSlot())); + data_ptr->push_back(reinterpret_cast(tensor_ptr->data_c(false))); + data_size->push_back(tensor_ptr->data().nbytes()); + + int condition_item = -1; + if (watchpoint_table[*it_hit_id].conditions.nan.enabled) { + condition_item = 0; + } else if (watchpoint_table[*it_hit_id].conditions.inf.enabled || + watchpoint_table[*it_hit_id].conditions.neg_inf.enabled) { + condition_item = 1; + } + condition->push_back(condition_item); + + wacthpoint_id->push_back(*it_hit_id); + + watchpoints_to_check_table.erase(*it_hit_id); + } + + hit_encountered.clear(); + } + + if (watchpoints_to_check_table.empty()) { + break; + } + } + } +} + +void DebugServices::read_nodes_tensors(std::vector name, std::vector *ret_name, + std::vector *data_ptr, std::vector *data_size, + std::vector *dtype, std::vector> *shape) { + std::vector>> result_list; + tensor_loader_->SearchTensors(name, &result_list); + + for (auto result : result_list) { + if (!std::get<1>(result)) { + continue; + } + ret_name->push_back(std::get<0>(result)); + data_ptr->push_back(reinterpret_cast(std::get<1>(result)->GetTensor()->data_c(false))); + data_size->push_back(std::get<1>(result)->GetTensor()->data().nbytes()); + dtype->push_back(std::get<1>(result)->GetTensor()->Dtype()); + shape->push_back(std::get<1>(result)->GetTensor()->shape()); + } +} + +TensorLoader *DebugServices::get_tensor_loader() const { return tensor_loader_; } + +} // namespace mindspore diff --git a/mindspore/ccsrc/debug/debug_services.h b/mindspore/ccsrc/debug/debug_services.h new file mode 100644 index 0000000000..b2fd41cd68 --- /dev/null +++ b/mindspore/ccsrc/debug/debug_services.h @@ -0,0 +1,95 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_DEBUG_DEBUG_SERVICES_H_ +#define MINDSPORE_CCSRC_DEBUG_DEBUG_SERVICES_H_ + +#include +#include +#include +#include +#include +#include +#include "debug/tensor_load.h" +#include "debug/tensor_data.h" +#include "ir/dtype.h" + +namespace mindspore { +class DebugServices { + public: + DebugServices(); + + DebugServices(const DebugServices &other); + + DebugServices &operator=(const DebugServices &other); + + ~DebugServices(); + + void add_watchpoint(unsigned int id, unsigned int watch_condition, + const std::vector> &check_node_list); + + void remove_watchpoint(unsigned int id); + + void check_watchpoints(std::vector *name, std::vector *slot, std::vector *data_ptr, + std::vector *data_size, std::vector *condition, + std::vector *wacthpoint_id); + + void read_nodes_tensors(std::vector name, std::vector *ret_name, + std::vector *data_ptr, std::vector *data_size, + std::vector *dtype, std::vector> *shape); + + TensorLoader *get_tensor_loader() const; + + private: + typedef struct condition_no_param { + bool enabled = false; + } condition_no_param_t; + + typedef struct condition_with_param { + bool enabled = false; + float parameter = 0; + } condition_with_param_t; + + typedef struct conditions { + condition_no_param_t inf; + condition_no_param_t neg_inf; + condition_no_param_t nan; + condition_with_param_t max_below; + condition_with_param_t max_above; + condition_with_param_t min_below; + condition_with_param_t min_above; + condition_with_param_t max_minus_min_below; + condition_with_param_t max_minus_min_above; + condition_with_param_t mean_below; + condition_with_param_t mean_above; + condition_with_param_t std_dev_below; + condition_with_param_t std_dev_above; + } conditions_t; + + typedef struct watchpoint { + unsigned int id; + conditions_t conditions; + std::vector> check_node_list; + } watchpoint_t; + + std::mutex lock_; + + std::unordered_map watchpoint_table; + + TensorLoader *tensor_loader_; +}; +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_DEBUG_DEBUG_SERVICES_H_ diff --git a/mindspore/ccsrc/debug/debugger/debug_graph.proto b/mindspore/ccsrc/debug/debugger/debug_graph.proto new file mode 100644 index 0000000000..042360fac3 --- /dev/null +++ b/mindspore/ccsrc/debug/debugger/debug_graph.proto @@ -0,0 +1,316 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = "proto2"; + +package debugger; + +// Versioning +enum Version { + // unknown version + UNKNOWWN_VERSION = 0; + + // Initial version (IR VERSION 1), published on Sep 23, 2019 + IR_VERSION = 0x0000000000000001; +} + +// Data type definition +enum DataType { + DT_UNDEFINED = 0; + // Basic types. + DT_BOOL = 1; // bool + + DT_INT8 = 2; // int8_t + DT_INT16 = 3; // int16_t + DT_INT32 = 4; // int32_t + DT_INT64 = 5; // int64_t + + DT_UINT8 = 6; // uint8_t + DT_UINT16 = 7; // uint16_t + DT_UINT32 = 8; // uint32_t + DT_UINT64 = 9; // uint64_t + + DT_FLOAT16 = 10; // float 16 + DT_FLOAT32 = 11; // float 32 + DT_FLOAT64 = 12; // float 64 + + DT_STRING = 13; // string + DT_TENSOR = 14; // tensor + DT_GRAPH = 15; // graph + + // list type + DT_BOOLS = 16; // list of bool + + DT_INTS8 = 17; // list of int8_t + DT_INTS16 = 18; // list of int16_t + DT_INTS32 = 19; // list of int32_t + DT_INTS64 = 20; // list of int64_t + + DT_UINTS8 = 21; // list of uint8_t + DT_UINTS16 = 22; // list of uint16_t + DT_UINTS32 = 23; // list of uint32_t + DT_UINTS64 = 24; // list of uint64_t + + DT_FLOATS16 = 25; // list of float16 + DT_FLOATS32 = 26; // list of float32 + DT_FLOATS64 = 27; // list of float64 + + DT_STRINGS = 28; // list of string + DT_TENSORS = 29; // list of tensor + DT_GRAPHS = 30; // list of graph + + DT_TUPLE = 31; // tuple + DT_LIST = 32; // list + DT_DICT = 33; // dictionary + + // other types + DT_NONE = 34; // None + DT_SYM_INST = 35; // Symbolic Key Instance + + // type related type + DT_BASE_INT = 36; // type generic int + DT_BASE_UINT = 37; // type generate unsigned int + DT_BASE_FLOAT = 38; // type generate float + DT_TYPE = 39; // type type + DT_ANYTHING = 40; // type anything + DT_REFKEY = 41; // type refkey + DT_REF = 42; // type ref +} + +// Value definition for attribute value or parameter default value +message ValueProto { + // data type of value + optional DataType dtype = 1; // discriminator that indicates which field below is in use + + // Exactly ONE of the following fields must be present for this version of the IR + optional bool bool_val = 2; // bool + optional int64 int_val = 3; // int + optional uint64 uint_val = 4; // uint + optional float float_val = 5; // float + optional double double_val = 6; // double + optional string str_val = 7; // string + optional TensorProto tensor_val = 8; // tensor value + optional GraphProto graph = 9; // graph + + repeated bool bool_vals = 10; // list of bool + repeated int64 int_vals = 11; // list of int + repeated uint64 uint_vals = 12; // list of uint + repeated float float_vals = 13; // list of float + repeated double double_vals = 14; // list of double + repeated string str_vals = 15; // list of string + repeated TensorProto tensor_vals = 16; // list of tensor value + repeated GraphProto graphs = 17; // list of graph + + // tuple or list + repeated ValueProto values = 18; // tuple, list of value + + // dictionary + repeated NamedValueProto dict_val = 19; // dictionary info + + // filed for type type + optional TypeProto type_val = 20; // type type info +} + +message AttributeProto { + optional string name = 1; // attribute name + optional ValueProto value = 2; // attribute value +} + +message NamedValueProto { + optional string key = 1; // attribute name + optional ValueProto value = 2; // attribute value +} + +// Defines a tensor shape. +message TensorShapeProto { + // One dimension of the tensor. + message Dimension { + // Size of the tensor in that dimension. + // This value must be >= -1, but values of -1 are reserved for "unknown" + // shapes (values of -1 mean "unknown" dimension). + optional int64 size = 1; + + // Optional name of the tensor dimension. + optional string name = 2; + }; + + repeated Dimension dim = 1; +} + +// Types for graph input(parameter) and output +message TypeProto { + + message Tensor { + // This field MUST have a valid DataType value except DT_TENSOR + optional DataType elem_type = 1; + optional TensorShapeProto shape = 2; // for scalar, this field is not set + } + + // tuple type + message Sequence { + // The type and optional shape of elements of the tuple. + repeated TypeProto elem_types = 1; + }; + + // data type + optional DataType data_type = 1; + + oneof value { + // The type of a tensor. + Tensor tensor_type = 2; + + // The type of a tuple. + Sequence sequence_type = 3; + } +} + +// Defines information on graph parameters, including the name, the type, and +// the default value of parameter if exists. +message ParameterProto { + optional string name = 1; // parameter name + optional TypeProto type = 2; // parameter type + optional ValueProto default_val = 3; // default value of parameter if exists +} + +// Defines graph output information +message OutputProto { + optional string name = 1; // output node name + optional TypeProto type = 2; // output node type +} + +// Define node input information +message InputProto { + enum EdgeType { + DATA_EDGE = 0; // data edge + CONTROL_EDGE = 1; // control edge + } + + optional string name = 1; + optional EdgeType type = 2; +} + +// Nodes +// +// Computation graphs are made up of a DAG of nodes, which represent what is +// commonly called a "layer" or "pipeline stage" in machine learning frameworks. +// +// For example, it can be a node of type "Conv" that takes in an image, a filter +// tensor and a bias tensor, and produces the convolved output. +message NodeProto { + repeated InputProto input = 1; // namespace Value + optional string name = 2; // namespace Value + + // The symbolic identifier of the Operator to execute. + optional string op_type = 3; // namespace Operator + // The domain of the OperatorSet that specifies the operator named by op_type. + optional string scope = 4; // namespace Domain + + // Additional named attributes. + repeated AttributeProto attribute = 5; + + // Optional type info of this node + optional TypeProto output_type = 6; + + // other fields for debug + optional uint64 output_i = 7; + + // for debugger, full name with scope + optional string debug_name = 8; +} + +// Models +// +// ModelProto is a top-level file/container format for bundling a ML model and +// associating its computation graph with metadata. +// +// The semantics of the model are described by the associated GraphProto. +message ModelProto { + // ir version + optional int64 ir_version = 1; + + // Domain name of the model. + // We use reverse domain names as name space indicators. For example: + // `com.facebook.fair` or `com.microsoft.cognitiveservices` + // + // Together with `model_version` and GraphProto.name, this forms the unique identity of + // the graph. + optional string domain = 2; + + // The version of the graph encoded. See Version enum below. + optional int64 model_version = 3; + + // The parameterized graph that is evaluated to execute the model. + optional GraphProto graph = 4; + + // metadata info of opeartors + optional OperatorSetProto metadata_operators = 5; +}; + +message OperatorProto { + optional string name = 1; // used as key, must be distinct + optional bytes config = 2; // operator config info + optional bytes obj_info = 3; // operator related object info, e.g. content of operator binary or name +}; + +message OperatorSetProto { + repeated OperatorProto operators = 1; + optional string summary = 2; // summary info of operators, e.g. file position of operators file +} + +// Graphs +// +// A graph defines the computational logic of a model and is comprised of a parameterized +// list of nodes that form a directed acyclic graph based on their inputs and outputs. +// This is the equivalent of the "network" or "graph" in many deep learning +// frameworks. +message GraphProto { + // The nodes in the graph, sorted topologically. + repeated NodeProto node = 1; + + // The name of the graph. + optional string name = 2; // namespace Graph + + // The parameters(inputs) and outputs of the graph. + repeated ParameterProto parameters = 3; + repeated OutputProto outputs = 4; + + // Constants used in this graph + repeated NamedValueProto const_vals = 5; +} + +// Tensors +// +// A serialized tensor value. +message TensorProto { + // The node name of the tensor. + optional string node_name = 1; + + // The slot of the tensor in its node. + optional string slot = 2; + + // The serialized tensor content. + optional bytes tensor_content = 3; + + // The shape of the tensor. + repeated int64 dims = 4; + + // The data type of the tensor. + // This field MUST have a valid DataType value except DT_TENSOR + optional DataType data_type = 5; + + // If the tensor content transferring is finished. + optional bool finished = 6; +} \ No newline at end of file diff --git a/mindspore/ccsrc/debug/debugger/debug_grpc.proto b/mindspore/ccsrc/debug/debugger/debug_grpc.proto new file mode 100644 index 0000000000..f742987a4e --- /dev/null +++ b/mindspore/ccsrc/debug/debugger/debug_grpc.proto @@ -0,0 +1,81 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = "proto3"; + +package debugger; + +import "debug_graph.proto"; + +service EventListener { + rpc WaitCMD (Metadata) returns (EventReply) {}; + rpc SendMetadata (Metadata) returns (EventReply) {}; + rpc SendGraph (GraphProto) returns (EventReply) {}; + rpc SendTensors (stream TensorProto) returns (EventReply) {}; + rpc SendWatchpointHits (stream WatchpointHit) returns (EventReply) {}; +} + +message Metadata { + string device_name = 1; + int32 cur_step = 2; +} + +message EventReply { + enum Status { + OK = 0; + FAILED = 1; + PENDING = 2; + } + + Status status = 1; + + oneof cmd { + bool exit = 2; + int32 run_cmd = 3; + SetCMD set_cmd = 4; + ViewCMD view_cmd = 5; + } +} + +message SetCMD { + repeated WatchNode watch_nodes = 1; + WatchCondition watch_condition = 2; + bool delete = 3; + int32 id = 4; +} + +message ViewCMD { + repeated TensorProto tensors = 1; +} + +message WatchCondition { + enum Condition { + nan = 0; + inf = 1; + } + Condition condition = 1; +} + +message WatchNode { + string node_name = 1; + string node_type = 2; +} + +message WatchpointHit { + TensorProto tensor = 1; + WatchCondition watch_condition = 2; + int32 id = 3; +} diff --git a/mindspore/ccsrc/debug/debugger/debugger.cc b/mindspore/ccsrc/debug/debugger/debugger.cc new file mode 100644 index 0000000000..ea147a929f --- /dev/null +++ b/mindspore/ccsrc/debug/debugger/debugger.cc @@ -0,0 +1,488 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include "debug/debugger/debugger.h" +#include "pipeline/pipeline.h" +#include "session/anf_runtime_algorithm.h" + +using debugger::EventReply; +using debugger::GraphProto; +using debugger::ModelProto; +using debugger::TensorProto; +using debugger::WatchCondition; +using debugger::WatchCondition_Condition_inf; +using debugger::WatchCondition_Condition_nan; +using debugger::WatchNode; +using debugger::WatchpointHit; + +namespace mindspore { + +DebuggerPtr Debugger::debugger_ = nullptr; +std::mutex Debugger::instance_lock_; + +Debugger::Debugger() + : grpc_client_(nullptr), + debug_services_(nullptr), + device_id_(0), + num_step_(0), + debugger_enabled_(false), + is_dataset_graph_(false) {} + +void Debugger::Init(const uint32_t device_id) { + // access lock for public method + std::lock_guard a_lock(access_lock_); + // save device_id + MS_LOG(INFO) << "Debugger got device_id: " << device_id; + device_id_ = device_id; +} + +void Debugger::EnableDebugger() { + // reset some of the class members + num_step_ = 0; + debugger_enabled_ = false; + grpc_client_ = nullptr; + debug_services_ = nullptr; + + // get env variables to configure debugger + const char *env_enable_str = std::getenv("ENABLE_MS_DEBUGGER"); + if (env_enable_str != nullptr) { + MS_LOG(INFO) << "Getenv ENABLE_MS_DEBUGGER: " << env_enable_str; + if (std::strcmp(env_enable_str, "1") == 0) { + debugger_enabled_ = true; + } + } + if (!debugger_enabled_) { + MS_LOG(WARNING) << "Not enabling debugger. Set environment variable ENABLE_MS_DEBUGGER=1 to enable debugger."; + return; + } + // configure host + const char *env_host_str = std::getenv("MS_DEBUGGER_HOST"); + std::string host; + if (env_host_str != nullptr) { + MS_LOG(INFO) << "Getenv MS_DEBUGGER_HOST: " << env_host_str; + host = std::string(env_host_str); + } else { + MS_LOG(WARNING) << "Environment variable MS_DEBUGGER_HOST doesn't exist. Using default debugger host: localhost"; + host = "localhost"; + } + // configure port + const char *env_port_str = std::getenv("MS_DEBUGGER_PORT"); + std::string port; + if (env_port_str != nullptr) { + MS_LOG(INFO) << "Getenv MS_DEBUGGER_PORT: " << env_port_str; + port = std::string(env_port_str); + } else { + MS_LOG(WARNING) << "Environment variable MS_DEBUGGER_PORT doesn't exist. Using default debugger port: 50051"; + port = "50051"; + } + + // initialize grpc client + grpc_client_ = std::make_unique(host, port); + debug_services_ = std::make_unique(); +} + +void Debugger::Reset() { + // access lock for public method + std::lock_guard a_lock(access_lock_); + // reset components + device_id_ = 0; + num_step_ = 0; + debugger_enabled_ = false; + is_dataset_graph_ = false; + graph_ptr_ = nullptr; + grpc_client_ = nullptr; + debug_services_ = nullptr; +} + +void Debugger::PreExecute(const KernelGraphPtr &graph_ptr) { + // access lock for public method + std::lock_guard a_lock(access_lock_); + // check and save graph_ptr, suspend if graph is new + CheckGraphPtr(graph_ptr); +} + +void Debugger::PostExecute() { + // access lock for public method + std::lock_guard a_lock(access_lock_); + // analyze tensor data and send the watchpoints been hit + if (debugger_enabled_ && !is_dataset_graph_) { + num_step_++; + MS_LOG(INFO) << "Debugger suspend at end of step; number of steps executed: " << num_step_; + SendWatchpointsAndSuspend(CheckWatchpoints()); + } +} + +void Debugger::PostDebugOp() { + // access lock for public method + std::lock_guard a_lock(access_lock_); + // suspend if debugger is enabled + if (debugger_enabled_ && !is_dataset_graph_) { + MS_LOG(INFO) << "Debugger suspend at debug_op"; + CommandLoop(); + } +} + +void Debugger::CheckGraphPtr(const KernelGraphPtr &graph_ptr) { + if (graph_ptr_ != graph_ptr) { + MS_LOG(INFO) << "Debugger got new graph: " << graph_ptr->graph_id(); + // save new graph_ptr + graph_ptr_ = graph_ptr; + // check if it is dataset graph + CheckDatasetGraph(); + if (!is_dataset_graph_) { + // only try to enable debugger if it is not a dataset graph + EnableDebugger(); + if (debugger_enabled_) { + // get graph proto and send to mindinsight + SendGraphAndSuspend(GetGraphProto()); + } + } + } +} + +void Debugger::CheckDatasetGraph() { + // print parameter node names + const auto ¶ms = graph_ptr_->inputs(); + for (const auto ¶m : params) { + MS_LOG(INFO) << "param: " << param->fullname_with_scope(); + } + // check if there is GetNext or InitDataSetQueue node + const auto &nodes = graph_ptr_->execution_order(); + for (const auto &node : nodes) { + auto node_name = AnfAlgo::GetCNodeName(node); + MS_LOG(INFO) << "node: " << node->fullname_with_scope(); + if (node_name == "GetNext" || node_name == "InitDataSetQueue") { + MS_LOG(WARNING) << "Not enabling debugger for graph " << graph_ptr_->graph_id() << ": found dataset graph node " + << node_name; + is_dataset_graph_ = true; + return; + } + } + is_dataset_graph_ = false; +} + +GraphProto Debugger::GetGraphProto() { + // convert kernel graph to debugger modelproto + ModelProto model = GetDebuggerFuncGraphProto(graph_ptr_); + return model.graph(); +} + +void Debugger::SendGraphAndSuspend(const GraphProto &graph_proto) { + // prepare metadata + std::string device_name = std::to_string(device_id_) + ":" + std::to_string(graph_ptr_->graph_id()); + Metadata metadata; + metadata.set_device_name(device_name); + metadata.set_cur_step(num_step_); + EventReply reply_metadata = grpc_client_->SendMetadata(metadata); + if (reply_metadata.status() != reply_metadata.OK) { + MS_LOG(ERROR) << "Error: SendMetadata failed"; + } + // send graph to mindinght server + EventReply reply = grpc_client_->SendGraph(graph_proto); + if (reply.status() != reply.OK) { + MS_LOG(ERROR) << "Error: SendGraph failed"; + } + // enter command loop, wait and process commands + CommandLoop(); +} + +void Debugger::CommandLoop() { + // prepare metadata + std::string device_name = std::to_string(device_id_) + ":" + std::to_string(graph_ptr_->graph_id()); + Metadata metadata; + metadata.set_device_name(device_name); + metadata.set_cur_step(num_step_); + + // loop exit flag + bool run = false; + int num_wait_fail = 0; + const int max_num_wait_fail = 5; + + while (!run) { + // wait for command + EventReply reply = grpc_client_->WaitForCommand(metadata); + if (reply.status() != reply.OK) { + MS_LOG(ERROR) << "Error: WaitForCommand failed"; + num_wait_fail++; + if (num_wait_fail > max_num_wait_fail) { + MS_LOG(ERROR) << "Maximum number of WaitForCommand retry reached: exiting training session"; + Exit(); + } + MS_LOG(ERROR) << "Number of consecutive WaitForCommand fail:" << num_wait_fail << "; Retry after " + << num_wait_fail << "s"; + std::this_thread::sleep_for(std::chrono::milliseconds(1000 * num_wait_fail)); + continue; + } + + // get type of the command in reply + DebuggerCommand cmd = GetCommand(reply); + if (cmd == DebuggerCommand::kUnknownCMD) { + MS_LOG(ERROR) << "Error: debugger recieved unknown command"; + continue; + } + + MS_LOG(INFO) << "recieved command: "; + switch (cmd) { + case DebuggerCommand::kUnknownCMD: + MS_LOG(INFO) << "UnknownCMD"; + break; + case DebuggerCommand::kExitCMD: + MS_LOG(INFO) << "ExitCMD"; + Exit(); + break; + case DebuggerCommand::kRunCMD: + MS_LOG(INFO) << "RunCMD"; + // exit loop + run = true; + break; + case DebuggerCommand::kSetCMD: + MS_LOG(INFO) << "SetCMD"; + { + // print set cmd content + ProtoVector recieved_nodes = GetWatchnodes(reply); + for (auto node : recieved_nodes) { + MS_LOG(INFO) << "node name: " << node.node_name(); + MS_LOG(INFO) << "node type: " << node.node_type(); + } + WatchCondition recieved_condition = GetWatchcondition(reply); + MS_LOG(INFO) << "condition: " << recieved_condition.condition(); + int32_t id = GetWatchpointID(reply); + MS_LOG(INFO) << "id: " << id; + bool delete_ = GetWatchpointDelete(reply); + MS_LOG(INFO) << "delete: " << delete_; + } + MS_LOG(INFO) << "Setting watchpoint"; + if (GetWatchpointDelete(reply)) { + RemoveWatchpoint(GetWatchpointID(reply)); + } else { + SetWatchpoint(GetWatchnodes(reply), GetWatchcondition(reply), GetWatchpointID(reply)); + } + break; + case DebuggerCommand::kViewCMD: + MS_LOG(INFO) << "ViewCMD"; + { + // print view cmd content + ProtoVector received_tensors = GetTensors(reply); + for (auto tensor : received_tensors) { + MS_LOG(INFO) << "tensor node name: " << tensor.node_name(); + MS_LOG(INFO) << "tensor slot: " << tensor.slot(); + MS_LOG(INFO) << "tensor finished: " << std::boolalpha << tensor.finished() << std::noboolalpha; + } + } + MS_LOG(INFO) << "Sending tensors"; + std::list tensors = LoadTensors(GetTensors(reply)); + { + for (auto tensor : tensors) { + MS_LOG(INFO) << "tensor node name: " << tensor.node_name(); + MS_LOG(INFO) << "tensor slot: " << tensor.slot(); + MS_LOG(INFO) << "tensor finished: " << std::boolalpha << tensor.finished() << std::noboolalpha; + MS_LOG(INFO) << "tensor dims: "; + for (auto dim : tensor.dims()) { + MS_LOG(INFO) << dim << ","; + } + MS_LOG(INFO) << "tensor dtype: " << tensor.data_type(); + } + } + EventReply send_tensors_reply = grpc_client_->SendTensors(tensors); + if (send_tensors_reply.status() != send_tensors_reply.OK) { + MS_LOG(ERROR) << "Error: SendTensors failed"; + } + break; + } + } +} + +DebuggerCommand Debugger::GetCommand(const EventReply &reply) { + DebuggerCommand cmd = DebuggerCommand::kUnknownCMD; + switch (reply.cmd_case()) { + case debugger::EventReply::CmdCase::kExit: + cmd = DebuggerCommand::kExitCMD; + break; + case debugger::EventReply::CmdCase::kRunCmd: + cmd = DebuggerCommand::kRunCMD; + break; + case debugger::EventReply::CmdCase::kSetCmd: + cmd = DebuggerCommand::kSetCMD; + break; + case debugger::EventReply::CmdCase::kViewCmd: + cmd = DebuggerCommand::kViewCMD; + break; + default: + MS_LOG(ERROR) << "Error: UnknownCMD"; + break; + } + return cmd; +} + +ProtoVector Debugger::GetWatchnodes(const EventReply &reply) { + if (!reply.has_set_cmd()) { + MS_LOG(ERROR) << "Error: Not SetCMD, can not get WatchNodes. Returning default value: ProtoVector()."; + return ProtoVector(); + } + return reply.set_cmd().watch_nodes(); +} + +WatchCondition Debugger::GetWatchcondition(const EventReply &reply) { + if (!reply.has_set_cmd() || !reply.set_cmd().has_watch_condition()) { + MS_LOG(ERROR) << "Error: Can not get WatchCondition from command. Returning default value: WatchCondition()."; + return WatchCondition(); + } + return reply.set_cmd().watch_condition(); +} + +int32_t Debugger::GetWatchpointID(const EventReply &reply) { + if (!reply.has_set_cmd()) { + MS_LOG(ERROR) << "Error: Not SetCMD, can not get Watchpoint ID. Returning default value: 0."; + return 0; + } + return reply.set_cmd().id(); +} + +bool Debugger::GetWatchpointDelete(const EventReply &reply) { + if (!reply.has_set_cmd()) { + MS_LOG(ERROR) << "Error: Not SetCMD, can not get Watchpoint delete flag. Returning default value: false."; + return false; + } + return reply.set_cmd().delete_(); +} + +ProtoVector Debugger::GetTensors(const EventReply &reply) { + if (!reply.has_view_cmd()) { + MS_LOG(ERROR) << "Error: Not ViewCMD, can not get Tensors. Returning default value: ProtoVector()."; + return ProtoVector(); + } + return reply.view_cmd().tensors(); +} + +void Debugger::SetWatchpoint(const ProtoVector &nodes, const WatchCondition &condition, const int32_t id) { + std::vector> check_node_list; + std::transform(nodes.begin(), nodes.end(), std::back_inserter(check_node_list), + [](WatchNode node) -> std::tuple { + return make_tuple(node.node_name(), node.node_type() == "scope"); + }); + + debug_services_->add_watchpoint(id, condition.condition(), check_node_list); +} + +void Debugger::RemoveWatchpoint(const int32_t id) { debug_services_->remove_watchpoint(id); } + +std::list Debugger::LoadTensors(const ProtoVector &tensors) { + std::vector name; + std::vector ret_name; + std::vector data_ptr; + std::vector data_size; + std::vector dtype; + std::vector> shape; + + std::transform(tensors.begin(), tensors.end(), std::back_inserter(name), + [](TensorProto tensor) -> std::string { return tensor.node_name() + ":" + tensor.slot(); }); + + debug_services_->read_nodes_tensors(name, &ret_name, &data_ptr, &data_size, &dtype, &shape); + + std::list tensor_list; + unsigned int result_index = 0; + TensorProto tensor_item; + + for (auto tensor : tensors) { + tensor_item.set_node_name(tensor.node_name()); + tensor_item.set_slot(tensor.slot()); + tensor_item.set_finished(true); + + // return empty tensor if didn't find the requested tensor + if (result_index >= ret_name.size() || ret_name[result_index] != tensor.node_name() + ":" + tensor.slot()) { + tensor_list.push_back(tensor_item); + continue; + } + + tensor_item.set_tensor_content(data_ptr[result_index], data_size[result_index]); + tensor_item.set_data_type(GetDebuggerNumberDataType(dtype[result_index])); + tensor_item.clear_dims(); + for (auto &elem : shape[result_index]) { + tensor_item.add_dims(elem); + } + + tensor_list.push_back(tensor_item); + + result_index++; + } + + return tensor_list; +} + +void Debugger::Exit() { + // clear resource before exit + pipeline::ClearResAtexit(); + std::exit(EXIT_FAILURE); +} + +std::list Debugger::CheckWatchpoints() { + std::vector name; + std::vector slot; + std::vector data_ptr; + std::vector data_size; + std::vector condition; + std::vector watchpoint_id; + + debug_services_->check_watchpoints(&name, &slot, &data_ptr, &data_size, &condition, &watchpoint_id); + + std::list points; + + for (unsigned int i = 0; i < name.size(); i++) { + TensorProto *tensor_item; + tensor_item = new TensorProto(); + tensor_item->set_node_name(name[i]); + tensor_item->set_slot(slot[i]); + tensor_item->set_tensor_content(data_ptr[i], data_size[i]); + + // finished in TensorProto will always be true before we implement big tensor splitting + tensor_item->set_finished(true); + + WatchCondition *condition_item; + condition_item = new WatchCondition(); + condition_item->set_condition(debugger::WatchCondition_Condition(condition[i])); + + WatchpointHit point; + point.set_allocated_tensor(tensor_item); + point.set_allocated_watch_condition(condition_item); + point.set_id(watchpoint_id[i]); + + points.push_back(point); + } + + return points; +} + +void Debugger::SendWatchpointsAndSuspend(const std::list &points) { + // send info about watchpoint + if (!points.empty()) { + EventReply reply = grpc_client_->SendWatchpointHits(points); + if (reply.status() != reply.OK) { + MS_LOG(ERROR) << "Error: SendWatchpointHits failed"; + } + } + // enter command loop + CommandLoop(); +} + +DebugServices *Debugger::get_debug_services() { return debug_services_.get(); } + +bool Debugger::debugger_enabled() { return debugger_enabled_; } + +} // namespace mindspore diff --git a/mindspore/ccsrc/debug/debugger/debugger.h b/mindspore/ccsrc/debug/debugger/debugger.h new file mode 100644 index 0000000000..6ce7d03625 --- /dev/null +++ b/mindspore/ccsrc/debug/debugger/debugger.h @@ -0,0 +1,159 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_DEBUG_DEBUGGER_DEBUGGER_H_ +#define MINDSPORE_CCSRC_DEBUG_DEBUGGER_DEBUGGER_H_ + +#include +#include +#include +#include "session/kernel_graph.h" +#include "debug/debugger/grpc_client.h" +#include "debug/debug_services.h" + +using debugger::DataType; +using debugger::EventReply; +using debugger::GraphProto; +using debugger::ModelProto; +using debugger::TensorProto; +using debugger::WatchCondition; +using debugger::WatchNode; +using debugger::WatchpointHit; + +template +using ProtoVector = google::protobuf::RepeatedPtrField; + +namespace mindspore { +// different types of command recieved by debugger +// need to keep sync with client-side proto and server-side proto +enum class DebuggerCommand { kExitCMD = 2, kRunCMD = 3, kSetCMD = 4, kViewCMD = 5, kUnknownCMD = -1 }; + +class Debugger : public std::enable_shared_from_this { + public: + static std::shared_ptr GetInstance() { + std::lock_guard i_lock(instance_lock_); + if (debugger_ == nullptr) { + debugger_ = std::shared_ptr(new (std::nothrow) Debugger()); + } + return debugger_; + } + + // deconstructor + ~Debugger() = default; + + // init + // only save device_id + void Init(const uint32_t device_id); + + // reset debugger + void Reset(); + + // enable debugger + // send graph and wait for command + // do nothing if graph is set already + void PreExecute(const KernelGraphPtr &graph_ptr); + + // analyze tensors and wait for command + // don't need a graph_ptr because it is saved during pre_execute + void PostExecute(); + + // suspend the execution after a debug_op + void PostDebugOp(); + + DebugServices *get_debug_services(); + + bool debugger_enabled(); + + private: + // private constructor for singleton + Debugger(); + + // enable debugger + // instantiate class members + // read env variable for grpc client + void EnableDebugger(); + + // check and save graph pointer + void CheckGraphPtr(const KernelGraphPtr &graph_ptr); + + // check if the graph is a dataset graph + void CheckDatasetGraph(); + + // serialize graph and get proto + GraphProto GetGraphProto(); + + // send graph and enter command wait loop + void SendGraphAndSuspend(const GraphProto &graph_proto); + + // wait for command and process command + // send command request and process reply in a loop + // break if RunCMD + void CommandLoop(); + + // process reply and command type + DebuggerCommand GetCommand(const EventReply &reply); + + // parse other data out of EventReply + ProtoVector GetWatchnodes(const EventReply &reply); + WatchCondition GetWatchcondition(const EventReply &reply); + int32_t GetWatchpointID(const EventReply &reply); + bool GetWatchpointDelete(const EventReply &reply); + ProtoVector GetTensors(const EventReply &reply); + + // set what nodes and conditions to watch + void SetWatchpoint(const ProtoVector &nodes, const WatchCondition &condition, const int32_t id); + + // remove watchpoint with id + void RemoveWatchpoint(const int32_t id); + + // load tensor for view command + std::list LoadTensors(const ProtoVector &tensors); + + // terminate training process + void Exit(); + + // analyze tensors and check watchpoint conditions + // return names of tensors and what condition they hit + std::list CheckWatchpoints(); + + // send watchpoints that hit and enter command wait loop + void SendWatchpointsAndSuspend(const std::list &points); + + // class members + std::unique_ptr grpc_client_; + std::unique_ptr debug_services_; + KernelGraphPtr graph_ptr_; + uint32_t device_id_; + int32_t num_step_; + bool debugger_enabled_; + bool is_dataset_graph_; + std::mutex access_lock_; + + // singleton + static std::mutex instance_lock_; + static std::shared_ptr debugger_; +}; + +using DebuggerPtr = std::shared_ptr; + +// get debugger ModelProto +std::string GetDebuggerFuncGraphProtoString(const FuncGraphPtr &func_graph); +ModelProto GetDebuggerFuncGraphProto(const FuncGraphPtr &func_graph); + +// for getting proto DataType from Type of Tensor +DataType GetDebuggerNumberDataType(const TypePtr &type); + +} // namespace mindspore +#endif // MINDSPORE_CCSRC_DEBUG_DEBUGGER_DEBUGGER_H_ diff --git a/mindspore/ccsrc/debug/debugger/grpc_client.cc b/mindspore/ccsrc/debug/debugger/grpc_client.cc new file mode 100644 index 0000000000..7709f4c0d1 --- /dev/null +++ b/mindspore/ccsrc/debug/debugger/grpc_client.cc @@ -0,0 +1,124 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include "debug/debugger/grpc_client.h" +#include "utils/log_adapter.h" + +using debugger::EventListener; +using debugger::EventReply; +using debugger::EventReply_Status_FAILED; +using debugger::GraphProto; +using debugger::Metadata; +using debugger::TensorProto; +using debugger::WatchpointHit; + +namespace mindspore { +GrpcClient::GrpcClient(const std::string &host, const std::string &port) : stub_(nullptr) { Init(host, port); } + +void GrpcClient::Init(const std::string &host, const std::string &port) { + std::string target_str = host + ":" + port; + MS_LOG(INFO) << "GrpcClient connecting to: " << target_str; + + std::shared_ptr channel = grpc::CreateChannel(target_str, grpc::InsecureChannelCredentials()); + stub_ = EventListener::NewStub(channel); +} + +void GrpcClient::Reset() { stub_ = nullptr; } + +EventReply GrpcClient::WaitForCommand(const Metadata &metadata) { + EventReply reply; + grpc::ClientContext context; + grpc::Status status = stub_->WaitCMD(&context, metadata, &reply); + + if (!status.ok()) { + MS_LOG(ERROR) << "RPC failed: WaitForCommand"; + MS_LOG(ERROR) << status.error_code() << ": " << status.error_message(); + reply.set_status(EventReply_Status_FAILED); + } + return reply; +} + +EventReply GrpcClient::SendMetadata(const Metadata &metadata) { + EventReply reply; + grpc::ClientContext context; + grpc::Status status = stub_->SendMetadata(&context, metadata, &reply); + + if (!status.ok()) { + MS_LOG(ERROR) << "RPC failed: SendMetadata"; + MS_LOG(ERROR) << status.error_code() << ": " << status.error_message(); + reply.set_status(EventReply_Status_FAILED); + } + return reply; +} + +EventReply GrpcClient::SendGraph(const GraphProto &graph) { + EventReply reply; + grpc::ClientContext context; + grpc::Status status = stub_->SendGraph(&context, graph, &reply); + + if (!status.ok()) { + MS_LOG(ERROR) << "RPC failed: SendGraph"; + MS_LOG(ERROR) << status.error_code() << ": " << status.error_message(); + reply.set_status(EventReply_Status_FAILED); + } + return reply; +} + +EventReply GrpcClient::SendTensors(const std::list &tensors) { + EventReply reply; + grpc::ClientContext context; + + std::unique_ptr > writer(stub_->SendTensors(&context, &reply)); + for (const auto &tensor : tensors) { + if (!writer->Write(tensor)) { + break; + } + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + writer->WritesDone(); + grpc::Status status = writer->Finish(); + + if (!status.ok()) { + MS_LOG(ERROR) << "RPC failed: SendTensors"; + MS_LOG(ERROR) << status.error_code() << ": " << status.error_message(); + reply.set_status(EventReply_Status_FAILED); + } + return reply; +} + +EventReply GrpcClient::SendWatchpointHits(const std::list &watchpoints) { + EventReply reply; + grpc::ClientContext context; + + std::unique_ptr > writer(stub_->SendWatchpointHits(&context, &reply)); + for (const auto &watchpoint : watchpoints) { + if (!writer->Write(watchpoint)) { + break; + } + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + writer->WritesDone(); + grpc::Status status = writer->Finish(); + + if (!status.ok()) { + MS_LOG(ERROR) << "RPC failed: SendWatchpointHits"; + MS_LOG(ERROR) << status.error_code() << ": " << status.error_message(); + reply.set_status(EventReply_Status_FAILED); + } + return reply; +} +} // namespace mindspore diff --git a/mindspore/ccsrc/debug/debugger/grpc_client.h b/mindspore/ccsrc/debug/debugger/grpc_client.h new file mode 100644 index 0000000000..0b5359e444 --- /dev/null +++ b/mindspore/ccsrc/debug/debugger/grpc_client.h @@ -0,0 +1,61 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_DEBUG_DEBUGGER_GRPC_CLIENT_H_ +#define MINDSPORE_CCSRC_DEBUG_DEBUGGER_GRPC_CLIENT_H_ + +#include +#include +#include +#include +#include "proto/debug_grpc.grpc.pb.h" + +using debugger::EventListener; +using debugger::EventReply; +using debugger::GraphProto; +using debugger::Metadata; +using debugger::TensorProto; +using debugger::WatchpointHit; + +namespace mindspore { +class GrpcClient { + public: + // constructor + GrpcClient(const std::string &host, const std::string &port); + + // deconstructor + ~GrpcClient() = default; + + // init + void Init(const std::string &host, const std::string &port); + + // reset + void Reset(); + + EventReply WaitForCommand(const Metadata &metadata); + + EventReply SendMetadata(const Metadata &metadata); + + EventReply SendGraph(const GraphProto &graph); + + EventReply SendTensors(const std::list &tensors); + + EventReply SendWatchpointHits(const std::list &watchpoints); + + private: + std::unique_ptr stub_; +}; +} // namespace mindspore +#endif // MINDSPORE_CCSRC_DEBUG_DEBUGGER_GRPC_CLIENT_H_ diff --git a/mindspore/ccsrc/debug/debugger/proto_exporter.cc b/mindspore/ccsrc/debug/debugger/proto_exporter.cc new file mode 100644 index 0000000000..b4b4de9d99 --- /dev/null +++ b/mindspore/ccsrc/debug/debugger/proto_exporter.cc @@ -0,0 +1,542 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "debug/debugger/debugger.h" +#include "proto/debug_graph.pb.h" +#include "utils/graph_utils.h" +#include "utils/symbolic.h" + +namespace mindspore { +class DebuggerProtoExporter { + public: + DebuggerProtoExporter() {} + ~DebuggerProtoExporter() {} + + std::string GetFuncGraphProtoString(const FuncGraphPtr &func_graph); + debugger::ModelProto GetFuncGraphProto(const FuncGraphPtr &func_graph); + + private: + void InitModelInfo(); + void GetOpNodeTypeAndAttrs(const FuncGraphPtr &func_graph, const AnfNodePtr &node, debugger::NodeProto *node_proto); + std::string GetOpNodeInputId(const FuncGraphPtr &func_graph, const AnfNodePtr &node, + const std::map &apply_map, + std::map *const_map_ptr); + void SetValueToProto(const ValuePtr &attr_value, debugger::ValueProto *value_proto); + void SetScalarToProto(const ScalarPtr &val, debugger::ValueProto *value_proto); + void SetSequenceToProto(const ValueSequeuePtr &val, debugger::ValueProto *value_proto); + void SetDictionaryToProto(const ValueDictionaryPtr &val, debugger::ValueProto *value_proto); + void SetNodeOutputType(const AnfNodePtr &node, debugger::TypeProto *type_proto); + void SetNodeOutputType(const TypePtr &node, const BaseShapePtr &shape, debugger::TypeProto *type_proto); + + void ExportFuncGraph(const FuncGraphPtr &func_graph, debugger::GraphProto *graph_proto); + void ExportParameters(const FuncGraphPtr &func_graph, debugger::GraphProto *graph_proto); + void ExportCNodes(const FuncGraphPtr &func_graph, debugger::GraphProto *graph_proto, + std::map *const_map_ptr); + void ExportCNode(const FuncGraphPtr &func_graph, const CNodePtr &node, std::map *apply_map_ptr, + std::map *const_map_ptr, debugger::GraphProto *graph_proto); + void ExportFuncGraphOutput(const FuncGraphPtr &func_graph, const CNodePtr &ret_node, + const std::map &apply_map, std::map *const_map_ptr, + debugger::GraphProto *graph_proto); + void ExportValueNodes(const std::map &const_map, debugger::GraphProto *graph_proto); + + static std::string GetConstNodeId(size_t idx) { return std::string("cst") + std::to_string(idx); } + + debugger::ModelProto model_; +}; + +void DebuggerProtoExporter::SetNodeOutputType(const TypePtr &type, const BaseShapePtr &shape, + debugger::TypeProto *type_proto) { + if (type_proto == nullptr) { + return; + } + + if (type == nullptr) { + type_proto->set_data_type(debugger::DT_UNDEFINED); + } else if (type->isa()) { + type_proto->set_data_type(GetDebuggerNumberDataType(type)); + } else if (type->isa()) { + TypePtr elem_type = dyn_cast(type)->element(); + type_proto->mutable_tensor_type()->set_elem_type(GetDebuggerNumberDataType(elem_type)); + type_proto->set_data_type(debugger::DT_TENSOR); + if (shape != nullptr && shape->isa()) { + abstract::ShapePtr shape_info = dyn_cast(shape); + for (const auto &elem : shape_info->shape()) { + type_proto->mutable_tensor_type()->mutable_shape()->add_dim()->set_size(elem); + } + } + } else if (type->isa()) { + TuplePtr tuple_type = dyn_cast(type); + type_proto->set_data_type(debugger::DT_TUPLE); + for (const auto &elem_type : tuple_type->elements()) { + SetNodeOutputType(elem_type, nullptr, type_proto->mutable_sequence_type()->add_elem_types()); + } + } else if (type->isa()) { + type_proto->set_data_type(debugger::DT_TYPE); + } else if (type->isa()) { + ListPtr list_type = dyn_cast(type); + type_proto->set_data_type(debugger::DT_LIST); + for (const auto &elem_type : list_type->elements()) { + SetNodeOutputType(elem_type, nullptr, type_proto->mutable_sequence_type()->add_elem_types()); + } + } else if (type->isa()) { + type_proto->set_data_type(debugger::DT_ANYTHING); + } else if (type->isa()) { + type_proto->set_data_type(debugger::DT_REFKEY); + } else if (type->isa()) { + type_proto->set_data_type(debugger::DT_REF); + } else if (type->isa()) { + type_proto->set_data_type(debugger::DT_GRAPH); + } else if (type->isa()) { + type_proto->set_data_type(debugger::DT_NONE); + } else if (type->isa()) { + type_proto->set_data_type(debugger::DT_STRING); + } else if (type->isa()) { + // Do Nothing. + } else { + MS_LOG(EXCEPTION) << "Unknown type: " << type->type_name(); + } +} + +void DebuggerProtoExporter::SetNodeOutputType(const AnfNodePtr &node, debugger::TypeProto *type_proto) { + if (node == nullptr || type_proto == nullptr) { + return; + } + SetNodeOutputType(node->Type(), node->Shape(), type_proto); +} + +void DebuggerProtoExporter::SetValueToProto(const ValuePtr &val, debugger::ValueProto *value_proto) { + if (val == nullptr || value_proto == nullptr) { + return; + } + + if (val->isa()) { + const StringImmPtr &value = dyn_cast(val); + value_proto->set_dtype(debugger::DT_STRING); + value_proto->set_str_val(value->value()); + } else if (val->isa()) { + SetScalarToProto(dyn_cast(val), value_proto); + } else if (val->isa()) { + value_proto->set_dtype(debugger::DT_TYPE); + value_proto->mutable_type_val()->set_data_type(debugger::DT_BOOL); + } else if (val->isa()) { + value_proto->set_dtype(debugger::DT_TYPE); + value_proto->mutable_type_val()->set_data_type(debugger::DT_BASE_INT); + } else if (val->isa()) { + value_proto->set_dtype(debugger::DT_TYPE); + value_proto->mutable_type_val()->set_data_type(debugger::DT_BASE_FLOAT); + } else if (val->isa()) { + SetSequenceToProto(dyn_cast(val), value_proto); + } else if (val->isa()) { + value_proto->set_dtype(debugger::DT_NONE); + value_proto->set_str_val("None"); + } else if (val->isa()) { + SymbolicKeyInstancePtr sym_inst = dyn_cast(val); + ParameterPtr sym_node = dyn_cast(sym_inst->node()); + value_proto->set_dtype(debugger::DT_SYM_INST); + value_proto->set_str_val(sym_node == nullptr ? std::string("nullptr") : sym_node->ToString()); + } else if (val->isa()) { + SetDictionaryToProto(dyn_cast(val), value_proto); + } else if (val->isa()) { + tensor::TensorPtr tensor_ptr = dyn_cast(val); + value_proto->set_dtype(debugger::DT_TENSOR); + debugger::TensorProto *tensor_proto = value_proto->mutable_tensor_val(); + tensor_proto->set_data_type(GetDebuggerNumberDataType(tensor_ptr->Dtype())); + for (auto &elem : tensor_ptr->shape()) { + tensor_proto->add_dims(elem); + } + tensor_proto->set_tensor_content(tensor_ptr->data_c(), tensor_ptr->data().nbytes()); + } else if (val->isa()) { + value_proto->set_dtype(debugger::DT_TYPE); + + debugger::TypeProto *type_proto = value_proto->mutable_type_val(); + type_proto->set_data_type(debugger::DT_TENSOR); + TypePtr elem_type = dyn_cast(val)->element(); + type_proto->mutable_tensor_type()->set_elem_type(GetDebuggerNumberDataType(elem_type)); + } else { + MS_LOG(WARNING) << "Unsupported type " << val->type_name(); + } +} + +void DebuggerProtoExporter::SetScalarToProto(const ScalarPtr &val, debugger::ValueProto *value_proto) { + if (val == nullptr || value_proto == nullptr) { + return; + } + + if (val->isa()) { + const BoolImmPtr &value = dyn_cast(val); + value_proto->set_dtype(debugger::DT_BOOL); + value_proto->set_bool_val(value->value()); + } else if (val->isa()) { + const Int8ImmPtr &value = dyn_cast(val); + value_proto->set_dtype(debugger::DT_INT8); + value_proto->set_int_val(value->value()); + } else if (val->isa()) { + const Int16ImmPtr &value = dyn_cast(val); + value_proto->set_dtype(debugger::DT_INT16); + value_proto->set_int_val(value->value()); + } else if (val->isa()) { + const Int32ImmPtr &value = dyn_cast(val); + value_proto->set_dtype(debugger::DT_INT32); + value_proto->set_int_val(value->value()); + } else if (val->isa()) { + const Int64ImmPtr &value = dyn_cast(val); + value_proto->set_dtype(debugger::DT_INT64); + value_proto->set_int_val(value->value()); + } else if (val->isa()) { + const UInt8ImmPtr &value = dyn_cast(val); + value_proto->set_dtype(debugger::DT_UINT8); + value_proto->set_uint_val(value->value()); + } else if (val->isa()) { + const UInt16ImmPtr &value = dyn_cast(val); + value_proto->set_dtype(debugger::DT_UINT16); + value_proto->set_uint_val(value->value()); + } else if (val->isa()) { + const UInt32ImmPtr &value = dyn_cast(val); + value_proto->set_dtype(debugger::DT_UINT32); + value_proto->set_uint_val(value->value()); + } else if (val->isa()) { + const UInt64ImmPtr &value = dyn_cast(val); + value_proto->set_dtype(debugger::DT_UINT64); + value_proto->set_uint_val(value->value()); + } else if (val->isa()) { + const FP32ImmPtr &value = dyn_cast(val); + value_proto->set_dtype(debugger::DT_FLOAT32); + value_proto->set_float_val(value->value()); + } else if (val->isa()) { + const FP64ImmPtr &value = dyn_cast(val); + value_proto->set_dtype(debugger::DT_FLOAT64); + value_proto->set_double_val(value->value()); + } else { + MS_LOG(EXCEPTION) << "Unknown scalar type " << val->ToString(); + } +} + +void DebuggerProtoExporter::SetSequenceToProto(const ValueSequeuePtr &val, debugger::ValueProto *value_proto) { + if (val == nullptr || value_proto == nullptr) { + return; + } + + if (val->isa()) { + const ValueTuplePtr &value = dyn_cast(val); + value_proto->set_dtype(debugger::DT_TUPLE); + for (const auto &item : value->value()) { + SetValueToProto(item, value_proto->add_values()); + } + } else if (val->isa()) { + const ValueListPtr &value = dyn_cast(val); + value_proto->set_dtype(debugger::DT_LIST); + for (const auto &item : value->value()) { + SetValueToProto(item, value_proto->add_values()); + } + } +} + +void DebuggerProtoExporter::SetDictionaryToProto(const ValueDictionaryPtr &val, debugger::ValueProto *value_proto) { + if (val == nullptr || value_proto == nullptr) { + return; + } + + value_proto->set_dtype(debugger::DT_DICT); + for (const auto &item : val->value()) { + debugger::NamedValueProto *named_val = value_proto->add_dict_val(); + named_val->set_key(item.first); + SetValueToProto(item.second, named_val->mutable_value()); + } +} + +void DebuggerProtoExporter::GetOpNodeTypeAndAttrs(const FuncGraphPtr &, const AnfNodePtr &node, + debugger::NodeProto *node_proto) { + if (node == nullptr || node_proto == nullptr) { + return; + } + + if (node->isa() || node->isa() || IsValueNode(node)) { + MS_LOG(EXCEPTION) << "Op node can not be CNode, Parameter or ValueNode Graph. But got " << node->ToString(); + } + + if (!IsValueNode(node)) { + MS_LOG(EXCEPTION) << "Op node is not primitive: " << node->ToString(); + } + + const PrimitivePtr &prim = GetValueNode(node); + node_proto->set_op_type(prim->name()); + for (const auto &attr : prim->attrs()) { + debugger::AttributeProto *attr_proto = node_proto->add_attribute(); + attr_proto->set_name(attr.first); + SetValueToProto(attr.second, attr_proto->mutable_value()); + } + node_proto->set_scope(node->scope()->name()); +} + +std::string DebuggerProtoExporter::GetOpNodeInputId(const FuncGraphPtr &, const AnfNodePtr &node, + const std::map &apply_map, + std::map *const_map_ptr) { + if (node == nullptr || const_map_ptr == nullptr) { + return ""; + } + + if (node->isa()) { + auto iter = apply_map.find(node); + if (iter == apply_map.end()) { + MS_LOG(EXCEPTION) << "Can not find node '" << node->ToString() << "' in apply_map"; + } + return std::to_string(iter->second); + } + + if (node->isa()) { + return node->ToString(); + } + + if (node->isa()) { + auto iter = const_map_ptr->find(node); + if (iter == const_map_ptr->end()) { + // Start index number from 1 + auto const_idx = const_map_ptr->size() + 1; + (*const_map_ptr)[node] = const_idx; + } + return GetConstNodeId((*const_map_ptr)[node]); + } + + MS_LOG(EXCEPTION) << "Unknown node type. node is '" << node->ToString() << "'"; +} + +std::string DebuggerProtoExporter::GetFuncGraphProtoString(const FuncGraphPtr &func_graph) { + if (func_graph == nullptr) { + return ""; + } + + InitModelInfo(); + debugger::GraphProto *graph_proto = model_.mutable_graph(); + ExportFuncGraph(func_graph, graph_proto); + return model_.SerializeAsString(); +} + +debugger::ModelProto DebuggerProtoExporter::GetFuncGraphProto(const FuncGraphPtr &func_graph) { + if (func_graph == nullptr) { + return ModelProto(); + } + + InitModelInfo(); + debugger::GraphProto *graph_proto = model_.mutable_graph(); + ExportFuncGraph(func_graph, graph_proto); + return model_; +} + +void DebuggerProtoExporter::ExportFuncGraph(const FuncGraphPtr &func_graph, debugger::GraphProto *graph_proto) { + if (func_graph == nullptr || graph_proto == nullptr) { + return; + } + + // map for store ValueNodes of this graph + std::map const_map; + + // set graph name + graph_proto->set_name(func_graph->ToString()); + + ExportParameters(func_graph, graph_proto); + + ExportCNodes(func_graph, graph_proto, &const_map); + + ExportValueNodes(const_map, graph_proto); +} + +void DebuggerProtoExporter::ExportParameters(const FuncGraphPtr &func_graph, debugger::GraphProto *graph_proto) { + if (func_graph == nullptr || graph_proto == nullptr) { + return; + } + + // cast FuncGraph to KernelGraph to access inputs() + std::vector parameters = static_cast(func_graph.get())->inputs(); + + for (auto ¶m : parameters) { + debugger::ParameterProto *param_proto = graph_proto->add_parameters(); + param_proto->set_name(param->ToString()); + + SetNodeOutputType(param, param_proto->mutable_type()); + + const ParameterPtr param_ptr = dyn_cast(param); + if (param_ptr == nullptr) { + MS_LOG(EXCEPTION) << "Parameter '" << param->ToString() << "' could not cast to parameter."; + } + } +} + +void DebuggerProtoExporter::ExportCNodes(const FuncGraphPtr &func_graph, debugger::GraphProto *graph_proto, + std::map *const_map_ptr) { + if (func_graph == nullptr || graph_proto == nullptr || const_map_ptr == nullptr) { + return; + } + // topo sort nodes + std::vector nodes = TopoSort(func_graph->get_return(), SuccIncoming, AlwaysInclude); + std::map apply_map; + for (const AnfNodePtr &node : nodes) { + MS_EXCEPTION_IF_NULL(node); + if (!node->isa()) { + continue; + } + auto cnode = node->cast(); + if (cnode != func_graph->get_return()) { + ExportCNode(func_graph, cnode, &apply_map, const_map_ptr, graph_proto); + } else { + ExportFuncGraphOutput(func_graph, cnode, apply_map, const_map_ptr, graph_proto); + } + } +} + +void DebuggerProtoExporter::ExportCNode(const FuncGraphPtr &func_graph, const CNodePtr &node, + std::map *apply_map_ptr, + std::map *const_map_ptr, + debugger::GraphProto *graph_proto) { + if (func_graph == nullptr || node == nullptr || apply_map_ptr == nullptr || const_map_ptr == nullptr || + graph_proto == nullptr) { + return; + } + + auto apply_idx = apply_map_ptr->size() + 1; + (*apply_map_ptr)[node] = apply_idx; + + auto &inputs = node->inputs(); + if (inputs.size() < 1) { + MS_LOG(EXCEPTION) << "Inputs of apply node is empty"; + } + AnfNodePtr op = inputs[0]; + debugger::NodeProto *node_proto = graph_proto->add_node(); + + // CNode/ConstGraph/Const/Parameter + if (op->isa() || IsValueNode(op) || op->isa()) { + MS_LOG(WARNING) << "Operator must be a primitive"; + } else { + GetOpNodeTypeAndAttrs(func_graph, op, node_proto); + node_proto->set_name(std::to_string(apply_idx)); + node_proto->set_scope(node->scope()->name()); + + // add debug_name for debugger + node_proto->set_debug_name(node->fullname_with_scope()); + + // process OP inputs + for (size_t i = 1; i < inputs.size(); ++i) { + debugger::InputProto *input_proto = node_proto->add_input(); + input_proto->set_type(debugger::InputProto_EdgeType_DATA_EDGE); + std::string id = GetOpNodeInputId(func_graph, inputs[i], *apply_map_ptr, const_map_ptr); + input_proto->set_name(id); + } + + // set node output type + SetNodeOutputType(node, node_proto->mutable_output_type()); + } +} + +void DebuggerProtoExporter::ExportFuncGraphOutput(const FuncGraphPtr &func_graph, const CNodePtr &ret_node, + const std::map &apply_map, + std::map *const_map_ptr, + debugger::GraphProto *graph_proto) { + if (ret_node == nullptr || !ret_node->isa()) { + MS_LOG(EXCEPTION) << "Graph return node is illegal"; + } + AnfNodePtr arg = ret_node->input(1); + if (graph_proto == nullptr) { + MS_LOG(EXCEPTION) << "graph_proto is nullptr"; + } + debugger::OutputProto *output_proto = graph_proto->add_outputs(); + if (output_proto == nullptr) { + MS_LOG(EXCEPTION) << "output_proto is nullptr"; + } + std::string id = GetOpNodeInputId(func_graph, arg, apply_map, const_map_ptr); + output_proto->set_name(id); + SetNodeOutputType(arg, output_proto->mutable_type()); +} + +static bool CompareValue(const std::pair &x, const std::pair &y) { + return x.second < y.second; +} + +void DebuggerProtoExporter::ExportValueNodes(const std::map &const_map, + debugger::GraphProto *graph_proto) { + std::vector> nodes; + (void)std::transform(const_map.cbegin(), const_map.cend(), std::back_inserter(nodes), + [](const std::pair &item) { return item; }); + + sort(nodes.begin(), nodes.end(), CompareValue); + + for (auto &item : nodes) { + if (graph_proto == nullptr) { + MS_LOG(EXCEPTION) << "graph_proto is nullptr"; + } + debugger::NamedValueProto *named_value = graph_proto->add_const_vals(); + MS_EXCEPTION_IF_NULL(named_value); + named_value->set_key(GetConstNodeId(item.second)); + SetValueToProto(GetValueNode(item.first), named_value->mutable_value()); + } +} + +void DebuggerProtoExporter::InitModelInfo() { model_.set_ir_version(debugger::IR_VERSION); } + +std::string GetDebuggerFuncGraphProtoString(const FuncGraphPtr &func_graph) { + DebuggerProtoExporter exporter; + return exporter.GetFuncGraphProtoString(func_graph); +} + +debugger::ModelProto GetDebuggerFuncGraphProto(const FuncGraphPtr &func_graph) { + DebuggerProtoExporter exporter; + return exporter.GetFuncGraphProto(func_graph); +} + +debugger::DataType GetDebuggerNumberDataType(const TypePtr &type) { + switch (type->type_id()) { + case kNumberTypeBool: + return debugger::DT_BOOL; + case kNumberTypeInt8: + return debugger::DT_INT8; + case kNumberTypeInt16: + return debugger::DT_INT16; + case kNumberTypeInt32: + return debugger::DT_INT32; + case kNumberTypeInt64: + return debugger::DT_INT64; + case kNumberTypeUInt8: + return debugger::DT_UINT8; + case kNumberTypeUInt16: + return debugger::DT_UINT16; + case kNumberTypeUInt32: + return debugger::DT_UINT32; + case kNumberTypeUInt64: + return debugger::DT_UINT64; + case kNumberTypeFloat16: + return debugger::DT_FLOAT16; + case kNumberTypeFloat32: + return debugger::DT_FLOAT32; + case kNumberTypeFloat64: + return debugger::DT_FLOAT64; + case kNumberTypeInt: + return debugger::DT_BASE_INT; + case kNumberTypeUInt: + return debugger::DT_BASE_UINT; + case kNumberTypeFloat: + return debugger::DT_BASE_FLOAT; + default: + MS_LOG(EXCEPTION) << "Unexpected type " << type->type_name(); + } +} + +} // namespace mindspore diff --git a/mindspore/ccsrc/debug/tensor_data.h b/mindspore/ccsrc/debug/tensor_data.h new file mode 100644 index 0000000000..9704d69089 --- /dev/null +++ b/mindspore/ccsrc/debug/tensor_data.h @@ -0,0 +1,75 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_DEBUG_TENSOR_DATA_H_ +#define MINDSPORE_CCSRC_DEBUG_TENSOR_DATA_H_ + +#include +#include +#include +#include +#include "ir/tensor.h" + +namespace mindspore { +class TensorData { + private: + mindspore::tensor::TensorPtr tensor_ptr; + std::string name; + size_t slot; + int execution_order; + + public: + TensorData() : slot(0), execution_order(-1) {} + + TensorData(const TensorData &obj) { + std::cout << "Copy Constructor" << std::endl; + this->name = obj.name; + this->execution_order = obj.execution_order; + this->slot = obj.slot; + this->tensor_ptr = obj.tensor_ptr; + } + + ~TensorData() {} + + std::string GetName() { return this->name; } + + mindspore::tensor::TensorPtr GetTensor() { return this->tensor_ptr; } + + size_t GetSlot() { return this->slot; } + + int GetExecutionOrder() { return this->execution_order; } + + int SetExecutionOrder(int execution_order) { + this->execution_order = execution_order; + return true; + } + + int SetName(const std::string &name) { + this->name = name; + return true; + } + + bool SetTensor(mindspore::tensor::TensorPtr out_tensor) { + this->tensor_ptr = out_tensor; + return true; + } + + bool SetSlot(size_t slot) { + this->slot = slot; + return true; + } +}; +} // namespace mindspore +#endif // MINDSPORE_CCSRC_DEBUG_TENSOR_DATA_H_ diff --git a/mindspore/ccsrc/debug/tensor_load.h b/mindspore/ccsrc/debug/tensor_load.h new file mode 100644 index 0000000000..6c3ea67a78 --- /dev/null +++ b/mindspore/ccsrc/debug/tensor_load.h @@ -0,0 +1,69 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_DEBUG_TENSOR_LOAD_H_ +#define MINDSPORE_CCSRC_DEBUG_TENSOR_LOAD_H_ + +#include +#include +#include +#include +#include +#include "debug/tensor_data.h" +namespace mindspore { +class TensorLoader { + public: + TensorLoader() : iter_num(-1) {} + + ~TensorLoader() {} + + bool LoadNewTensor(std::shared_ptr tensor) { + tensor_list.push_back(tensor); + tensor_list_map.insert({tensor->GetName(), tensor}); + return true; + } + std::vector> GetTensor() { return tensor_list; } + + uint32_t GetIterNum() { return iter_num; } + + std::map> GetTensorMap() { return tensor_list_map; } + void SearchTensors(const std::vector &search_list, + std::vector>> *result_list) { + for (auto i : search_list) { + std::map>::iterator iter; + iter = tensor_list_map.find(i); + if (iter != tensor_list_map.end()) { + result_list->push_back(std::make_tuple(i, iter->second)); + } else { + result_list->push_back(std::make_tuple(i, nullptr)); + } + } + } + + bool EmptyTensor() { + tensor_list_map.clear(); + tensor_list.clear(); + return true; + } + + void set_iter_num(uint32_t iter_num) { this->iter_num = iter_num; } + + private: + std::vector> tensor_list; + std::map> tensor_list_map; + uint32_t iter_num; +}; +} // namespace mindspore +#endif // MINDSPORE_CCSRC_DEBUG_TENSOR_LOAD_H_ diff --git a/mindspore/ccsrc/device/ascend/ascend_device_address.cc b/mindspore/ccsrc/device/ascend/ascend_device_address.cc index a47c482c0e..71a16607ef 100644 --- a/mindspore/ccsrc/device/ascend/ascend_device_address.cc +++ b/mindspore/ccsrc/device/ascend/ascend_device_address.cc @@ -30,6 +30,10 @@ #ifdef ENABLE_DUMP_E2E #include "debug/e2e_dump.h" #endif +#ifdef ENABLE_DEBUGGER +#include "debug/tensor_load.h" +#endif + namespace mindspore { namespace device { namespace ascend { @@ -346,6 +350,52 @@ bool AscendDeviceAddress::DumpMemToFile(bool trans_flag, const std::string &file return ret; } #endif + +#ifdef ENABLE_DEBUGGER +bool AscendDeviceAddress::LoadMemToHost(bool trans_flag, const std::string &tensor_name, int execution_order, + const std::string &host_fmt, const std::vector &host_shape, + TypeId host_type, size_t slot, Debugger *debugger) const { + bool ret = false; + + DebugServices *debug_services = debugger->get_debug_services(); + TensorLoader *tensor_loader = debug_services->get_tensor_loader(); + + if (trans_flag) { + MS_LOG(INFO) << "E2E tensor name is " << tensor_name; + mindspore::tensor::TensorPtr out_tensor = std::make_shared(host_type, host_shape); + size_t host_size = out_tensor->data().nbytes(); + ret = SyncDeviceToHost(host_shape, host_size, host_type, out_tensor->data_c(true)); + if (!ret) { + MS_LOG(ERROR) << "Copy device mem to host failed"; + return ret; + } + auto tensor_data = std::make_shared(); + tensor_data->SetName(tensor_name); + tensor_data->SetExecutionOrder(execution_order); + tensor_data->SetTensor(out_tensor); + tensor_data->SetSlot(slot); + ret = tensor_loader->LoadNewTensor(tensor_data); + + } else { + mindspore::tensor::TensorPtr out_tensor = std::make_shared(type_id_, host_shape); + size_t host_size = out_tensor->data().nbytes(); + auto ret_rt_memcpy = rtMemcpy(out_tensor->data_c(true), host_size, ptr_, host_size, RT_MEMCPY_DEVICE_TO_HOST); + + auto tensor_data = std::make_shared(); + tensor_data->SetName(tensor_name); + tensor_data->SetExecutionOrder(execution_order); + tensor_data->SetTensor(out_tensor); + tensor_data->SetSlot(slot); + ret = tensor_loader->LoadNewTensor(tensor_data); + if (ret_rt_memcpy != RT_ERROR_NONE) { + MS_LOG(ERROR) << "SyncDeviceToHost: rtMemcpy mem size[" << size_ << "] fail, ret[" << ret_rt_memcpy << "]"; + } + MS_LOG(INFO) << "E2E tensor name is " << tensor_name; + } + return ret; +} +#endif + } // namespace ascend } // namespace device } // namespace mindspore diff --git a/mindspore/ccsrc/device/ascend/ascend_device_address.h b/mindspore/ccsrc/device/ascend/ascend_device_address.h index 364f9e95fd..6871abfe1b 100644 --- a/mindspore/ccsrc/device/ascend/ascend_device_address.h +++ b/mindspore/ccsrc/device/ascend/ascend_device_address.h @@ -25,6 +25,9 @@ #include "ir/dtype.h" namespace mindspore { +#ifdef ENABLE_DEBUGGER +class Debugger; +#endif namespace device { namespace ascend { class AscendDeviceAddress : public DeviceAddress { @@ -39,6 +42,10 @@ class AscendDeviceAddress : public DeviceAddress { #ifdef ENABLE_DUMP_E2E bool DumpMemToFile(bool dump_mode, const std::string &filepath, const std::string &host_fmt, const std::vector &host_shape, TypeId host_type) const; +#endif +#ifdef ENABLE_DEBUGGER + bool LoadMemToHost(bool dump_mode, const std::string &tensor_name, int execution_order, const std::string &host_fmt, + const std::vector &host_shape, TypeId host_type, size_t slot, Debugger *debugger) const; #endif private: bool SyncDeviceToHostAndConvertFormat(const std::vector &shape, size_t size, TypeId type, void *host_ptr) const; diff --git a/mindspore/ccsrc/device/ascend/ascend_kernel_runtime.cc b/mindspore/ccsrc/device/ascend/ascend_kernel_runtime.cc index fb2a3f350b..5ec1e90a61 100644 --- a/mindspore/ccsrc/device/ascend/ascend_kernel_runtime.cc +++ b/mindspore/ccsrc/device/ascend/ascend_kernel_runtime.cc @@ -41,6 +41,7 @@ #include "kernel/tbe/tbe_python_funcs.h" #include "pre_activate/mem_reuse/mem_reuse_checker.h" #include "device/ascend/ascend_memory_manager.h" +#include "debug/tensor_load.h" using mindspore::device::ascend::ProfilingManager; using mindspore::device::ascend::ProfilingUtils; @@ -293,6 +294,91 @@ bool AscendKernelRuntime::DumpData(mindspore::session::KernelGraph *graph) { return true; } +#ifdef ENABLE_DEBUGGER +namespace { +void LoadOutput(mindspore::session::KernelGraph *graph, Debugger *debugger) { + MS_EXCEPTION_IF_NULL(graph); + bool trans_flag = false; + const auto &apply_kernels = graph->execution_order(); + // for kernels, execution order starts from 1 + int exec_order = 1; + for (const auto &node : apply_kernels) { + MS_EXCEPTION_IF_NULL(node); + auto node_name = AnfAlgo::GetCNodeName(node); + std::string kernel_name = node->fullname_with_scope(); + auto output_size = AnfAlgo::GetOutputTensorNum(node); + for (size_t j = 0; j < output_size; ++j) { + auto addr = AnfAlgo::GetOutputAddr(node, j); + auto type = AnfAlgo::GetOutputInferDataType(node, j); + auto format = kOpFormat_DEFAULT; + string tensor_name = kernel_name + ':' + std::to_string(j); + auto ascend_addr = dynamic_cast(addr); + std::vector int_shapes; + if (trans_flag) { + int_shapes = trans::GetRuntimePaddingShape(node, j); + } else { + auto shape = AnfAlgo::GetOutputDeviceShape(node, j); + (void)std::transform(shape.begin(), shape.end(), std::back_inserter(int_shapes), + [](size_t inner_item) { return SizeToInt(inner_item); }); + } + auto ret = ascend_addr->LoadMemToHost(trans_flag, tensor_name, exec_order, format, int_shapes, type, j, debugger); + if (!ret) { + MS_LOG(ERROR) << "LoadMemToHost: flag:" << trans_flag << ", tensor_name:" << tensor_name + << ", host_format:" << format << ".!"; + } + } + exec_order = exec_order + 1; + } +} + +void LoadParameters(mindspore::session::KernelGraph *graph, Debugger *debugger) { + MS_EXCEPTION_IF_NULL(graph); + bool trans_flag = false; + const auto ¶meters = graph->inputs(); + // for parameters, set its execution order to be 0; + int exec_order = 0; + for (auto &item : parameters) { + if (!item->isa()) { + continue; + } + std::string parameter_name = item->fullname_with_scope(); + auto addr = AnfAlgo::GetOutputAddr(item, PRAMATER_OUTPUT_INDEX); + auto type = AnfAlgo::GetOutputInferDataType(item, PRAMATER_OUTPUT_INDEX); + auto format = kOpFormat_DEFAULT; + string tensor_name = parameter_name + ':' + "0"; + auto ascend_addr = dynamic_cast(addr); + std::vector int_shapes; + if (trans_flag) { + int_shapes = trans::GetRuntimePaddingShape(item, PRAMATER_OUTPUT_INDEX); + } else { + auto shape = AnfAlgo::GetOutputDeviceShape(item, PRAMATER_OUTPUT_INDEX); + (void)std::transform(shape.begin(), shape.end(), std::back_inserter(int_shapes), + [](size_t inner_item) { return SizeToInt(inner_item); }); + } + auto ret = ascend_addr->LoadMemToHost(trans_flag, tensor_name, exec_order, format, int_shapes, type, 0, debugger); + if (!ret) { + MS_LOG(ERROR) << "LoadMemToHost Failed: flag:" << trans_flag << ", path:" << tensor_name + << ", host_format:" << format << ".!"; + } + } +} +} // namespace +#endif + +bool AscendKernelRuntime::LoadData(mindspore::session::KernelGraph *graph, Debugger *debugger) { + MS_EXCEPTION_IF_NULL(graph); +#ifdef ENABLE_DEBUGGER + MS_LOG(INFO) << "start load step"; + uint32_t cur_iter = 0; + MS_LOG(INFO) << "cur iter is " << cur_iter; + // load output + LoadOutput(graph, debugger); + // load parameters + LoadParameters(graph, debugger); +#endif + return true; +} + bool AscendKernelRuntime::NodeOutputDeviceAddressExist(const AnfNodePtr &kernel, size_t index) { if (AnfAlgo::OutputAddrExist(kernel, index)) { auto address = AnfAlgo::GetOutputAddr(kernel, index); diff --git a/mindspore/ccsrc/device/ascend/ascend_kernel_runtime.h b/mindspore/ccsrc/device/ascend/ascend_kernel_runtime.h index 28076f95b7..69ba8b295a 100644 --- a/mindspore/ccsrc/device/ascend/ascend_kernel_runtime.h +++ b/mindspore/ccsrc/device/ascend/ascend_kernel_runtime.h @@ -37,6 +37,7 @@ class AscendKernelRuntime : public KernelRuntime { ~AscendKernelRuntime() override; bool Init() override; bool DumpData(session::KernelGraph *graph) override; + bool LoadData(session::KernelGraph *graph, Debugger *debugger) override; bool GenTask(const session::KernelGraph *graph) override; bool RunTask(const session::KernelGraph *graph) override; bool LoadTask(const session::KernelGraph *graph) override; diff --git a/mindspore/ccsrc/device/kernel_runtime.cc b/mindspore/ccsrc/device/kernel_runtime.cc index aae21aac72..9fb6a88076 100644 --- a/mindspore/ccsrc/device/kernel_runtime.cc +++ b/mindspore/ccsrc/device/kernel_runtime.cc @@ -79,6 +79,14 @@ bool KernelRuntime::DumpData(mindspore::session::KernelGraph *graph) { return false; } +// for D to impl +bool KernelRuntime::LoadData(mindspore::session::KernelGraph *graph, Debugger *debugger) { + if (graph != nullptr) { + return true; + } + return false; +} + // for D to impl bool KernelRuntime::GenTask(const session::KernelGraph *graph) { if (graph != nullptr) { diff --git a/mindspore/ccsrc/device/kernel_runtime.h b/mindspore/ccsrc/device/kernel_runtime.h index bfe857f61b..8442342e32 100644 --- a/mindspore/ccsrc/device/kernel_runtime.h +++ b/mindspore/ccsrc/device/kernel_runtime.h @@ -27,6 +27,9 @@ #ifdef ENABLE_DUMP_E2E #include "debug/e2e_dump.h" #endif +#ifdef ENABLE_DEBUGGER +#include "debug/debugger/debugger.h" +#endif #include "session/kernel_graph.h" #include "session/anf_runtime_algorithm.h" #include "kernel/kernel.h" @@ -34,11 +37,15 @@ #include "device/memory_manager.h" using mindspore::tensor::Tensor; +using std::vector; using TensorPtr = std::shared_ptr; using mindspore::kernel::AddressPtr; using AddressPtrList = std::vector; namespace mindspore { +#ifndef ENABLE_DEBUGGER +class Debugger; +#endif namespace device { class KernelRuntime { public: @@ -50,6 +57,7 @@ class KernelRuntime { void RunOpClearMemory(session::KernelGraph *graph); virtual bool Run(session::KernelGraph *graph); virtual bool DumpData(session::KernelGraph *graph); + virtual bool LoadData(session::KernelGraph *graph, Debugger *debugger); virtual bool RunTask(const session::KernelGraph *graph); virtual bool GenTask(const session::KernelGraph *graph); bool LaunchKernel(const session::KernelGraph *graph); diff --git a/mindspore/ccsrc/kernel/cpu/debug_cpu_kernel.cc b/mindspore/ccsrc/kernel/cpu/debug_cpu_kernel.cc new file mode 100644 index 0000000000..a1dcaca3f3 --- /dev/null +++ b/mindspore/ccsrc/kernel/cpu/debug_cpu_kernel.cc @@ -0,0 +1,50 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "kernel/cpu/debug_cpu_kernel.h" +#include "device/cpu/cpu_device_address.h" +#include "common/utils.h" +#ifdef ENABLE_DEBUGGER +#include "debug/debugger/debugger.h" +#endif + +namespace mindspore { +namespace kernel { +void DebugCPUKernel::InitKernel(const CNodePtr &kernel_node) { MS_EXCEPTION_IF_NULL(kernel_node); } + +bool DebugCPUKernel::Launch(const std::vector &inputs, + const std::vector & /*workspace*/, + const std::vector &outputs) { + if (inputs.size() < 1 || outputs.empty()) { + MS_LOG(EXCEPTION) << " input or output empty!"; + } + auto val = reinterpret_cast(inputs[0]->addr); + MS_LOG(DEBUG) << " launch DebugCountCPUKernel val " << *val; + + auto output = reinterpret_cast(outputs[0]->addr); + size_t elem_num = inputs[0]->size / sizeof(int); + for (size_t i = 0; i < elem_num; i++) { + output[i] = val[i]; + } + +#ifdef ENABLE_DEBUGGER + // debugger will suspend execution is neccessary + Debugger::GetInstance()->PostDebugOp(); +#endif + + return true; +} +} // namespace kernel +} // namespace mindspore diff --git a/mindspore/ccsrc/kernel/cpu/debug_cpu_kernel.h b/mindspore/ccsrc/kernel/cpu/debug_cpu_kernel.h new file mode 100644 index 0000000000..da9f3286b9 --- /dev/null +++ b/mindspore/ccsrc/kernel/cpu/debug_cpu_kernel.h @@ -0,0 +1,41 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_KERNEL_CPU_DEBUG_CPU_KERNEL_H_ +#define MINDSPORE_CCSRC_KERNEL_CPU_DEBUG_CPU_KERNEL_H_ + +#include +#include +#include "kernel/cpu/cpu_kernel.h" +#include "kernel/cpu/cpu_kernel_factory.h" + +namespace mindspore { +namespace kernel { +class DebugCPUKernel : public CPUKernel { + public: + DebugCPUKernel() = default; + ~DebugCPUKernel() override = default; + + void InitKernel(const CNodePtr &kernel_node) override; + + bool Launch(const std::vector &inputs, const std::vector &workspace, + const std::vector &outputs) override; +}; + +MS_REG_CPU_KERNEL(Debug, KernelAttr().AddInputAttr(kNumberTypeFloat32).AddOutputAttr(kNumberTypeInt32), DebugCPUKernel); +} // namespace kernel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_KERNEL_CPU_DEBUG_CPU_KERNEL_H_ diff --git a/mindspore/ccsrc/operator/ops.cc b/mindspore/ccsrc/operator/ops.cc index f86cbd7fd2..e6545d311c 100755 --- a/mindspore/ccsrc/operator/ops.cc +++ b/mindspore/ccsrc/operator/ops.cc @@ -275,5 +275,6 @@ const PrimitivePtr kPrimScalarSummary = std::make_shared("ScalarSumma const PrimitivePtr kPrimImageSummary = std::make_shared("ImageSummary"); const PrimitivePtr kPrimTensorSummary = std::make_shared("TensorSummary"); const PrimitivePtr kPrimHistogramSummary = std::make_shared("HistogramSummary"); +const PrimitivePtr kPrimDebug = std::make_shared("Debug"); } // namespace prim } // namespace mindspore diff --git a/mindspore/ccsrc/operator/ops.h b/mindspore/ccsrc/operator/ops.h index 65327cf407..01812a5529 100755 --- a/mindspore/ccsrc/operator/ops.h +++ b/mindspore/ccsrc/operator/ops.h @@ -276,6 +276,7 @@ extern const PrimitivePtr kPrimNotInDict; extern const PrimitivePtr kPrimMixedPrecisionCast; extern const PrimitivePtr kPrimIsConsant; extern const PrimitivePtr kPrimEquivFormat; +extern const PrimitivePtr kPrimDebug; // Comm ops extern const PrimitivePtr kPrimAllReduce; diff --git a/mindspore/ccsrc/operator/prim_debug.cc b/mindspore/ccsrc/operator/prim_debug.cc index a9962c6d14..5e6cdcc318 100644 --- a/mindspore/ccsrc/operator/prim_debug.cc +++ b/mindspore/ccsrc/operator/prim_debug.cc @@ -21,5 +21,21 @@ #include "utils/symbolic.h" namespace mindspore { -namespace abstract {} // namespace abstract +namespace abstract { +AbstractBasePtr InferImplDebug(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list) { + // Inputs: a tensor(value) + const std::string op_name = primitive->name(); + + CheckArgsSize(op_name, args_spec_list, 1); + auto tensor_value = CheckArg(op_name, args_spec_list, 0); + + int tensor_rank = SizeToInt(tensor_value->shape()->shape().size()); + if (tensor_rank == 0) { + MS_LOG(EXCEPTION) << op_name << " summary evaluator second arg should be an tensor, but got a scalar, rank is 0"; + } + + return std::make_shared(AbstractBasePtrList({tensor_value->Broaden()})); +} +} // namespace abstract } // namespace mindspore diff --git a/mindspore/ccsrc/parallel/node_check.cc b/mindspore/ccsrc/parallel/node_check.cc index 6f30a8ec1c..6b920f82ec 100644 --- a/mindspore/ccsrc/parallel/node_check.cc +++ b/mindspore/ccsrc/parallel/node_check.cc @@ -66,6 +66,7 @@ const std::set BLACK_LIST = {TUPLE_GETITEM, SCALARSUMMARY, IMAGESUMMARY, TENSORSUMMARY, + DEBUG, HISTOGRAMSUMMARY, COL2IMV1, RESOLVE, diff --git a/mindspore/ccsrc/parallel/ops_info/ops_utils.h b/mindspore/ccsrc/parallel/ops_info/ops_utils.h index 4b8f61bb2e..f9af7e2626 100644 --- a/mindspore/ccsrc/parallel/ops_info/ops_utils.h +++ b/mindspore/ccsrc/parallel/ops_info/ops_utils.h @@ -269,6 +269,7 @@ constexpr char SCALARSUMMARY[] = "ScalarSummary"; constexpr char IMAGESUMMARY[] = "ImageSummary"; constexpr char TENSORSUMMARY[] = "TensorSummary"; constexpr char HISTOGRAMSUMMARY[] = "HistogramSummary"; +constexpr char DEBUG[] = "Debug"; constexpr char BROADCASTGRADIENTARGS[] = "BroadcastGradientArgs"; constexpr char INVERTPERMUTATION[] = "InvertPermutation"; constexpr char CONTROLDEPEND[] = "ControlDepend"; diff --git a/mindspore/ccsrc/pipeline/pipeline.cc b/mindspore/ccsrc/pipeline/pipeline.cc index bb1f693c6b..d346c980ea 100644 --- a/mindspore/ccsrc/pipeline/pipeline.cc +++ b/mindspore/ccsrc/pipeline/pipeline.cc @@ -445,7 +445,10 @@ bool ExecutorPy::CompileInner(const py::object &obj, const py::tuple &args, cons std::string backend = MsContext::GetInstance()->backend_policy(); if (use_vm && backend != "ge") { // Create backend and session - resource->results()[kBackend] = compile::CreateBackend(); + auto backend_ptr = compile::CreateBackend(); + // Connect session to debugger + backend_ptr->SetDebugger(); + resource->results()[kBackend] = backend_ptr; p_actions = VmPipeline(); } else { p_actions = GePipeline(); diff --git a/mindspore/ccsrc/pipeline/static_analysis/prim.cc b/mindspore/ccsrc/pipeline/static_analysis/prim.cc index 82b8395933..bf1f319ae2 100644 --- a/mindspore/ccsrc/pipeline/static_analysis/prim.cc +++ b/mindspore/ccsrc/pipeline/static_analysis/prim.cc @@ -130,6 +130,8 @@ PrimitiveEvalImplMap &GetPrimitiveToEvalImplMap() { {prim::kPrimDepend, {InferImplDepend, true}}, {prim::kPrimBroadcastGradientArgs, {InferImplBroadcastGradientArgs, false}}, {prim::kPrimControlDepend, {InferImplControlDepend, true}}, + // Debug + {prim::kPrimDebug, {InferImplDebug, true}}, }; return prim_eval_implement_map; } diff --git a/mindspore/ccsrc/pipeline/static_analysis/prim.h b/mindspore/ccsrc/pipeline/static_analysis/prim.h index 5b910f8194..5b3972088a 100644 --- a/mindspore/ccsrc/pipeline/static_analysis/prim.h +++ b/mindspore/ccsrc/pipeline/static_analysis/prim.h @@ -346,6 +346,9 @@ AbstractBasePtr InferImplBroadcastGradientArgs(const AnalysisEnginePtr &, const const AbstractBasePtrList &args_spec_list); AbstractBasePtr InferImplControlDepend(const AnalysisEnginePtr &, const PrimitivePtr &primitive, const AbstractBasePtrList &args_spec_list); + +AbstractBasePtr InferImplDebug(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list); } // namespace abstract } // namespace mindspore diff --git a/mindspore/ccsrc/session/ascend_session.cc b/mindspore/ccsrc/session/ascend_session.cc index bae10ed943..7ef6551f2b 100644 --- a/mindspore/ccsrc/session/ascend_session.cc +++ b/mindspore/ccsrc/session/ascend_session.cc @@ -18,6 +18,7 @@ #include #include #include +#include #include #include "operator/ops.h" #include "ir/tensor.h" @@ -45,6 +46,7 @@ #include "kernel/tbe/tbe_python_funcs.h" #include "utils/config_manager.h" #include "utils/base_ref_extends.h" +#include "debug/tensor_load.h" namespace mindspore { namespace session { @@ -450,6 +452,12 @@ void AscendSession::RunGraph(const GraphId &graph_id, const std::vectorPreExecute(kernel_graph); + } +#endif { py::gil_scoped_release release; // run task on device @@ -459,8 +467,20 @@ void AscendSession::RunGraph(const GraphId &graph_id, const std::vectordebugger_enabled()) { + LoadTensor(kernel_graph); + } +#endif // dump used for debug Dump(kernel_graph); +#ifdef ENABLE_DEBUGGER + // debugger post-execution processing + if (debugger_) { + debugger_->PostExecute(); + } +#endif MS_LOG(INFO) << "Finish!"; } @@ -757,6 +777,22 @@ void AscendSession::ExportChildGraphs(const GraphId graph_id) { #endif } +void AscendSession::LoadTensor(const std::shared_ptr &kernel_graph) const { + MS_LOG(INFO) << "Start!"; + MS_EXCEPTION_IF_NULL(kernel_graph); +#ifdef ENABLE_DEBUGGER + auto runtime_instance = device::KernelRuntimeManager::Instance().GetKernelRuntime(kAscendDevice, device_id_); + MS_EXCEPTION_IF_NULL(runtime_instance); + DebugServices *debug_services = debugger_->get_debug_services(); + TensorLoader *tensor_loader = debug_services->get_tensor_loader(); + tensor_loader->EmptyTensor(); + uint32_t iter_num = tensor_loader->GetIterNum(); + tensor_loader->set_iter_num(++iter_num); + (void)runtime_instance->LoadData(kernel_graph.get(), debugger_.get()); +#endif + MS_LOG(INFO) << "Finish!"; +} + GraphId AscendSession::SetFinalGraphInput(const std::vector &args) { MS_LOG(INFO) << "Start! Args size " << args.size(); auto final_graph = NewKernelGraph(); diff --git a/mindspore/ccsrc/session/ascend_session.h b/mindspore/ccsrc/session/ascend_session.h index 7857330115..eaa01b8f80 100755 --- a/mindspore/ccsrc/session/ascend_session.h +++ b/mindspore/ccsrc/session/ascend_session.h @@ -87,6 +87,7 @@ class AscendSession : public SessionBasic { void ExecTask(const std::shared_ptr &kernel_graph) const; void Dump(const std::shared_ptr &kernel_graph) const; void ExportChildGraphs(const GraphId graph_id); + void LoadTensor(const std::shared_ptr &kernel_graph) const; // below functions are used for run op void RunOpHardwareOptimize(const std::shared_ptr &kernel_graph) const; void RunOpExecTask(const std::shared_ptr &kernel_graph) const; diff --git a/mindspore/ccsrc/session/cpu_session.cc b/mindspore/ccsrc/session/cpu_session.cc index e70e551022..49728bc4c2 100644 --- a/mindspore/ccsrc/session/cpu_session.cc +++ b/mindspore/ccsrc/session/cpu_session.cc @@ -25,6 +25,9 @@ #include "predict/predict.h" #include "kernel/cpu/cpu_kernel_factory.h" #include "device/cpu/kernel_select_cpu.h" +#ifdef ENABLE_DEBUGGER +#include "debug/debugger/debugger.h" +#endif namespace mindspore { namespace session { @@ -78,7 +81,12 @@ void CPUSession::RunGraph(const GraphId &graph_id, const std::vectorsummary_nodes(); runtime_.IncreaseSummaryRefCount(summary_outputs); } - +#ifdef ENABLE_DEBUGGER + // debugger pre-execution processing + if (debugger_) { + debugger_->PreExecute(kernel_graph); + } +#endif bool ret = runtime_.Run(kernel_graph.get()); if (!ret) { MS_LOG(EXCEPTION) << "Run graph failed"; @@ -92,6 +100,12 @@ void CPUSession::RunGraph(const GraphId &graph_id, const std::vectorPostExecute(); + } +#endif MS_LOG(INFO) << "Run graph end"; } diff --git a/mindspore/ccsrc/session/session_basic.h b/mindspore/ccsrc/session/session_basic.h index 27171b7589..1ea62d8df9 100755 --- a/mindspore/ccsrc/session/session_basic.h +++ b/mindspore/ccsrc/session/session_basic.h @@ -32,6 +32,9 @@ #include "utils/contract.h" #include "pynative/pynative_execute.h" #include "device/kernel_info.h" +#ifdef ENABLE_DEBUGGER +#include "debug/debugger/debugger.h" +#endif namespace mindspore { using GraphId = uint32_t; @@ -48,7 +51,11 @@ using OpRunInfoPtr = std::shared_ptr; class SessionBasic { public: - SessionBasic() : context_(nullptr), summary_callback_(nullptr), device_id_(0) {} + SessionBasic() : context_(nullptr), summary_callback_(nullptr), device_id_(0) { +#ifdef ENABLE_DEBUGGER + debugger_ = nullptr; +#endif + } virtual void Init(uint32_t device_id) { device_id_ = device_id; } @@ -92,6 +99,14 @@ class SessionBasic { virtual void SetActive(GraphId, GraphId) {} virtual void GetSummaryNodes(KernelGraph *graph); +#ifdef ENABLE_DEBUGGER + // set debugger + void SetDebugger() { + debugger_ = Debugger::GetInstance(); + debugger_->Init(device_id_); + } +#endif + protected: virtual void LoadInputData(const std::shared_ptr &kernel_graph, const std::vector &inputs_const) const; @@ -123,6 +138,9 @@ class SessionBasic { CallBackFunc summary_callback_; static GraphId graph_sum_; uint32_t device_id_; +#ifdef ENABLE_DEBUGGER + std::shared_ptr debugger_; +#endif }; using SessionPtr = std::shared_ptr; diff --git a/mindspore/ccsrc/transform/convert.cc b/mindspore/ccsrc/transform/convert.cc index a5726b078a..1d61b0050f 100644 --- a/mindspore/ccsrc/transform/convert.cc +++ b/mindspore/ccsrc/transform/convert.cc @@ -371,6 +371,7 @@ std::unordered_map &DfGraphConvertor::get_adpt_ma {prim::kPrimImageSummary->name(), ADPT_DESC(Summary)}, {prim::kPrimTensorSummary->name(), ADPT_DESC(Summary)}, {prim::kPrimHistogramSummary->name(), ADPT_DESC(Summary)}, + {prim::kPrimDebug->name(), ADPT_DESC(Summary)}, {prim::kPrimTensorAdd->name(), std::make_shared(std::make_shared>(ExtraAttr({{"mode", MakeValue(1)}})), std::make_shared>(ExtraAttr({{"mode", MakeValue(1)}})))}, diff --git a/mindspore/ccsrc/utils/context/ms_context.cc b/mindspore/ccsrc/utils/context/ms_context.cc index 9f283319a7..d385ec7a3f 100644 --- a/mindspore/ccsrc/utils/context/ms_context.cc +++ b/mindspore/ccsrc/utils/context/ms_context.cc @@ -69,7 +69,11 @@ MsContext::MsContext(const std::string &policy, const std::string &target) { enable_task_sink_ = true; ir_fusion_flag_ = true; enable_hccl_ = false; +#ifdef ENABLE_DEBUGGER + enable_mem_reuse_ = false; +#else enable_mem_reuse_ = true; +#endif enable_gpu_summary_ = true; precompile_only_ = false; auto_mixed_precision_flag_ = false; diff --git a/mindspore/ccsrc/vm/backend.cc b/mindspore/ccsrc/vm/backend.cc index 3fde263c9d..1a27fcb63a 100644 --- a/mindspore/ccsrc/vm/backend.cc +++ b/mindspore/ccsrc/vm/backend.cc @@ -362,5 +362,9 @@ GraphId MsBackend::CompileGraph(NotNull fg) { return target_sess_- VectorRef MsBackend::RunGraph(GraphId graph_id, const VectorRef &args) { return MsRunGraph(graph_id, args); } +#ifdef ENABLE_DEBUGGER +void MsBackend::SetDebugger() { target_sess_->SetDebugger(); } +#endif + } // namespace compile } // namespace mindspore diff --git a/mindspore/ccsrc/vm/backend.h b/mindspore/ccsrc/vm/backend.h index 0e0b02c055..3a93cf930f 100644 --- a/mindspore/ccsrc/vm/backend.h +++ b/mindspore/ccsrc/vm/backend.h @@ -69,6 +69,8 @@ class Backend { bool is_switch_call() const { return is_switch_call_; } void set_simu_flag(bool simu) { simu_flag_ = simu; } + virtual void SetDebugger() {} + protected: std::string name_; LinkFuncType convert_fn_; @@ -109,6 +111,10 @@ class MsBackend : public Backend { VectorRef RunGraph(GraphId graph_id, const VectorRef &args); void CreateOtherSession(const std::string &target); +#ifdef ENABLE_DEBUGGER + void SetDebugger() override; +#endif + private: session::SessionPtr target_sess_; session::SessionPtr other_sess_; diff --git a/mindspore/ops/_grad/grad_debug_ops.py b/mindspore/ops/_grad/grad_debug_ops.py index 1cb756219a..6e31556b14 100644 --- a/mindspore/ops/_grad/grad_debug_ops.py +++ b/mindspore/ops/_grad/grad_debug_ops.py @@ -66,3 +66,12 @@ def get_bprop_insert_gradient_of(self): def bprop(x, out, dout): return (f(dout),) return bprop + + +@bprop_getters.register(P.Debug) +def get_bprop_debug(self): + """Generate bprop for Debug""" + + def bprop(x, out, dout): + return dout + return bprop diff --git a/mindspore/ops/operations/__init__.py b/mindspore/ops/operations/__init__.py index 792381a15f..6193292316 100644 --- a/mindspore/ops/operations/__init__.py +++ b/mindspore/ops/operations/__init__.py @@ -37,7 +37,7 @@ from .comm_ops import (AllGather, AllReduce, _AlltoAll, ReduceScatter, Broadcast _VirtualDiv, _GetTensorSlice, HostAllGather, HostReduceScatter) from .debug_ops import (ImageSummary, InsertGradientOf, HookBackward, ScalarSummary, - TensorSummary, HistogramSummary, Print) + TensorSummary, HistogramSummary, Debug, Print) from .control_ops import ControlDepend, GeSwitch, Merge from .inner_ops import ScalarCast @@ -173,6 +173,7 @@ __all__ = [ 'ImageSummary', 'TensorSummary', 'HistogramSummary', + "Debug", "Print", 'InsertGradientOf', 'HookBackward', diff --git a/mindspore/ops/operations/debug_ops.py b/mindspore/ops/operations/debug_ops.py index c6b635a69f..4bb51f2564 100644 --- a/mindspore/ops/operations/debug_ops.py +++ b/mindspore/ops/operations/debug_ops.py @@ -17,7 +17,7 @@ from types import FunctionType, MethodType from ..._checkparam import Validator as validator from ...common import dtype as mstype -from ..primitive import prim_attr_register, PrimitiveWithInfer +from ..primitive import prim_attr_register, PrimitiveWithInfer, Primitive def _check_summary_param(name, value, class_name): @@ -340,3 +340,29 @@ class Print(PrimitiveWithInfer): for dtype in inputs: validator.check_subclass("input", dtype, (mstype.tensor, mstype.string), self.name) return mstype.int32 + + +class Debug(Primitive): + """ + Print tensor value. + + Inputs: + - **value** (Tensor) - The value of tensor. + + Examples: + >>> class DebugNN(nn.Cell): + >>> def __init__(self,): + >>> self.debug = nn.Debug() + >>> + >>> def construct(self, x, y): + >>> x = self.add(x, y) + >>> self.debug(x) + >>> return x + """ + + @prim_attr_register + def __init__(self): + """init""" + + def __call__(self, *args, **kwargs): + pass diff --git a/tests/ut/cpp/CMakeLists.txt b/tests/ut/cpp/CMakeLists.txt index 13f961fa24..0ba778f5c5 100644 --- a/tests/ut/cpp/CMakeLists.txt +++ b/tests/ut/cpp/CMakeLists.txt @@ -114,6 +114,12 @@ list(REMOVE_ITEM MINDSPORE_SRC_LIST "../../../mindspore/ccsrc/utils/node_strateg list(REMOVE_ITEM MINDSPORE_SRC_LIST "../../../mindspore/ccsrc/utils/load_onnx/anf_model_parser.cc") list(REMOVE_ITEM MINDSPORE_SRC_LIST "../../../mindspore/ccsrc/utils/load_onnx/anf_converter.cc") +# remove files for debugger +list(REMOVE_ITEM MINDSPORE_SRC_LIST "../../../mindspore/ccsrc/debug/debugger/debugger.cc") +list(REMOVE_ITEM MINDSPORE_SRC_LIST "../../../mindspore/ccsrc/debug/debugger/grpc_client.cc") +list(REMOVE_ITEM MINDSPORE_SRC_LIST "../../../mindspore/ccsrc/debug/debug_services.cc") +list(REMOVE_ITEM MINDSPORE_SRC_LIST "../../../mindspore/ccsrc/debug/debugger/proto_exporter.cc") + file(GLOB_RECURSE UT_SUTB_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "stub/aicpu/*.cc" "stub/cce/*.cc" From 3c4f621d752f4d94735f685bdd46fbcdb40d7348 Mon Sep 17 00:00:00 2001 From: ougongchang Date: Mon, 22 Jun 2020 17:09:45 +0800 Subject: [PATCH 004/254] fix the summary operator is not work in constant folding scene The summary operator will be optimized when it return the origin value in constant folding scene. So I return a None value to avoid this. --- mindspore/ops/operations/debug_ops.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/mindspore/ops/operations/debug_ops.py b/mindspore/ops/operations/debug_ops.py index c6b635a69f..91f56e0e19 100644 --- a/mindspore/ops/operations/debug_ops.py +++ b/mindspore/ops/operations/debug_ops.py @@ -32,6 +32,13 @@ def _check_summary_param(name, value, class_name): validator.check_value_type('value', v_type, [type(mstype.tensor)], class_name) +# Note: The return value of the summary operator is not used, +# so there's nothing special about the return `dtype` or `shape`, any value is ok. +# The `value` should be set to None, else summary operators may be optimized at compile graph phase, +# it cause summary operators can not record data in constant folding scene. +SUMMARY_RETURN_VALUE = {'dtype': mstype.int32, 'shape': [1], 'value': None} + + class ScalarSummary(PrimitiveWithInfer): """ Output scalar to protocol buffer through scalar summary operator. @@ -67,7 +74,7 @@ class ScalarSummary(PrimitiveWithInfer): raise ValueError(f"For 'value' the type should be scalar, " f"shape should be [] or [1] in {self.__class__.__name__}, but got {v_shape}.") - return value + return SUMMARY_RETURN_VALUE class ImageSummary(PrimitiveWithInfer): @@ -104,7 +111,7 @@ class ImageSummary(PrimitiveWithInfer): raise ValueError(f"For 'value' the dim should be {image_dim} in {self.__class__.__name__}," f" but got {len(v_shape)}.") - return value + return SUMMARY_RETURN_VALUE class TensorSummary(PrimitiveWithInfer): @@ -142,7 +149,7 @@ class TensorSummary(PrimitiveWithInfer): raise ValueError(f"For 'value' the type should be tensor in {self.__class__.__name__}, " f"shape should not be [].") - return value + return SUMMARY_RETURN_VALUE class HistogramSummary(PrimitiveWithInfer): @@ -180,7 +187,7 @@ class HistogramSummary(PrimitiveWithInfer): raise ValueError(f"For 'value' the type should be tensor in {self.__class__.__name__}, " f"shape should not be [].") - return value + return SUMMARY_RETURN_VALUE class InsertGradientOf(PrimitiveWithInfer): From 5977fe79365d7efbbcab30df5a8fccc075518356 Mon Sep 17 00:00:00 2001 From: duxiutao Date: Mon, 22 Jun 2020 21:55:26 +0800 Subject: [PATCH 005/254] add description for graph_kernel switch --- mindspore/context.py | 4 +++- mindspore/nn/cell.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/mindspore/context.py b/mindspore/context.py index ad601f8fab..b955e19b49 100644 --- a/mindspore/context.py +++ b/mindspore/context.py @@ -511,7 +511,7 @@ def reset_auto_parallel_context(): save_graphs_path=str, save_ms_model=bool, save_ms_model_path=str, enable_dump=bool, save_dump_path=str, enable_reduce_precision=bool, variable_memory_max_size=str, enable_profiling=bool, profiling_options=str, enable_auto_mixed_precision=bool, - check_bprop=bool, max_device_memory=str, print_file_path=str) + enable_graph_kernel=bool, check_bprop=bool, max_device_memory=str, print_file_path=str) def set_context(**kwargs): """ Sets context for running environment. @@ -539,6 +539,8 @@ def set_context(**kwargs): save_ms_model_path (str): Path to save converted lite model. Default: "." save_graphs_path (str): Path to save graphs. Default: "." enable_auto_mixed_precision (bool): Whether to enable auto mixed precision. Default: True. + enable_graph_kernel (bool): Whether to enable composition of basic primitives. These primitives would be + compiled into a fused kernel automatically. Default: False. reserve_class_name_in_scope (bool) : Whether to save the network class name in the scope. Default: True. enable_reduce_precision (bool): Whether to enable precision reduction. Default: True. enable_dump (bool): Whether to enable dump. Default: False. diff --git a/mindspore/nn/cell.py b/mindspore/nn/cell.py index c046c2e1bf..446e804b52 100755 --- a/mindspore/nn/cell.py +++ b/mindspore/nn/cell.py @@ -822,7 +822,7 @@ class GraphKernel(Cell): """ Base class for GraphKernel. - A `GraphKernel` a composite of basic primitives and can be compiled into a fused kernel automaticly when + A `GraphKernel` a composite of basic primitives and can be compiled into a fused kernel automatically when context.set_context(enable_graph_kernel=True). Examples: From 41ddc153a62ebadf7da45f8f2c27faf36cfe27aa Mon Sep 17 00:00:00 2001 From: Ziyan Date: Thu, 18 Jun 2020 20:15:03 +0800 Subject: [PATCH 006/254] modify lars interface --- mindspore/nn/optim/lars.py | 83 +++++++++---------- .../models/resnet50/test_resnet50_imagenet.py | 10 +-- tests/ut/python/nn/optim/test_lars.py | 4 +- .../parallel/test_loss_and_optimizer.py | 2 +- 4 files changed, 47 insertions(+), 52 deletions(-) diff --git a/mindspore/nn/optim/lars.py b/mindspore/nn/optim/lars.py index b55d1c5574..7b05b372eb 100755 --- a/mindspore/nn/optim/lars.py +++ b/mindspore/nn/optim/lars.py @@ -13,22 +13,18 @@ # limitations under the License. # ============================================================================ """lars optimizer""" -from typing import Iterable -from mindspore.common import dtype as mstype -from mindspore.common import Tensor -from mindspore.common.initializer import initializer -from mindspore.common.parameter import Parameter from mindspore.ops import operations as P from mindspore.ops import composite as C from mindspore.ops import functional as F from mindspore._checkparam import Validator as validator +from mindspore.common import Tensor, Parameter, dtype as mstype from .optimizer import _grad_scale, Optimizer _lars_opt = C.MultitypeFuncGraph("lars_opt") -@_lars_opt.register("Function", "Number", "Tensor", "Tensor", "Tensor", "Bool", "Bool") -def _tensor_run_opt(lars, weight_decay, learning_rate, gradient, weight, decay_flag, lars_flag): +@_lars_opt.register("Function", "Tensor", "Number", "Tensor", "Tensor", "Bool", "Bool") +def _tensor_run_opt(lars, learning_rate, weight_decay, gradient, weight, decay_flag, lars_flag): """Apply lars optimizer to the weight parameter.""" if lars_flag: op_reduce_sum = P.SquareSumAll() @@ -42,10 +38,12 @@ def _tensor_run_opt(lars, weight_decay, learning_rate, gradient, weight, decay_f return gradient -def _check_param_value(optimizer, epsilon, hyperpara, use_clip, prim_name): +def _check_param_value(optimizer, epsilon, coefficient, use_clip, prim_name): validator.check_value_type("optimizer", optimizer, Optimizer, prim_name) + if "Adam" in optimizer.cls_name or "Lamb" in optimizer.cls_name: + raise TypeError("LARS can not be used with ", optimizer.cls_name) validator.check_value_type("epsilon", epsilon, [float], prim_name) - validator.check_value_type("hyperpara", hyperpara, [float], prim_name) + validator.check_value_type("coefficient", coefficient, [float], prim_name) validator.check_value_type("use_clip", use_clip, [bool], prim_name) class LARS(Optimizer): @@ -58,14 +56,10 @@ class LARS(Optimizer): Args: optimizer (Optimizer): MindSpore optimizer for which to wrap and modify gradients. epsilon (float): Term added to the denominator to improve numerical stability. Default: 1e-05. - hyperpara (float): Trust coefficient for calculating the local learning rate. Default: 0.001. - weight_decay (float): Weight decay (L2 penalty). It should be equal to or greater than 0. Default: 0.0. + coefficient (float): Trust coefficient for calculating the local learning rate. Default: 0.001. use_clip (bool): Whether to use clip operation for calculating the local learning rate. Default: False. - decay_filter (Function): A function to determine whether apply weight decay on parameters. Default: - lambda x: 'LayerNorm' not in x.name and 'bias' not in x.name. lars_filter (Function): A function to determine whether apply lars algorithm. Default: lambda x: 'LayerNorm' not in x.name and 'bias' not in x.name. - loss_scale (float): A floating point value for the loss scale. It should be greater than 0. Default: 1.0. Inputs: - **gradients** (tuple[Tensor]) - The gradients of `params` in optimizer, the shape is @@ -78,51 +72,54 @@ class LARS(Optimizer): >>> net = Net() >>> loss = nn.SoftmaxCrossEntropyWithLogits() >>> opt = nn.Momentum(net.trainable_params(), 0.1, 0.9) - >>> opt_lars = nn.LARS(opt, epsilon=1e-08, hyperpara=0.02) + >>> opt_lars = nn.LARS(opt, epsilon=1e-08, coefficient=0.02) >>> model = Model(net, loss_fn=loss, optimizer=opt_lars, metrics=None) """ - def __init__(self, optimizer, epsilon=1e-05, hyperpara=0.001, weight_decay=0.0, use_clip=False, - decay_filter=lambda x: 'LayerNorm' not in x.name and 'bias' not in x.name, - lars_filter=lambda x: 'LayerNorm' not in x.name and 'bias' not in x.name, loss_scale=1.0): - super(LARS, self).__init__(0.0, [Parameter(Tensor(0.0), name="trivial")], weight_decay, loss_scale) - if optimizer.is_group: - raise RuntimeError(f"The {self.cls_name} optimizer cannot support group setting.") - _check_param_value(optimizer, epsilon, hyperpara, use_clip, self.cls_name) + def __init__(self, optimizer, epsilon=1e-05, coefficient=0.001, use_clip=False, + lars_filter=lambda x: 'LayerNorm' not in x.name and 'bias' not in x.name): + super(LARS, self).__init__(0.0, [Parameter(Tensor(0.0), name="fake_param")]) + _check_param_value(optimizer, epsilon, coefficient, use_clip, self.cls_name) self.opt = optimizer - self.parameters = optimizer.parameters - self.learning_rate = optimizer.learning_rate - self.lars = P.LARSUpdate(epsilon, hyperpara, use_clip) - self.reciprocal_scale = 1.0 / loss_scale - self.weight_decay = weight_decay + self.lars = P.LARSUpdate(epsilon, coefficient, use_clip) self.cast = P.Cast() - self.decay_flag = tuple(decay_filter(x) for x in self.parameters) + self.parameters = optimizer.parameters + if use_clip is True: + self.learning_rate = optimizer.learning_rate + self.dynamic_lr = optimizer.dynamic_lr + self.gather = optimizer.gather + self.assignadd = optimizer.assignadd + self.global_step = optimizer.global_step + else: + self.learning_rate = Parameter(Tensor(0.0, dtype=mstype.float32), name="fake_lr") + self.reciprocal_scale = optimizer.reciprocal_scale + optimizer.reciprocal_scale = 1.0 + self.is_group = optimizer.is_group + if self.is_group: + self.weight_decay = tuple(map(lambda x: x / optimizer.loss_scale, optimizer.weight_decay)) + else: + self.weight_decay = optimizer.weight_decay / optimizer.loss_scale + optimizer.exec_weight_decay = False + optimizer.weight_decay = 0.0 + self.decay_flags = optimizer.decay_flags self.lars_flag = tuple(lars_filter(x) for x in self.parameters) self.hyper_map = C.HyperMap() - self.dynamic_lr = False - self.gather = None - self.global_step = None - self.axis = None - if isinstance(self.learning_rate.default_input, Iterable) or \ - (isinstance(self.learning_rate.default_input, Tensor) and self.learning_rate.default_input.dim() == 1): - self.dynamic_lr = True - self.assignadd = P.AssignAdd() - self.gather = P.GatherV2() - self.global_step = Parameter(initializer(0, [1], mstype.int32), name="lars_global_step") - self.axis = 0 def construct(self, gradients): params = self.parameters if self.dynamic_lr: - lr = self.gather(self.learning_rate, self.global_step, self.axis) + lr = self.gather(self.learning_rate, self.global_step, 0) F.control_depend(lr, self.assignadd(self.global_step, 1)) else: lr = self.learning_rate if self.reciprocal_scale != 1.0: gradients = self.hyper_map(F.partial(_grad_scale, self.reciprocal_scale), gradients) - - grad_t = self.hyper_map(F.partial(_lars_opt, self.lars, self.weight_decay, lr), - gradients, params, self.decay_flag, self.lars_flag) + if self.is_group: + grad_t = self.hyper_map(F.partial(_lars_opt, self.lars, lr), self.weight_decay, + gradients, params, self.decay_flags, self.lars_flag) + else: + grad_t = self.hyper_map(F.partial(_lars_opt, self.lars, lr, self.weight_decay), + gradients, params, self.decay_flags, self.lars_flag) success = self.opt(grad_t) return success diff --git a/tests/st/networks/models/resnet50/test_resnet50_imagenet.py b/tests/st/networks/models/resnet50/test_resnet50_imagenet.py index c991b469ee..64f4adda99 100644 --- a/tests/st/networks/models/resnet50/test_resnet50_imagenet.py +++ b/tests/st/networks/models/resnet50/test_resnet50_imagenet.py @@ -182,13 +182,11 @@ def train_process(q, device_id, epoch_size, device_num, enable_hccl): {'order_params': net.trainable_params()}] if config.use_lars: - momentum = nn.Momentum(filter(lambda x: x.requires_grad, net.get_parameters()), lr, config.momentum, + momentum = nn.Momentum(group_params, lr, config.momentum, + weight_decay=config.weight_decay, loss_scale=config.loss_scale, use_nesterov=config.use_nesterov) - opt = nn.LARS(momentum, epsilon=config.lars_epsilon, hyperpara=config.lars_coefficient, - weight_decay=config.weight_decay, - decay_filter=lambda x: 'beta' not in x.name and 'gamma' not in x.name and 'bias' not in x.name, - lars_filter=lambda x: 'beta' not in x.name and 'gamma' not in x.name and 'bias' not in x.name, - loss_scale=config.loss_scale) + opt = nn.LARS(momentum, epsilon=config.lars_epsilon, coefficient=config.lars_coefficient, + lars_filter=lambda x: 'beta' not in x.name and 'gamma' not in x.name and 'bias' not in x.name) else: opt = nn.Momentum(group_params, lr, config.momentum, diff --git a/tests/ut/python/nn/optim/test_lars.py b/tests/ut/python/nn/optim/test_lars.py index d6fd4cd90e..1373691b72 100644 --- a/tests/ut/python/nn/optim/test_lars.py +++ b/tests/ut/python/nn/optim/test_lars.py @@ -56,7 +56,7 @@ def test_lars_multi_step_lr(): lr = multisteplr(10, [2, 6]) SGD = Momentum(net.trainable_params(), lr, 0.9) - optimizer = LARS(SGD, epsilon=1e-08, hyperpara=0.02, decay_filter=lambda x: 'bn' not in x.name, + optimizer = LARS(SGD, epsilon=1e-08, coefficient=0.02, use_clip=True, lars_filter=lambda x: 'bn' not in x.name) net_with_loss = WithLossCell(net, loss) @@ -73,7 +73,7 @@ def test_lars_float_lr(): lr = 0.1 SGD = Momentum(net.trainable_params(), lr, 0.9) - optimizer = LARS(SGD, epsilon=1e-08, hyperpara=0.02, decay_filter=lambda x: 'bn' not in x.name, + optimizer = LARS(SGD, epsilon=1e-08, coefficient=0.02, lars_filter=lambda x: 'bn' not in x.name) net_with_loss = WithLossCell(net, loss) diff --git a/tests/ut/python/parallel/test_loss_and_optimizer.py b/tests/ut/python/parallel/test_loss_and_optimizer.py index b4cf62c29e..91be7682ab 100644 --- a/tests/ut/python/parallel/test_loss_and_optimizer.py +++ b/tests/ut/python/parallel/test_loss_and_optimizer.py @@ -205,7 +205,7 @@ def test_lars(): lr = Tensor(np.ones([6]), dtype=ms.float32) sgd = Momentum(net.trainable_params(), lr, 0.9) - optimizer = LARS(sgd, epsilon=1e-08, hyperpara=0.02, decay_filter=lambda x: 'bn' not in x.name, + optimizer = LARS(sgd, epsilon=1e-08, coefficient=0.02, lars_filter=lambda x: 'bn' not in x.name) net_with_loss = NetWithLoss(net, strategy3) train_net = TrainOneStepCell(net_with_loss, optimizer) From 6a7b974346c63d5619900fc71aa9aebacf66ef22 Mon Sep 17 00:00:00 2001 From: caojian05 Date: Tue, 23 Jun 2020 11:28:27 +0800 Subject: [PATCH 007/254] change lstm import pacakge location --- model_zoo/lstm/eval.py | 2 +- model_zoo/lstm/train.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/model_zoo/lstm/eval.py b/model_zoo/lstm/eval.py index a9b81199c1..6d731fbd0d 100644 --- a/model_zoo/lstm/eval.py +++ b/model_zoo/lstm/eval.py @@ -23,8 +23,8 @@ import numpy as np from src.config import lstm_cfg as cfg from src.dataset import lstm_create_dataset, convert_to_mindrecord +from src.lstm import SentimentNet from mindspore import Tensor, nn, Model, context -from mindspore.model_zoo.lstm import SentimentNet from mindspore.nn import Accuracy from mindspore.train.callback import LossMonitor from mindspore.train.serialization import load_checkpoint, load_param_into_net diff --git a/model_zoo/lstm/train.py b/model_zoo/lstm/train.py index 732655f1de..51ae12c685 100644 --- a/model_zoo/lstm/train.py +++ b/model_zoo/lstm/train.py @@ -24,8 +24,8 @@ import numpy as np from src.config import lstm_cfg as cfg from src.dataset import convert_to_mindrecord from src.dataset import lstm_create_dataset +from src.lstm import SentimentNet from mindspore import Tensor, nn, Model, context -from mindspore.model_zoo.lstm import SentimentNet from mindspore.nn import Accuracy from mindspore.train.callback import LossMonitor, CheckpointConfig, ModelCheckpoint, TimeMonitor from mindspore.train.serialization import load_param_into_net, load_checkpoint From 304427cd93ba5d2819e4e4d809da4f520d00421e Mon Sep 17 00:00:00 2001 From: yujianfeng Date: Tue, 23 Jun 2020 11:54:51 +0800 Subject: [PATCH 008/254] Add an output to apply_proximal_adagrad op register --- .../ccsrc/kernel/cpu/sparse_apply_proximal_adagrad_cpu_kernel.h | 1 + tests/st/ops/cpu/test_sparse_apply_proximal_adagrad_op.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/mindspore/ccsrc/kernel/cpu/sparse_apply_proximal_adagrad_cpu_kernel.h b/mindspore/ccsrc/kernel/cpu/sparse_apply_proximal_adagrad_cpu_kernel.h index 082809a9c2..00ca5bc693 100644 --- a/mindspore/ccsrc/kernel/cpu/sparse_apply_proximal_adagrad_cpu_kernel.h +++ b/mindspore/ccsrc/kernel/cpu/sparse_apply_proximal_adagrad_cpu_kernel.h @@ -48,6 +48,7 @@ MS_REG_CPU_KERNEL(SparseApplyProximalAdagrad, .AddInputAttr(kNumberTypeFloat32) .AddInputAttr(kNumberTypeFloat32) .AddInputAttr(kNumberTypeInt32) + .AddOutputAttr(kNumberTypeFloat32) .AddOutputAttr(kNumberTypeFloat32), SparseApplyProximalAdagradCPUKernel); } // namespace kernel diff --git a/tests/st/ops/cpu/test_sparse_apply_proximal_adagrad_op.py b/tests/st/ops/cpu/test_sparse_apply_proximal_adagrad_op.py index 0eaa11a201..f90e11ec3d 100644 --- a/tests/st/ops/cpu/test_sparse_apply_proximal_adagrad_op.py +++ b/tests/st/ops/cpu/test_sparse_apply_proximal_adagrad_op.py @@ -44,4 +44,4 @@ def test_net(): context.set_context(mode=context.GRAPH_MODE, device_target="CPU") sparse_apply_proximal_adagrad = Net() output = sparse_apply_proximal_adagrad(gradient, indices) - print(output.asnumpy()[0]) + print(output[0].asnumpy()) From 82599adc3fef660f10196823592a49d6b87d2d7e Mon Sep 17 00:00:00 2001 From: chenfei Date: Tue, 23 Jun 2020 14:33:41 +0800 Subject: [PATCH 009/254] check depend mode of control depend --- mindspore/ccsrc/session/kernel_graph.cc | 11 ++++++++--- mindspore/ccsrc/utils/utils.h | 1 + 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/mindspore/ccsrc/session/kernel_graph.cc b/mindspore/ccsrc/session/kernel_graph.cc index 7e9bb62aab..5c05b3fdcc 100644 --- a/mindspore/ccsrc/session/kernel_graph.cc +++ b/mindspore/ccsrc/session/kernel_graph.cc @@ -538,11 +538,16 @@ void KernelGraph::UpdateControlDependRelations(const std::vector &de MS_EXCEPTION_IF_NULL(depend_node); std::vector prior_nodes = {prior_node}; std::vector depend_nodes = {depend_node}; - MS_LOG(INFO) << "Prior node[" << prior_node->DebugString() << "], depend node[" << depend_node->DebugString(); - if (prior_node->isa()) { + int depend_mode = 0; + if (AnfAlgo::HasNodeAttr(kControlDependMode, cnode)) { + depend_mode = AnfAlgo::GetNodeAttr(cnode, kControlDependMode); + } + MS_LOG(INFO) << "Prior node[" << prior_node->DebugString() << "], depend node[" << depend_node->DebugString() + << "], depend_mode :" << depend_mode << "."; + if (prior_node->isa() && depend_mode == 1) { prior_nodes = GetOutputNodes(prior_node); } - if (depend_node->isa()) { + if (depend_node->isa() && depend_mode == 1) { depend_nodes = GetOutputNodes(depend_node); } for (auto &first_node : prior_nodes) { diff --git a/mindspore/ccsrc/utils/utils.h b/mindspore/ccsrc/utils/utils.h index 972d8df319..7380ef501f 100644 --- a/mindspore/ccsrc/utils/utils.h +++ b/mindspore/ccsrc/utils/utils.h @@ -246,6 +246,7 @@ constexpr auto kTupleGetItemInputSize = 3; // index define of control depend constexpr auto kControlDependPriorIndex = 1; constexpr auto kControlDependBehindIndex = 2; +constexpr auto kControlDependMode = "depend_mode"; // index define of depend constexpr auto kRealInputIndexInDepend = 1; constexpr auto kDependAttachNodeIndex = 2; From 6c9d32d446030a32d598226f5a02ed7fdef4642e Mon Sep 17 00:00:00 2001 From: jiangjinsheng Date: Tue, 23 Jun 2020 14:21:26 +0800 Subject: [PATCH 010/254] fix StridedSliceGrad --- mindspore/ops/operations/_grad_ops.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/mindspore/ops/operations/_grad_ops.py b/mindspore/ops/operations/_grad_ops.py index 583005f78d..187a17fd4c 100644 --- a/mindspore/ops/operations/_grad_ops.py +++ b/mindspore/ops/operations/_grad_ops.py @@ -1066,8 +1066,18 @@ class StridedSliceGrad(PrimitiveWithInfer): self.init_prim_io_names(inputs=['dy', 'shapex', 'begin', 'end', 'strides'], outputs=['output']) def __infer__(self, dy, shapex, begin, end, strides): - args = {"shapex": shapex['dtype'],"begin": begin['dtype'],"end": end['dtype'],"strides": strides['dtype']} + args = {"dy": dy['dtype']} validator.check_tensor_type_same(args, mstype.number_type, self.name) + + for idx, item in enumerate(shapex['value']): + validator.check_value_type("shapex[%d]" % idx, item, [int], self.name) + for idx, item in enumerate(begin['value']): + validator.check_value_type("begin[%d]" % idx, item, [int], self.name) + for idx, item in enumerate(end['value']): + validator.check_value_type("end[%d]" % idx, item, [int], self.name) + for idx, item in enumerate(strides['value']): + validator.check_value_type("strides[%d]" % idx, item, [int], self.name) + return {'shape': shapex['value'], 'dtype': dy['dtype'], 'value': None} From 8873f9dc7e355a0b939ec7cb161a88e9bd601650 Mon Sep 17 00:00:00 2001 From: chenzomi Date: Mon, 22 Jun 2020 15:02:26 +0800 Subject: [PATCH 011/254] add fake quant test case for gpu --- .../cuda_impl/fake_quant_perchannel_impl.cu | 47 +- .../cuda_impl/fake_quant_perchannel_impl.cuh | 27 +- .../gpu/cuda_impl/fake_quant_perlayer_impl.cu | 74 +-- .../cuda_impl/fake_quant_perlayer_impl.cuh | 21 +- .../quant/fake_quant_perchannel_gpu_kernel.cc | 6 +- .../fake_quant_perchannel_grad_gpu_kernel.cc | 6 +- .../quant/fake_quant_perlayer_gpu_kernel.cc | 16 +- .../fake_quant_perlayer_grad_gpu_kernel.cc | 8 +- mindspore/train/quant/quant.py | 16 +- .../st/ops/gpu/test_fake_quant_perchannel.py | 625 ++++++++++++++++++ .../gpu/test_fake_quant_perchannel_grad.py | 373 +++++++++++ tests/st/ops/gpu/test_fake_quant_perlayer.py | 386 +++++++++++ .../ops/gpu/test_fake_quant_perlayer_grad.py | 221 +++++++ 13 files changed, 1703 insertions(+), 123 deletions(-) create mode 100644 tests/st/ops/gpu/test_fake_quant_perchannel.py create mode 100644 tests/st/ops/gpu/test_fake_quant_perchannel_grad.py create mode 100644 tests/st/ops/gpu/test_fake_quant_perlayer.py create mode 100644 tests/st/ops/gpu/test_fake_quant_perlayer_grad.py diff --git a/mindspore/ccsrc/kernel/gpu/cuda_impl/fake_quant_perchannel_impl.cu b/mindspore/ccsrc/kernel/gpu/cuda_impl/fake_quant_perchannel_impl.cu index 75c5eacb25..0e762b7dc4 100644 --- a/mindspore/ccsrc/kernel/gpu/cuda_impl/fake_quant_perchannel_impl.cu +++ b/mindspore/ccsrc/kernel/gpu/cuda_impl/fake_quant_perchannel_impl.cu @@ -20,7 +20,6 @@ #include #include #include "fake_quant_perchannel_impl.cuh" -#include "device/gpu/cuda_common.h" /** * Find the nudge min, max and scale value as output. @@ -34,13 +33,17 @@ * @param channel_num * @return */ -__global__ void NudgeMinMaxPerChannel(const float *input_min, const float *input_max, const float quant_min, - const float quant_max, float *nudge_min, float *nudge_max, float *scale, - int channel_num) { +__global__ void NudgeMinMaxPerChannel(float *input_min, float *input_max, const float quant_min, const float quant_max, + float *nudge_min, float *nudge_max, float *scale, int channel_num, + const bool symmetric) { float zp_from_min = 0.f; float nudge_zp = 0.f; for (int i = blockIdx.x * blockDim.x + threadIdx.x; i < channel_num; i += blockDim.x * gridDim.x) { + if (symmetric) { + input_max[i] = abs(input_min[0]) < input_max[i] ? input_max[i] : -input_min[i]; + input_min[i] = abs(input_min[i]) < input_max[i] ? -input_max[i] : input_min[i]; + } if ((quant_max - quant_min) == 0 || (input_max[i] - input_min[i]) == 0) { scale[i] = 0.f; zp_from_min = 0.f; @@ -62,11 +65,11 @@ __global__ void NudgeMinMaxPerChannel(const float *input_min, const float *input } } -void CalNudgePerChannel(const float *input_min, const float *input_max, const float quant_min, const float quant_max, - float *nudge_min, float *nudge_max, float *scale, const int channel_num, +void CalNudgePerChannel(float *input_min, float *input_max, const float quant_min, const float quant_max, + float *nudge_min, float *nudge_max, float *scale, const int channel_num, const bool symmetric, cudaStream_t cuda_stream) { NudgeMinMaxPerChannel<<>>( - input_min, input_max, quant_min, quant_max, nudge_min, nudge_max, scale, channel_num); + input_min, input_max, quant_min, quant_max, nudge_min, nudge_max, scale, channel_num, symmetric); } /** @@ -80,9 +83,8 @@ void CalNudgePerChannel(const float *input_min, const float *input_max, const fl * @param scale - array * @return */ -__global__ void FakeQuantizePerChannel(const float *input, float *output, const int total_size, const int channel_size, - const float *nudge_min, const float *nudge_max, const float *scale, - bool symmetric) { +__global__ void FakeQuantPerChannel(const float *input, float *output, const int total_size, const int channel_size, + const float *nudge_min, const float *nudge_max, const float *scale) { float input_x = 0.f; int nudge_input = 0; int channel_idx = 0; @@ -106,16 +108,15 @@ __global__ void FakeQuantizePerChannel(const float *input, float *output, const } } -void CalFakeQuantizePerChannel(const float *input, float *output, const int total_size, const int channel_size, - const float *nudge_min, const float *nudge_max, const float *scale, bool symmetric, - cudaStream_t cuda_stream) { - FakeQuantizePerChannel<<>>( - input, output, total_size, channel_size, nudge_min, nudge_max, scale, symmetric); +void CalFakeQuantPerChannel(const float *input, float *output, const int total_size, const int channel_size, + const float *nudge_min, const float *nudge_max, const float *scale, + cudaStream_t cuda_stream) { + FakeQuantPerChannel<<>>(input, output, total_size, channel_size, + nudge_min, nudge_max, scale); } -__global__ void FakeQuantizePerChannelGrad(const float *input, const float *gradient, float *output, - const int total_size, const int channel_size, const float *nudge_min, - const float *nudge_max) { +__global__ void FakeQuantPerChannelGrad(const float *input, const float *gradient, float *output, const int total_size, + const int channel_size, const float *nudge_min, const float *nudge_max) { int channel_idx = 0; int per_channel_num = total_size / channel_size; @@ -129,9 +130,9 @@ __global__ void FakeQuantizePerChannelGrad(const float *input, const float *grad } } -void CalFakeQuantizePerChannelGrad(const float *input, const float *gradient, float *output, const int total_num, - const int channel_num, const float *nudge_min, const float *nudge_max, - cudaStream_t cuda_stream) { - FakeQuantizePerChannelGrad<<>>( - input, gradient, output, total_num, channel_num, nudge_min, nudge_max); +void CalFakeQuantPerChannelGrad(const float *input, const float *gradient, float *output, const int total_num, + const int channel_num, const float *nudge_min, const float *nudge_max, + cudaStream_t cuda_stream) { + FakeQuantPerChannelGrad<<>>(input, gradient, output, total_num, + channel_num, nudge_min, nudge_max); } diff --git a/mindspore/ccsrc/kernel/gpu/cuda_impl/fake_quant_perchannel_impl.cuh b/mindspore/ccsrc/kernel/gpu/cuda_impl/fake_quant_perchannel_impl.cuh index 3dff7156a7..ad2e387b08 100644 --- a/mindspore/ccsrc/kernel/gpu/cuda_impl/fake_quant_perchannel_impl.cuh +++ b/mindspore/ccsrc/kernel/gpu/cuda_impl/fake_quant_perchannel_impl.cuh @@ -14,22 +14,21 @@ * limitations under the License. */ -#ifndef MINDSPORE_CCSRC_KERNEL_GPU_CUDA_IMP_FAKEQUANTIZE_H_ -#define MINDSPORE_CCSRC_KERNEL_GPU_CUDA_IMP_FAKEQUANTIZE_H_ +#ifndef MINDSPORE_CCSRC_KERNEL_GPU_CUDA_IMP_FAKE_QUANT_PERCHANNEL_H_ +#define MINDSPORE_CCSRC_KERNEL_GPU_CUDA_IMP_FAKE_QUANT_PERCHANNEL_H_ -void CalNudgePerChannel(const float* input_min, const float* input_max, const float quant_min, const float quant_max, - float* nudge_min, float* nudge_max, float* scale, const int channel_num, - cudaStream_t cuda_stream); +#include "device/gpu/cuda_common.h" -void CalFakeQuantizePerChannel(const float* input, float* output, const int total_num, const int channel_num, - const float* nudge_min, const float* nudge_max, const float* scale, bool symmetric, - cudaStream_t cuda_stream); +void CalNudgePerChannel(float *input_min, float *input_max, const float quant_min, const float quant_max, + float *nudge_min, float *nudge_max, float *scale, const int channel_num, const bool symmetric, + cudaStream_t cuda_stream); -void CalMinMaxPerChannel(float* input, float* input_min, float* input_max, const int total_num, const int channel_num, - const float ema_decay, const bool ema, cudaStream_t cuda_stream); +void CalFakeQuantPerChannel(const float *input, float *output, const int total_num, const int channel_num, + const float *nudge_min, const float *nudge_max, const float *scale, + cudaStream_t cuda_stream); -void CalFakeQuantizePerChannelGrad(const float* input, const float* gradient, float* output, const int total_num, - const int channel_num, const float* nudge_min, const float* nudge_max, - cudaStream_t cuda_stream); +void CalFakeQuantPerChannelGrad(const float *input, const float *gradient, float *output, const int total_num, + const int channel_num, const float *nudge_min, const float *nudge_max, + cudaStream_t cuda_stream); -#endif // MINDSPORE_CCSRC_KERNEL_GPU_CUDA_IMP_FAKEQUANTIZE_H_ +#endif // MINDSPORE_CCSRC_KERNEL_GPU_CUDA_IMP_FAKE_QUANT_PERCHANNEL_H_ diff --git a/mindspore/ccsrc/kernel/gpu/cuda_impl/fake_quant_perlayer_impl.cu b/mindspore/ccsrc/kernel/gpu/cuda_impl/fake_quant_perlayer_impl.cu index 11a25ba294..f527d33df9 100644 --- a/mindspore/ccsrc/kernel/gpu/cuda_impl/fake_quant_perlayer_impl.cu +++ b/mindspore/ccsrc/kernel/gpu/cuda_impl/fake_quant_perlayer_impl.cu @@ -17,11 +17,10 @@ #include #include #include -#include "device/gpu/cuda_common.h" #include "fake_quant_perlayer_impl.cuh" -__global__ void FakeQuantize(const float *input, float *output, const int size, const float *nudge_min, - const float *nudge_max, const float *scale) { +__global__ void FakeQuantPerLayer(const float *input, float *output, const int size, const float *nudge_min, + const float *nudge_max, const float *scale) { float input_x = 0.f; int nudge_input = 0; @@ -43,8 +42,8 @@ __global__ void FakeQuantize(const float *input, float *output, const int size, return; } -__global__ void FakeQuantizeGrad(const float *input, const float *gradient, float *output, const int size, - const float *nudge_min, const float *nudge_max) { +__global__ void FakeQuantPerLayerGrad(const float *input, const float *gradient, float *output, const int size, + const float *nudge_min, const float *nudge_max) { for (int i = blockIdx.x * blockDim.x + threadIdx.x; i < size; i += blockDim.x * gridDim.x) { if (input[i] < nudge_min[0] || input[i] > nudge_max[0]) { output[i] = 0; @@ -55,12 +54,18 @@ __global__ void FakeQuantizeGrad(const float *input, const float *gradient, floa return; } -__global__ void NudgeMinMax(const float *input_min, const float *input_max, const float quant_min, - const float quant_max, float *nudge_min, float *nudge_max, float *scale) { +__global__ void NudgeMinMaxPerLayer(float *input_min, float *input_max, const float quant_min, const float quant_max, + float *nudge_min, float *nudge_max, float *scale, const bool symmetric) { float zp_from_min = 0.f; scale[0] = 0.f; nudge_max[0] = 0.f; nudge_min[0] = 0.f; + + if (symmetric) { + input_max[0] = abs(input_min[0]) < input_max[0] ? input_max[0] : -input_min[0]; + input_min[0] = abs(input_min[0]) < input_max[0] ? -input_max[0] : input_min[0]; + } + if ((quant_max - quant_min) == 0 || (input_max[0] - input_min[0]) == 0) { scale[0] = 0.f; zp_from_min = 0.f; @@ -83,53 +88,24 @@ __global__ void NudgeMinMax(const float *input_min, const float *input_max, cons return; } -__global__ void UpdateInputMinMaxWithEMA(float *input_min, float *input_max, const float min, const float max, - const float decay) { - input_min[0] = decay * (min) + (1 - decay) * (input_min[0]); - input_min[0] = input_min[0] > 0 ? 0 : input_min[0]; - input_max[0] = decay * (max) + (1 - decay) * (input_max[0]); - input_max[0] = input_max[0] < 0 ? 0 : input_max[0]; - return; -} - -__global__ void UpdateInputMinMax(float *input_min, float *input_max, const float min, const float max) { - input_min[0] = min > 0 ? 0 : min; - input_max[0] = max < 0 ? 0 : max; -} - -void CalFakeQuantize(const float *input, float *output, const int size, const float *nudge_min, const float *nudge_max, - const float *scale, bool symmetric, cudaStream_t cuda_stream) { - FakeQuantize<<>>(input, output, size, nudge_min, nudge_max, scale); - return; -} - -void CalFakeQuantizeGrad(const float *input, const float *gradient, float *output, const int size, - const float *nudge_min, const float *nudge_max, cudaStream_t cuda_stream) { - FakeQuantizeGrad<<>>(input, gradient, output, size, nudge_min, - nudge_max); +void CalFakeQuantPerLayer(const float *input, float *output, const int size, const float *nudge_min, + const float *nudge_max, const float *scale, cudaStream_t cuda_stream) { + FakeQuantPerLayer<<>>(input, output, size, nudge_min, nudge_max, + scale); return; } -void CalNudge(const float *input_min, const float *input_max, const float quant_min, const float quant_max, - float *nudge_min, float *nudge_max, float *scale, cudaStream_t cuda_stream) { - NudgeMinMax<<<1, 1, 0, cuda_stream>>>(input_min, input_max, quant_min, quant_max, nudge_min, nudge_max, scale); +void CalFakeQuantPerLayerGrad(const float *input, const float *gradient, float *output, const int size, + const float *nudge_min, const float *nudge_max, cudaStream_t cuda_stream) { + FakeQuantPerLayerGrad<<>>(input, gradient, output, size, nudge_min, + nudge_max); return; } -void CalMinMax(float *input, float *input_min, float *input_max, const int size, const float ema_decay, const bool ema, - cudaStream_t cuda_stream) { - float minel = 0.f; - float maxel = 0.f; - auto policy = thrust::cuda::par.on(cuda_stream); - thrust::pair, thrust::device_ptr> tuple; - tuple = thrust::minmax_element(policy, thrust::device_pointer_cast(input), thrust::device_pointer_cast(input) + size); - minel = tuple.first[0]; - maxel = tuple.second[0]; - - if (ema) { - UpdateInputMinMaxWithEMA<<<1, 1, 0, cuda_stream>>>(input_min, input_max, minel, maxel, ema_decay); - } else { - UpdateInputMinMax<<<1, 1, 0, cuda_stream>>>(input_min, input_max, minel, maxel); - } +void CalNudgePerLayer(float *input_min, float *input_max, const float quant_min, const float quant_max, + float *nudge_min, float *nudge_max, float *scale, const bool symmetric, + cudaStream_t cuda_stream) { + NudgeMinMaxPerLayer<<<1, 1, 0, cuda_stream>>>(input_min, input_max, quant_min, quant_max, nudge_min, nudge_max, scale, + symmetric); return; } diff --git a/mindspore/ccsrc/kernel/gpu/cuda_impl/fake_quant_perlayer_impl.cuh b/mindspore/ccsrc/kernel/gpu/cuda_impl/fake_quant_perlayer_impl.cuh index 27c39dead1..dda95ed781 100644 --- a/mindspore/ccsrc/kernel/gpu/cuda_impl/fake_quant_perlayer_impl.cuh +++ b/mindspore/ccsrc/kernel/gpu/cuda_impl/fake_quant_perlayer_impl.cuh @@ -14,19 +14,18 @@ * limitations under the License. */ -#ifndef MINDSPORE_CCSRC_KERNEL_GPU_CUDA_IMP_FAKEQUANTIZE_H_ -#define MINDSPORE_CCSRC_KERNEL_GPU_CUDA_IMP_FAKEQUANTIZE_H_ +#ifndef MINDSPORE_CCSRC_KERNEL_GPU_CUDA_IMP_FAKE_QUANT_PERLAYER_H_ +#define MINDSPORE_CCSRC_KERNEL_GPU_CUDA_IMP_FAKE_QUANT_PERLAYER_H_ -void CalFakeQuantize(const float *input, float *output, const int size, const float *nudge_min, const float *nudge_max, - const float *scale, bool symmetric, cudaStream_t cuda_stream); +#include "device/gpu/cuda_common.h" -void CalFakeQuantizeGrad(const float *input, const float *gradient, float *output, const int size, - const float *nudge_min, const float *nudge_max, cudaStream_t cuda_stream); +void CalNudgePerLayer(float *input_min, float *input_max, const float quant_min, const float quant_max, + float *nudge_min, float *nudge_max, float *scale, const bool symmetric, cudaStream_t cuda_stream); -void CalNudge(const float *input_min, const float *input_max, const float quant_min, const float quant_max, - float *nudge_min, float *nudge_max, float *scale, cudaStream_t cuda_stream); +void CalFakeQuantPerLayer(const float *input, float *output, const int size, const float *nudge_min, + const float *nudge_max, const float *scale, cudaStream_t cuda_stream); -void CalMinMax(float *input, float *input_min, float *input_max, const int size, const float ema_decay, const bool ema, - cudaStream_t cuda_stream); +void CalFakeQuantPerLayerGrad(const float *input, const float *gradient, float *output, const int size, + const float *nudge_min, const float *nudge_max, cudaStream_t cuda_stream); -#endif // MINDSPORE_CCSRC_KERNEL_GPU_CUDA_IMP_FAKEQUANTIZE_H_ +#endif // MINDSPORE_CCSRC_KERNEL_GPU_CUDA_IMP_FAKE_QUANT_PERLAYER_H_ diff --git a/mindspore/ccsrc/kernel/gpu/quant/fake_quant_perchannel_gpu_kernel.cc b/mindspore/ccsrc/kernel/gpu/quant/fake_quant_perchannel_gpu_kernel.cc index ffed550fbb..8db6ddd848 100644 --- a/mindspore/ccsrc/kernel/gpu/quant/fake_quant_perchannel_gpu_kernel.cc +++ b/mindspore/ccsrc/kernel/gpu/quant/fake_quant_perchannel_gpu_kernel.cc @@ -102,9 +102,9 @@ void FakeQuantPerChannelGpuKernel::InitSizeLists() { void FakeQuantPerChannelGpuKernel::CalFakeQuantize(float *input, float *output, float *input_min, float *input_max, float *nudge_min, float *nudge_max, float *scale, void *stream_ptr) { CalNudgePerChannel(input_min, input_max, quant_min_, quant_max_, nudge_min, nudge_max, scale, num_channels_, - reinterpret_cast(stream_ptr)); - CalFakeQuantizePerChannel(input, output, input_size_ / sizeof(float), num_channels_, nudge_min, nudge_max, scale, - symmetric_, reinterpret_cast(stream_ptr)); + symmetric_, reinterpret_cast(stream_ptr)); + CalFakeQuantPerChannel(input, output, input_size_ / sizeof(float), num_channels_, nudge_min, nudge_max, scale, + reinterpret_cast(stream_ptr)); } bool FakeQuantPerChannelGpuKernel::Launch(const std::vector &inputs, diff --git a/mindspore/ccsrc/kernel/gpu/quant/fake_quant_perchannel_grad_gpu_kernel.cc b/mindspore/ccsrc/kernel/gpu/quant/fake_quant_perchannel_grad_gpu_kernel.cc index a57516eb2c..5c774c05ed 100644 --- a/mindspore/ccsrc/kernel/gpu/quant/fake_quant_perchannel_grad_gpu_kernel.cc +++ b/mindspore/ccsrc/kernel/gpu/quant/fake_quant_perchannel_grad_gpu_kernel.cc @@ -119,9 +119,9 @@ bool FakeQuantPerChannelGradGpuKernel::Launch(const std::vector &inp int total_size = input_size_ / sizeof(float); if (global_step_ >= quant_delay_) { CalNudgePerChannel(input_min, input_max, quant_min_, quant_max_, nudge_min, nudge_max, scale, num_channels_, - reinterpret_cast(stream_ptr)); - CalFakeQuantizePerChannelGrad(input, gradient, output, total_size, num_channels_, nudge_min, nudge_max, - reinterpret_cast(stream_ptr)); + symmetric_, reinterpret_cast(stream_ptr)); + CalFakeQuantPerChannelGrad(input, gradient, output, total_size, num_channels_, nudge_min, nudge_max, + reinterpret_cast(stream_ptr)); } else { CHECK_CUDA_RET_WITH_ERROR(cudaMemcpyAsync(output, gradient, input_size_, cudaMemcpyDeviceToDevice, reinterpret_cast(stream_ptr)), diff --git a/mindspore/ccsrc/kernel/gpu/quant/fake_quant_perlayer_gpu_kernel.cc b/mindspore/ccsrc/kernel/gpu/quant/fake_quant_perlayer_gpu_kernel.cc index 845fb5b923..44869983eb 100644 --- a/mindspore/ccsrc/kernel/gpu/quant/fake_quant_perlayer_gpu_kernel.cc +++ b/mindspore/ccsrc/kernel/gpu/quant/fake_quant_perlayer_gpu_kernel.cc @@ -117,10 +117,10 @@ bool FakeQuantPerLayerGpuKernel::Launch(const std::vector &inputs, c // control flow for quant_delay if (global_step_ >= quant_delay_) { // real launch - CalNudge(input_min, input_max, quant_min_, quant_max_, nudge_min, nudge_max, scale, - reinterpret_cast(stream_ptr)); - CalFakeQuantize(input, output, quant_num_, nudge_min, nudge_max, scale, symmetric_, - reinterpret_cast(stream_ptr)); + CalNudgePerLayer(input_min, input_max, quant_min_, quant_max_, nudge_min, nudge_max, scale, symmetric_, + reinterpret_cast(stream_ptr)); + CalFakeQuantPerLayer(input, output, quant_num_, nudge_min, nudge_max, scale, + reinterpret_cast(stream_ptr)); } else { CHECK_CUDA_RET_WITH_ERROR(cudaMemcpyAsync(output, input, input_size_, cudaMemcpyDeviceToDevice, reinterpret_cast(stream_ptr)), @@ -129,10 +129,10 @@ bool FakeQuantPerLayerGpuKernel::Launch(const std::vector &inputs, c global_step_++; } else { // real launch - CalNudge(input_min, input_max, quant_min_, quant_max_, nudge_min, nudge_max, scale, - reinterpret_cast(stream_ptr)); - CalFakeQuantize(input, output, quant_num_, nudge_min, nudge_max, scale, symmetric_, - reinterpret_cast(stream_ptr)); + CalNudgePerLayer(input_min, input_max, quant_min_, quant_max_, nudge_min, nudge_max, scale, symmetric_, + reinterpret_cast(stream_ptr)); + CalFakeQuantPerLayer(input, output, quant_num_, nudge_min, nudge_max, scale, + reinterpret_cast(stream_ptr)); } return true; diff --git a/mindspore/ccsrc/kernel/gpu/quant/fake_quant_perlayer_grad_gpu_kernel.cc b/mindspore/ccsrc/kernel/gpu/quant/fake_quant_perlayer_grad_gpu_kernel.cc index 9c6584e239..c8d57b2bb1 100644 --- a/mindspore/ccsrc/kernel/gpu/quant/fake_quant_perlayer_grad_gpu_kernel.cc +++ b/mindspore/ccsrc/kernel/gpu/quant/fake_quant_perlayer_grad_gpu_kernel.cc @@ -115,10 +115,10 @@ bool FakeQuantPerLayerGradGpuKernel::Launch(const std::vector &input } if (global_step_ >= quant_delay_) { - CalNudge(input_min, input_max, quant_min_, quant_max_, nudge_min, nudge_max, scale, - reinterpret_cast(stream_ptr)); - CalFakeQuantizeGrad(input, gradient, output, quant_num_, nudge_min, nudge_max, - reinterpret_cast(stream_ptr)); + CalNudgePerLayer(input_min, input_max, quant_min_, quant_max_, nudge_min, nudge_max, scale, symmetric_, + reinterpret_cast(stream_ptr)); + CalFakeQuantPerLayerGrad(input, gradient, output, quant_num_, nudge_min, nudge_max, + reinterpret_cast(stream_ptr)); } else { CHECK_CUDA_RET_WITH_ERROR(cudaMemcpyAsync(output, gradient, input_size_, cudaMemcpyDeviceToDevice, reinterpret_cast(stream_ptr)), diff --git a/mindspore/train/quant/quant.py b/mindspore/train/quant/quant.py index 937e54a7e4..46b3cd1934 100644 --- a/mindspore/train/quant/quant.py +++ b/mindspore/train/quant/quant.py @@ -150,7 +150,7 @@ class ConvertToQuantNetwork: prefix = name add_quant = _AddFakeQuantAfterSubCell(prim_op, num_bits=self.act_bits, - quant_delay=self.act_delay, + quant_delay=self.act_qdelay, per_channel=self.act_channel, symmetric=self.act_symmetric, narrow_range=self.act_range) @@ -408,19 +408,19 @@ def convert_quant_network(network, Args: network (Cell): Obtain a pipeline through network for saving graph summary. - quant_delay (int or tuple): Number of steps after which weights and activations are quantized during - eval. The first element represent weights and second element represent data flow. Default: (0, 0) bn_fold (bool): Flag to used bn fold ops for simulation inference operation. Default: False. freeze_bn (int): Number of steps after which BatchNorm OP parameters used total mean and variance. Default: 0. - num_bits (int or tuple): Number of bits to use for quantizing weights and activations. The first + quant_delay (int, list or tuple): Number of steps after which weights and activations are quantized during + eval. The first element represent weights and second element represent data flow. Default: (0, 0) + num_bits (int, list or tuple): Number of bits to use for quantizing weights and activations. The first element represent weights and second element represent data flow. Default: (8, 8) - per_channel (int or tuple): Quantization granularity based on layer or on channel. If `True` + per_channel (bool, list or tuple): Quantization granularity based on layer or on channel. If `True` then base on per channel otherwise base on per layer. The first element represent weights and second element represent data flow. Default: (False, False) - symmetric (int or tuple): Quantization algorithm use symmetric or not. If `True` then base on - symmetric otherwise base on assymmetric. The first element represent weights and second + symmetric (bool, list or tuple): Quantization algorithm use symmetric or not. If `True` then base on + symmetric otherwise base on asymmetric. The first element represent weights and second element represent data flow. Default: (False, False) - narrow_range (int or tuple): Quantization algorithm use narrow range or not. If `True` then base + narrow_range (bool, list or tuple): Quantization algorithm use narrow range or not. If `True` then base on narrow range otherwise base on off narrow range. The first element represent weights and second element represent data flow. Default: (False, False) diff --git a/tests/st/ops/gpu/test_fake_quant_perchannel.py b/tests/st/ops/gpu/test_fake_quant_perchannel.py new file mode 100644 index 0000000000..caa7d4b7e8 --- /dev/null +++ b/tests/st/ops/gpu/test_fake_quant_perchannel.py @@ -0,0 +1,625 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +import numpy as np +import pytest + +import mindspore.context as context +from mindspore.common.tensor import Tensor +from mindspore import nn +from mindspore.ops.operations import _quant_ops as Q + +context.set_context(device_target='GPU', device_id=0) + + +class Net(nn.Cell): + def __init__(self, num_bits=8, symmetric=False, narrow_range=False, channel_axis=1): + super(Net, self).__init__() + self.op = Q.FakeQuantPerChannel(num_bits=num_bits, + symmetric=symmetric, + narrow_range=narrow_range, + channel_axis=channel_axis) + + def construct(self, x, minq, maxq): + return self.op(x, minq, maxq) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_fake_quant_perchannel1(): + # WithVarsPerChannel_ZeroMinAndMax + x = np.array([0.0, 0.0, 0.0, 0.0]).astype(np.float32) + min_val = np.array([0.0, 0.0, 0.0, 0.0]).astype(np.float32) + max_val = np.array([0.0, 0.0, 0.0, 0.0]).astype(np.float32) + expect = np.array([0.0, 0.0, 0.0, 0.0]).astype(np.float32) + + net = Net(num_bits=8, narrow_range=False, channel_axis=0) + output = net(Tensor(x), Tensor(min_val), Tensor(max_val)) + + error = np.ones(shape=expect.shape) * 1.0e-5 + diff = output.asnumpy() - expect + print("output: ", output) + print("expect: ", expect) + assert np.all(np.abs(diff) < error) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_fake_quant_perchannel2(): + # WithVarsPerChannelDim1NudgedDown_RegularRange + # scale 1/4, zp 0.4, nudge 0. nudged ranges [0.0, 63.75] + x = np.array([-0.1, 0.0, 63.75, 63.8]).astype(np.float32) + min_val = np.array([-0.1, -0.1, -0.1, -0.1]).astype(np.float32) + max_val = np.array([63.65, 63.65, 63.65, 63.65]).astype(np.float32) + expect = np.array([0.0, 0.0, 63.75, 63.75]).astype(np.float32) + + net = Net(num_bits=8, narrow_range=False, channel_axis=0) + output = net(Tensor(x), Tensor(min_val), Tensor(max_val)) + + error = np.ones(shape=expect.shape) * 1.0e-5 + diff = output.asnumpy() - expect + print("output: ", output) + print("expect: ", expect) + assert np.all(np.abs(diff) < error) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_fake_quant_perchannel3(): + # WithVarsPerChannelDim1NudgedDown_NarrowRange + # scale 1/4, zp 1.4, nudge 1. nudged ranges[0.0, 63.5] + x = np.array([-0.1, 0.0, 63.5, 63.6]).astype(np.float32) + min_val = np.array([-0.1, -0.1, -0.1, -0.1]).astype(np.float32) + max_val = np.array([63.4, 63.4, 63.4, 63.4]).astype(np.float32) + expect = np.array([0.0, 0.0, 63.5, 63.5]).astype(np.float32) + + net = Net(num_bits=8, narrow_range=True, channel_axis=0) + output = net(Tensor(x), Tensor(min_val), Tensor(max_val)) + + error = np.ones(shape=expect.shape) * 1.0e-5 + diff = output.asnumpy() - expect + print("output: ", output) + print("expect: ", expect) + assert np.all(np.abs(diff) < error) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_fake_quant_perchannel4(): + # WithVarsPerChannelDim1NudgedUp_RegularRange + # [-0.125, 63.625] + # scale 1/4, zp: 0.5, nudge 0. nudged range [-0.25, 63.5] + x = np.array([-0.26, -0.25, -0.24, 63.6]).astype(np.float32) + expect = np.array([-0.25, -0.25, -0.25, 63.5]).astype(np.float32) + min_val = np.array([-0.125, -0.125, -0.125, -0.125]).astype(np.float32) + max_val = np.array([63.625, 63.625, 63.625, 63.625]).astype(np.float32) + + net = Net(num_bits=8, narrow_range=False, channel_axis=0) + output = net(Tensor(x), Tensor(min_val), Tensor(max_val)) + + error = np.ones(shape=expect.shape) * 1.0e-5 + diff = output.asnumpy() - expect + print("output: ", output) + print("expect: ", expect) + assert np.all(np.abs(diff) < error) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_fake_quant_perchannel5(): + # WithVarsPerChannelDim1NudgedUp_NarrowRange + # scale 1/4, zp: 1.5, nudge 2. nudged range [-0.25, 63.25] + x = np.array([-0.26, -0.25, -0.24, 63.3]).astype(np.float32) + expect = np.array([-0.25, -0.25, -0.25, 63.25]).astype(np.float32) + min_val = np.array([-0.125, -0.125, -0.125, -0.125]).astype(np.float32) + max_val = np.array([63.375, 63.375, 63.375, 63.375]).astype(np.float32) + + net = Net(num_bits=8, narrow_range=True, channel_axis=0) + output = net(Tensor(x), Tensor(min_val), Tensor(max_val)) + + error = np.ones(shape=expect.shape) * 1.0e-5 + diff = output.asnumpy() - expect + print("output: ", output) + print("expect: ", expect) + assert np.all(np.abs(diff) < error) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_fake_quant_perchannel6(): + # WithVarsPerChannelDim2NudgedDown_RegularRange + # scale 1/4, zp: 0.4, nudge 0. nudged range [-0.25, 63.75] + x = np.array([-0.1, 0.0, 0.1, 0.25, 63.75, 63.80] + ).reshape(2, 3).astype(np.float32) + expect = np.array([-0.0, 0.0, 0.0, 0.25, 63.75, 63.75]).astype(np.float32) + min_val = np.array([-0.1, -0.1, -0.1]).reshape(3).astype(np.float32) + max_val = np.array([63.65, 63.65, 63.65]).reshape(3).astype(np.float32) + + net = Net(num_bits=8, narrow_range=False, channel_axis=1) + output = net(Tensor(x), Tensor(min_val), Tensor(max_val)) + + error = np.ones(shape=expect.shape) * 1.0e-5 + diff = output.asnumpy().flatten() - expect + print("output: ", output) + print("expect: ", expect) + assert np.all(np.abs(diff) < error) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_fake_quant_perchannel7(): + # WithVarsPerChannelDim2NudgedDown_NarrowRange + # scale 1/4, zp: 1.4, nudge 1. nudged range [-0.25, 63.5] + x = np.array([-0.1, 0.0, 0.1, 0.25, 63.5, 63.6] + ).reshape(2, 3).astype(np.float32) + expect = np.array([0.0, 0.0, 0.0, 0.25, 63.5, 63.5]).astype(np.float32) + min_val = np.array([-0.1, -0.1, -0.1]).reshape(3).astype(np.float32) + max_val = np.array([63.4, 63.4, 63.4]).reshape(3).astype(np.float32) + + net = Net(num_bits=8, narrow_range=True, channel_axis=1) + output = net(Tensor(x), Tensor(min_val), Tensor(max_val)) + + error = np.ones(shape=expect.shape) * 1.0e-5 + diff = output.asnumpy().flatten() - expect + print("output: ", output) + print("expect: ", expect) + assert np.all(np.abs(diff) < error) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_fake_quant_perchannel8(): + # WithVarsPerChannelDim2NudgedUp_RegularRange + # scale 1/4, zp: 0.5, nudge 1. nudged range [-0.25, 63.5] + x = np.array([-0.26, -0.25, -0.24, 0.0, 63.5, 63.6] + ).reshape(2, 3).astype(np.float32) + expect = np.array([-0.25, -0.25, -0.25, 0.0, 63.5, 63.5] + ).astype(np.float32) + min_val = np.array([-0.125, -0.125, -0.125]).reshape(3).astype(np.float32) + max_val = np.array([63.625, 63.625, 63.625]).reshape(3).astype(np.float32) + + net = Net(num_bits=8, narrow_range=False, channel_axis=1) + output = net(Tensor(x), Tensor(min_val), Tensor(max_val)) + + error = np.ones(shape=expect.shape) * 1.0e-5 + diff = output.asnumpy().flatten() - expect + print("output: ", output) + print("expect: ", expect) + assert np.all(np.abs(diff) < error) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_fake_quant_perchannel9(): + # WithVarsPerChannelDim2NudgedUp_NarrowRange + # scale 1/4, zp: 0.5, nudge 2. nudged range [-0.25, 63.25] + x = np.array([-0.26, -0.25, -0.24, 0.0, 63.25, 63.3] + ).reshape(2, 3).astype(np.float32) + expect = np.array( + [-0.25, -0.25, -0.25, 0.0, 63.25, 63.25]).astype(np.float32) + min_val = np.array([-0.125, -0.125, -0.125]).reshape(3).astype(np.float32) + max_val = np.array([63.375, 63.375, 63.375]).reshape(3).astype(np.float32) + + net = Net(num_bits=8, narrow_range=True, channel_axis=1) + output = net(Tensor(x), Tensor(min_val), Tensor(max_val)) + + error = np.ones(shape=expect.shape) * 1.0e-5 + diff = output.asnumpy().flatten() - expect + print("output: ", output) + print("expect: ", expect) + assert np.all(np.abs(diff) < error) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_fake_quant_perchannel10(): + # WithVarsPerChannelDim4NudgedDown_RegularRange + # scale 1/4, zp: 0.4, nudge 0. nudged range [-0.25, 63.25] + x = np.array([-0.1, 0.0, 0.1, 0.25, 0.5, 0.75, + 1.0, 1.25, 1.5, 1.75, 2.0, 2.25, + 63.0, 63.25, 63.5, 63.7, 63.75, 63.8, + 63.9, 100.0, 100.0, 100.0, 100.0, 1000.0]).reshape(1, 4, 2, 3).astype(np.float32) + expect = np.array([0.0, 0.0, 0.0, 0.25, 0.5, 0.75, + 1.0, 1.25, 1.5, 1.75, 2.0, 2.25, + 63.0, 63.25, 63.5, 63.75, 63.75, 63.75, + 63.75, 63.75, 63.75, 63.75, 63.75, 63.75]).astype(np.float32) + min_val = np.array([-0.1, -0.1, -0.1, -0.1]).reshape(4).astype(np.float32) + max_val = np.array([63.65, 63.65, 63.65, 63.65] + ).reshape(4).astype(np.float32) + + net = Net(num_bits=8, narrow_range=False, channel_axis=1) + output = net(Tensor(x), Tensor(min_val), Tensor(max_val)) + + error = np.ones(shape=expect.shape) * 1.0e-5 + diff = output.asnumpy().flatten() - expect + + print("output: ", output) + print("expect: ", expect) + assert np.all(np.abs(diff) < error) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_fake_quant_perchannel11(): + # WithVarsPerChannelDim4NudgedDown_NarrowRange + # scale 1/4, zp: 1.4, nudge 1. nudged range [0.0, 63.25] + x = np.array([-0.1, 0.0, 0.1, 0.25, 0.5, 0.75, + 1.0, 1.25, 1.5, 1.75, 2.0, 2.25, + 63.0, 63.25, 63.3, 63.4, 63.5, 63.6, + 63.7, 100.0, 100.0, 100.0, 100.0, 1000.0]).reshape(1, 4, 2, 3).astype(np.float32) + expect = np.array([0.0, 0.0, 0.0, 0.25, 0.5, 0.75, + 1.0, 1.25, 1.5, 1.75, 2.0, 2.25, + 63.0, 63.25, 63.25, 63.5, 63.5, 63.5, + 63.5, 63.5, 63.5, 63.5, 63.5, 63.5]).astype(np.float32) + min_val = np.array([-0.1, -0.1, -0.1, -0.1]).reshape(4).astype(np.float32) + max_val = np.array([63.4, 63.4, 63.4, 63.4]).reshape(4).astype(np.float32) + + net = Net(num_bits=8, narrow_range=True, channel_axis=1) + output = net(Tensor(x), Tensor(min_val), Tensor(max_val)) + + error = np.ones(shape=expect.shape) * 1.0e-5 + diff = output.asnumpy().flatten() - expect + print("output: ", output) + print("expect: ", expect) + assert np.all(np.abs(diff) < error) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_fake_quant_perchannel12(): + # WithVarsPerChannelDim4NudgedUp_RegularRange + # scale 1/4, zp: 0.5, nudge 2. nudged range [-0.25, 63.25] + x = np.array([-0.3, -0.25, -0.2, 0.0, 0.25, 0.5, + 0.75, 1.0, 1.25, 1.5, 1.75, 2.0, + 63.0, 63.25, 63.4, 63.5, 63.6, 63.7, + 100.0, 100.0, 100.0, 100.0, 100.0, 1000.0]).reshape(1, 4, 2, 3).astype(np.float32) + expect = np.array([-0.25, -0.25, -0.25, 0.0, 0.25, 0.5, + 0.75, 1.0, 1.25, 1.5, 1.75, 2.0, + 63.0, 63.25, 63.5, 63.5, 63.5, 63.5, + 63.5, 63.5, 63.5, 63.5, 63.5, 63.5]).astype(np.float32) + min_val = np.array([-0.125, -0.125, -0.125, -0.125] + ).reshape(4).astype(np.float32) + max_val = np.array([63.625, 63.625, 63.625, 63.625] + ).reshape(4).astype(np.float32) + + net = Net(num_bits=8, narrow_range=False, channel_axis=1) + output = net(Tensor(x), Tensor(min_val), Tensor(max_val)) + + error = np.ones(shape=expect.shape) * 1.0e-5 + diff = output.asnumpy().flatten() - expect + print("output: ", output) + print("expect: ", expect) + assert np.all(np.abs(diff) < error) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_fake_quant_perchannel13(): + # WithVarsPerChannelDim4NudgedUp_NarrowRange + # scale 1/4, zp: 0.5, nudge 2. nudged range [-0.25, 63.25] + x = np.array([-0.3, -0.25, -0.2, 0.0, 0.25, 0.5, + 0.75, 1.0, 1.25, 1.5, 1.75, 2.0, + 63.0, 63.2, 63.25, 63.3, 63.4, 63.5, + 100.0, 100.0, 100.0, 100.0, 100.0, 1000.0]).reshape(1, 4, 2, 3).astype(np.float32) + expect = np.array([-0.25, -0.25, -0.25, 0.0, 0.25, 0.5, + 0.75, 1.0, 1.25, 1.5, 1.75, 2.0, + 63.0, 63.25, 63.25, 63.25, 63.25, 63.25, + 63.25, 63.25, 63.25, 63.25, 63.25, 63.25]).astype(np.float32) + min_val = np.array([-0.125, -0.125, -0.125, -0.125] + ).reshape(4).astype(np.float32) + max_val = np.array([63.375, 63.375, 63.375, 63.375] + ).reshape(4).astype(np.float32) + + net = Net(num_bits=8, narrow_range=True, channel_axis=1) + output = net(Tensor(x), Tensor(min_val), Tensor(max_val)) + + error = np.ones(shape=expect.shape) * 1.0e-5 + diff = output.asnumpy().flatten() - expect + print("output: ", output) + print("expect: ", expect) + assert np.all(np.abs(diff) < error) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_fake_quant_perchannel14(): + # WithVarsPerChannelDim1NudgedDown_4Bits_RegularRange + # scale 1/4, zp: 0.5, nudge 2. nudged range [-0.25, 63.25] + x = np.array([-0.1, 0.0, 7.5, 7.6]).reshape(4).astype(np.float32) + expect = np.array([0.0, 0.0, 7.5, 7.5]).astype(np.float32) + min_val = np.array([-0.1, -0.1, -0.1, -0.1]).reshape(4).astype(np.float32) + max_val = np.array([7.4, 7.4, 7.4, 7.4]).reshape(4).astype(np.float32) + + net = Net(num_bits=4, narrow_range=False, channel_axis=0) + output = net(Tensor(x), Tensor(min_val), Tensor(max_val)) + + error = np.ones(shape=expect.shape) * 1.0e-5 + diff = output.asnumpy().flatten() - expect + print("output: ", output) + print("expect: ", expect) + assert np.all(np.abs(diff) < error) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_fake_quant_perchannel15(): + # WithVarsPerChannelDim1NudgedDown_4Bits_NarrowRange + # scale 1/4, zp: 0.5, nudge 2. nudged range [-0.25, 63.25] + x = np.array([-0.1, 0.0, 7.0, 7.1]).reshape(4).astype(np.float32) + expect = np.array([0.0, 0.0, 7.0, 7.0]).astype(np.float32) + min_val = np.array([-0.1, -0.1, -0.1, -0.1]).reshape(4).astype(np.float32) + max_val = np.array([6.9, 6.9, 6.9, 6.9]).reshape(4).astype(np.float32) + + net = Net(num_bits=4, narrow_range=True, channel_axis=0) + output = net(Tensor(x), Tensor(min_val), Tensor(max_val)) + + error = np.ones(shape=expect.shape) * 1.0e-5 + diff = output.asnumpy() - expect + print("output: ", output) + print("expect: ", expect) + assert np.all(np.abs(diff) < error) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_fake_quant_perchannel16(): + # WithVarsPerChannelDim1NudgedUp_4Bits_RegularRange + # scale 1/4, zp: 0.5, nudge 2. nudged range [-0.25, 63.25] + x = np.array([-0.6, -0.5, 7.0, 7.1]).reshape(4).astype(np.float32) + expect = np.array([-0.5, -0.5, 7.0, 7.0]).astype(np.float32) + min_val = np.array([-0.4, -0.4, -0.4, -0.4]).reshape(4).astype(np.float32) + max_val = np.array([7.1, 7.1, 7.1, 7.1]).reshape(4).astype(np.float32) + + net = Net(num_bits=4, narrow_range=False, channel_axis=0) + output = net(Tensor(x), Tensor(min_val), Tensor(max_val)) + + error = np.ones(shape=expect.shape) * 1.0e-5 + diff = output.asnumpy() - expect + print("output: ", output) + print("expect: ", expect) + assert np.all(np.abs(diff) < error) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_fake_quant_perchannel17(): + # WithVarsPerChannelDim1NudgedUp_4Bits_NarrowRange + # scale 1/4, zp: 0.5, nudge 2. nudged range [-0.25, 63.25] + x = np.array([-0.6, -0.5, 6.5, 6.6]).reshape(4).astype(np.float32) + expect = np.array([-0.5, -0.5, 6.5, 6.5]).astype(np.float32) + min_val = np.array([-0.4, -0.4, -0.4, -0.4]).reshape(4).astype(np.float32) + max_val = np.array([6.6, 6.6, 6.6, 6.6]).reshape(4).astype(np.float32) + + net = Net(num_bits=4, narrow_range=True, channel_axis=0) + output = net(Tensor(x), Tensor(min_val), Tensor(max_val)) + + error = np.ones(shape=expect.shape) * 1.0e-5 + diff = output.asnumpy() - expect + print("output: ", output) + print("expect: ", expect) + assert np.all(np.abs(diff) < error) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_fake_quant_perchannel18(): + # WithVarsPerChannelDim2NudgedDown_4Bits_RegularRange + # scale 1/4, zp: 0.5, nudge 2. nudged range [-0.25, 63.25] + x = np.array([-0.1, 0.0, 0.1, 0.5, 7.5, 7.6] + ).reshape(2, 3).astype(np.float32) + expect = np.array([0.0, 0.0, 0.0, 0.5, 7.5, 7.5]).astype(np.float32) + min_val = np.array([-0.1, -0.1, -0.1]).reshape(3).astype(np.float32) + max_val = np.array([7.4, 7.4, 7.4]).reshape(3).astype(np.float32) + + net = Net(num_bits=4, narrow_range=False, channel_axis=1) + output = net(Tensor(x), Tensor(min_val), Tensor(max_val)) + + error = np.ones(shape=expect.shape) * 1.0e-5 + diff = output.asnumpy().flatten() - expect + print("output: ", output) + print("expect: ", expect) + assert np.all(np.abs(diff) < error) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_fake_quant_perchannel19(): + # WithVarsPerChannelDim2NudgedDown_4Bits_NarrowRange + # scale 1/4, zp: 0.5, nudge 2. nudged range [-0.25, 63.25] + x = np.array([-0.1, 0.0, 0.1, 0.5, 7.0, 7.1] + ).reshape(2, 3).astype(np.float32) + expect = np.array([0.0, 0.0, 0.0, 0.5, 7.0, 7.0]).astype(np.float32) + min_val = np.array([-0.1, -0.1, -0.1]).reshape(3).astype(np.float32) + max_val = np.array([6.9, 6.9, 6.9]).reshape(3).astype(np.float32) + + net = Net(num_bits=4, narrow_range=True, channel_axis=1) + output = net(Tensor(x), Tensor(min_val), Tensor(max_val)) + + error = np.ones(shape=expect.shape) * 1.0e-5 + diff = output.asnumpy().flatten() - expect + print("output: ", output) + print("expect: ", expect) + assert np.all(np.abs(diff) < error) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_fake_quant_perchannel20(): + # WithVarsPerChannelDim2NudgedUp_4Bits_RegularRange + # scale 1/4, zp: 0.5, nudge 2. nudged range [-0.25, 63.25] + x = np.array([-0.51, -0.5, -0.24, 0.0, 7.0, 7.1] + ).reshape(2, 3).astype(np.float32) + expect = np.array([-0.5, -0.5, 0.0, 0.0, 7.0, 7.0]).astype(np.float32) + min_val = np.array([-0.4, -0.4, -0.4]).reshape(3).astype(np.float32) + max_val = np.array([7.1, 7.1, 7.1]).reshape(3).astype(np.float32) + + net = Net(num_bits=4, narrow_range=False, channel_axis=1) + output = net(Tensor(x), Tensor(min_val), Tensor(max_val)) + + error = np.ones(shape=expect.shape) * 1.0e-5 + diff = output.asnumpy().flatten() - expect + print("output: ", output) + print("expect: ", expect) + assert np.all(np.abs(diff) < error) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_fake_quant_perchannel21(): + # WithVarsPerChannelDim2NudgedUp_4Bits_NarrowRange + # scale 1/4, zp: 0.5, nudge 2. nudged range [-0.25, 63.25] + x = np.array([-0.6, -0.5, -0.24, 0.0, 6.5, 6.6] + ).reshape(2, 3).astype(np.float32) + expect = np.array([-0.5, -0.5, 0.0, 0.0, 6.5, 6.5]).astype(np.float32) + min_val = np.array([-0.4, -0.4, -0.4]).reshape(3).astype(np.float32) + max_val = np.array([6.6, 6.6, 6.6]).reshape(3).astype(np.float32) + + net = Net(num_bits=4, narrow_range=True, channel_axis=1) + output = net(Tensor(x), Tensor(min_val), Tensor(max_val)) + + error = np.ones(shape=expect.shape) * 1.0e-5 + diff = output.asnumpy().flatten() - expect + print("output: ", output) + print("expect: ", expect) + assert np.all(np.abs(diff) < error) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_fake_quant_perchannel22(): + # WithVarsPerChannelDim4NudgedDown_4Bits_RegularRange + # scale 1/4, zp: 0.5, nudge 2. nudged range [-0.25, 63.25] + x = np.array([-0.1, 0.0, 0.1, 0.5, 1.0, 1.5, + 1.5, 2.0, 2.5, 3.0, 3.5, 4.0, + 6.0, 6.5, 7.0, 7.4, 7.5, 7.7, + 7.8, 100.0, 100.0, 100.0, 100.0, 1000.0]).reshape(1, 4, 2, 3).astype(np.float32) + expect = np.array([0.0, 0.0, 0.0, 0.5, 1.0, 1.5, + 1.5, 2.0, 2.5, 3.0, 3.5, 4.0, + 6.0, 6.5, 7.0, 7.5, 7.5, 7.5, + 7.5, 7.5, 7.5, 7.5, 7.5, 7.5]).astype(np.float32) + min_val = np.array([-0.1, -0.1, -0.1, -0.1]).reshape(4).astype(np.float32) + max_val = np.array([7.4, 7.4, 7.4, 7.4]).reshape(4).astype(np.float32) + + net = Net(num_bits=4, narrow_range=False, channel_axis=1) + output = net(Tensor(x), Tensor(min_val), Tensor(max_val)) + + error = np.ones(shape=expect.shape) * 1.0e-5 + diff = output.asnumpy().flatten() - expect + print("output: ", output) + print("expect: ", expect) + assert np.all(np.abs(diff) < error) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_fake_quant_perchannel23(): + # WithVarsPerChannelDim4NudgedDown_4Bits_NarrowRange + # scale 1/4, zp: 0.5, nudge 2. nudged range [-0.25, 63.25] + x = np.array([-0.1, 0.0, 0.1, 0.5, 1.0, 1.5, + 1.5, 2.0, 2.5, 3.0, 3.5, 4.0, + 6.0, 6.5, 6.8, 6.9, 7.0, 7.1, + 7.2, 100.0, 100.0, 100.0, 100.0, 1000.0]).reshape(1, 4, 2, 3).astype(np.float32) + expect = np.array([0.0, 0.0, 0.0, 0.5, 1.0, 1.5, + 1.5, 2.0, 2.5, 3.0, 3.5, 4.0, + 6.0, 6.5, 7.0, 7.0, 7.0, 7.0, + 7.0, 7.0, 7.0, 7.0, 7.0, 7.0]).astype(np.float32) + min_val = np.array([-0.1, -0.1, -0.1, -0.1]).reshape(4).astype(np.float32) + max_val = np.array([6.9, 6.9, 6.9, 6.9]).reshape(4).astype(np.float32) + + net = Net(num_bits=4, narrow_range=True, channel_axis=1) + output = net(Tensor(x), Tensor(min_val), Tensor(max_val)) + + error = np.ones(shape=expect.shape) * 1.0e-5 + diff = output.asnumpy().flatten() - expect + print("output: ", output) + print("expect: ", expect) + assert np.all(np.abs(diff) < error) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_fake_quant_perchannel24(): + # WithVarsPerChannelDim4NudgedUp_4Bits_RegularRange + # scale 1/4, zp: 0.5, nudge 2. nudged range [-0.25, 63.25] + x = np.array([-0.6, -0.5, -0.4, 0.0, 0.5, 1.0, + 1.5, 2.0, 2.5, 3.0, 3.5, 4.0, + 6.0, 6.5, 6.9, 7.0, 7.1, 7.7, + 100.0, 100.0, 100.0, 100.0, 100.0, 1000.0]).reshape(1, 4, 2, 3).astype(np.float32) + expect = np.array([-0.5, -0.5, -0.5, 0.0, 0.5, 1.0, + 1.5, 2.0, 2.5, 3.0, 3.5, 4.0, + 6.0, 6.5, 7.0, 7.0, 7.0, 7.0, + 7.0, 7.0, 7.0, 7.0, 7.0, 7.0]).astype(np.float32) + min_val = np.array([-0.4, -0.4, -0.4, -0.4]).reshape(4).astype(np.float32) + max_val = np.array([7.1, 7.1, 7.1, 7.1]).reshape(4).astype(np.float32) + + net = Net(num_bits=4, narrow_range=False, channel_axis=1) + output = net(Tensor(x), Tensor(min_val), Tensor(max_val)) + + error = np.ones(shape=expect.shape) * 1.0e-5 + diff = output.asnumpy().flatten() - expect + print("output: ", output) + print("expect: ", expect) + assert np.all(np.abs(diff) < error) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_fake_quant_perchannel25(): + # WithVarsPerChannelDim4NudgedUp_4Bits_NarrowRange + # scale 1/4, zp: 0.5, nudge 2. nudged range [-0.25, 63.25] + x = np.array([-0.6, -0.5, -0.4, 0.0, 0.5, 1.0, + 1.5, 2.0, 2.5, 3.0, 3.5, 4.0, + 5.5, 6.0, 6.4, 6.5, 6.6, 6.7, + 100.0, 100.0, 100.0, 100.0, 100.0, 1000.0]).reshape(1, 4, 2, 3).astype(np.float32) + expect = np.array([-0.5, -0.5, -0.5, 0.0, 0.5, 1.0, + 1.5, 2.0, 2.5, 3.0, 3.5, 4.0, + 5.5, 6.0, 6.5, 6.5, 6.5, 6.5, + 6.5, 6.5, 6.5, 6.5, 6.5, 6.5]).astype(np.float32) + min_val = np.array([-0.4, -0.4, -0.4, -0.4]).reshape(4).astype(np.float32) + max_val = np.array([6.6, 6.6, 6.6, 6.6]).reshape(4).astype(np.float32) + + net = Net(num_bits=4, narrow_range=True, channel_axis=1) + output = net(Tensor(x), Tensor(min_val), Tensor(max_val)) + + error = np.ones(shape=expect.shape) * 1.0e-5 + diff = output.asnumpy().flatten() - expect + print("output: ", output) + print("expect: ", expect) + assert np.all(np.abs(diff) < error) diff --git a/tests/st/ops/gpu/test_fake_quant_perchannel_grad.py b/tests/st/ops/gpu/test_fake_quant_perchannel_grad.py new file mode 100644 index 0000000000..aeb90b6804 --- /dev/null +++ b/tests/st/ops/gpu/test_fake_quant_perchannel_grad.py @@ -0,0 +1,373 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +import numpy as np +import pytest +from mindspore import Tensor +import mindspore.nn as nn +import mindspore.context as context +from mindspore.ops.operations import _quant_ops as Q + +context.set_context(device_target='GPU', device_id=0) + + +class Net(nn.Cell): + def __init__(self, num_bits=8, narrow_range=False): + super(Net, self).__init__() + self.op = Q.FakeQuantPerChannelGrad( + num_bits=num_bits, narrow_range=narrow_range) + + def construct(self, dout, x, minq, maxq): + return self.op(dout, x, minq, maxq) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_fake_quant_grad1(): + # WithVarsPerChannelDim1GradientNudgedDown_ZeroMinAndMax + dout = np.random.uniform(-1, 1, size=[4]).astype('float32') + x = np.array([0.0, 0.0, 0.0, 0.0]).astype(np.float32) + min_val = np.array([0.0, 0.0, 0.0, 0.0]).astype(np.float32) + max_val = np.array([0.0, 0.0, 0.0, 0.0]).astype(np.float32) + expect = dout + + net = Net(num_bits=8, narrow_range=False) + output = net(Tensor(dout), Tensor(x), Tensor(min_val), Tensor(max_val)) + + error = np.ones(shape=expect.shape) * 1.0e-5 + diff = output.asnumpy().flatten() - expect + print("=" * 40) + print("output: ", output) + print("expect: ", expect) + assert np.all(np.abs(diff) < error) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_fake_quant_grad2(): + # WithVarsPerChannelDim1GradientNudgedDown_RegularRange + dout = np.random.uniform(-1, 1, size=[4]).astype('float32') + x = np.array([-0.1, 0.0, 63.75, 63.8]).astype(np.float32) + min_val = np.array([-0.1, -0.1, -0.1, -0.1]).astype(np.float32) + max_val = np.array([63.65, 63.65, 63.65, 63.65]).astype(np.float32) + expect = np.array([0.0, dout[1], dout[2], 0.0]).astype(np.float32) + + net = Net(num_bits=8, narrow_range=False) + output = net(Tensor(dout), Tensor(x), Tensor(min_val), Tensor(max_val)) + + error = np.ones(shape=expect.shape) * 1.0e-5 + diff = output.asnumpy().flatten() - expect + print("=" * 40) + print("output: ", output) + print("expect: ", expect) + assert np.all(np.abs(diff) < error) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_fake_quant_grad3(): + # WithVarsPerChannelDim1GradientNudgedDown_NarrowRange + dout = np.random.uniform(-1, 1, size=[4]).astype('float32') + x = np.array([-0.1, 0.0, 63.5, 63.6]).astype(np.float32) + min_val = np.array([-0.1, -0.1, -0.1, -0.1]).astype(np.float32) + max_val = np.array([63.4, 63.4, 63.4, 63.4]).astype(np.float32) + expect = np.array([0.0, dout[1], dout[2], 0.0]).astype(np.float32) + + net = Net(num_bits=8, narrow_range=True) + output = net(Tensor(dout), Tensor(x), Tensor(min_val), Tensor(max_val)) + + error = np.ones(shape=expect.shape) * 1.0e-5 + diff = output.asnumpy().flatten() - expect + print("=" * 40) + print("output: ", output) + print("expect: ", expect) + assert np.all(np.abs(diff) < error) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_fake_quant_grad4(): + # WithVarsPerChannelDim1GradientNudgedUp_RegularRange + dout = np.random.uniform(-1, 1, size=[4]).astype('float32') + x = np.array([-0.3, -0.25, 63.5, 63.6]).astype(np.float32) + min_val = np.array([-0.125, -0.125, -0.125, -0.125]).astype(np.float32) + max_val = np.array([63.625, 63.625, 63.625, 63.625]).astype(np.float32) + expect = np.array([0.0, dout[1], dout[2], 0.0]).astype(np.float32) + + net = Net(num_bits=8, narrow_range=False) + output = net(Tensor(dout), Tensor(x), Tensor(min_val), Tensor(max_val)) + + error = np.ones(shape=expect.shape) * 1.0e-5 + diff = output.asnumpy().flatten() - expect + print("=" * 40) + print("output: ", output) + print("expect: ", expect) + assert np.all(np.abs(diff) < error) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_fake_quant_grad5(): + # WithVarsPerChannelDim1GradientNudgedUp_NarrowRange + dout = np.random.uniform(-1, 1, size=[4]).astype('float32') + x = np.array([-0.3, -0.25, 63.25, 63.3]).astype(np.float32) + min_val = np.array([-0.125, -0.125, -0.125, -0.125]).astype(np.float32) + max_val = np.array([63.375, 63.375, 63.375, 63.375]).astype(np.float32) + expect = np.array([0.0, dout[1], dout[2], 0.0]).astype(np.float32) + + net = Net(num_bits=8, narrow_range=True) + output = net(Tensor(dout), Tensor(x), Tensor(min_val), Tensor(max_val)) + + error = np.ones(shape=expect.shape) * 1.0e-5 + diff = output.asnumpy().flatten() - expect + print("=" * 40) + print("output: ", output) + print("expect: ", expect) + assert np.all(np.abs(diff) < error) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_fake_quant_grad6(): + # WithVarsPerChannelDim2GradientNudgedDown_RegularRange + read_dout = np.random.uniform(-1, 1, size=[3, 2]).astype('float32') + x = np.array([-0.1, 0.0, 0.1, 0.25, 63.75, 63.8] + ).reshape(3, 2).astype(np.float32) + min_val = np.array([-0.1, -0.1, -0.1]).astype(np.float32) + max_val = np.array([63.65, 63.65, 63.65]).astype(np.float32) + dout = read_dout.flatten() + expect = np.array([0.0, dout[1], dout[2], dout[3], + dout[4], 0.0]).astype(np.float32) + + net = Net(num_bits=8, narrow_range=True) + output = net(Tensor(read_dout), Tensor( + x), Tensor(min_val), Tensor(max_val)) + + error = np.ones(shape=expect.shape) * 1.0e-5 + diff = output.asnumpy().flatten() - expect + print("=" * 40) + print("output: ", output) + print("expect: ", expect) + assert np.all(np.abs(diff) < error) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_fake_quant_grad7(): + # WithVarsPerChannelDim2GradientNudgedDown_NarrowRange + read_dout = np.random.uniform(-1, 1, size=[3, 2]).astype('float32') + x = np.array([-0.1, 0.0, 0.1, 0.25, 63.5, 63.6] + ).reshape(3, 2).astype(np.float32) + min_val = np.array([-0.1, -0.1, -0.1]).astype(np.float32) + max_val = np.array([63.4, 63.4, 63.4]).astype(np.float32) + dout = read_dout.flatten() + expect = np.array([0.0, dout[1], dout[2], dout[3], + dout[4], 0.0]).astype(np.float32) + + net = Net(num_bits=8, narrow_range=True) + output = net(Tensor(read_dout), Tensor( + x), Tensor(min_val), Tensor(max_val)) + + error = np.ones(shape=expect.shape) * 1.0e-5 + diff = output.asnumpy().flatten() - expect + print("=" * 40) + print("output: ", output) + print("expect: ", expect) + assert np.all(np.abs(diff) < error) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_fake_quant_grad8(): + # WithVarsPerChannelDim2GradientNudgedUp_RegularRange + read_dout = np.random.uniform(-1, 1, size=[3, 2]).astype('float32') + x = np.array([-0.3, -0.25, -0.2, 0.0, 63.5, 63.6] + ).reshape(3, 2).astype(np.float32) + min_val = np.array([-0.125, -0.125, -0.125]).astype(np.float32) + max_val = np.array([63.625, 63.625, 63.625]).astype(np.float32) + dout = read_dout.flatten() + expect = np.array([0.0, dout[1], dout[2], dout[3], + dout[4], 0.0]).astype(np.float32) + + net = Net(num_bits=8, narrow_range=False) + output = net(Tensor(read_dout), Tensor( + x), Tensor(min_val), Tensor(max_val)) + + error = np.ones(shape=expect.shape) * 1.0e-5 + diff = output.asnumpy().flatten() - expect + print("=" * 40) + print("output: ", output) + print("expect: ", expect) + assert np.all(np.abs(diff) < error) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_fake_quant_grad9(): + # WithVarsPerChannelDim2GradientNudgedUp_NarrowRange + read_dout = np.random.uniform(-1, 1, size=[3, 2]).astype('float32') + x = np.array([-0.3, -0.25, -0.2, 0.0, 63.25, 63.3] + ).reshape(3, 2).astype(np.float32) + min_val = np.array([-0.125, -0.125, -0.125]).astype(np.float32) + max_val = np.array([63.375, 63.375, 63.375]).astype(np.float32) + dout = read_dout.flatten() + expect = np.array([0.0, dout[1], dout[2], dout[3], + dout[4], 0.0]).astype(np.float32) + + net = Net(num_bits=8, narrow_range=True) + output = net(Tensor(read_dout), Tensor( + x), Tensor(min_val), Tensor(max_val)) + + error = np.ones(shape=expect.shape) * 1.0e-5 + diff = output.asnumpy().flatten() - expect + print("=" * 40) + print("output: ", output) + print("expect: ", expect) + assert np.all(np.abs(diff) < error) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_fake_quant_grad10(): + # WithVarsPerChannelDim4GradientNudgedDown_RegularRange + read_dout = np.random.uniform(-1, 1, size=[4, 3, 2, 1]).astype('float32') + x = np.array([-0.1, 0.0, 63.75, 63.8, -0.1, 0.0, + 63.75, 63.8, -0.1, 0.0, 63.75, 63.8, + -0.1, 0.0, 63.75, 63.8, -0.1, 0.0, + 63.75, 63.8, -0.1, 0.0, 63.75, 63.8]).reshape(4, 3, 2, 1).astype(np.float32) + min_val = np.array([-0.1, -0.1, -0.1, -0.1]).astype(np.float32) + max_val = np.array([63.65, 63.65, 63.65, 63.65]).astype(np.float32) + dout = read_dout.flatten() + expect = np.array([0.0, dout[1], dout[2], 0.0, + 0.0, dout[5], dout[6], 0.0, + 0.0, dout[9], dout[10], 0.0, + 0.0, dout[13], dout[14], 0.0, + 0.0, dout[17], dout[18], 0.0, + 0.0, dout[21], dout[22], 0.0]).astype(np.float32) + + net = Net(num_bits=8, narrow_range=False) + output = net(Tensor(read_dout), Tensor( + x), Tensor(min_val), Tensor(max_val)) + + error = np.ones(shape=expect.shape) * 1.0e-5 + diff = output.asnumpy().flatten() - expect + print("=" * 40) + print("output: ", output) + print("expect: ", expect) + assert np.all(np.abs(diff) < error) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_fake_quant_grad11(): + # WithVarsPerChannelDim4GradientNudgedDown_NarrowRange + read_dout = np.random.uniform(-1, 1, size=[4, 3, 2, 1]).astype('float32') + x = np.array([-0.1, 0.0, 63.5, 63.6, -0.1, 0.0, 63.5, 63.6, -0.1, 0.0, 63.5, 63.6, -0.1, 0.0, 63.5, + 63.6, -0.1, 0.0, 63.5, 63.6, -0.1, 0.0, 63.5, 63.6]).reshape(4, 3, 2, 1).astype(np.float32) + min_val = np.array([-0.1, -0.1, -0.1, -0.1]).astype(np.float32) + max_val = np.array([63.4, 63.4, 63.4, 63.4]).astype(np.float32) + dout = read_dout.flatten() + expect = np.array([0.0, dout[1], dout[2], 0.0, + 0.0, dout[5], dout[6], 0.0, + 0.0, dout[9], dout[10], 0.0, + 0.0, dout[13], dout[14], 0.0, + 0.0, dout[17], dout[18], 0.0, + 0.0, dout[21], dout[22], 0.0]).astype(np.float32) + + net = Net(num_bits=8, narrow_range=True) + output = net(Tensor(read_dout), Tensor( + x), Tensor(min_val), Tensor(max_val)) + + error = np.ones(shape=expect.shape) * 1.0e-5 + diff = output.asnumpy().flatten() - expect + print("=" * 40) + print("output: ", output) + print("expect: ", expect) + assert np.all(np.abs(diff) < error) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_fake_quant_grad12(): + # WithVarsPerChannelDim4GradientNudgedUp_RegularRange + read_dout = np.random.uniform(-1, 1, size=[4, 3, 2, 1]).astype('float32') + x = np.array([-0.3, -0.25, 63.5, 63.6, -0.3, -0.25, + 63.5, 63.6, -0.3, -0.25, 63.5, 63.6, + -0.3, -0.25, 63.5, 63.6, -0.3, -0.25, + 63.5, 63.6, -0.3, -0.25, 63.5, 63.6]).reshape(4, 3, 2, 1).astype(np.float32) + min_val = np.array([-0.125, -0.125, -0.125, -0.125]).astype(np.float32) + max_val = np.array([63.625, 63.625, 63.625, 63.625]).astype(np.float32) + dout = read_dout.flatten() + expect = np.array([0.0, dout[1], dout[2], 0.0, + 0.0, dout[5], dout[6], 0.0, + 0.0, dout[9], dout[10], 0.0, + 0.0, dout[13], dout[14], 0.0, + 0.0, dout[17], dout[18], 0.0, + 0.0, dout[21], dout[22], 0.0]).astype(np.float32) + + net = Net(num_bits=8, narrow_range=False) + output = net(Tensor(read_dout), Tensor( + x), Tensor(min_val), Tensor(max_val)) + + error = np.ones(shape=expect.shape) * 1.0e-5 + diff = output.asnumpy().flatten() - expect + print("=" * 40) + print("output: ", output) + print("expect: ", expect) + assert np.all(np.abs(diff) < error) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_fake_quant_grad13(): + # WithVarsPerChannelDim4GradientNudgedUp_NarrowRange + read_dout = np.random.uniform(-1, 1, size=[4, 3, 2, 1]).astype('float32') + x = np.array([-0.3, -0.25, 63.25, 63.3, -0.3, -0.25, + 63.25, 63.3, -0.3, -0.25, 63.25, 63.3, + -0.3, -0.25, 63.25, 63.3, -0.3, -0.25, + 63.25, 63.3, -0.3, -0.25, 63.25, 63.3]).reshape(4, 3, 2, 1).astype(np.float32) + min_val = np.array([-0.125, -0.125, -0.125, -0.125]).astype(np.float32) + max_val = np.array([63.375, 63.375, 63.375, 63.375]).astype(np.float32) + dout = read_dout.flatten() + expect = np.array([0.0, dout[1], dout[2], 0.0, + 0.0, dout[5], dout[6], 0.0, + 0.0, dout[9], dout[10], 0.0, + 0.0, dout[13], dout[14], 0.0, + 0.0, dout[17], dout[18], 0.0, + 0.0, dout[21], dout[22], 0.0]).astype(np.float32) + + net = Net(num_bits=8, narrow_range=True) + output = net(Tensor(read_dout), Tensor( + x), Tensor(min_val), Tensor(max_val)) + + error = np.ones(shape=expect.shape) * 1.0e-5 + diff = output.asnumpy().flatten() - expect + print("=" * 40) + print("output: ", output) + print("expect: ", expect) + assert np.all(np.abs(diff) < error) diff --git a/tests/st/ops/gpu/test_fake_quant_perlayer.py b/tests/st/ops/gpu/test_fake_quant_perlayer.py new file mode 100644 index 0000000000..661cea0925 --- /dev/null +++ b/tests/st/ops/gpu/test_fake_quant_perlayer.py @@ -0,0 +1,386 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +import numpy as np +import pytest + +import mindspore.context as context +from mindspore.common.tensor import Tensor +import mindspore.nn as nn +from mindspore.ops.operations import _quant_ops as Q + +context.set_context(device_target='GPU', device_id=0) + + +class Net(nn.Cell): + def __init__(self, + num_bits=8, + quant_delay=0, + symmetric=False, + narrow_range=False, + training=True): + super(Net, self).__init__() + self.fake_quant = Q.FakeQuantPerLayer(num_bits=num_bits, + quant_delay=quant_delay, + symmetric=symmetric, + narrow_range=narrow_range, + training=training) + + def construct(self, x, minq, maxq): + return self.fake_quant(x, minq, maxq) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_fake_quant1(): + # (8, false, 0.0f, 0.0f, TensorShape({2, 3}), + # {0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f}, + # {0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f}); + x = np.array([0.0, 0.0, 0.0, 0.0, 0.0, 0.0]).reshape(2, 3).astype(np.float32) + min_val = np.array([0]).reshape(1).astype(np.float32) + max_val = np.array([0]).reshape(1).astype(np.float32) + expect = np.array([0.0, 0.0, 0.0, 0.0, 0.0, 0.0]).astype(np.float32) + + net = Net(num_bits=8, narrow_range=False) + output = net(Tensor(x), Tensor(min_val), Tensor(max_val)) + + error = np.ones(shape=expect.shape) * 1.0e-5 + diff = output.asnumpy().flatten() - expect + print("output: ", output) + print("expect: ", expect) + assert np.all(np.abs(diff) < error) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_fake_quant2(): + # 8, false, -10.0f, 53.75f, TensorShape({2, 3}), + # {-10.1f, -10.0f, -9.9f, -9.75f, 53.75f, 53.8f}, + # {-10.0f, -10.0f, -10.0f, -9.75f, 53.75f, 53.75f}); + x = np.array([-10.1, -10.0, -9.9, -9.75, 53.75, 53.8]).reshape(2, 3).astype(np.float32) + min_val = np.array([-10.0]).reshape(1).astype(np.float32) + max_val = np.array([53.75]).reshape(1).astype(np.float32) + expect = np.array([-10.0, -10.0, -10.0, -9.75, 53.75, 53.75]).astype(np.float32) + + net = Net(num_bits=8, narrow_range=False) + output = net(Tensor(x), Tensor(min_val), Tensor(max_val)) + + error = np.ones(shape=expect.shape) * 1.0e-5 + diff = output.asnumpy().flatten() - expect + print("output: ", output) + print("expect: ", expect) + assert np.all(np.abs(diff) < error) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_fake_quant3(): + # WithVarsNoNudging_NarrowRange + x = np.array([-10.1, -10.0, -9.90, -9.75, 53.5, 53.6]).reshape(2, 3).astype(np.float32) + min_val = np.array([-10.0]).reshape(1).astype(np.float32) + max_val = np.array([53.5]).reshape(1).astype(np.float32) + expect = np.array([-10.0, -10.0, -10.0, -9.75, 53.5, 53.5]).astype(np.float32) + + net = Net(num_bits=8, narrow_range=True) + output = net(Tensor(x), Tensor(min_val), Tensor(max_val)) + + error = np.ones(shape=expect.shape) * 1.0e-5 + diff = output.asnumpy().flatten() - expect + print("output: ", output) + print("expect: ", expect) + assert np.all(np.abs(diff) < error) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_fake_quant4(): + # WithVarsNudgedDown_RegularRange + x = np.array([-0.1, 0.0, 0.1, 0.25, 63.75, 63.8]).reshape(2, 3).astype(np.float32) + min_val = np.array([-0.1]).reshape(1).astype(np.float32) + max_val = np.array([63.65]).reshape(1).astype(np.float32) + expect = np.array([-0.0, 0.0, 0.0, 0.25, 63.75, 63.75]).astype(np.float32) + + net = Net(num_bits=8, narrow_range=False) + output = net(Tensor(x), Tensor(min_val), Tensor(max_val)) + + error = np.ones(shape=expect.shape) * 1.0e-5 + diff = output.asnumpy().flatten() - expect + print("output: ", output) + print("expect: ", expect) + assert np.all(np.abs(diff) < error) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_fake_quant5(): + # WithVarsNudgedDown_NarrowRange + x = np.array([-0.1, 0.0, 0.1, 0.25, 63.5, 63.6]).reshape(2, 3).astype(np.float32) + min_val = np.array([-0.1]).reshape(1).astype(np.float32) + max_val = np.array([63.4]).reshape(1).astype(np.float32) + expect = np.array([-0.0, 0.0, 0.0, 0.25, 63.5, 63.5]).astype(np.float32) + + net = Net(num_bits=8, narrow_range=True) + output = net(Tensor(x), Tensor(min_val), Tensor(max_val)) + + error = np.ones(shape=expect.shape) * 1.0e-5 + diff = output.asnumpy().flatten() - expect + print("output: ", output) + print("expect: ", expect) + assert np.all(np.abs(diff) < error) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_fake_quant6(): + # WithVarsNudgedUp_RegularRange + x = np.array([-0.26, -0.25, -0.24, 0.0, 63.5, 63.6]).reshape(2, 3).astype(np.float32) + min_val = np.array([-0.125]).reshape(1).astype(np.float32) + max_val = np.array([63.625]).reshape(1).astype(np.float32) + expect = np.array([-0.25, -0.25, -0.25, 0.0, 63.5, 63.5]).astype(np.float32) + + net = Net(num_bits=8, narrow_range=False) + output = net(Tensor(x), Tensor(min_val), Tensor(max_val)) + + error = np.ones(shape=expect.shape) * 1.0e-5 + diff = output.asnumpy().flatten() - expect + print("output: ", output) + print("expect: ", expect) + assert np.all(np.abs(diff) < error) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_fake_quant7(): + # WithVarsNudgedUp_NarrowRange + x = np.array([-0.26, -0.25, -0.24, 0.0, 63.25, 63.3]).reshape(2, 3).astype(np.float32) + min_val = np.array([-0.125]).reshape(1).astype(np.float32) + max_val = np.array([63.375]).reshape(1).astype(np.float32) + expect = np.array([-0.25, -0.25, -0.25, 0.0, 63.25, 63.25]).astype(np.float32) + + net = Net(num_bits=8, narrow_range=True) + output = net(Tensor(x), Tensor(min_val), Tensor(max_val)) + + error = np.ones(shape=expect.shape) * 1.0e-5 + diff = output.asnumpy().flatten() - expect + print("output: ", output) + print("expect: ", expect) + assert np.all(np.abs(diff) < error) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_fake_quant8(): + # WithVarsNudgedZeroIs255_RegularRange + x = np.array([-63.80, -63.75, -63.70, -63.5, 0.0, 0.1]).reshape(2, 3).astype(np.float32) + min_val = np.array([-63.65]).reshape(1).astype(np.float32) + max_val = np.array([0.1]).reshape(1).astype(np.float32) + expect = np.array([-63.75, -63.75, -63.75, -63.5, 0.0, 0.0]).astype(np.float32) + + net = Net(num_bits=8, narrow_range=False) + output = net(Tensor(x), Tensor(min_val), Tensor(max_val)) + + error = np.ones(shape=expect.shape) * 1.0e-5 + diff = output.asnumpy().flatten() - expect + print("output: ", output) + print("expect: ", expect) + assert np.all(np.abs(diff) < error) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_fake_quant9(): + # WithVarsNudgedZeroIs255_NarrowRange + x = np.array([-63.6, -63.5, -63.4, -63.25, 0.0, 0.1]).reshape(2, 3).astype(np.float32) + min_val = np.array([-63.4]).reshape(1).astype(np.float32) + max_val = np.array([0.1]).reshape(1).astype(np.float32) + expect = np.array([-63.5, -63.5, -63.5, -63.25, 0.0, 0.0]).astype(np.float32) + + net = Net(num_bits=8, narrow_range=True) + output = net(Tensor(x), Tensor(min_val), Tensor(max_val)) + + error = np.ones(shape=expect.shape) * 1.0e-5 + diff = output.asnumpy().flatten() - expect + print("output: ", output) + print("expect: ", expect) + assert np.all(np.abs(diff) < error) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_fake_quant10(): + # WithVarsNoNudging_4Bits_RegularRange + x = np.array([-6.1, -6.0, -5.9, -5.5, 1.5, 1.6]).reshape(2, 3).astype(np.float32) + min_val = np.array([-6.0]).reshape(1).astype(np.float32) + max_val = np.array([1.5]).reshape(1).astype(np.float32) + expect = np.array([-6.0, -6.0, -6.0, -5.5, 1.5, 1.5]).astype(np.float32) + + net = Net(num_bits=4, narrow_range=False) + output = net(Tensor(x), Tensor(min_val), Tensor(max_val)) + + error = np.ones(shape=expect.shape) * 1.0e-5 + diff = output.asnumpy().flatten() - expect + print("output: ", output) + print("expect: ", expect) + assert np.all(np.abs(diff) < error) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_fake_quant11(): + # WithVarsNoNudging_4Bits_NarrowRange + x = np.array([-6.1, -6.0, -5.9, -5.5, 1.0, 1.1]).reshape(2, 3).astype(np.float32) + min_val = np.array([-6.0]).reshape(1).astype(np.float32) + max_val = np.array([1.0]).reshape(1).astype(np.float32) + expect = np.array([-6.0, -6.0, -6.0, -5.5, 1.0, 1.0]).astype(np.float32) + + net = Net(num_bits=4, narrow_range=True) + output = net(Tensor(x), Tensor(min_val), Tensor(max_val)) + + error = np.ones(shape=expect.shape) * 1.0e-5 + diff = output.asnumpy().flatten() - expect + print("output: ", output) + print("expect: ", expect) + assert np.all(np.abs(diff) < error) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_fake_quant12(): + # WithVarsNudgedDown_4Bits_RegularRange + x = np.array([-0.1, 0.0, 0.1, 0.5, 7.5, 7.6]).reshape(2, 3).astype(np.float32) + min_val = np.array([-0.1]).reshape(1).astype(np.float32) + max_val = np.array([7.4]).reshape(1).astype(np.float32) + expect = np.array([-0.0, 0.0, 0.0, 0.5, 7.5, 7.5]).astype(np.float32) + + net = Net(num_bits=4, narrow_range=False) + output = net(Tensor(x), Tensor(min_val), Tensor(max_val)) + + error = np.ones(shape=expect.shape) * 1.0e-5 + diff = output.asnumpy().flatten() - expect + print("output: ", output) + print("expect: ", expect) + assert np.all(np.abs(diff) < error) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_fake_quant13(): + # WithVarsNudgedDown_4Bits_NarrowRange + x = np.array([-0.1, 0.0, 0.1, 0.5, 7.0, 7.1]).reshape(2, 3).astype(np.float32) + min_val = np.array([-0.1]).reshape(1).astype(np.float32) + max_val = np.array([6.9]).reshape(1).astype(np.float32) + expect = np.array([-0.0, 0.0, 0.0, 0.5, 7.0, 7.0]).astype(np.float32) + + net = Net(num_bits=4, narrow_range=True) + output = net(Tensor(x), Tensor(min_val), Tensor(max_val)) + + error = np.ones(shape=expect.shape) * 1.0e-5 + diff = output.asnumpy().flatten() - expect + print("output: ", output) + print("expect: ", expect) + assert np.all(np.abs(diff) < error) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_fake_quant14(): + # WithVarsNudgedUp_4Bits_RegularRange + x = np.array([-0.6, -0.5, -0.24, 0.0, 7.0, 7.1]).reshape(2, 3).astype(np.float32) + min_val = np.array([-0.4]).reshape(1).astype(np.float32) + max_val = np.array([7.1]).reshape(1).astype(np.float32) + expect = np.array([-0.5, -0.5, -0.00, 0.0, 7.0, 7.0]).astype(np.float32) + + net = Net(num_bits=4, narrow_range=False) + output = net(Tensor(x), Tensor(min_val), Tensor(max_val)) + + error = np.ones(shape=expect.shape) * 1.0e-5 + diff = output.asnumpy().flatten() - expect + print("output: ", output) + print("expect: ", expect) + assert np.all(np.abs(diff) < error) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_fake_quant15(): + # WithVarsNudgedUp_4Bits_NarrowRange + x = np.array([-0.6, -0.5, -0.24, 0.0, 6.5, 6.6]).reshape(2, 3).astype(np.float32) + min_val = np.array([-0.4]).reshape(1).astype(np.float32) + max_val = np.array([6.6]).reshape(1).astype(np.float32) + expect = np.array([-0.5, -0.5, -0.00, 0.0, 6.5, 6.5]).astype(np.float32) + + net = Net(num_bits=4, narrow_range=True) + output = net(Tensor(x), Tensor(min_val), Tensor(max_val)) + + error = np.ones(shape=expect.shape) * 1.0e-5 + diff = output.asnumpy().flatten() - expect + print("output: ", output) + print("expect: ", expect) + assert np.all(np.abs(diff) < error) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_fake_quant16(): + # WithVarsNudgedZero15_4Bits_RegularRange + x = np.array([-7.6, -7.5, -7.4, -7.2, 0.0, 0.1]).reshape(2, 3).astype(np.float32) + min_val = np.array([-7.3]).reshape(1).astype(np.float32) + max_val = np.array([0.2]).reshape(1).astype(np.float32) + expect = np.array([-7.5, -7.5, -7.5, -7.0, 0.0, 0.0]).astype(np.float32) + + net = Net(num_bits=4, narrow_range=False) + output = net(Tensor(x), Tensor(min_val), Tensor(max_val)) + + error = np.ones(shape=expect.shape) * 1.0e-5 + diff = output.asnumpy().flatten() - expect + print("output: ", output) + print("expect: ", expect) + assert np.all(np.abs(diff) < error) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_fake_quant17(): + # WithVarsNudgedZero15_4Bits_NarrowRange + x = np.array([-7.1, -7.0, -6.9, -6.5, 0.0, 0.1]).reshape(2, 3).astype(np.float32) + min_val = np.array([-6.8]).reshape(1).astype(np.float32) + max_val = np.array([0.2]).reshape(1).astype(np.float32) + expect = np.array([-7.0, -7.0, -7.0, -6.5, 0.0, 0.0]).astype(np.float32) + + net = Net(num_bits=4, narrow_range=True) + output = net(Tensor(x), Tensor(min_val), Tensor(max_val)) + + error = np.ones(shape=expect.shape) * 1.0e-5 + diff = output.asnumpy().flatten() - expect + print("output: ", output) + print("expect: ", expect) + assert np.all(np.abs(diff) < error) diff --git a/tests/st/ops/gpu/test_fake_quant_perlayer_grad.py b/tests/st/ops/gpu/test_fake_quant_perlayer_grad.py new file mode 100644 index 0000000000..f8330eff9f --- /dev/null +++ b/tests/st/ops/gpu/test_fake_quant_perlayer_grad.py @@ -0,0 +1,221 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +import numpy as np +import pytest +from mindspore import Tensor +import mindspore.nn as nn +import mindspore.context as context +from mindspore.ops.operations import _quant_ops as Q + +context.set_context(device_target='GPU', device_id=0) + + +class Net(nn.Cell): + def __init__(self, num_bits=8, narrow_range=False): + super(Net, self).__init__() + self.op = Q.FakeQuantPerLayerGrad(num_bits=num_bits, narrow_range=narrow_range) + + def construct(self, dout, x, minq, maxq): + return self.op(dout, x, minq, maxq) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_fake_quant_grad1(): + # WithArgsGradient RegularRange + dout = np.random.uniform(-1, 1, size=[6]).astype('float32') + x = np.array([-0.26, -0.25, -0.24, 0.0, 63.5, 63.6]).astype(np.float32) + min_val = np.array([-0.125]).reshape(1).astype(np.float32) + max_val = np.array([63.625]).reshape(1).astype(np.float32) + expect = np.array([0.0, dout[1], dout[2], dout[3], dout[4], 0.0]).astype(np.float32) + + net = Net(num_bits=8, narrow_range=False) + output = net(Tensor(dout), Tensor(x), Tensor(min_val), Tensor(max_val)) + + error = np.ones(shape=expect.shape) * 1.0e-5 + diff = output.asnumpy().flatten() - expect + print("output: ", output) + print("expect: ", expect) + assert np.all(np.abs(diff) < error) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_fake_quant_grad2(): + # WithArgsGradient NarrowRange + dout = np.random.uniform(-1, 1, size=[6]).astype('float32') + x = np.array([-0.26, -0.25, -0.24, 0.0, 63.25, 63.3]).astype(np.float32) + min_val = np.array([-0.125]).reshape(1).astype(np.float32) + max_val = np.array([63.375]).reshape(1).astype(np.float32) + expect = np.array([0.0, dout[1], dout[2], dout[3], dout[4], 0.0]).astype(np.float32) + + net = Net(num_bits=8, narrow_range=True) + output = net(Tensor(dout), Tensor(x), Tensor(min_val), Tensor(max_val)) + + error = np.ones(shape=expect.shape) * 1.0e-5 + diff = output.asnumpy().flatten() - expect + print("output: ", output) + print("expect: ", expect) + assert np.all(np.abs(diff) < error) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_fake_quant_grad3(): + # WithArgsGradient_4Bits_RegularRange + dout = np.random.uniform(-1, 1, size=[6]).astype('float32') + x = np.array([-0.6, -0.5, -0.4, 0.0, 7.0, 7.1]).astype(np.float32) + min_val = np.array([-0.4]).reshape(1).astype(np.float32) + max_val = np.array([7.1]).reshape(1).astype(np.float32) + expect = np.array([0.0, dout[1], dout[2], dout[3], dout[4], 0.0]).astype(np.float32) + + net = Net(num_bits=4, narrow_range=False) + output = net(Tensor(dout), Tensor(x), Tensor(min_val), Tensor(max_val)) + + error = np.ones(shape=expect.shape) * 1.0e-5 + diff = output.asnumpy().flatten() - expect + print("output: ", output) + print("expect: ", expect) + assert np.all(np.abs(diff) < error) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_fake_quant_grad4(): + # WithArgsGradient_4Bits_NarrowRange + dout = np.random.uniform(-1, 1, size=[6]).astype('float32') + x = np.array([-0.6, -0.5, -0.4, 0.0, 6.5, 6.6]).astype(np.float32) + min_val = np.array([-0.4]).reshape(1).astype(np.float32) + max_val = np.array([6.6]).reshape(1).astype(np.float32) + expect = np.array([0.0, dout[1], dout[2], dout[3], dout[4], 0.0]).astype(np.float32) + + net = Net(num_bits=4, narrow_range=True) + output = net(Tensor(dout), Tensor(x), Tensor(min_val), Tensor(max_val)) + + error = np.ones(shape=expect.shape) * 1.0e-5 + diff = output.asnumpy().flatten() - expect + print("output: ", output) + print("expect: ", expect) + assert np.all(np.abs(diff) < error) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_fake_quant_grad5(): + # FakeQuantWithMinMaxVarsGradient + dout = np.random.uniform(-1, 1, size=[6]).astype('float32') + x = np.array([0.0, 0.0, 0.0, 0.0, 0.0, 0.0]).astype(np.float32) + min_val = np.array([0.0]).reshape(1).astype(np.float32) + max_val = np.array([0.0]).reshape(1).astype(np.float32) + expect = dout + + net = Net(num_bits=8, narrow_range=True) + output = net(Tensor(dout), Tensor(x), Tensor(min_val), Tensor(max_val)) + + error = np.ones(shape=expect.shape) * 1.0e-5 + diff = output.asnumpy().flatten() - expect + print("output: ", output) + print("expect: ", expect) + assert np.all(np.abs(diff) < error) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_fake_quant_grad6(): + # WithVarsGradient_RegularRange + dout = np.random.uniform(-1, 1, size=[6]).astype('float32') + x = np.array([-0.26, -0.25, -0.24, 0.0, 63.5, 63.6]).astype(np.float32) + min_val = np.array([-0.125]).reshape(1).astype(np.float32) + max_val = np.array([63.625]).reshape(1).astype(np.float32) + expect = np.array([0.0, dout[1], dout[2], dout[3], dout[4], 0.0]).astype(np.float32) + + net = Net(num_bits=8, narrow_range=False) + output = net(Tensor(dout), Tensor(x), Tensor(min_val), Tensor(max_val)) + + error = np.ones(shape=expect.shape) * 1.0e-5 + diff = output.asnumpy().flatten() - expect + print("output: ", output) + print("expect: ", expect) + assert np.all(np.abs(diff) < error) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_fake_quant_grad7(): + # WithVarsGradient_NarrowRange + dout = np.random.uniform(-1, 1, size=[6]).astype('float32') + x = np.array([-0.26, -0.25, -0.24, 0.0, 63.25, 63.3]).astype(np.float32) + min_val = np.array([-0.125]).reshape(1).astype(np.float32) + max_val = np.array([63.375]).reshape(1).astype(np.float32) + expect = np.array([0.0, dout[1], dout[2], dout[3], dout[4], 0.0]).astype(np.float32) + + net = Net(num_bits=8, narrow_range=True) + output = net(Tensor(dout), Tensor(x), Tensor(min_val), Tensor(max_val)) + + error = np.ones(shape=expect.shape) * 1.0e-5 + diff = output.asnumpy().flatten() - expect + print("output: ", output) + print("expect: ", expect) + assert np.all(np.abs(diff) < error) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_fake_quant_grad8(): + # WithVarsGradient_4Bits_RegularRange + dout = np.random.uniform(-1, 1, size=[6]).astype('float32') + x = np.array([-0.6, -0.5, -0.4, 0.0, 7.0, 7.1]).astype(np.float32) + min_val = np.array([-0.4]).reshape(1).astype(np.float32) + max_val = np.array([7.1]).reshape(1).astype(np.float32) + expect = np.array([0.0, dout[1], dout[2], dout[3], dout[4], 0.0]).astype(np.float32) + + net = Net(num_bits=4, narrow_range=False) + output = net(Tensor(dout), Tensor(x), Tensor(min_val), Tensor(max_val)) + + error = np.ones(shape=expect.shape) * 1.0e-5 + diff = output.asnumpy().flatten() - expect + print("output: ", output) + print("expect: ", expect) + assert np.all(np.abs(diff) < error) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test_fake_quant_grad9(): + # WithVarsGradient_4Bits_NarrowRange + dout = np.random.uniform(-1, 1, size=[6]).astype('float32') + x = np.array([-0.6, -0.5, -0.4, 0.0, 6.5, 6.6]).astype(np.float32) + min_val = np.array([-0.4]).reshape(1).astype(np.float32) + max_val = np.array([6.6]).reshape(1).astype(np.float32) + expect = np.array([0.0, dout[1], dout[2], dout[3], dout[4], 0.0]).astype(np.float32) + + net = Net(num_bits=4, narrow_range=True) + output = net(Tensor(dout), Tensor(x), Tensor(min_val), Tensor(max_val)) + + error = np.ones(shape=expect.shape) * 1.0e-5 + diff = output.asnumpy().flatten() - expect + print("output: ", output) + print("expect: ", expect) + assert np.all(np.abs(diff) < error) From b2ec296f9954bf53ca4bfb0d6a40a630275d84be Mon Sep 17 00:00:00 2001 From: zhousiyi Date: Mon, 22 Jun 2020 01:17:50 +0000 Subject: [PATCH 012/254] add opt pass for tuple_getitem with constant input --- mindspore/ccsrc/optimizer/irpass.cc | 1 + mindspore/ccsrc/optimizer/irpass.h | 1 + .../optimizer/irpass/arithmetic_simplify.h | 220 ++++++++++++------ .../optimizer/irpass/item_tuple_eliminate.h | 47 +++- mindspore/ccsrc/pipeline/pass.cc | 1 + tests/ut/cpp/optimizer/lib_test.cc | 20 ++ 6 files changed, 214 insertions(+), 76 deletions(-) diff --git a/mindspore/ccsrc/optimizer/irpass.cc b/mindspore/ccsrc/optimizer/irpass.cc index 72177ccb06..0033e386d8 100644 --- a/mindspore/ccsrc/optimizer/irpass.cc +++ b/mindspore/ccsrc/optimizer/irpass.cc @@ -51,6 +51,7 @@ OptimizeIRPassLib::OptimizeIRPassLib() { arithmetic_simplify_ = MakeSubstitution(ArithmeticSimplify(), "arithmetic_simplify", {prim::kPrimScalarAdd, prim::kPrimScalarMul, prim::kPrimTensorAdd, prim::kPrimIdentity, prim::kPrimMomentum, prim::kPrimMul, prim::kPrimPow}); + arithmetic_simplify2_ = MakeSubstitution(ArithmeticSimplify2(), "arithmetic_simplify2", {prim::kPrimMul}); special_op_eliminate_ = MakeSubstitution(SpecialOpEliminater(), "special_op_eliminate", {prim::kPrimInsertGradientOf, prim::kPrimStopGradient, prim::kPrimHookBackward, diff --git a/mindspore/ccsrc/optimizer/irpass.h b/mindspore/ccsrc/optimizer/irpass.h index 5e1550c883..fa4d1e4cae 100644 --- a/mindspore/ccsrc/optimizer/irpass.h +++ b/mindspore/ccsrc/optimizer/irpass.h @@ -33,6 +33,7 @@ class OptimizeIRPassLib { ~OptimizeIRPassLib() = default; SubstitutionPtr arithmetic_simplify_; + SubstitutionPtr arithmetic_simplify2_; SubstitutionPtr special_op_eliminate_; SubstitutionPtr zero_like_fill_zero_; SubstitutionPtr adjust_all_reduce_mul_add_; diff --git a/mindspore/ccsrc/optimizer/irpass/arithmetic_simplify.h b/mindspore/ccsrc/optimizer/irpass/arithmetic_simplify.h index 1836a88dbc..ae44ec1f7d 100644 --- a/mindspore/ccsrc/optimizer/irpass/arithmetic_simplify.h +++ b/mindspore/ccsrc/optimizer/irpass/arithmetic_simplify.h @@ -139,76 +139,8 @@ class CheckTensorConstant { int check_value_; }; -// {prim::kPrimMul, 0, X}, {prim::kPrimMul, X, 0} -// {prim::kPrimMul, 1, X}, {prim::kPrimMul, X, 1} -class TensorMultiplyByZeroOrOne : public AnfVisitor { - public: - TensorMultiplyByZeroOrOne() : zero_(MakeValue(0)) {} - ~TensorMultiplyByZeroOrOne() override = default; - AnfNodePtr operator()(const OptimizerPtr &, const AnfNodePtr &node) override { - Reset(); - AnfVisitor::Match(prim::kPrimMul)(node); - - if (is_zero_) { - if (x_->func_graph() != node->func_graph()) { - return nullptr; - } - return NewTensorFilledWithData(node); - } - if (is_one_) { - return NewTensorFilledWithData(node, x_); - } - return nullptr; - } - - void Visit(const AnfNodePtr &node) override { - if (is_zero_ || is_one_) { - x_ = node; - return; - } - - if (IsParam(node)) { - x_ = node; - return; - } - - if (IsCNode(node)) { - CNodePtr cnode = node->cast(); - if (IsPrimitive(cnode->input(0), prim::kPrimZerosLike)) { - is_zero_ = true; - return; - } - x_ = node; - return; - } - auto value = node->cast()->value(); - if (CheckTensorConstant(0).IsTensorConstant(value)) { - is_zero_ = true; - return; - } else if (CheckTensorConstant(1).IsTensorConstant(value)) { - is_one_ = true; - return; - } - x_ = node; - } - - void Visit(const ValueNodePtr &vnode) override { - auto value = vnode->value(); - if (CheckTensorConstant(0).IsTensorConstant(value)) { - is_zero_ = true; - return; - } else if (CheckTensorConstant(1).IsTensorConstant(value)) { - is_one_ = true; - return; - } - x_ = vnode; - } - void Reset() { - x_ = nullptr; - is_one_ = false; - is_zero_ = false; - } - +class TensorMultiplyBase : public AnfVisitor { + protected: void *GetPointerToTensorData(const AnfNodePtr &node, bool writable = false) { if (!node->isa()) { return nullptr; @@ -287,10 +219,122 @@ class TensorMultiplyByZeroOrOne : public AnfVisitor { return new_vnode; } + AnfNodePtr x_{nullptr}; +}; + +// {prim::kPrimMul, 0, X}, {prim::kPrimMul, X, 0} +class TensorMultiplyByZero : public TensorMultiplyBase { + public: + TensorMultiplyByZero() : zero_(MakeValue(0)) {} + ~TensorMultiplyByZero() override = default; + AnfNodePtr operator()(const OptimizerPtr &, const AnfNodePtr &node) override { + Reset(); + AnfVisitor::Match(prim::kPrimMul)(node); + + if (is_zero_) { + if (x_->func_graph() != node->func_graph()) { + return nullptr; + } + return NewTensorFilledWithData(node); + } + return nullptr; + } + + void Visit(const AnfNodePtr &node) override { + if (is_zero_) { + x_ = node; + return; + } + + if (IsParam(node)) { + x_ = node; + return; + } + + if (IsCNode(node)) { + CNodePtr cnode = node->cast(); + if (IsPrimitive(cnode->input(0), prim::kPrimZerosLike)) { + is_zero_ = true; + return; + } + x_ = node; + return; + } + auto value = node->cast()->value(); + if (CheckTensorConstant(0).IsTensorConstant(value)) { + is_zero_ = true; + return; + } + x_ = node; + } + + void Visit(const ValueNodePtr &vnode) override { + auto value = vnode->value(); + if (CheckTensorConstant(0).IsTensorConstant(value)) { + is_zero_ = true; + return; + } + x_ = vnode; + } + void Reset() { + x_ = nullptr; + is_zero_ = false; + } + private: - bool is_zero_{false}, is_one_{false}; + bool is_zero_{false}; ValuePtr zero_; - AnfNodePtr x_{nullptr}; +}; + +// {prim::kPrimMul, 1, X}, {prim::kPrimMul, X, 1} +class TensorMultiplyByOne : public TensorMultiplyBase { + public: + TensorMultiplyByOne() {} + ~TensorMultiplyByOne() override = default; + AnfNodePtr operator()(const OptimizerPtr &, const AnfNodePtr &node) override { + Reset(); + AnfVisitor::Match(prim::kPrimMul)(node); + + if (is_one_) { + return NewTensorFilledWithData(node, x_); + } + return nullptr; + } + + void Visit(const AnfNodePtr &node) override { + if (is_one_) { + x_ = node; + return; + } + + if (IsParam(node) || IsCNode(node)) { + x_ = node; + return; + } + + auto value = node->cast()->value(); + if (CheckTensorConstant(1).IsTensorConstant(value)) { + is_one_ = true; + return; + } + x_ = node; + } + + void Visit(const ValueNodePtr &vnode) override { + auto value = vnode->value(); + if (CheckTensorConstant(1).IsTensorConstant(value)) { + is_one_ = true; + return; + } + x_ = vnode; + } + void Reset() { + x_ = nullptr; + is_one_ = false; + } + + private: + bool is_one_{false}; }; // {prim::kPrimScalarAdd, X, 0} @@ -699,7 +743,7 @@ class ArithmeticSimplify { public: ArithmeticSimplify() : multiply_by_zero_or_one_(), - tensor_multiply_by_zero_or_one_(), + tensor_multiply_by_one_(), add_by_zero_(), tensor_add_by_zero_(), identity_(prim::kPrimIdentity), @@ -707,7 +751,7 @@ class ArithmeticSimplify { constant_duplicate_mul_(), power_one_() { eliminaters_.emplace_back(multiply_by_zero_or_one_); - eliminaters_.emplace_back(tensor_multiply_by_zero_or_one_); + eliminaters_.emplace_back(tensor_multiply_by_one_); eliminaters_.emplace_back(add_by_zero_); eliminaters_.emplace_back(tensor_add_by_zero_); eliminaters_.emplace_back(identity_); @@ -730,7 +774,7 @@ class ArithmeticSimplify { private: MultiplyByZeroOrOne multiply_by_zero_or_one_; - TensorMultiplyByZeroOrOne tensor_multiply_by_zero_or_one_; + TensorMultiplyByOne tensor_multiply_by_one_; AddByZero add_by_zero_; TensorAddByZero tensor_add_by_zero_; PrimEliminater identity_; @@ -739,6 +783,32 @@ class ArithmeticSimplify { PowerOneEliminate power_one_; std::vector eliminaters_{}; }; + +// Arithmetic Simplifications should be done after step_parallel. +// eg: Mul(0, weight) where weight is a parameter will be simplified to a constant tensor +// with shape(weight), but after step_parallel, shape of weight may be changed, so the +// shape of the constant tensor should also be changed. So this pass is seperated from +// ArithmeticSimplify and deferred until step_parallel. +class ArithmeticSimplify2 { + public: + ArithmeticSimplify2() : tensor_multiply_by_zero_() { eliminaters_.emplace_back(tensor_multiply_by_zero_); } + ~ArithmeticSimplify2() = default; + + AnfNodePtr operator()(const OptimizerPtr &optimizer, const AnfNodePtr &node) { + AnfNodePtr new_node; + for (auto &eliminater : eliminaters_) { + new_node = eliminater(optimizer, node); + if (new_node != nullptr) { + return new_node; + } + } + return nullptr; + } + + private: + TensorMultiplyByZero tensor_multiply_by_zero_; + std::vector eliminaters_{}; +}; } // namespace irpass } // namespace opt } // namespace mindspore diff --git a/mindspore/ccsrc/optimizer/irpass/item_tuple_eliminate.h b/mindspore/ccsrc/optimizer/irpass/item_tuple_eliminate.h index 2693aec1c9..21cdff51ad 100644 --- a/mindspore/ccsrc/optimizer/irpass/item_tuple_eliminate.h +++ b/mindspore/ccsrc/optimizer/irpass/item_tuple_eliminate.h @@ -70,6 +70,45 @@ class GetitemEliminater : public AnfVisitor { CNodePtr tuple_{nullptr}; }; +// (a, b, c, ...)[0] => a +// (a, b, c, ...)[1] => b +// {prim::kPrimTupleGetItem, C1, C} +class GetitemConstEliminater : public AnfVisitor { + public: + AnfNodePtr operator()(const OptimizerPtr &, const AnfNodePtr &node) override { + Reset(); + AnfVisitor::Match(prim::kPrimTupleGetItem, {IsVNode, IsVNode})(node); + + if (is_match_) { + return NewValueNode((*tuple_)[id_]); + } + return nullptr; + } + + void Visit(const ValueNodePtr &vnode) override { + if (IsValueNode(vnode)) { + tuple_ = GetValueNode(vnode); + } + if (tuple_ != nullptr && IsValueNode(vnode)) { + id_ = IntToSize(GetValue(vnode->value())); + if (tuple_->size() > id_) { + is_match_ = true; + } + } + } + + void Reset() { + id_ = 0; + tuple_ = nullptr; + is_match_ = false; + } + + private: + bool is_match_{false}; + size_t id_{0}; + ValueTuplePtr tuple_{nullptr}; +}; + // setitem((a, b, c, ...), 0, z) => (z, b, c, ...) // setitem((a, b, c, ...), 1, z) => (a, z, c, ...) // {prim::kPrimTupleSetItem, {prim::kPrimMakeTuple, Xs}, C, Z} @@ -225,8 +264,13 @@ class GetitemDependReorder : public AnfVisitor { class ItemTupleEliminater { public: ItemTupleEliminater() - : get_item_eliminater_(), set_item_eliminater_(), get_set_item_eliminater_(), get_item_depend_reorder_() { + : get_item_eliminater_(), + get_item_const_eliminater_(), + set_item_eliminater_(), + get_set_item_eliminater_(), + get_item_depend_reorder_() { eliminaters_.emplace_back(get_item_eliminater_); + eliminaters_.emplace_back(get_item_const_eliminater_); eliminaters_.emplace_back(set_item_eliminater_); eliminaters_.emplace_back(get_set_item_eliminater_); eliminaters_.emplace_back(get_item_depend_reorder_); @@ -246,6 +290,7 @@ class ItemTupleEliminater { private: GetitemEliminater get_item_eliminater_; + GetitemConstEliminater get_item_const_eliminater_; SetitemEliminater set_item_eliminater_; GetSetitemEliminater get_set_item_eliminater_; GetitemDependReorder get_item_depend_reorder_; diff --git a/mindspore/ccsrc/pipeline/pass.cc b/mindspore/ccsrc/pipeline/pass.cc index 94063fb780..9876c0280a 100644 --- a/mindspore/ccsrc/pipeline/pass.cc +++ b/mindspore/ccsrc/pipeline/pass.cc @@ -114,6 +114,7 @@ OptPassGroupMap GetOptPassesA(const opt::irpass::OptimizeIRPassLib &irpass) { irpass.depend_value_elim_, }); opt::OptPassConfig a_3 = opt::OptPassConfig({ + irpass.arithmetic_simplify2_, irpass.same_eliminate_, irpass.check_bprop_eliminate_, irpass.replace_applicator_, diff --git a/tests/ut/cpp/optimizer/lib_test.cc b/tests/ut/cpp/optimizer/lib_test.cc index ed4497f9a5..ebbcdf6f7c 100644 --- a/tests/ut/cpp/optimizer/lib_test.cc +++ b/tests/ut/cpp/optimizer/lib_test.cc @@ -20,9 +20,12 @@ #include "common/py_func_graph_fetcher.h" #include "ir/anf.h" +#include "ir/func_graph.h" #include "ir/func_graph_cloner.h" #include "ir/manager.h" +#include "ir/value.h" #include "ir/visitor.h" +#include "operator/ops.h" #include "optimizer/irpass.h" #include "pipeline/resource.h" #include "debug/draw.h" @@ -343,9 +346,26 @@ TEST_F(TestOptLib, test_tuple_getitem) { FuncGraphPtr after_0 = getPyFun.CallAndParseRet("test_tuple_getitem", "after_0"); FuncGraphPtr after_1 = getPyFun.CallAndParseRet("test_tuple_getitem", "after_1"); + FuncGraphPtr make_get_const = std::make_shared(); + auto value_node_1 = NewValueNode(1); + auto value_node_2 = NewValueNode(2); + std::vector vec{1, 2}; + auto value_node_tuple = NewValueNode(MakeValue(vec)); + std::vector node_list{ + NewValueNode(prim::kPrimTupleGetItem), + value_node_tuple, + value_node_1 + }; + auto get_item = make_get_const->NewCNode(node_list); + make_get_const->set_output(get_item); + + FuncGraphPtr after_2 = std::make_shared(); + after_2->set_output(value_node_2); + auto patterns = std::vector({irpass.item_tuple_eliminate_}); ASSERT_TRUE(CheckOpt(make_get_0, after_0, patterns)); ASSERT_TRUE(CheckOpt(make_get_1, after_1, patterns)); + ASSERT_TRUE(CheckOpt(make_get_const, after_2, patterns)); } TEST_F(TestOptLib, test_tuple_setitem) { From a1e148cb4dfca361f3c0b62ed86fca89339a4e5d Mon Sep 17 00:00:00 2001 From: jiangjinsheng Date: Wed, 17 Jun 2020 11:54:46 +0800 Subject: [PATCH 013/254] vm for LRN and LRNGrad --- mindspore/ccsrc/kernel/tbe/tbe_adapter.cc | 2 ++ mindspore/ops/_grad/grad_nn_ops.py | 12 +++++++ mindspore/ops/_op_impl/tbe/__init__.py | 2 ++ mindspore/ops/_op_impl/tbe/lrn.py | 41 ++++++++++++++++++++++ mindspore/ops/_op_impl/tbe/lrn_grad.py | 42 +++++++++++++++++++++++ mindspore/ops/operations/__init__.py | 5 +-- mindspore/ops/operations/_grad_ops.py | 19 ++++++++++ mindspore/ops/operations/nn_ops.py | 41 ++++++++++++++++++++++ tests/ut/python/ops/test_nn_ops.py | 33 ++++++++++++++++++ 9 files changed, 195 insertions(+), 2 deletions(-) create mode 100644 mindspore/ops/_op_impl/tbe/lrn.py create mode 100644 mindspore/ops/_op_impl/tbe/lrn_grad.py diff --git a/mindspore/ccsrc/kernel/tbe/tbe_adapter.cc b/mindspore/ccsrc/kernel/tbe/tbe_adapter.cc index b7bad4fff8..deb858ff39 100644 --- a/mindspore/ccsrc/kernel/tbe/tbe_adapter.cc +++ b/mindspore/ccsrc/kernel/tbe/tbe_adapter.cc @@ -107,6 +107,8 @@ static std::map tbe_func_adapter_map = { {"r_oi_align_grad", "roi_align_grad"}, {"i_ou", "iou"}, {"s_gd", "sgd"}, + {"l_rn", "lrn"}, + {"l_rn_grad", "lrn_grad"}, {"l_ars_update", "lars_v2_update"}, {"n_ms_with_mask", "nms_with_mask"}, {"square_sum_all", "square_sum_all"}, diff --git a/mindspore/ops/_grad/grad_nn_ops.py b/mindspore/ops/_grad/grad_nn_ops.py index 13fb89b23f..00b9e3051b 100755 --- a/mindspore/ops/_grad/grad_nn_ops.py +++ b/mindspore/ops/_grad/grad_nn_ops.py @@ -721,3 +721,15 @@ def get_bprop_basic_lstm_cell(self): dw, db = basic_lstm_cell_weight_grad(F.depend(x, dxt), h, dgate) return dxt, dht, dct_1, dw, db return bprop + + +@bprop_getters.register(P.LRN) +def get_bprop_lrn(self): + """Grad definition for `LRN` operation.""" + grad = G.LRNGrad(self.depth_radius, self.bias, self.alpha, self.beta) + + def bprop(x, out, dout): + dx = grad(dout, x, out) + return (dx,) + + return bprop diff --git a/mindspore/ops/_op_impl/tbe/__init__.py b/mindspore/ops/_op_impl/tbe/__init__.py index 631ec1bf44..7207e5ee69 100644 --- a/mindspore/ops/_op_impl/tbe/__init__.py +++ b/mindspore/ops/_op_impl/tbe/__init__.py @@ -267,3 +267,5 @@ from .lin_space import _lin_space_tbe from .matrix_diag import _matrix_diag_tbe from .matrix_diag_part import _matrix_diag_part_tbe from .matrix_set_diag import _matrix_set_diag_tbe +from .lrn import _lrn_tbe +from .lrn_grad import _lrn_grad_tbe diff --git a/mindspore/ops/_op_impl/tbe/lrn.py b/mindspore/ops/_op_impl/tbe/lrn.py new file mode 100644 index 0000000000..2f22684f09 --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/lrn.py @@ -0,0 +1,41 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""LRN op""" +from mindspore.ops.op_info_register import op_info_register, TBERegOp, DataType + +lrn_op_info = TBERegOp("LRN") \ + .fusion_type("OPAQUE") \ + .async_flag(False) \ + .binfile_name("lrn.so") \ + .compute_cost(10) \ + .kernel_name("lrn") \ + .partial_flag(True) \ + .attr("depth_radius", "optional", "int", "all", "5") \ + .attr("bias", "optional", "float", "all", "1.0") \ + .attr("alpha", "optional", "float", "all", "1.0") \ + .attr("beta", "optional", "float", "all", "0.5") \ + .attr("norm_region", "optional", "str", "all", "ACROSS_CHANNELS") \ + .input(0, "x", False, "required", "all") \ + .output(0, "y", False, "required", "all") \ + .dtype_format(DataType.F16_NCHW, DataType.F16_NCHW) \ + .dtype_format(DataType.F32_NCHW, DataType.F32_NCHW) \ + .get_op_info() + + +@op_info_register(lrn_op_info) +def _lrn_tbe(): + """LRN TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/lrn_grad.py b/mindspore/ops/_op_impl/tbe/lrn_grad.py new file mode 100644 index 0000000000..4d37cf741b --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/lrn_grad.py @@ -0,0 +1,42 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""LRNGrad op""" +from mindspore.ops.op_info_register import op_info_register, TBERegOp, DataType + +lrn_grad_op_info = TBERegOp("LRNGrad") \ + .fusion_type("OPAQUE") \ + .async_flag(False) \ + .binfile_name("lrn_grad.so") \ + .compute_cost(10) \ + .kernel_name("lrn_grad") \ + .partial_flag(True) \ + .attr("depth_radius", "optional", "int", "all") \ + .attr("bias", "optional", "float", "all") \ + .attr("alpha", "optional", "float", "all") \ + .attr("beta", "optional", "float", "all") \ + .input(0, "grads", False, "required", "all") \ + .input(1, "x", False, "required", "all") \ + .input(2, "y", False, "required", "all") \ + .output(0, "z", False, "required", "all") \ + .dtype_format(DataType.F16_NCHW, DataType.F16_NCHW, DataType.F16_NCHW, DataType.F16_NCHW) \ + .dtype_format(DataType.F32_NCHW, DataType.F32_NCHW, DataType.F32_NCHW, DataType.F32_NCHW) \ + .get_op_info() + + +@op_info_register(lrn_grad_op_info) +def _lrn_grad_tbe(): + """LRNGrad TBE register""" + return diff --git a/mindspore/ops/operations/__init__.py b/mindspore/ops/operations/__init__.py index 6193292316..f9a6ee39a9 100644 --- a/mindspore/ops/operations/__init__.py +++ b/mindspore/ops/operations/__init__.py @@ -68,7 +68,7 @@ from .nn_ops import (LSTM, SGD, Adam, SparseApplyAdam, SparseApplyLazyAdam, Appl MaxPoolWithArgmax, OneHot, Pad, MirrorPad, PReLU, ReLU, ReLU6, ReLUV2, HSwish, HSigmoid, ResizeBilinear, Sigmoid, SigmoidCrossEntropyWithLogits, - SmoothL1Loss, Softmax, Softplus, + SmoothL1Loss, Softmax, Softplus, LRN, SoftmaxCrossEntropyWithLogits, ROIAlign, SparseSoftmaxCrossEntropyWithLogits, Tanh, TopK, BinaryCrossEntropy, SparseApplyAdagrad, LARSUpdate, ApplyFtrl, SparseApplyFtrl, @@ -316,7 +316,8 @@ __all__ = [ "DataFormatDimMap", "ApproximateEqual", "InplaceUpdate", - "InTopK" + "InTopK", + "LRN" ] __all__.sort() diff --git a/mindspore/ops/operations/_grad_ops.py b/mindspore/ops/operations/_grad_ops.py index c3f97b9f33..94b514cd0f 100644 --- a/mindspore/ops/operations/_grad_ops.py +++ b/mindspore/ops/operations/_grad_ops.py @@ -1364,3 +1364,22 @@ class InvGrad(PrimitiveWithInfer): validator.check_type_name("dgate", x, [mstype.float16, mstype.float32, mstype.int32, mstype.int8], self.name) validator.check_type_name("grad", grad, [mstype.float16, mstype.float32, mstype.int32, mstype.int8], self.name) return x + + +class LRNGrad(PrimitiveWithInfer): + """Computes gradients for LRN operation.""" + @prim_attr_register + def __init__(self, depth_radius=5, bias=1.0, alpha=1.0, beta=0.5): + self.init_prim_io_names(inputs=['grads', 'x', 'y'], outputs=['z']) + validator.check_value_type("depth_radius", depth_radius, [int], self.name) + validator.check_value_type("bias", bias, [float], self.name) + validator.check_value_type("alpha", alpha, [float], self.name) + validator.check_value_type("beta", beta, [float], self.name) + + def infer_dtype(self, grads, x, y): + args = {"grads": grads, "x": x, "y": y} + validator.check_tensor_type_same(args, (mstype.float16, mstype.float32,), self.name) + return x + + def infer_shape(self, grads, x, y): + return x diff --git a/mindspore/ops/operations/nn_ops.py b/mindspore/ops/operations/nn_ops.py index 521607ffb9..28944f8b4e 100644 --- a/mindspore/ops/operations/nn_ops.py +++ b/mindspore/ops/operations/nn_ops.py @@ -4252,3 +4252,44 @@ class InTopK(PrimitiveWithInfer): validator.check("x2", len(x2_shape), "", 1, Rel.EQ, self.name) validator.check("size of x2", x2_shape[0], "x1's first dimension", x1_shape[0], Rel.EQ, self.name) return x2_shape + + +class LRN(PrimitiveWithInfer): + r""" + Local Response Normalization + + Args: + depth_radius (int): Half-width of the 1-D normalization window. Shape of 0-D. + bias (float): An offset (usually positive to avoid dividing by 0). + alpha (float): A scale factor, usually positive. + beta (float): An exponent. + norm_region (str): Specify normalization region. Options: "ACROSS_CHANNELS", "WITHIN_CHANNEL". + Default: "ACROSS_CHANNELS". + + Inputs: + - **x** (Tensor) - A 4D Tensor with float16 or float32 data type. + + Outputs: + Tensor, With shape and data type same as the input tensor. + + Examples: + >>> x = Tensor(np.random.rand(1, 10, 4, 4)), mindspore.float32) + >>> lrn = P.LRN() + >>> lrn(x) + """ + @prim_attr_register + def __init__(self, depth_radius=5, bias=1.0, alpha=1.0, beta=0.5, norm_region="ACROSS_CHANNELS"): + """Init LRN""" + self.init_prim_io_names(inputs=['x'], outputs=['y']) + validator.check_value_type("depth_radius", depth_radius, [int], self.name) + validator.check_value_type("bias", bias, [float], self.name) + validator.check_value_type("alpha", alpha, [float], self.name) + validator.check_value_type("beta", beta, [float], self.name) + validator.check_value_type("norm_region", norm_region, [str], self.name) + + def infer_dtype(self, x_dtype): + validator.check_tensor_type_same({"x": x_dtype}, (mstype.float16, mstype.float32,), self.name) + return x_dtype + + def infer_shape(self, x_shape): + return x_shape diff --git a/tests/ut/python/ops/test_nn_ops.py b/tests/ut/python/ops/test_nn_ops.py index e950707234..ed7a8e695e 100644 --- a/tests/ut/python/ops/test_nn_ops.py +++ b/tests/ut/python/ops/test_nn_ops.py @@ -482,6 +482,29 @@ class PReLUGradNet(nn.Cell): def construct(self, dout, x, w): return self.prelu_grad(dout, x, w) + +class LRNNet(nn.Cell): + """ LRNNet definition """ + + def __init__(self): + super(LRNNet, self).__init__() + self.lrn = P.LRN() + + def construct(self, x): + return self.lrn(x) + + +class LRNGradNet(nn.Cell): + """ LRNGradNet definition """ + + def __init__(self): + super(LRNGradNet, self).__init__() + self.lrn_grad = G.LRNGrad() + + def construct(self, dout, x, out): + return self.lrn_grad(dout, x, out) + + test_cases = [ ('SoftMaxGrad', { 'block': SoftMaxGrad(VirtualNetWithLoss(P.Softmax())), @@ -593,6 +616,16 @@ test_cases = [ Tensor(np.array([1, 2]).astype(np.float32))], 'skip': ['backward'] }), + ('LRNNet', { + 'block': LRNNet(), + 'desc_inputs': [Tensor(np.ones([1, 5, 4, 4], np.float32))], + }), + ('LRNGradNet', { + 'block': LRNGradNet(), + 'desc_inputs': [Tensor(np.ones([1, 5, 4, 4], np.float32)), + Tensor(np.ones([1, 5, 4, 4], np.float32)), + Tensor(np.ones([1, 5, 4, 4], np.float32))], + }), ] test_cases_for_verify_exception = [ From dc29cfcbf707a5ae6d020ff259d537bcc33e3514 Mon Sep 17 00:00:00 2001 From: kswang Date: Tue, 23 Jun 2020 11:31:20 +0800 Subject: [PATCH 014/254] add cpu profile time --- .../ccsrc/device/cpu/cpu_kernel_runtime.cc | 8 ++++++++ .../cpu/sparse_apply_ftrl_cpu_kernel.cc | 19 ++++++++----------- mindspore/ccsrc/session/gpu_session.cc | 4 ++-- 3 files changed, 18 insertions(+), 13 deletions(-) diff --git a/mindspore/ccsrc/device/cpu/cpu_kernel_runtime.cc b/mindspore/ccsrc/device/cpu/cpu_kernel_runtime.cc index 6725dff524..cfcc1b7c79 100644 --- a/mindspore/ccsrc/device/cpu/cpu_kernel_runtime.cc +++ b/mindspore/ccsrc/device/cpu/cpu_kernel_runtime.cc @@ -26,6 +26,7 @@ #include "device/cpu/cpu_device_address.h" #include "utils/context/ms_context.h" #include "utils/config_manager.h" +#include "utils/profile.h" #include "common/utils.h" #include "session/anf_runtime_algorithm.h" #include "session/session_basic.h" @@ -270,6 +271,9 @@ bool CPUKernelRuntime::Run(session::KernelGraph *kernel_graph) { auto kernels = kernel_graph->execution_order(); for (const auto &kernel : kernels) { +#ifdef ENABLE_PROFILE + double start_time = GetTime(); +#endif std::vector kernel_inputs; std::vector kernel_workspaces; std::vector kernel_outputs; @@ -297,6 +301,10 @@ bool CPUKernelRuntime::Run(session::KernelGraph *kernel_graph) { if (!ret) { MS_LOG(EXCEPTION) << "Launch kernel failed."; } +#ifdef ENABLE_PROFILE + double cost_time = GetTime() - start_time; + MS_LOG(INFO) << "cpu kernel: " << kernel->fullname_with_scope() << " costs " << cost_time * 1e6 << " us"; +#endif } return true; } diff --git a/mindspore/ccsrc/kernel/cpu/sparse_apply_ftrl_cpu_kernel.cc b/mindspore/ccsrc/kernel/cpu/sparse_apply_ftrl_cpu_kernel.cc index 005195ea33..af014022d1 100644 --- a/mindspore/ccsrc/kernel/cpu/sparse_apply_ftrl_cpu_kernel.cc +++ b/mindspore/ccsrc/kernel/cpu/sparse_apply_ftrl_cpu_kernel.cc @@ -29,7 +29,7 @@ void ComputeFtrl(MultiThreadComputeParams *input_params, size_t start, size_t en auto linear = input_params->linear_; auto lr = input_params->lr_; auto l1 = input_params->l1_; - auto l2 = input_params->l2_; + auto l2_plus = 2 * input_params->l2_; auto lr_power = input_params->lr_power_; auto unique_sparse_grad = input_params->sparse_grad_; auto var_first_dim_size = input_params->var_first_dim_size_; @@ -44,21 +44,18 @@ void ComputeFtrl(MultiThreadComputeParams *input_params, size_t start, size_t en for (size_t j = start_index, k = var_outer_dim_size * i; j < end_index; ++j, ++k) { auto summed_grad = unique_sparse_grad.value_[k]; auto accum_new = accum[j] + summed_grad * summed_grad; - if (lr_power == -0.5) { - linear[j] += summed_grad - (std::sqrt(accum_new) - std::sqrt(accum[j])) / lr * var[j]; - } else { - linear[j] += summed_grad - (std::pow(accum_new, -lr_power) - std::pow(accum[j], -lr_power)) / lr * var[j]; - } - auto x = Sign(linear[j]) * l1 - linear[j]; float y; if (lr_power == -0.5) { - y = std::sqrt(accum_new) / lr + 2 * l2; + y = std::sqrt(accum_new); + linear[j] += summed_grad - (y - std::sqrt(accum[j])) / lr * var[j]; } else { - y = std::pow(accum_new, -lr_power) / lr + 2 * l2; + y = std::pow(accum_new, -lr_power); + linear[j] += summed_grad - (y - std::pow(accum[j], -lr_power)) / lr * var[j]; } - auto pre_shrink = x / y; - var[j] = std::fabs(linear[j]) > l1 ? pre_shrink : 0; accum[j] = accum_new; + auto x = Sign(linear[j]) * l1 - linear[j]; + y = y / lr + l2_plus; + var[j] = std::fabs(linear[j]) > l1 ? x / y : 0; } } } diff --git a/mindspore/ccsrc/session/gpu_session.cc b/mindspore/ccsrc/session/gpu_session.cc index e67a922567..a0a43f2edd 100644 --- a/mindspore/ccsrc/session/gpu_session.cc +++ b/mindspore/ccsrc/session/gpu_session.cc @@ -112,10 +112,10 @@ void GPUSession::LoadInputData(const std::shared_ptr &kernel_graph, auto tensor_address = tensor->device_address(); bool need_sync = false; if (ms_context->enable_pynative_infer()) { - if (tensor_address.get() == nullptr || tensor_address != device_address) { + if (tensor_address == nullptr || tensor_address != device_address) { need_sync = true; } - } else if (tensor->is_dirty()) { + } else if (tensor->is_dirty() || tensor_address == nullptr) { need_sync = true; } else if (tensor_address != device_address) { if (tensor_address->DeviceType() == device_address->DeviceType()) { From 2eb739de6e924db97154b3b0ded7e55535dc5773 Mon Sep 17 00:00:00 2001 From: Yi Huaijie Date: Tue, 23 Jun 2020 16:05:01 +0800 Subject: [PATCH 015/254] change HostAllGather and HostReduceScatter to internal interface --- .../ccsrc/kernel/cpu/allgather_cpu_kernel.h | 2 +- .../kernel/cpu/reduce_scatter_cpu_kernel.h | 2 +- mindspore/ccsrc/parallel/ops_info/ops_utils.h | 2 +- mindspore/ccsrc/transform/convert.cc | 2 - mindspore/ops/_grad/grad_comm_ops.py | 16 ++-- mindspore/ops/operations/__init__.py | 4 +- mindspore/ops/operations/_grad_ops.py | 6 +- mindspore/ops/operations/comm_ops.py | 51 +------------ tests/st/ops/cpu/test_reduce_scatter.py | 76 ------------------- tests/ut/python/communication/test_comm.py | 61 --------------- 10 files changed, 19 insertions(+), 203 deletions(-) delete mode 100644 tests/st/ops/cpu/test_reduce_scatter.py diff --git a/mindspore/ccsrc/kernel/cpu/allgather_cpu_kernel.h b/mindspore/ccsrc/kernel/cpu/allgather_cpu_kernel.h index 94180fa89b..1dddf810ef 100644 --- a/mindspore/ccsrc/kernel/cpu/allgather_cpu_kernel.h +++ b/mindspore/ccsrc/kernel/cpu/allgather_cpu_kernel.h @@ -36,7 +36,7 @@ class AllGatherCPUKernel : public CPUKernel { std::vector ranks_group_; }; -MS_REG_CPU_KERNEL(HostAllGather, KernelAttr().AddInputAttr(kNumberTypeFloat32).AddOutputAttr(kNumberTypeFloat32), +MS_REG_CPU_KERNEL(_HostAllGather, KernelAttr().AddInputAttr(kNumberTypeFloat32).AddOutputAttr(kNumberTypeFloat32), AllGatherCPUKernel); } // namespace kernel } // namespace mindspore diff --git a/mindspore/ccsrc/kernel/cpu/reduce_scatter_cpu_kernel.h b/mindspore/ccsrc/kernel/cpu/reduce_scatter_cpu_kernel.h index c3bfe571a4..5c6907602a 100644 --- a/mindspore/ccsrc/kernel/cpu/reduce_scatter_cpu_kernel.h +++ b/mindspore/ccsrc/kernel/cpu/reduce_scatter_cpu_kernel.h @@ -37,7 +37,7 @@ class ReduceScatterCPUKernel : public CPUKernel { std::vector ranks_group_; }; -MS_REG_CPU_KERNEL(HostReduceScatter, KernelAttr().AddInputAttr(kNumberTypeFloat32).AddOutputAttr(kNumberTypeFloat32), +MS_REG_CPU_KERNEL(_HostReduceScatter, KernelAttr().AddInputAttr(kNumberTypeFloat32).AddOutputAttr(kNumberTypeFloat32), ReduceScatterCPUKernel); } // namespace kernel } // namespace mindspore diff --git a/mindspore/ccsrc/parallel/ops_info/ops_utils.h b/mindspore/ccsrc/parallel/ops_info/ops_utils.h index f9af7e2626..9cb3c7040a 100644 --- a/mindspore/ccsrc/parallel/ops_info/ops_utils.h +++ b/mindspore/ccsrc/parallel/ops_info/ops_utils.h @@ -145,7 +145,7 @@ constexpr char MIRROR_OPERATOR[] = "_MirrorOperator"; constexpr char STRIDED_SLICE[] = "StridedSlice"; constexpr char ALL_GATHER[] = "AllGather"; constexpr char REDUCE_SCATTER[] = "ReduceScatter"; -constexpr char HOST_REDUCE_SCATTER[] = "HostReduceScatter"; +constexpr char HOST_REDUCE_SCATTER[] = "_HostReduceScatter"; constexpr char EMBEDDING_LOOKUP[] = "EmbeddingLookup"; constexpr char CONCAT[] = "Concat"; constexpr char SOFTMAX_CROSS_ENTROPY_WITH_LOGITS[] = "SoftmaxCrossEntropyWithLogits"; diff --git a/mindspore/ccsrc/transform/convert.cc b/mindspore/ccsrc/transform/convert.cc index 1d61b0050f..32333a06ae 100644 --- a/mindspore/ccsrc/transform/convert.cc +++ b/mindspore/ccsrc/transform/convert.cc @@ -55,9 +55,7 @@ const char kNameSimpleMeanGrad[] = "SimpleMeanGrad"; const char kNameAllReduce[] = "AllReduce"; const char kNameBroadcast[] = "Broadcast"; const char kNameAllgather[] = "AllGather"; -const char kNameHostAllgather[] = "HostAllGather"; const char kNameReduceScatter[] = "ReduceScatter"; -const char kNameHostReduceScatter[] = "HostReduceScatter"; const char kNameReduceSum[] = "ReduceSum"; const char kNameIsFinite[] = "isFinite"; const char kNameReciprocal[] = "Reciprocal"; diff --git a/mindspore/ops/_grad/grad_comm_ops.py b/mindspore/ops/_grad/grad_comm_ops.py index 7477d50895..34df18beba 100644 --- a/mindspore/ops/_grad/grad_comm_ops.py +++ b/mindspore/ops/_grad/grad_comm_ops.py @@ -18,9 +18,9 @@ import mindspore.common.dtype as mstype from mindspore.ops import functional as F from .. import operations as P from ..composite.multitype_ops.zeros_like_impl import zeros_like -from ..operations.comm_ops import (AllGather, HostAllGather, AllReduce, _AlltoAll, Broadcast, +from ..operations.comm_ops import (AllGather, _HostAllGather, AllReduce, _AlltoAll, Broadcast, _GetTensorSlice, _MirrorOperator, ReduceOp, - ReduceScatter, HostReduceScatter, _VirtualDiv) + ReduceScatter, _HostReduceScatter, _VirtualDiv) from .grad_base import bprop_getters @@ -93,10 +93,10 @@ def get_bprop_all_gather(self): return bprop -@bprop_getters.register(HostAllGather) +@bprop_getters.register(_HostAllGather) def get_bprop_host_all_gather(self): - """Generate bprop for HostAllGather""" - host_all_gather_grad = HostReduceScatter(ReduceOp.SUM, self.group) + """Generate bprop for _HostAllGather""" + host_all_gather_grad = _HostReduceScatter(ReduceOp.SUM, self.group) if self.instance_name: instance_name = "grad" + self.instance_name host_all_gather_grad.set_prim_instance_name(instance_name) @@ -126,10 +126,10 @@ def get_bprop_reduce_scatter(self): return bprop -@bprop_getters.register(HostReduceScatter) +@bprop_getters.register(_HostReduceScatter) def get_bprop_host_reduce_scatter(self): - """Generate bprop for HostReduceScatter""" - host_reduce_scatter_grad = HostAllGather(self.group) + """Generate bprop for _HostReduceScatter""" + host_reduce_scatter_grad = _HostAllGather(self.group) if self.instance_name: instance_name = "grad" + self.instance_name host_reduce_scatter_grad.set_prim_instance_name(instance_name) diff --git a/mindspore/ops/operations/__init__.py b/mindspore/ops/operations/__init__.py index 6193292316..92d32c06e3 100644 --- a/mindspore/ops/operations/__init__.py +++ b/mindspore/ops/operations/__init__.py @@ -35,7 +35,7 @@ from .array_ops import (Argmax, Argmin, Cast, Concat, Pack, Unpack, from .comm_ops import (AllGather, AllReduce, _AlltoAll, ReduceScatter, Broadcast, _MirrorOperator, ReduceOp, _VirtualDataset, _VirtualDiv, _GetTensorSlice, - HostAllGather, HostReduceScatter) + _HostAllGather, _HostReduceScatter) from .debug_ops import (ImageSummary, InsertGradientOf, HookBackward, ScalarSummary, TensorSummary, HistogramSummary, Debug, Print) from .control_ops import ControlDepend, GeSwitch, Merge @@ -244,10 +244,8 @@ __all__ = [ 'UnsortedSegmentSum', 'UnsortedSegmentMin', "AllGather", - "HostAllGather", "AllReduce", "ReduceScatter", - "HostReduceScatter", "Broadcast", "ReduceOp", 'ScalarCast', diff --git a/mindspore/ops/operations/_grad_ops.py b/mindspore/ops/operations/_grad_ops.py index c3f97b9f33..da073aedf6 100644 --- a/mindspore/ops/operations/_grad_ops.py +++ b/mindspore/ops/operations/_grad_ops.py @@ -1166,7 +1166,7 @@ class EmbeddingLookupCommGrad(PrimitiveWithInfer): Perform the gradient for the communication part of EmbeddingLookup operator. This works ONLY when 'reduce_scatter_flag' is True in 'EmbeddingLookup'. Roughly speaking, - this primitive is implemented by StridedSlice --> HostAllGather --> Concat. This primitive runs on host. + this primitive is implemented by StridedSlice --> _HostAllGather --> Concat. This primitive runs on host. """ @prim_attr_register def __init__(self): @@ -1177,8 +1177,8 @@ class EmbeddingLookupCommGrad(PrimitiveWithInfer): """ This primitive is implemented by three steps: 1) Split the 'dy' along dimension 0 into 'split_num' parts. - 2) For each part, perform HostAllGather((0, 1, 2, 3, 4, 5, 6, 7)) on the host. - 3) After HostAllGather, there are still 'split_num' parts in each process. Then, perform Concat on them + 2) For each part, perform _HostAllGather((0, 1, 2, 3, 4, 5, 6, 7)) on the host. + 3) After _HostAllGather, there are still 'split_num' parts in each process. Then, perform Concat on them along dimension 0. The output shape of this primitive: shape(output)[0] == shape(dy)[0] * 8 diff --git a/mindspore/ops/operations/comm_ops.py b/mindspore/ops/operations/comm_ops.py index dc690b5f6e..1b212d161a 100644 --- a/mindspore/ops/operations/comm_ops.py +++ b/mindspore/ops/operations/comm_ops.py @@ -176,13 +176,13 @@ class AllGather(PrimitiveWithInfer): raise NotImplementedError -class HostAllGather(PrimitiveWithInfer): +class _HostAllGather(PrimitiveWithInfer): """ Gathers tensors from the specified communication group on host. Note: Tensor must have the same shape and format in all processes participating in the collective. - HostAllGather is a host-side operator, it depends on OpenMPI and must use build option -M on + _HostAllGather is a host-side operator, it depends on OpenMPI and must use build option -M on to enable it. Using mpirun command to run it: mpirun -output-filename log -merge-stderr-to-stdout -np 3 python test_host_all_gather.py @@ -199,27 +199,6 @@ class HostAllGather(PrimitiveWithInfer): Outputs: Tensor. If the number of devices in the group is N, then the shape of output is :math:`(N, x_1, x_2, ..., x_R)`. - - Examples: - >>> import mindspore.nn as nn - >>> import mindspore.context as context - >>> import mindspore.ops.operations as P - >>> from mindspore import Tensor - >>> - >>> context.set_context(mode=context.GRAPH_MODE, device_target='CPU') - >>> context.set_mpi_config(enable_mpi=True) - >>> - >>> class Net(nn.Cell): - >>> def __init__(self): - >>> super(Net, self).__init__() - >>> self.hostallgather = P.HostAllGather(group=(0, 1, 2, 3)) - >>> - >>> def construct(self, x): - >>> return self.hostallgather(x) - >>> - >>> input_ = Tensor(np.ones([2, 8]).astype(np.float32)) - >>> net = Net() - >>> output = net(input_) """ @prim_attr_register @@ -308,13 +287,13 @@ class ReduceScatter(PrimitiveWithInfer): raise NotImplementedError -class HostReduceScatter(PrimitiveWithInfer): +class _HostReduceScatter(PrimitiveWithInfer): """ Reduces and scatters tensors from the specified communication group on host. Note: Tensor must have the same shape and format in all processes participating in the collective. - HostReduceScatter is a host-side operator, it depends on OpenMPI and must use build option + _HostReduceScatter is a host-side operator, it depends on OpenMPI and must use build option -M on to enable it. Using mpirun command to run it: mpirun -output-filename log -merge-stderr-to-stdout -np 3 python test_host_reduce_scatter.py @@ -328,28 +307,6 @@ class HostReduceScatter(PrimitiveWithInfer): or elements of group are not int. ValueError: If the first dimension of input can not be divided by group size, or group is not set, or rank_id not in [0, 7]. - - Examples: - >>> import mindspore.nn as nn - >>> import mindspore.context as context - >>> import mindspore.ops.operations as P - >>> from mindspore import Tensor - >>> from mindspore.ops.operations.comm_ops import ReduceOp - >>> - >>> context.set_context(mode=context.GRAPH_MODE, device_target='CPU') - >>> context.set_mpi_config(enable_mpi=True) - >>> - >>> class Net(nn.Cell): - >>> def __init__(self): - >>> super(Net, self).__init__() - >>> self.hostreducescatter = P.HostReduceScatter(ReduceOp.SUM, group=[0, 1, 2, 3]) - >>> - >>> def construct(self, x): - >>> return self.hostreducescatter(x) - >>> - >>> input_ = Tensor(np.ones([8, 8]).astype(np.float32)) - >>> net = Net() - >>> output = net(input_) """ @prim_attr_register def __init__(self, op=ReduceOp.SUM, group=None): diff --git a/tests/st/ops/cpu/test_reduce_scatter.py b/tests/st/ops/cpu/test_reduce_scatter.py deleted file mode 100644 index 6b21efe89c..0000000000 --- a/tests/st/ops/cpu/test_reduce_scatter.py +++ /dev/null @@ -1,76 +0,0 @@ -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ - -import numpy as np -import pytest - -import mindspore.context as context -import mindspore.nn as nn -from mindspore import Tensor -from mindspore.common import dtype as mstype -from mindspore.ops import operations as P -import mindspore._ms_mpi as mpi -# run comand: -# mpirun -output-filename log -merge-stderr-to-stdout -np 3 python test_reduce_scatter.py - -context.set_context(mode=context.GRAPH_MODE, device_target='CPU') -context.set_mpi_config(enable_mpi=True) - -class Net(nn.Cell): - def __init__(self): - super(Net, self).__init__() - self.op = "sum" - - self.reducescatter = P.HostReduceScatter(op=self.op, group=[0,1,2]) - - def construct(self, x): - return self.reducescatter(x) - -class AllGatherNet(nn.Cell): - def __init__(self): - super(AllGatherNet, self).__init__() - self.hostallgather = P.HostAllGather(group=(0, 1, 2)) - - def construct(self, x): - return self.hostallgather(x) - -def test_net_reduce_scatter(): - x = np.arange(12).astype(np.float32) * 0.1 - - reducescatter = Net() - rankid = mpi.get_rank_id() - print("self rankid:", rankid) - output = reducescatter(Tensor(x, mstype.float32)) - print("output:\n", output) - if rankid == 0: - expect_result = np.arange(4).astype(np.float32) * 0.3 - if rankid == 1: - expect_result = np.arange(4, 8).astype(np.float32) * 0.3 - if rankid == 2: - expect_result = np.arange(8, 12).astype(np.float32) * 0.3 - diff = abs(output.asnumpy() - expect_result) - error = np.ones(shape=expect_result.shape) * 1.0e-6 - assert np.all(diff < error) - - allgather = AllGatherNet() - allgather_output = allgather(output) - print("allgather result:\n", allgather_output) - expect_allgather_result = np.arange(12).astype(np.float32) * 0.3 - diff = abs(allgather_output.asnumpy() - expect_allgather_result) - error = np.ones(shape=expect_allgather_result.shape) * 1.0e-6 - assert np.all(diff < error) - -if __name__ == '__main__': - test_net_reduce_scatter() diff --git a/tests/ut/python/communication/test_comm.py b/tests/ut/python/communication/test_comm.py index f3530cb261..7688adb41a 100644 --- a/tests/ut/python/communication/test_comm.py +++ b/tests/ut/python/communication/test_comm.py @@ -26,7 +26,6 @@ from mindspore.nn import Momentum from mindspore.nn import ReLU from mindspore.nn import TrainOneStepCell, WithLossCell from mindspore.ops.operations.comm_ops import AllReduce, AllGather, _AlltoAll, ReduceOp, ReduceScatter -from mindspore.ops.operations.comm_ops import HostAllGather, HostReduceScatter from mindspore.ops.operations.comm_ops import Broadcast # pylint: disable=W0212 @@ -87,21 +86,6 @@ class AllGatherNet(nn.Cell): return self.relu(x) -class HostAllGatherNet(nn.Cell): - """HostAllGatherNet definition""" - - def __init__(self, input_channel, output_channel): - super(HostAllGatherNet, self).__init__() - self.dense = Dense(input_channel, output_channel) - self.hostallgather = HostAllGather((0, 1)) - self.relu = ReLU() - - def construct(self, x): - x = self.dense(x) - x = self.hostallgather(x) - return self.relu(x) - - class ReduceScatterNet(nn.Cell): """ReduceScatterNet definition""" @@ -117,21 +101,6 @@ class ReduceScatterNet(nn.Cell): return self.relu(x) -class HostReduceScatterNet(nn.Cell): - """HostReduceScatterNet definition""" - - def __init__(self, input_channel, out_channel, op): - super(HostReduceScatterNet, self).__init__() - self.dense = Dense(input_channel, out_channel) - self.hostreducescatter = HostReduceScatter(op, (0, 1)) - self.relu = ReLU() - - def construct(self, x): - x = self.dense(x) - x = self.hostreducescatter(x) - return self.relu(x) - - class AlltoAllNet(nn.Cell): """AlltoAllNet definition""" @@ -185,21 +154,6 @@ def test_allgather(): _executor.compile(network, input_tensor, label_tensor) -def test_hostallgather(): - """test_hostallgather""" - context.set_context(mode=context.GRAPH_MODE) - input_tensor = Tensor(np.array([[1.2, 2.1], [2.2, 3.2]], dtype=np.float32)) - label_tensor = Tensor(np.array([[1.2], [2.2], [3.2], [4.2]], dtype=np.float32)) - network = HostAllGatherNet(2, 1) - loss_fn = nn.SoftmaxCrossEntropyWithLogits() - optimizer = Momentum(filter(lambda x: x.requires_grad, network.get_parameters()), - learning_rate=0.1, - momentum=0.9) - network = WithLossCell(network, loss_fn) - network = TrainOneStepCell(network, optimizer) - _executor.compile(network, input_tensor, label_tensor) - - def run_reducescatter(op): """run_reducescatter""" context.set_context(mode=context.GRAPH_MODE) @@ -221,21 +175,6 @@ def test_reducescatter(): run_reducescatter(ReduceOp.SUM) -def test_hostreducescatter(): - """test_hostreducescatter""" - context.set_context(mode=context.GRAPH_MODE) - input_tensor = Tensor(np.array([[1.2, 2.1], [2.2, 3.2]], dtype=np.float32)) - label_tensor = Tensor(np.array([[1.2]], dtype=np.float32)) - network = HostReduceScatterNet(2, 1, ReduceOp.SUM) - loss_fn = nn.SoftmaxCrossEntropyWithLogits() - optimizer = Momentum(filter(lambda x: x.requires_grad, network.get_parameters()), - learning_rate=0.1, - momentum=0.9) - network = WithLossCell(network, loss_fn) - network = TrainOneStepCell(network, optimizer) - _executor.compile(network, input_tensor, label_tensor) - - def test_broadcast(): """test_broadcast""" context.set_context(mode=context.GRAPH_MODE) From ad06ab4049a8a3b09c69d2dc5cbc1ca7db9ff605 Mon Sep 17 00:00:00 2001 From: yao_yf Date: Tue, 23 Jun 2020 11:29:04 +0800 Subject: [PATCH 016/254] dont create hccl group in auto parallel strategy search --- .../parallel/auto_parallel/graph_costmodel.cc | 4 +++- .../ccsrc/parallel/ops_info/reshape_info.cc | 16 +++++++++++++--- mindspore/ccsrc/parallel/ops_info/reshape_info.h | 1 + 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/mindspore/ccsrc/parallel/auto_parallel/graph_costmodel.cc b/mindspore/ccsrc/parallel/auto_parallel/graph_costmodel.cc index 043d3a9e23..05be097e6a 100644 --- a/mindspore/ccsrc/parallel/auto_parallel/graph_costmodel.cc +++ b/mindspore/ccsrc/parallel/auto_parallel/graph_costmodel.cc @@ -1406,7 +1406,9 @@ Status CostGraph::InitSelectedStrategy() { int32_t next_index = reshape_info->next_operator_index(); reshape_info->SetOutputLayout((*next_iter)->next_operator()->inputs_tensor_info()[next_index].tensor_layout()); } - return reshape_info->Init(nullptr); + if (reshape_info->Init(nullptr) != SUCCESS) { + return FAILED; + } } } return SUCCESS; diff --git a/mindspore/ccsrc/parallel/ops_info/reshape_info.cc b/mindspore/ccsrc/parallel/ops_info/reshape_info.cc index c470c379ff..57e1a76d0a 100644 --- a/mindspore/ccsrc/parallel/ops_info/reshape_info.cc +++ b/mindspore/ccsrc/parallel/ops_info/reshape_info.cc @@ -133,9 +133,13 @@ Status ReshapeInfo::GetParameterInput() { Status ReshapeInfo::ComputeReplaceOp() { RankList dev_list = global_device_list(); - TensorRedistribution tensor_redistribution(true, true); + TensorRedistribution tensor_redistribution(!is_generating_costs_, true); if (tensor_redistribution.Init(input_layout_, output_layout_, dev_list) == FAILED) { - MS_LOG(ERROR) << name_ << ": tensor_redistribution init failed."; + if (is_generating_costs_) { + MS_LOG(DEBUG) << name_ << ": tensor_redistribution init failed."; + } else { + MS_LOG(ERROR) << name_ << ": tensor_redistribution init failed."; + } return FAILED; } MS_LOG(DEBUG) << name_ << ": input " << input_layout_.ToString(); @@ -143,7 +147,11 @@ Status ReshapeInfo::ComputeReplaceOp() { MS_LOG(DEBUG) << name_ << ": dev_list " << dev_list.size(); RedistributionOpListPtr redistribution_oplist_ptr = tensor_redistribution.InferTensorRedistributionOperatorList(); if (redistribution_oplist_ptr == nullptr) { - MS_LOG(ERROR) << name_ << "InferTensorRedistribution failed."; + if (is_generating_costs_) { + MS_LOG(DEBUG) << name_ << "InferTensorRedistribution failed."; + } else { + MS_LOG(ERROR) << name_ << "InferTensorRedistribution failed."; + } return FAILED; } replace_op_ = redistribution_oplist_ptr->first; @@ -444,6 +452,7 @@ Status ReshapeInfo::GenerateStrategies(int32_t stage_id) { Status ReshapeInfo::GenetateStrategyCosts(const std::vector> &pre_stra_costs, const std::vector> &next_stra_costs, int32_t out_index, int32_t in_index, bool is_prev_param) { + is_generating_costs_ = true; for (auto pre_stra_cost : pre_stra_costs) { std::vector pre_out_tensor_infos; if (is_prev_param) { @@ -488,6 +497,7 @@ Status ReshapeInfo::GenetateStrategyCosts(const std::vector Date: Tue, 23 Jun 2020 18:36:12 +0800 Subject: [PATCH 017/254] code review stage 2 --- mindspore/ccsrc/kernel/kash/kernel_pack.cc | 2 ++ mindspore/ccsrc/kernel/oplib/oplib.cc | 2 ++ .../ccsrc/kernel/tbe/tbe_kernel_build.cc | 28 +++++++++---------- .../kernel/tbe/tbe_kernel_parallel_build.cc | 2 +- .../kernel/tbe/tbe_kernel_parallel_build.h | 2 +- .../tbe_kernel_broadcast_selecter.cc | 5 ++-- .../tbe_kernel_reduce_selecter.cc | 5 ++-- .../tbe_kernel_select/tbe_kernel_select.cc | 5 ++-- mindspore/ccsrc/kernel/tbe/tbe_utils.cc | 27 +++++++++--------- 9 files changed, 40 insertions(+), 38 deletions(-) diff --git a/mindspore/ccsrc/kernel/kash/kernel_pack.cc b/mindspore/ccsrc/kernel/kash/kernel_pack.cc index 79e2ab9dbb..a87441031b 100644 --- a/mindspore/ccsrc/kernel/kash/kernel_pack.cc +++ b/mindspore/ccsrc/kernel/kash/kernel_pack.cc @@ -50,6 +50,8 @@ bool CheckHash(const std::string &json_file, const std::string &bin_file, const } // namespace const std::string KernelPack::Serialize() const { + MS_EXCEPTION_IF_NULL(json_); + MS_EXCEPTION_IF_NULL(kernel_); std::string buffer; (void)buffer.append((const char *)json_, json_->len + sizeof(json_->len)); (void)buffer.append((const char *)kernel_, kernel_->len + sizeof(kernel_->len)); diff --git a/mindspore/ccsrc/kernel/oplib/oplib.cc b/mindspore/ccsrc/kernel/oplib/oplib.cc index 35bc407026..e01bbe9162 100644 --- a/mindspore/ccsrc/kernel/oplib/oplib.cc +++ b/mindspore/ccsrc/kernel/oplib/oplib.cc @@ -293,8 +293,10 @@ bool OpLib::GetRefInfo(const std::shared_ptr &op_info) { const auto &output_infos = op_info->outputs_ptr(); const auto &input_infos = op_info->inputs_ptr(); for (size_t out_index = 0; out_index < output_infos.size(); out_index++) { + MS_EXCEPTION_IF_NULL(output_infos[out_index]); const auto &out_name = output_infos[out_index]->name(); for (size_t in_index = 0; in_index < input_infos.size(); in_index++) { + MS_EXCEPTION_IF_NULL(input_infos[in_index]); const auto &in_name = input_infos[in_index]->name(); if (out_name == in_name) { if (op_info->has_ref_index(out_index)) { diff --git a/mindspore/ccsrc/kernel/tbe/tbe_kernel_build.cc b/mindspore/ccsrc/kernel/tbe/tbe_kernel_build.cc index 76df819043..645a195f5e 100644 --- a/mindspore/ccsrc/kernel/tbe/tbe_kernel_build.cc +++ b/mindspore/ccsrc/kernel/tbe/tbe_kernel_build.cc @@ -189,7 +189,7 @@ bool TbeKernelJsonCreator::GenInputList(const std::shared_ptr &anf_node input_list->emplace_back(input_desc_json); continue; } - MS_LOG(ERROR) << "input num: " << *real_input_index << " is not match op inputs"; + MS_LOG(ERROR) << "Input num: " << *real_input_index << " is not match op inputs"; return false; } if (op_name == "BatchNorm") { @@ -197,7 +197,7 @@ bool TbeKernelJsonCreator::GenInputList(const std::shared_ptr &anf_node auto attr = primitive->GetAttr("is_training"); MS_EXCEPTION_IF_NULL(attr); bool is_training = GetValue(attr); - MS_LOG(INFO) << "op_name" << op_name << ", tensor_name " << input_ptr->name() << ", is_training " + MS_LOG(INFO) << "Op_name" << op_name << ", tensor_name " << input_ptr->name() << ", is_training " << is_training; if (is_training) { (*real_input_index)++; @@ -230,7 +230,7 @@ bool GetInputNameAndRealNum(const std::shared_ptr &anf_node, const std: if (input_ptr->param_type() == kParamDynamic) { if (*dyn_input_index >= dyn_input_sizes.size()) { - MS_LOG(ERROR) << "dyn input index" << *dyn_input_index << "is over dyn input num" << dyn_input_sizes.size(); + MS_LOG(ERROR) << "Dyn input index" << *dyn_input_index << "is over dyn input num" << dyn_input_sizes.size(); return false; } *input_num = IntToSize(dyn_input_sizes[*dyn_input_index]); @@ -314,7 +314,7 @@ bool TbeKernelJsonCreator::GenOutputDescJson( output_obj_num = real_output_num; } else { if (output_idx >= real_output_num) { - MS_LOG(INFO) << "op:" << op_name << ", output" << output_ptr->name() << " is optional, output is none."; + MS_LOG(INFO) << "Op:" << op_name << ", output" << output_ptr->name() << " is optional, output is none."; std::vector output_list; nlohmann::json output_obj; output_obj[kJName] = output_ptr->name(); @@ -389,7 +389,7 @@ bool TbeKernelJsonCreator::GenTbeAttrJson(const std::shared_ptr &anf_no attr_obj[kJValid] = false; } else { if (attr_ptr->param_type() == kParamRequred && creater_type_ == SINGLE_BUILD) { - MS_LOG(EXCEPTION) << "op name: " << op_info->op_name() << " attr: " << attr_name + MS_LOG(EXCEPTION) << "Op name: " << op_info->op_name() << " attr: " << attr_name << " is required, but not set."; } else { attr_obj[kJValid] = false; @@ -451,7 +451,7 @@ void TbeKernelJsonCreator::ParseAttrValue(const std::string &type, const mindspo auto attr_value = GetValue>>(value); (*attr_obj)[kJValue] = attr_value; } else { - MS_LOG(EXCEPTION) << "type: " << type << "not support"; + MS_LOG(EXCEPTION) << "Type: " << type << "not support"; } } @@ -536,7 +536,7 @@ std::string TbeKernelJsonCreator::GetDeviceOutputFormat(const AnfNodePtr &anf_no bool TbeKernelBuild::GetIOSize(const nlohmann::json &kernel_json, std::vector *input_size_list, std::vector *output_size_list) { if (input_size_list == nullptr || output_size_list == nullptr) { - MS_LOG(ERROR) << "input size or output size is nullptr"; + MS_LOG(ERROR) << "Input size or output size is nullptr"; return false; } input_size_list->clear(); @@ -750,7 +750,7 @@ bool TbeKernelBuild::GenFusionDataInputJson(const std::shared_ptr output_desc_list; if (!data_input) { - MS_LOG(INFO) << "data input is optional node"; + MS_LOG(INFO) << "Data input is optional node"; auto name = std::string(kOptional) + std::to_string(*index); (*data_str)[kJName] = name; nlohmann::json output_desc; @@ -766,7 +766,7 @@ bool TbeKernelBuild::GenFusionDataInputJson(const std::shared_ptrfullname_with_scope() << " index:" << real_idx; + MS_LOG(INFO) << "Real name " << real_node->fullname_with_scope() << " index:" << real_idx; // kJOutputDesc nlohmann::json output_desc; GenDescJson(real_node, real_idx, real_idx, &output_desc, fusion_data_type); @@ -842,18 +842,18 @@ bool TbeKernelBuild::GenFusionComputeInputJson(const mindspore::CNodePtr &cnode, auto kernel_idx = AnfAlgo::VisitKernel(input, 0); auto real_node = kernel_idx.first; size_t real_idx = kernel_idx.second; - MS_LOG(INFO) << "real name" << real_node->fullname_with_scope() << "index:" << real_idx; + MS_LOG(INFO) << "Real name" << real_node->fullname_with_scope() << "index:" << real_idx; nlohmann::json input_desc; GenDescJson(real_node, real_idx, real_idx, &input_desc); if (is_dynamic_input) { - MS_LOG(INFO) << "node has dynamic input."; + MS_LOG(INFO) << "Node has dynamic input."; input_desc[kJDynIndex] = (i - 1); } input_desc_list_tmp.emplace_back(input_desc); } size_t optional_num = GetOptionalInput(cnode, is_dynamic_input); if (optional_num > 0) { - MS_LOG(INFO) << "node has optional input."; + MS_LOG(INFO) << "Node has optional input."; for (size_t i = 0; i < optional_num; ++i) { nlohmann::json optional_input_desc; optional_input_desc[kJName] = std::string(kOptional) + std::to_string(*index); @@ -871,7 +871,7 @@ std::vector TbeKernelBuild::GetDescOutputIndex(const std::vector &o std::vector desc_output_index = {}; for (size_t idx = 0; idx < output_used_nums.size(); ++idx) { auto output_use_num_item = output_used_nums[idx]; - MS_LOG(INFO) << "output used num[" << idx << "] = " << output_use_num_item; + MS_LOG(INFO) << "Output used num[" << idx << "] = " << output_use_num_item; desc_output_index.emplace_back(idx); if (output_use_num_item > 1) { desc_output_index.emplace_back(idx); @@ -990,7 +990,7 @@ bool TbeKernelBuild::GetIOSize(const nlohmann::json &fusion_op_list, auto op_output_desces = op[kJOutputDesc]; if (output_node != real_node) { // tuple_get item - MS_LOG(INFO) << "output is a tuple getitem node"; + MS_LOG(INFO) << "Output is a tuple getitem node"; auto output_desc = op_output_desces[real_idx]; if (output_desc[kJShape].empty()) { MS_LOG(INFO) << "Fusion error: output_desc's shape is empty. real_index " << real_idx; diff --git a/mindspore/ccsrc/kernel/tbe/tbe_kernel_parallel_build.cc b/mindspore/ccsrc/kernel/tbe/tbe_kernel_parallel_build.cc index 79e5e0e109..43d492f397 100644 --- a/mindspore/ccsrc/kernel/tbe/tbe_kernel_parallel_build.cc +++ b/mindspore/ccsrc/kernel/tbe/tbe_kernel_parallel_build.cc @@ -77,7 +77,7 @@ bool TbeOpParallelPreBuild(const std::vector &anf_nodes) { return true; } -bool TbeOpParallelBuild(std::vector anf_nodes) { +bool TbeOpParallelBuild(const std::vector &anf_nodes) { auto build_manger = std::make_shared(); MS_EXCEPTION_IF_NULL(build_manger); set processed_kernel; diff --git a/mindspore/ccsrc/kernel/tbe/tbe_kernel_parallel_build.h b/mindspore/ccsrc/kernel/tbe/tbe_kernel_parallel_build.h index c900baf036..637c03bce3 100644 --- a/mindspore/ccsrc/kernel/tbe/tbe_kernel_parallel_build.h +++ b/mindspore/ccsrc/kernel/tbe/tbe_kernel_parallel_build.h @@ -27,7 +27,7 @@ namespace mindspore { namespace kernel { bool TbeOpParallelPreBuild(const std::vector &anf_nodes); -bool TbeOpParallelBuild(std::vector anf_nodes); +bool TbeOpParallelBuild(const std::vector &anf_nodes); struct KernelBuildTaskInfo { AnfNode *node; diff --git a/mindspore/ccsrc/kernel/tbe/tbe_kernel_select/tbe_kernel_broadcast_selecter.cc b/mindspore/ccsrc/kernel/tbe/tbe_kernel_select/tbe_kernel_broadcast_selecter.cc index 9d28af3f3f..8050f02f95 100644 --- a/mindspore/ccsrc/kernel/tbe/tbe_kernel_select/tbe_kernel_broadcast_selecter.cc +++ b/mindspore/ccsrc/kernel/tbe/tbe_kernel_select/tbe_kernel_broadcast_selecter.cc @@ -20,7 +20,6 @@ namespace mindspore { namespace kernel { -constexpr char kDynInputKey[] = "dyn_input_sizes"; constexpr size_t kInputIndex_0 = 0; constexpr size_t kChannelN = 0; constexpr size_t kChannelC = 1; @@ -34,9 +33,9 @@ bool TbeKernelBroadCastSelecter::GetShapeInfo(SupportFormat *support_format) { output_num_ = 0; input_shapes_.clear(); output_shapes_.clear(); - if (AnfAlgo::HasNodeAttr(kDynInputKey, cnode_ptr_)) { + if (AnfAlgo::HasNodeAttr(kAttrDynInputSizes, cnode_ptr_)) { MS_LOG(INFO) << "This broadcast node has dynamic input."; - auto dynamic_size_vec = AnfAlgo::GetNodeAttr>(cnode_ptr_, kDynInputKey); + auto dynamic_size_vec = AnfAlgo::GetNodeAttr>(cnode_ptr_, kAttrDynInputSizes); if (dynamic_size_vec.empty() || dynamic_size_vec[0] < 2) { MS_LOG(EXCEPTION) << "dynamic attr set error, please check."; } diff --git a/mindspore/ccsrc/kernel/tbe/tbe_kernel_select/tbe_kernel_reduce_selecter.cc b/mindspore/ccsrc/kernel/tbe/tbe_kernel_select/tbe_kernel_reduce_selecter.cc index da0466feaa..3f8e5b85c3 100644 --- a/mindspore/ccsrc/kernel/tbe/tbe_kernel_select/tbe_kernel_reduce_selecter.cc +++ b/mindspore/ccsrc/kernel/tbe/tbe_kernel_select/tbe_kernel_reduce_selecter.cc @@ -23,7 +23,6 @@ namespace mindspore { namespace kernel { -constexpr char kKeepDims[] = "keep_dims"; constexpr char kAxis[] = "axis"; constexpr char kTypeInt32[] = "Int32"; constexpr size_t kInputIndex_0 = 0; @@ -148,12 +147,12 @@ void TbeKernelReduceSelecter::GetReduceAttrAxis() { } void TbeKernelReduceSelecter::GetReduceAttrKeepDim() { - if (!AnfAlgo::HasNodeAttr(kKeepDims, cnode_ptr_)) { + if (!AnfAlgo::HasNodeAttr(kAttrKeepDims, cnode_ptr_)) { MS_LOG(INFO) << "This node does't have keep_attr."; keep_dims_ = false; return; } - keep_dims_ = AnfAlgo::GetNodeAttr(cnode_ptr_, kKeepDims); + keep_dims_ = AnfAlgo::GetNodeAttr(cnode_ptr_, kAttrKeepDims); } void TbeKernelReduceSelecter::AssignSupportFormat(const std::string &support_format_str, diff --git a/mindspore/ccsrc/kernel/tbe/tbe_kernel_select/tbe_kernel_select.cc b/mindspore/ccsrc/kernel/tbe/tbe_kernel_select/tbe_kernel_select.cc index 573ad176cf..5ef5d50e9c 100644 --- a/mindspore/ccsrc/kernel/tbe/tbe_kernel_select/tbe_kernel_select.cc +++ b/mindspore/ccsrc/kernel/tbe/tbe_kernel_select/tbe_kernel_select.cc @@ -39,7 +39,6 @@ constexpr auto kDtype = "dtype"; constexpr auto kFormat = "format"; constexpr auto kPrefixInput = "input"; constexpr auto kPrefixOutput = "output"; -constexpr char kDynInputKey[] = "dyn_input_sizes"; constexpr char kParamTypeDynamic[] = "dynamic"; constexpr char kParamTypeRequre[] = "required"; constexpr char kParamTypeOptional[] = "optional"; @@ -87,8 +86,8 @@ void TbeKernelSelect::GetCommonPatternKernelInfo(const OpInfo &op_info) { auto primitive = AnfAlgo::GetCNodePrimitive(cnode_ptr_); MS_EXCEPTION_IF_NULL(primitive); std::vector dyn_input_sizes; - if (primitive->HasAttr(kDynInputKey)) { - dyn_input_sizes = GetValue>(primitive->GetAttr(kDynInputKey)); + if (primitive->HasAttr(kAttrDynInputSizes)) { + dyn_input_sizes = GetValue>(primitive->GetAttr(kAttrDynInputSizes)); } // get real input/output num size_t real_input_tensor_num = AnfAlgo::GetInputTensorNum(cnode_ptr_); diff --git a/mindspore/ccsrc/kernel/tbe/tbe_utils.cc b/mindspore/ccsrc/kernel/tbe/tbe_utils.cc index a930fd3dca..ae7e5cb6d5 100644 --- a/mindspore/ccsrc/kernel/tbe/tbe_utils.cc +++ b/mindspore/ccsrc/kernel/tbe/tbe_utils.cc @@ -59,14 +59,14 @@ void TbeUtils::SaveJsonInfo(const std::string &json_name, const std::string &inf MS_LOG(INFO) << "json file exist, no need to create."; return; } - std::ofstream filewrite; - filewrite.open(path); - if (!filewrite.is_open()) { + std::ofstream file_write; + file_write.open(path); + if (!file_write.is_open()) { return; } - filewrite << info << std::endl; - filewrite.close(); - if (nullptr == realpath(path.c_str(), real_path)) { + file_write << info << std::endl; + file_write.close(); + if (realpath(path.c_str(), real_path) == nullptr) { MS_LOG(INFO) << "dir: " << path << "does not exit."; return; } @@ -144,12 +144,12 @@ uintptr_t KernelManager::GenFuncStub(const mindspore::kernel::KernelPack &kernel auto kernel_json_info = kernel_pack.kernel_json_info(); *block_dim = kernel_json_info.block_dim; - string funcname = kernel_json_info.kernel_name; + string func_name = kernel_json_info.kernel_name; string magic = kernel_json_info.magic; if (!force_reload) { // use the cached object. - auto iter = info_table_.find(funcname); + auto iter = info_table_.find(func_name); if (iter != info_table_.end()) { auto kernelmeta = iter->second; *block_dim = kernelmeta->block_dim_; @@ -157,23 +157,24 @@ uintptr_t KernelManager::GenFuncStub(const mindspore::kernel::KernelPack &kernel } } void *module = nullptr; - if (0 != BinaryRegister((*kernel_pack.GetKernel()), &module, magic)) { + if (BinaryRegister((*kernel_pack.GetKernel()), &module, magic) != 0) { MS_LOG(INFO) << "Call runtime BinaryRegister error."; return 0; } // to diff different funcs. - uintptr_t funcstub = ++kernel_stub_gen_; + uintptr_t func_stub = ++kernel_stub_gen_; if (RT_ERROR_NONE != - rtFunctionRegister(module, reinterpret_cast(funcstub), funcname.c_str(), funcname.c_str(), 0)) { + rtFunctionRegister(module, reinterpret_cast(func_stub), func_name.c_str(), func_name.c_str(), 0)) { MS_LOG(INFO) << "Call runtime rtFunctionRegister error."; return 0; } // cache the registered kernelmeta. - info_table_[funcname] = std::make_shared(KernelMetaInfo{funcstub, *block_dim}); - return funcstub; + info_table_[func_name] = std::make_shared(KernelMetaInfo{func_stub, *block_dim}); + return func_stub; } std::string KernelManager::GetStubFuncName(const KernelPackPtr &kernel_pack) { + MS_EXCEPTION_IF_NULL(kernel_pack); auto kernel_json_info = kernel_pack->kernel_json_info(); return kernel_json_info.kernel_name; } From 09c50ee43577c104c45b663c3b1893b7abba7f9d Mon Sep 17 00:00:00 2001 From: zhoufeng Date: Tue, 23 Jun 2020 19:14:39 +0800 Subject: [PATCH 018/254] Graph kernel use control sink Signed-off-by: zhoufeng --- mindspore/ccsrc/session/session_basic.cc | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/mindspore/ccsrc/session/session_basic.cc b/mindspore/ccsrc/session/session_basic.cc index ff6fa8ff94..730c20d699 100644 --- a/mindspore/ccsrc/session/session_basic.cc +++ b/mindspore/ccsrc/session/session_basic.cc @@ -436,7 +436,12 @@ CNodePtr SessionBasic::CreateNewCNode(const CNodePtr &cnode, KernelGraph *graph) std::vector cnode_inputs; auto attr_input = cnode->input(kAnfPrimitiveIndex); MS_EXCEPTION_IF_NULL(attr_input); - if (IsValueNode(attr_input)) { + if (AnfAlgo::IsGraphKernel(cnode)) { + auto fg = AnfAlgo::GetCNodeFuncGraphPtr(cnode); + MS_EXCEPTION_IF_NULL(fg); + auto new_fg = BasicClone(fg); + cnode_inputs.push_back(std::make_shared(new_fg)); + } else if (IsValueNode(attr_input)) { // create primitive of cnode:call cnode_inputs = {graph->NewValueNode(NewValueNode(std::make_shared(prim::kPrimCall->name())))}; // create a ValueNode as input of cnode:call From 2f1cec7225698a967f1285c1b3f1d26a30ea47fa Mon Sep 17 00:00:00 2001 From: liuwenhao4 Date: Fri, 19 Jun 2020 15:33:33 +0800 Subject: [PATCH 019/254] Fix some mistakes of TransData vm ops --- mindspore/ops/_op_impl/tbe/trans_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mindspore/ops/_op_impl/tbe/trans_data.py b/mindspore/ops/_op_impl/tbe/trans_data.py index c0cce302cd..91b4bc85f5 100644 --- a/mindspore/ops/_op_impl/tbe/trans_data.py +++ b/mindspore/ops/_op_impl/tbe/trans_data.py @@ -50,7 +50,7 @@ trans_data_op_info = TBERegOp("TransData") \ .dtype_format(DataType.F16_HWCN, DataType.F16_FracZ) \ .dtype_format(DataType.F16_FracZ, DataType.F16_HWCN) \ .dtype_format(DataType.F16_C1HWNCoC0, DataType.F16_HWCN) \ - .dtype_format(DataType.F16_HWCN, DataType.F16_5HD) \ + .dtype_format(DataType.F16_HWCN, DataType.F16_C1HWNCoC0) \ .dtype_format(DataType.F16_Default, DataType.F16_FracNZ) \ .dtype_format(DataType.F32_Default, DataType.F32_FracNZ) \ .dtype_format(DataType.F16_FracNZ, DataType.F16_Default) \ From 8f56528f8c918b5e6688a89e34fb8e695b698d3d Mon Sep 17 00:00:00 2001 From: Wei Luning Date: Mon, 22 Jun 2020 23:21:01 +0800 Subject: [PATCH 020/254] add flags on function --- .gitignore | 1 + mindspore/ccsrc/ir/primitive.cc | 2 +- mindspore/ccsrc/ir/tensor.cc | 4 ++-- mindspore/ccsrc/pipeline/parse/parse.cc | 13 ++++++---- mindspore/ccsrc/pipeline/parse/parse.h | 6 ++--- mindspore/nn/layer/basic.py | 3 +-- mindspore/nn/wrap/loss_scale.py | 2 +- mindspore/ops/composite/base.py | 24 ++++++++++++------- .../Transformer/src/transformer_for_train.py | 2 +- model_zoo/bert/src/bert_for_pre_training.py | 7 +++--- model_zoo/deeplabv3/src/deeplabv3.py | 4 +++- .../models/bert/src/bert_for_pre_training.py | 2 +- tests/ut/python/keep_order/test_keep_order.py | 2 +- tests/ut/python/ops/test_math_ops.py | 2 +- .../ge/model/test_lenet_model.py | 4 ++-- 15 files changed, 44 insertions(+), 34 deletions(-) diff --git a/.gitignore b/.gitignore index 77ff222a1a..057169ec42 100644 --- a/.gitignore +++ b/.gitignore @@ -26,6 +26,7 @@ cmake-build-debug *_pb2.py *.pb.h *.pb.cc +*.pb # Object files *.o diff --git a/mindspore/ccsrc/ir/primitive.cc b/mindspore/ccsrc/ir/primitive.cc index 59497affd5..6ec27c2567 100644 --- a/mindspore/ccsrc/ir/primitive.cc +++ b/mindspore/ccsrc/ir/primitive.cc @@ -86,7 +86,7 @@ void PrimitivePy::AddPyAttr(const py::str &name, const py::object &obj) { } bool converted = parse::ConvertData(obj, &converted_ret); if (!converted) { - MS_LOG(EXCEPTION) << "Attribute convert error with type:" << std::string(py::str(obj)); + MS_LOG(EXCEPTION) << "Attribute convert error with type: " << std::string(py::str(obj)); } (void)this->AddAttr(attr_name, converted_ret); } diff --git a/mindspore/ccsrc/ir/tensor.cc b/mindspore/ccsrc/ir/tensor.cc index e5212e922d..4b02fdf2a7 100644 --- a/mindspore/ccsrc/ir/tensor.cc +++ b/mindspore/ccsrc/ir/tensor.cc @@ -345,14 +345,14 @@ abstract::AbstractBasePtr Tensor::ToAbstract() { std::string Tensor::GetShapeAndDataTypeInfo() const { std::ostringstream buf; - buf << "Tensor \nshape:[" << shape() << "]" << this->Dtype()->ToString(); + buf << "Tensor shape:[" << shape() << "]" << this->Dtype()->ToString(); return buf.str(); } std::string Tensor::ToString() const { const int small_tensor_size = 30; std::ostringstream buf; - buf << "Tensor \nshape:[" << shape() << "]" << this->Dtype()->ToString(); + buf << "Tensor shape:[" << shape() << "]" << this->Dtype()->ToString(); // only print small tensor if (DataSize() < small_tensor_size) { buf << "val:" << std::string(py::str(data())); diff --git a/mindspore/ccsrc/pipeline/parse/parse.cc b/mindspore/ccsrc/pipeline/parse/parse.cc index 6d5c28c98c..66908240cb 100644 --- a/mindspore/ccsrc/pipeline/parse/parse.cc +++ b/mindspore/ccsrc/pipeline/parse/parse.cc @@ -234,7 +234,11 @@ FunctionBlockPtr Parser::ParseFunction(const py::object &node, const FunctionBlo current_fg->debug_info()->set_deco_location(GetLocation(deco_list)); } - bool set_flag = ast_->UpdateFuncGraphFlags(current_fg); + bool set_flag = UpdateFuncGraphFlags(ast_->function(), current_fg); + if (ast_->obj() != ast_->function()) { + set_flag = set_flag && UpdateFuncGraphFlags(ast_->obj(), current_fg); + } + if (!set_flag) { MS_LOG(ERROR) << "Set flags failed"; return nullptr; @@ -1436,17 +1440,17 @@ bool ParseAst::IsClassMember(const py::object &node) { return ret.cast(); } -bool ParseAst::UpdateFuncGraphFlags(const FuncGraphPtr &func_graph) { +bool UpdateFuncGraphFlags(py::object obj, const FuncGraphPtr &func_graph) { if (func_graph == nullptr) { MS_LOG(ERROR) << "FuncGraph is null"; return false; } - if (!py::hasattr(obj_, PYTHON_EXTERN_MINDSPORE_FLAG)) { + if (!py::hasattr(obj, PYTHON_EXTERN_MINDSPORE_FLAG)) { MS_LOG(DEBUG) << "No flags"; return true; } - py::dict flags = python_adapter::GetPyObjAttr(obj_, PYTHON_EXTERN_MINDSPORE_FLAG); + py::dict flags = python_adapter::GetPyObjAttr(obj, PYTHON_EXTERN_MINDSPORE_FLAG); for (auto &item : flags) { if (!py::isinstance(item.first)) { MS_LOG(ERROR) << "Type error in flags dict convert"; @@ -1466,7 +1470,6 @@ bool ParseAst::UpdateFuncGraphFlags(const FuncGraphPtr &func_graph) { return false; } } - return true; } diff --git a/mindspore/ccsrc/pipeline/parse/parse.h b/mindspore/ccsrc/pipeline/parse/parse.h index 0a56ccaed9..19c503c6d0 100644 --- a/mindspore/ccsrc/pipeline/parse/parse.h +++ b/mindspore/ccsrc/pipeline/parse/parse.h @@ -327,9 +327,6 @@ class ParseAst { bool IsClassMember(const py::object &node); - // update the graph flags - bool UpdateFuncGraphFlags(const FuncGraphPtr &func_graph); - private: // save obj,eg: class instance or function py::object obj_; @@ -350,6 +347,9 @@ class ParseAst { int function_line_offset_; }; +// update the graph flags +bool UpdateFuncGraphFlags(py::object obj, const FuncGraphPtr &func_graph); + AnfNodePtr GetMixedPrecisionCastHelp(const FuncGraphPtr &func_graph, const AnfNodePtr ¶m); } // namespace parse diff --git a/mindspore/nn/layer/basic.py b/mindspore/nn/layer/basic.py index b1d5af48c9..d0e6904ec5 100644 --- a/mindspore/nn/layer/basic.py +++ b/mindspore/nn/layer/basic.py @@ -284,7 +284,6 @@ class ClipByNorm(Cell): self.reduce_sum = P.ReduceSum(keep_dims=True) self.select_ = P.Select() self.greater_ = P.Greater() - self.axis = () self.cast = P.Cast() self.zero = Tensor(np.array([0.0]).astype(np.float32)) self.sqrt = P.Sqrt() @@ -299,7 +298,7 @@ class ClipByNorm(Cell): def construct(self, x, clip_norm): """add ms_function decorator for pynative mode""" mul_x = F.square(x) - l2sum = self.cast(self.reduce_sum(mul_x, self.axis), mstype.float32) + l2sum = self.cast(self.reduce_sum(mul_x), mstype.float32) cond = self.greater_(l2sum, self.zero) ones_ = self.fill(self.dtype(cond), self.shape(cond), 1.0) diff --git a/mindspore/nn/wrap/loss_scale.py b/mindspore/nn/wrap/loss_scale.py index 2bae6bbc5c..a9aa4d781b 100644 --- a/mindspore/nn/wrap/loss_scale.py +++ b/mindspore/nn/wrap/loss_scale.py @@ -234,8 +234,8 @@ class TrainOneStepWithLossScaleCell(Cell): if scale_update_cell: self.loss_scale = Parameter(Tensor(scale_update_cell.get_loss_scale(), dtype=mstype.float32), name="loss_scale") - self.add_flags(has_effect=True) + @C.add_flags(has_effect=True) def construct(self, data, label, sens=None): weights = self.weights loss = self.network(data, label) diff --git a/mindspore/ops/composite/base.py b/mindspore/ops/composite/base.py index e283867684..b0f16d82bf 100644 --- a/mindspore/ops/composite/base.py +++ b/mindspore/ops/composite/base.py @@ -30,16 +30,16 @@ from ...common.parameter import Parameter __all__ = [EnvInstance_, TupleAdd_, TupleSlice_, UnpackCall_, TupleGetItemTensor_] -def add_flags(fn, **flags): +def add_flags(fn=None, **flags): """ - An interface to add flag for a function. + An decorator to add flag for a function. Note: Only supports bool value. Args: - fn (Function): Function or cell to add flag. - flags (bool): Flags use kwargs. + fn (Function): Function or cell to add flag. Default: None. + flags (dict): Flags use kwargs. Default: None. Returns: Function, the fn added flags. @@ -47,11 +47,17 @@ def add_flags(fn, **flags): Examples: >>> add_flags(net, predit=True) """ - # need set the attr and access on c++ - if not hasattr(fn, "_mindspore_flags"): - fn._mindspore_flags = {} - fn._mindspore_flags.update({**flags}) - return fn + def deco(fn): + # need set the attr and access on c++ + if not hasattr(fn, "_mindspore_flags"): + fn._mindspore_flags = {} + + fn._mindspore_flags.update({**flags}) + return fn + ret = deco + if fn is not None: + ret = deco(fn) + return ret def core(fn=None, **flags): diff --git a/model_zoo/Transformer/src/transformer_for_train.py b/model_zoo/Transformer/src/transformer_for_train.py index 758ac65ab5..76237bee96 100644 --- a/model_zoo/Transformer/src/transformer_for_train.py +++ b/model_zoo/Transformer/src/transformer_for_train.py @@ -277,8 +277,8 @@ class TransformerTrainOneStepWithLossScaleCell(nn.Cell): if scale_update_cell: self.loss_scale = Parameter(Tensor(scale_update_cell.get_loss_scale(), dtype=mstype.float32), name="loss_scale") - self.add_flags(has_effect=True) + @C.add_flags(has_effect=True) def construct(self, source_eos_ids, source_eos_mask, diff --git a/model_zoo/bert/src/bert_for_pre_training.py b/model_zoo/bert/src/bert_for_pre_training.py index 5e014f02ba..802391ee86 100644 --- a/model_zoo/bert/src/bert_for_pre_training.py +++ b/model_zoo/bert/src/bert_for_pre_training.py @@ -132,9 +132,9 @@ class GetNextSentenceOutput(nn.Cell): def __init__(self, config): super(GetNextSentenceOutput, self).__init__() self.log_softmax = _selected_ops.LogSoftmax() - self.weight_init = TruncatedNormal(config.initializer_range) + weight_init = TruncatedNormal(config.initializer_range) self.dense = nn.Dense(config.hidden_size, 2, - weight_init=self.weight_init, has_bias=True).to_float(config.compute_type) + weight_init=weight_init, has_bias=True).to_float(config.compute_type) self.dtype = config.dtype self.cast = P.Cast() @@ -321,7 +321,6 @@ class BertTrainOneStepCell(nn.Cell): if self.reducer_flag: # apply grad reducer on grads grads = self.grad_reducer(grads) - succ = self.optimizer(grads) return F.depend(loss, succ) @@ -380,8 +379,8 @@ class BertTrainOneStepWithLossScaleCell(nn.Cell): if scale_update_cell: self.loss_scale = Parameter(Tensor(scale_update_cell.get_loss_scale(), dtype=mstype.float32), name="loss_scale") - self.add_flags(has_effect=True) + @C.add_flags(has_effect=True) def construct(self, input_ids, input_mask, diff --git a/model_zoo/deeplabv3/src/deeplabv3.py b/model_zoo/deeplabv3/src/deeplabv3.py index 906a207302..03bb03ad14 100644 --- a/model_zoo/deeplabv3/src/deeplabv3.py +++ b/model_zoo/deeplabv3/src/deeplabv3.py @@ -17,6 +17,7 @@ import numpy as np import mindspore.nn as nn from mindspore.ops import operations as P +from mindspore.ops.composite import add_flags from .backbone.resnet_deeplab import _conv_bn_relu, resnet50_dl, _deep_conv_bn_relu, \ DepthwiseConv2dNative, SpaceToBatch, BatchToSpace @@ -121,6 +122,7 @@ class ASPP(nn.Cell): self.feature_shape = feature_shape self.concat = P.Concat(axis=1) + @add_flags(loop_can_unroll=True) def construct(self, x, scale_index=0): aspp0 = self.aspp0(x) aspp1 = self.global_poolings[scale_index](x) @@ -276,7 +278,7 @@ class SingleDeepLabV3(nn.Cell): atrous_rates=atrous_rates, output_stride=output_stride, fine_tune_batch_norm=fine_tune_batch_norm) - self.aspp.add_flags(loop_can_unroll=True) + atrous_rates_len = 0 if atrous_rates is not None: atrous_rates_len = len(atrous_rates) diff --git a/tests/st/networks/models/bert/src/bert_for_pre_training.py b/tests/st/networks/models/bert/src/bert_for_pre_training.py index 976f1a3c43..7c557a49c9 100644 --- a/tests/st/networks/models/bert/src/bert_for_pre_training.py +++ b/tests/st/networks/models/bert/src/bert_for_pre_training.py @@ -379,8 +379,8 @@ class BertTrainOneStepWithLossScaleCell(nn.Cell): if scale_update_cell: self.loss_scale = Parameter(Tensor(scale_update_cell.get_loss_scale(), dtype=mstype.float32), name="loss_scale") - self.add_flags(has_effect=True) + @C.add_flags(has_effect=True) def construct(self, input_ids, input_mask, diff --git a/tests/ut/python/keep_order/test_keep_order.py b/tests/ut/python/keep_order/test_keep_order.py index 1cf2b8e19a..fa0df6dd5d 100644 --- a/tests/ut/python/keep_order/test_keep_order.py +++ b/tests/ut/python/keep_order/test_keep_order.py @@ -133,8 +133,8 @@ def test_keep_order_io_effect_exception_return_dtype(): self.dtype = P.DType() self.sub = P.Sub() self.neg = P.Neg() - self.add_flags(has_effect=True) + @C.add_flags(has_effect=True) def construct(self, x): init = self.alloc_status() self.clear_status(init) diff --git a/tests/ut/python/ops/test_math_ops.py b/tests/ut/python/ops/test_math_ops.py index 09f113204e..21a3a3c9e1 100755 --- a/tests/ut/python/ops/test_math_ops.py +++ b/tests/ut/python/ops/test_math_ops.py @@ -268,8 +268,8 @@ class NpuFloatNet(nn.Cell): self.reduce_sum = P.ReduceSum(keep_dims=True) self.sub = P.Sub() self.neg = P.Neg() - self.add_flags(has_effect=True) + @C.add_flags(has_effect=True) def construct(self, x): init = self.alloc_status() self.clear_status(init) diff --git a/tests/ut/python/pynative_mode/ge/model/test_lenet_model.py b/tests/ut/python/pynative_mode/ge/model/test_lenet_model.py index f684a175c8..23999c398e 100644 --- a/tests/ut/python/pynative_mode/ge/model/test_lenet_model.py +++ b/tests/ut/python/pynative_mode/ge/model/test_lenet_model.py @@ -14,13 +14,13 @@ # ============================================================================ """ test_lenet_model """ import numpy as np +import pytest import mindspore.nn as nn from mindspore.common.tensor import Tensor from mindspore.nn import WithGradCell, WithLossCell from mindspore.nn.optim import Momentum from mindspore.ops import operations as P -from ....ut_filter import non_graph_engine class LeNet5(nn.Cell): @@ -47,7 +47,7 @@ class LeNet5(nn.Cell): return x -@non_graph_engine +@pytest.mark.skip(reason="need ge backend") def test_lenet_pynative_train_net(): """ test_lenet_pynative_train_net """ data = Tensor(np.ones([1, 1, 32, 32]).astype(np.float32) * 0.01) From eb37669e3b37b076c18bbbe4006bc9ddd13cd976 Mon Sep 17 00:00:00 2001 From: laiyongqiang Date: Tue, 23 Jun 2020 20:02:08 +0800 Subject: [PATCH 021/254] fix bug to remove reshape when reshape is depend's input --- .../ccsrc/pre_activate/mem_reuse/mem_reuse.cc | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/mindspore/ccsrc/pre_activate/mem_reuse/mem_reuse.cc b/mindspore/ccsrc/pre_activate/mem_reuse/mem_reuse.cc index 2927b1204f..210bc232f5 100644 --- a/mindspore/ccsrc/pre_activate/mem_reuse/mem_reuse.cc +++ b/mindspore/ccsrc/pre_activate/mem_reuse/mem_reuse.cc @@ -223,13 +223,21 @@ KernelRefCountPtr MemReuseUtil::GetRef(const AnfNodePtr &node, int output_idx) { } KernelRefCountPtr MemReuseUtil::GetKernelInputRef(const CNodePtr &kernel, size_t input_idx) { + auto is_all_nop_node = opt::IsAllNopNode(graph_); if (input_idx >= AnfAlgo::GetInputTensorNum(kernel)) { MS_LOG(EXCEPTION) << "Input index " << input_idx << " is larger than input number " << AnfAlgo::GetInputTensorNum(kernel); } auto input_node = kernel->input(input_idx + 1); // Graph may be all nop nodes and not remove nop node, so this can not skip nop node. - auto kernel_input = AnfAlgo::VisitKernelWithReturnType(input_node, 0, false); + session::KernelWithIndex kernel_input; + if (is_all_nop_node) { + // The graph does not remove the nop node. + kernel_input = AnfAlgo::VisitKernelWithReturnType(input_node, 0, false); + } else { + // The graph removes the nop node. + kernel_input = AnfAlgo::VisitKernelWithReturnType(input_node, 0, true); + } if (IsPrimitive(kernel_input.first, prim::kPrimMakeTuple)) { MS_LOG(EXCEPTION) << "Input node [" << input_node->DebugString() << "]'s input " << input_idx << " is MakeTuple"; } @@ -257,6 +265,7 @@ void MemReuseUtil::SetKernelDefMap() { } void MemReuseUtil::SetKernelDefInputs() { + auto is_all_nop_node = opt::IsAllNopNode(graph_); for (const auto &kernel : graph_->execution_order()) { MS_EXCEPTION_IF_NULL(kernel); auto key = kernel.get(); @@ -272,7 +281,14 @@ void MemReuseUtil::SetKernelDefInputs() { // set the inputs of this kernel_def auto input_node = AnfAlgo::GetInputNode(kernel, i); // Graph may be all nop nodes and not remove nop node, so this can not skip nop node. - auto input = AnfAlgo::VisitKernelWithReturnType(input_node, 0, false); + session::KernelWithIndex input; + if (is_all_nop_node) { + // The graph does not remove the nop node. + input = AnfAlgo::VisitKernelWithReturnType(input_node, 0, false); + } else { + // The graph removes the nop node. + input = AnfAlgo::VisitKernelWithReturnType(input_node, 0, true); + } if (IsPrimitive(input.first, prim::kPrimMakeTuple)) { MS_LOG(EXCEPTION) << "Input node [" << input_node->DebugString() << "]'s input " << i << " is MakeTuple"; } From dcd5773f640bf84157a947df17f52dbaaac2354f Mon Sep 17 00:00:00 2001 From: Wei Luning Date: Tue, 23 Jun 2020 20:34:44 +0800 Subject: [PATCH 022/254] fix bug in quant deploy export --- mindspore/train/quant/quant.py | 4 +++- mindspore/train/quant/quant_utils.py | 2 +- tests/ut/python/train/quant/test_quant.py | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/mindspore/train/quant/quant.py b/mindspore/train/quant/quant.py index 46b3cd1934..cb4cb39e66 100644 --- a/mindspore/train/quant/quant.py +++ b/mindspore/train/quant/quant.py @@ -154,7 +154,9 @@ class ConvertToQuantNetwork: per_channel=self.act_channel, symmetric=self.act_symmetric, narrow_range=self.act_range) - prefix = '.'.join([network.param_prefix, self._convert_op_name(prim_op.name)]) + prefix = self._convert_op_name(prim_op.name) + if network.param_prefix: + prefix = '.'.join([network.param_prefix, self._convert_op_name(prim_op.name)]) add_quant.update_parameters_name(prefix + '.') del network.__dict__[name] network.insert_child_to_cell(name, add_quant) diff --git a/mindspore/train/quant/quant_utils.py b/mindspore/train/quant/quant_utils.py index c9e6ac92e1..c4a8004012 100644 --- a/mindspore/train/quant/quant_utils.py +++ b/mindspore/train/quant/quant_utils.py @@ -125,7 +125,7 @@ def scale_zp_from_fack_quant_cell(cell, data_type): """ minq = cell.minq.data.asnumpy() maxq = cell.maxq.data.asnumpy() - op = cell.fake_quant + op = cell.fake_quant_infer scale, zp = cal_quantization_params( minq, maxq, data_type, diff --git a/tests/ut/python/train/quant/test_quant.py b/tests/ut/python/train/quant/test_quant.py index c9398be456..54563d86eb 100644 --- a/tests/ut/python/train/quant/test_quant.py +++ b/tests/ut/python/train/quant/test_quant.py @@ -67,7 +67,7 @@ def test_qat_lenet(): img = Tensor(np.ones((32, 1, 32, 32)).astype(np.float32)) net = LeNet5() net = qat.convert_quant_network( - net, quant_delay=0, bn_fold=False, freeze_bn=10000, num_bits=8) + net, freeze_bn=10000, num_bits=8) # should load the checkpoint. mock here for param in net.get_parameters(): param.init_data() From 034d2ea2aa08ad0b55432c305b57fd9ac22c23fb Mon Sep 17 00:00:00 2001 From: wilfChen Date: Tue, 23 Jun 2020 20:40:45 +0800 Subject: [PATCH 023/254] Gpu Adam Fusion --- .../gpu/cuda_impl/adam_weight_decay_impl.cu | 50 ++++++++ .../gpu/cuda_impl/adam_weight_decay_impl.cuh | 24 ++++ .../kernel/gpu/nn/fused_adam_weight_decay.cc | 52 ++++++++ .../kernel/gpu/nn/fused_adam_weight_decay.h | 103 +++++++++++++++ mindspore/ccsrc/operator/ops.h | 2 + .../ccsrc/pre_activate/gpu/adam_fusion.cc | 112 +++++++++++++++++ .../ccsrc/pre_activate/gpu/adam_fusion.h | 56 +++++++++ .../gpu/adam_weight_decay_fusion.cc | 117 ++++++++++++++++++ .../gpu/adam_weight_decay_fusion.h | 58 +++++++++ mindspore/ccsrc/session/gpu_session.cc | 18 ++- mindspore/ccsrc/session/gpu_session.h | 2 + mindspore/ccsrc/utils/utils.h | 2 + tests/st/ops/gpu/test_adam_fusion.py | 87 +++++++++++++ tests/st/ops/gpu/test_batch_matmul.py | 22 ++-- 14 files changed, 694 insertions(+), 11 deletions(-) create mode 100644 mindspore/ccsrc/kernel/gpu/cuda_impl/adam_weight_decay_impl.cu create mode 100644 mindspore/ccsrc/kernel/gpu/cuda_impl/adam_weight_decay_impl.cuh create mode 100644 mindspore/ccsrc/kernel/gpu/nn/fused_adam_weight_decay.cc create mode 100644 mindspore/ccsrc/kernel/gpu/nn/fused_adam_weight_decay.h create mode 100644 mindspore/ccsrc/pre_activate/gpu/adam_fusion.cc create mode 100644 mindspore/ccsrc/pre_activate/gpu/adam_fusion.h create mode 100644 mindspore/ccsrc/pre_activate/gpu/adam_weight_decay_fusion.cc create mode 100644 mindspore/ccsrc/pre_activate/gpu/adam_weight_decay_fusion.h create mode 100644 tests/st/ops/gpu/test_adam_fusion.py diff --git a/mindspore/ccsrc/kernel/gpu/cuda_impl/adam_weight_decay_impl.cu b/mindspore/ccsrc/kernel/gpu/cuda_impl/adam_weight_decay_impl.cu new file mode 100644 index 0000000000..dfadaa09d6 --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/cuda_impl/adam_weight_decay_impl.cu @@ -0,0 +1,50 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "adam_weight_decay_impl.cuh" +#include "device/gpu/cuda_common.h" + +template +__global__ void AdamWeightDecayKernel(const int element_num_, const bool need_decay, const float *beta1, + const float *one_sub_beta1, const float *beta2, const float *one_sub_beta2, + const float *epsilon, const float *lr, const float *weight_decay, T *m, T *v, + T *param, T *gradient) { + for (int i = blockIdx.x * blockDim.x + threadIdx.x; i < element_num_; i += blockDim.x * gridDim.x) { + float next_m = beta1[0] * m[i] + one_sub_beta1[0] * gradient[i]; + float next_v = beta2[0] * v[i] + one_sub_beta2[0] * gradient[i] * gradient[i]; + float update = next_m / (sqrt(next_v) + epsilon[0]); + if (need_decay && weight_decay != nullptr) { + update += weight_decay[0] * param[i]; + } + param[i] -= lr[0] * update; + m[i] = next_m; + v[i] = next_v; + } +} + +template +void AdamWeightDecay(const int &element_num_, const bool &need_decay, const float *beta1, const float *one_sub_beta1, + const float *beta2, const float *one_sub_beta2, const float *epsilon, const float *lr, + const float *weight_decay, T *m, T *v, T *param, T *gradient, cudaStream_t stream) { + AdamWeightDecayKernel<<>>( + element_num_, need_decay, beta1, one_sub_beta1, beta2, one_sub_beta2, epsilon, lr, weight_decay, m, v, param, + gradient); +} + +template void AdamWeightDecay(const int &element_num_, const bool &need_decay, const float *beta1, + const float *one_sub_beta1, const float *beta2, const float *one_sub_beta2, + const float *epsilon, const float *lr, const float *weight_decay, float *m, float *v, + float *param, float *gradient, cudaStream_t stream); diff --git a/mindspore/ccsrc/kernel/gpu/cuda_impl/adam_weight_decay_impl.cuh b/mindspore/ccsrc/kernel/gpu/cuda_impl/adam_weight_decay_impl.cuh new file mode 100644 index 0000000000..2addffbf00 --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/cuda_impl/adam_weight_decay_impl.cuh @@ -0,0 +1,24 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_KERNEL_GPU_CUDA_IMPL_ADAM_WEIGHT_DECAY_H_ +#define MINDSPORE_CCSRC_KERNEL_GPU_CUDA_IMPL_ADAM_WEIGHT_DECAY_H_ +template +void AdamWeightDecay(const int &element_num_, const bool &need_decay, const float *beta1, const float *one_sub_beta1, + const float *beta2, const float *one_sub_beta2, const float *epsilon, const float *lr, + const float *weight_decay, T *m, T *v, T *param, T *gradient, cudaStream_t stream); + +#endif // MINDSPORE_CCSRC_KERNEL_GPU_CUDA_IMPL_ADAM_WEIGHT_DECAY_H_ diff --git a/mindspore/ccsrc/kernel/gpu/nn/fused_adam_weight_decay.cc b/mindspore/ccsrc/kernel/gpu/nn/fused_adam_weight_decay.cc new file mode 100644 index 0000000000..77cb7f8608 --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/nn/fused_adam_weight_decay.cc @@ -0,0 +1,52 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kernel/gpu/nn/fused_adam_weight_decay.h" + +namespace mindspore { +namespace kernel { +MS_REG_GPU_KERNEL_ONE(FusedAdamWeightDecay, + KernelAttr() + .AddInputAttr(kNumberTypeFloat32) + .AddInputAttr(kNumberTypeFloat32) + .AddInputAttr(kNumberTypeFloat32) + .AddInputAttr(kNumberTypeFloat32) + .AddInputAttr(kNumberTypeFloat32) + .AddInputAttr(kNumberTypeFloat32) + .AddInputAttr(kNumberTypeFloat32) + .AddInputAttr(kNumberTypeFloat32) + .AddInputAttr(kNumberTypeFloat32) + .AddInputAttr(kNumberTypeFloat32) + .AddInputAttr(kNumberTypeFloat32) + .AddOutputAttr(kNumberTypeFloat32), + FusedAdamWeightDecayGpuKernel, float) +MS_REG_GPU_KERNEL_ONE(FusedAdam, + KernelAttr() + .AddInputAttr(kNumberTypeFloat32) + .AddInputAttr(kNumberTypeFloat32) + .AddInputAttr(kNumberTypeFloat32) + .AddInputAttr(kNumberTypeFloat32) + .AddInputAttr(kNumberTypeFloat32) + .AddInputAttr(kNumberTypeFloat32) + .AddInputAttr(kNumberTypeFloat32) + .AddInputAttr(kNumberTypeFloat32) + .AddInputAttr(kNumberTypeFloat32) + .AddInputAttr(kNumberTypeFloat32) + .AddOutputAttr(kNumberTypeFloat32), + FusedAdamWeightDecayGpuKernel, float) + +} // namespace kernel +} // namespace mindspore diff --git a/mindspore/ccsrc/kernel/gpu/nn/fused_adam_weight_decay.h b/mindspore/ccsrc/kernel/gpu/nn/fused_adam_weight_decay.h new file mode 100644 index 0000000000..f13f6ed59f --- /dev/null +++ b/mindspore/ccsrc/kernel/gpu/nn/fused_adam_weight_decay.h @@ -0,0 +1,103 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_KERNEL_GPU_NN_FUSED_ADAM_WEIGHT_DECAY_KERNEL_H_ +#define MINDSPORE_CCSRC_KERNEL_GPU_NN_FUSED_ADAM_WEIGHT_DECAY_KERNEL_H_ + +#include +#include "kernel/gpu/gpu_kernel.h" +#include "kernel/gpu/gpu_kernel_factory.h" +#include "kernel/gpu/kernel_constants.h" +#include "kernel/gpu/cuda_impl/adam_weight_decay_impl.cuh" + +namespace mindspore { +namespace kernel { +template +class FusedAdamWeightDecayGpuKernel : public GpuKernel { + public: + FusedAdamWeightDecayGpuKernel() : element_nums_(0), weight_decay_(false) {} + ~FusedAdamWeightDecayGpuKernel() override = default; + + bool Init(const CNodePtr &kernel_node) override { + auto node_name = AnfAlgo::GetCNodeName(kernel_node); + if (node_name == "AdamWeighDecay") { + weight_decay_ = true; + } + + auto shape = AnfAlgo::GetPrevNodeOutputInferShape(kernel_node, 7); + element_nums_ = 1; + for (auto i : shape) { + element_nums_ *= i; + } + + InitSizeLists(); + return true; + } + + const std::vector &GetInputSizeList() const override { return input_size_list_; } + const std::vector &GetOutputSizeList() const override { return output_size_list_; } + const std::vector &GetWorkspaceSizeList() const override { return workspace_size_list_; } + + bool Launch(const std::vector &inputs, const std::vector &, + const std::vector &outputs, void *stream_ptr) override { + float *beta1 = GetDeviceAddress(inputs, 0); + float *one_sub_beta1 = GetDeviceAddress(inputs, 1); + float *beta2 = GetDeviceAddress(inputs, 2); + float *one_sub_beta2 = GetDeviceAddress(inputs, 3); + float *epsilon = GetDeviceAddress(inputs, 4); + float *lr = GetDeviceAddress(inputs, 5); + T *param = GetDeviceAddress(inputs, 6); + T *m = GetDeviceAddress(inputs, 7); + T *v = GetDeviceAddress(inputs, 8); + T *gradient = GetDeviceAddress(inputs, 9); + float *weight_decay = nullptr; + if (weight_decay_) { + weight_decay = GetDeviceAddress(inputs, 10); + } + AdamWeightDecay(element_nums_, true, beta1, one_sub_beta1, beta2, one_sub_beta2, epsilon, lr, weight_decay, m, v, + param, gradient, reinterpret_cast(stream_ptr)); + return true; + } + + protected: + void InitResource() override{}; + void InitSizeLists() override { + input_size_list_.push_back(sizeof(float)); + input_size_list_.push_back(sizeof(float)); + input_size_list_.push_back(sizeof(float)); + input_size_list_.push_back(sizeof(float)); + input_size_list_.push_back(element_nums_ * sizeof(T)); + input_size_list_.push_back(sizeof(float)); + input_size_list_.push_back(sizeof(float)); + input_size_list_.push_back(element_nums_ * sizeof(T)); + if (weight_decay_) { + input_size_list_.push_back(sizeof(float)); + } + output_size_list_.push_back(element_nums_ * sizeof(T)); + } + + private: + std::vector input_size_list_; + std::vector output_size_list_; + std::vector workspace_size_list_; + + int element_nums_; + bool weight_decay_; +}; +} // namespace kernel +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_KERNEL_GPU_NN_FUSED_ADAM_WEIGHT_DECAY_KERNEL_H_ diff --git a/mindspore/ccsrc/operator/ops.h b/mindspore/ccsrc/operator/ops.h index 01812a5529..522b80def6 100755 --- a/mindspore/ccsrc/operator/ops.h +++ b/mindspore/ccsrc/operator/ops.h @@ -182,9 +182,11 @@ extern const PrimitivePtr kPrimReduceMin; extern const PrimitivePtr kPrimNeg; extern const PrimitivePtr kPrimSub; extern const PrimitivePtr kPrimMul; +extern const PrimitivePtr kPrimRealDiv; extern const PrimitivePtr kPrimMinimum; extern const PrimitivePtr kPrimMaximum; extern const PrimitivePtr kPrimSquare; +extern const PrimitivePtr kPrimSqrt; extern const PrimitivePtr kPrimEqual; extern const PrimitivePtr kPrimLess; extern const PrimitivePtr kPrimLessEqual; diff --git a/mindspore/ccsrc/pre_activate/gpu/adam_fusion.cc b/mindspore/ccsrc/pre_activate/gpu/adam_fusion.cc new file mode 100644 index 0000000000..8111ee429d --- /dev/null +++ b/mindspore/ccsrc/pre_activate/gpu/adam_fusion.cc @@ -0,0 +1,112 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "pre_activate/gpu/adam_fusion.h" + +#include +#include +#include + +#include "session/anf_runtime_algorithm.h" +#include "ir/primitive.h" +#include "utils/utils.h" +#include "pre_activate/common/helper.h" + +namespace mindspore { +namespace opt { +namespace { +kernel::KernelBuildInfoPtr GenerateKernelBuildInfo(CNodePtr node) { + std::vector inputs_format; + std::vector outputs_format; + std::vector inputs_type; + std::vector outputs_type; + kernel::KernelBuildInfo::KernelBuildInfoBuilder builder; + + for (size_t input_index = 0; input_index < AnfAlgo::GetInputTensorNum(node); ++input_index) { + inputs_type.push_back(AnfAlgo::GetPrevNodeOutputInferDataType(node, input_index)); + inputs_format.push_back(kOpFormat_DEFAULT); + } + for (size_t output_index = 0; output_index < AnfAlgo::GetOutputTensorNum(node); ++output_index) { + outputs_type.push_back(AnfAlgo::GetOutputInferDataType(node, output_index)); + outputs_format.push_back(kOpFormat_DEFAULT); + } + builder.SetInputsDeviceType(inputs_type); + builder.SetInputsFormat(inputs_format); + builder.SetOutputsDeviceType(outputs_type); + builder.SetOutputsFormat(outputs_format); + return builder.Build(); +} +} // namespace + +const BaseRef AdamFusion::DefinePattern() const { + VectorRef next_m = VectorRef({prim::kPrimTensorAdd, VectorRef({prim::kPrimMul, beta1_, m_}), + VectorRef({prim::kPrimMul, one_sub_beta1_, gradient_})}); + VectorRef next_v = + VectorRef({prim::kPrimTensorAdd, VectorRef({prim::kPrimMul, beta2_, v_}), + VectorRef({prim::kPrimMul, one_sub_beta2_, VectorRef({prim::kPrimSquare, gradient_})})}); + VectorRef update = VectorRef( + {prim::kPrimRealDiv, next_m, VectorRef({prim::kPrimTensorAdd, eps_, VectorRef({prim::kPrimSqrt, next_v})})}); + VectorRef update_with_lr = VectorRef({prim::kPrimMul, lr_, update}); + VectorRef next_param = VectorRef({prim::kPrimSub, param_, update_with_lr}); + VectorRef depend1 = VectorRef({prim::kPrimDepend, next_v, VectorRef({prim::kPrimAssign, param_, next_param})}); + VectorRef depend2 = VectorRef({prim::kPrimDepend, depend1, VectorRef({prim::kPrimAssign, m_, next_m})}); + VectorRef depend3 = VectorRef({prim::kPrimDepend, depend2, VectorRef({prim::kPrimAssign, v_, depend2})}); + return depend3; +} + +const AnfNodePtr AdamFusion::Process(const FuncGraphPtr &graph, const AnfNodePtr &node, const EquivPtr &equiv) const { + MS_EXCEPTION_IF_NULL(graph); + MS_EXCEPTION_IF_NULL(node); + MS_EXCEPTION_IF_NULL(equiv); + auto beta1_input = utils::cast((*equiv)[beta1_]); + auto one_sub_beta1_input = utils::cast((*equiv)[one_sub_beta1_]); + auto beta2_input = utils::cast((*equiv)[beta2_]); + auto one_sub_beta2_input = utils::cast((*equiv)[one_sub_beta2_]); + auto eps_input = utils::cast((*equiv)[eps_]); + auto lr_input = utils::cast((*equiv)[lr_]); + auto param_input = utils::cast((*equiv)[param_]); + auto m_input = utils::cast((*equiv)[m_]); + auto v_input = utils::cast((*equiv)[v_]); + auto gradient_input = utils::cast((*equiv)[gradient_]); + MS_EXCEPTION_IF_NULL(beta1_input); + MS_EXCEPTION_IF_NULL(one_sub_beta1_input); + MS_EXCEPTION_IF_NULL(beta2_input); + MS_EXCEPTION_IF_NULL(one_sub_beta2_input); + MS_EXCEPTION_IF_NULL(eps_input); + MS_EXCEPTION_IF_NULL(lr_input); + MS_EXCEPTION_IF_NULL(param_input); + MS_EXCEPTION_IF_NULL(m_input); + MS_EXCEPTION_IF_NULL(v_input); + MS_EXCEPTION_IF_NULL(gradient_input); + + auto prim = std::make_shared(kFusedAdamName); + MS_EXCEPTION_IF_NULL(prim); + std::vector inputs = { + NewValueNode(prim), beta1_input, one_sub_beta1_input, beta2_input, one_sub_beta2_input, + eps_input, lr_input, param_input, m_input, v_input, + gradient_input}; + auto adam = graph->NewCNode(inputs); + MS_EXCEPTION_IF_NULL(adam); + auto types = {AnfAlgo::GetOutputInferDataType(node, 0)}; + auto shapes = {AnfAlgo::GetOutputInferShape(node, 0)}; + AnfAlgo::SetOutputInferTypeAndShape(types, shapes, adam.get()); + adam->set_scope(node->scope()); + + auto build_info = GenerateKernelBuildInfo(adam); + AnfAlgo::SetSelectKernelBuildInfo(build_info, adam.get()); + return adam; +} +} // namespace opt +} // namespace mindspore diff --git a/mindspore/ccsrc/pre_activate/gpu/adam_fusion.h b/mindspore/ccsrc/pre_activate/gpu/adam_fusion.h new file mode 100644 index 0000000000..d8c10a0986 --- /dev/null +++ b/mindspore/ccsrc/pre_activate/gpu/adam_fusion.h @@ -0,0 +1,56 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_PRE_ACTIVATE_GPU_IR_FUSION_ADAM_FUSION_H_ +#define MINDSPORE_CCSRC_PRE_ACTIVATE_GPU_IR_FUSION_ADAM_FUSION_H_ + +#include +#include "pre_activate/common/optimizer.h" + +namespace mindspore { +namespace opt { +class AdamFusion : public PatternProcessPass { + public: + explicit AdamFusion(bool multigraph = true) : PatternProcessPass("adam_fusion", multigraph) { + beta1_ = std::make_shared(); + one_sub_beta1_ = std::make_shared(); + beta2_ = std::make_shared(); + one_sub_beta2_ = std::make_shared(); + eps_ = std::make_shared(); + lr_ = std::make_shared(); + param_ = std::make_shared(); + m_ = std::make_shared(); + v_ = std::make_shared(); + gradient_ = std::make_shared(); + } + ~AdamFusion() override = default; + const BaseRef DefinePattern() const override; + const AnfNodePtr Process(const FuncGraphPtr &, const AnfNodePtr &, const EquivPtr &) const override; + + private: + VarPtr beta1_; + VarPtr one_sub_beta1_; + VarPtr beta2_; + VarPtr one_sub_beta2_; + VarPtr eps_; + VarPtr lr_; + VarPtr param_; + VarPtr m_; + VarPtr v_; + VarPtr gradient_; +}; +} // namespace opt +} // namespace mindspore +#endif // MINDSPORE_CCSRC_PRE_ACTIVATE_GPU_IR_FUSION_ADAM_FUSION_H_ diff --git a/mindspore/ccsrc/pre_activate/gpu/adam_weight_decay_fusion.cc b/mindspore/ccsrc/pre_activate/gpu/adam_weight_decay_fusion.cc new file mode 100644 index 0000000000..c950cbd56f --- /dev/null +++ b/mindspore/ccsrc/pre_activate/gpu/adam_weight_decay_fusion.cc @@ -0,0 +1,117 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "pre_activate/gpu/adam_weight_decay_fusion.h" + +#include +#include +#include + +#include "session/anf_runtime_algorithm.h" +#include "ir/primitive.h" +#include "utils/utils.h" +#include "pre_activate/common/helper.h" + +namespace mindspore { +namespace opt { +namespace { +kernel::KernelBuildInfoPtr GenerateKernelBuildInfo(CNodePtr node) { + std::vector inputs_format; + std::vector outputs_format; + std::vector inputs_type; + std::vector outputs_type; + kernel::KernelBuildInfo::KernelBuildInfoBuilder builder; + + for (size_t input_index = 0; input_index < AnfAlgo::GetInputTensorNum(node); ++input_index) { + inputs_type.push_back(AnfAlgo::GetPrevNodeOutputInferDataType(node, input_index)); + inputs_format.push_back(kOpFormat_DEFAULT); + } + for (size_t output_index = 0; output_index < AnfAlgo::GetOutputTensorNum(node); ++output_index) { + outputs_type.push_back(AnfAlgo::GetOutputInferDataType(node, output_index)); + outputs_format.push_back(kOpFormat_DEFAULT); + } + builder.SetInputsDeviceType(inputs_type); + builder.SetInputsFormat(inputs_format); + builder.SetOutputsDeviceType(outputs_type); + builder.SetOutputsFormat(outputs_format); + return builder.Build(); +} +} // namespace + +const BaseRef AdamWeightDecayFusion::DefinePattern() const { + VectorRef next_m = VectorRef({prim::kPrimTensorAdd, VectorRef({prim::kPrimMul, beta1_, m_}), + VectorRef({prim::kPrimMul, one_sub_beta1_, gradient_})}); + VectorRef next_v = + VectorRef({prim::kPrimTensorAdd, VectorRef({prim::kPrimMul, beta2_, v_}), + VectorRef({prim::kPrimMul, one_sub_beta2_, VectorRef({prim::kPrimSquare, gradient_})})}); + VectorRef update = VectorRef( + {prim::kPrimRealDiv, next_m, VectorRef({prim::kPrimTensorAdd, eps_, VectorRef({prim::kPrimSqrt, next_v})})}); + VectorRef new_update = VectorRef({prim::kPrimTensorAdd, VectorRef({prim::kPrimMul, weight_decay_, param_}), update}); + + VectorRef update_with_lr = VectorRef({prim::kPrimMul, lr_, new_update}); + VectorRef next_param = VectorRef({prim::kPrimSub, param_, update_with_lr}); + VectorRef depend1 = VectorRef({prim::kPrimDepend, next_v, VectorRef({prim::kPrimAssign, param_, next_param})}); + VectorRef depend2 = VectorRef({prim::kPrimDepend, depend1, VectorRef({prim::kPrimAssign, m_, next_m})}); + VectorRef depend3 = VectorRef({prim::kPrimDepend, depend2, VectorRef({prim::kPrimAssign, v_, depend2})}); + return depend3; +} + +const AnfNodePtr AdamWeightDecayFusion::Process(const FuncGraphPtr &graph, const AnfNodePtr &node, + const EquivPtr &equiv) const { + MS_EXCEPTION_IF_NULL(graph); + MS_EXCEPTION_IF_NULL(node); + MS_EXCEPTION_IF_NULL(equiv); + auto beta1_input = utils::cast((*equiv)[beta1_]); + auto one_sub_beta1_input = utils::cast((*equiv)[one_sub_beta1_]); + auto beta2_input = utils::cast((*equiv)[beta2_]); + auto one_sub_beta2_input = utils::cast((*equiv)[one_sub_beta2_]); + auto eps_input = utils::cast((*equiv)[eps_]); + auto lr_input = utils::cast((*equiv)[lr_]); + auto weight_decay_input = utils::cast((*equiv)[weight_decay_]); + auto param_input = utils::cast((*equiv)[param_]); + auto m_input = utils::cast((*equiv)[m_]); + auto v_input = utils::cast((*equiv)[v_]); + auto gradient_input = utils::cast((*equiv)[gradient_]); + MS_EXCEPTION_IF_NULL(beta1_input); + MS_EXCEPTION_IF_NULL(one_sub_beta1_input); + MS_EXCEPTION_IF_NULL(beta2_input); + MS_EXCEPTION_IF_NULL(one_sub_beta2_input); + MS_EXCEPTION_IF_NULL(eps_input); + MS_EXCEPTION_IF_NULL(lr_input); + MS_EXCEPTION_IF_NULL(weight_decay_input); + MS_EXCEPTION_IF_NULL(param_input); + MS_EXCEPTION_IF_NULL(m_input); + MS_EXCEPTION_IF_NULL(v_input); + MS_EXCEPTION_IF_NULL(gradient_input); + + auto prim = std::make_shared(kFusedAdamWeightDecayName); + MS_EXCEPTION_IF_NULL(prim); + std::vector inputs = { + NewValueNode(prim), beta1_input, one_sub_beta1_input, beta2_input, one_sub_beta2_input, + eps_input, lr_input, param_input, m_input, v_input, + gradient_input, weight_decay_input}; + auto adam_weight_decay = graph->NewCNode(inputs); + MS_EXCEPTION_IF_NULL(adam_weight_decay); + auto types = {AnfAlgo::GetOutputInferDataType(node, 0)}; + auto shapes = {AnfAlgo::GetOutputInferShape(node, 0)}; + AnfAlgo::SetOutputInferTypeAndShape(types, shapes, adam_weight_decay.get()); + adam_weight_decay->set_scope(node->scope()); + + auto build_info = GenerateKernelBuildInfo(adam_weight_decay); + AnfAlgo::SetSelectKernelBuildInfo(build_info, adam_weight_decay.get()); + return adam_weight_decay; +} +} // namespace opt +} // namespace mindspore diff --git a/mindspore/ccsrc/pre_activate/gpu/adam_weight_decay_fusion.h b/mindspore/ccsrc/pre_activate/gpu/adam_weight_decay_fusion.h new file mode 100644 index 0000000000..0ada5756e3 --- /dev/null +++ b/mindspore/ccsrc/pre_activate/gpu/adam_weight_decay_fusion.h @@ -0,0 +1,58 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_PRE_ACTIVATE_GPU_IR_FUSION_ADAM_WEIGHT_DECAY_FUSION_H_ +#define MINDSPORE_CCSRC_PRE_ACTIVATE_GPU_IR_FUSION_ADAM_WEIGHT_DECAY_FUSION_H_ + +#include +#include "pre_activate/common/optimizer.h" + +namespace mindspore { +namespace opt { +class AdamWeightDecayFusion : public PatternProcessPass { + public: + explicit AdamWeightDecayFusion(bool multigraph = true) : PatternProcessPass("adam_weight_decay_fusion", multigraph) { + beta1_ = std::make_shared(); + one_sub_beta1_ = std::make_shared(); + beta2_ = std::make_shared(); + one_sub_beta2_ = std::make_shared(); + eps_ = std::make_shared(); + lr_ = std::make_shared(); + weight_decay_ = std::make_shared(); + param_ = std::make_shared(); + m_ = std::make_shared(); + v_ = std::make_shared(); + gradient_ = std::make_shared(); + } + ~AdamWeightDecayFusion() override = default; + const BaseRef DefinePattern() const override; + const AnfNodePtr Process(const FuncGraphPtr &, const AnfNodePtr &, const EquivPtr &) const override; + + private: + VarPtr beta1_; + VarPtr one_sub_beta1_; + VarPtr beta2_; + VarPtr one_sub_beta2_; + VarPtr eps_; + VarPtr lr_; + VarPtr weight_decay_; + VarPtr param_; + VarPtr m_; + VarPtr v_; + VarPtr gradient_; +}; +} // namespace opt +} // namespace mindspore +#endif // MINDSPORE_CCSRC_PRE_ACTIVATE_GPU_IR_FUSION_ADAM_WEIGHT_DECAY_FUSION_H_ diff --git a/mindspore/ccsrc/session/gpu_session.cc b/mindspore/ccsrc/session/gpu_session.cc index a0a43f2edd..85ad2f3d1e 100644 --- a/mindspore/ccsrc/session/gpu_session.cc +++ b/mindspore/ccsrc/session/gpu_session.cc @@ -23,6 +23,8 @@ #include "pre_activate/common/helper.h" #include "pre_activate/pass/communication_op_fusion.h" #include "pre_activate/pass/getitem_tuple.h" +#include "pre_activate/gpu/adam_weight_decay_fusion.h" +#include "pre_activate/gpu/adam_fusion.h" #include "device/kernel_runtime_manager.h" #include "predict/predict.h" #include "common/utils.h" @@ -53,6 +55,16 @@ void GPUSession::StartKernelRT() const { void GPUSession::Optimize(const std::shared_ptr &kernel_graph) { MS_EXCEPTION_IF_NULL(kernel_graph); + auto optimizer = std::make_shared(); + auto pm = std::make_shared(); + pm->AddPass(std::make_shared()); + pm->AddPass(std::make_shared()); + optimizer->AddPassManager(pm); + (void)optimizer->Optimize(kernel_graph); + kernel_graph->SetExecOrderByDefault(); +} + +void GPUSession::HardwareOptimize(const std::shared_ptr &kernel_graph) { auto optimizer = std::make_shared(); auto pm = std::make_shared(); pm->AddPass(std::make_shared()); @@ -151,14 +163,16 @@ GraphId GPUSession::CompileGraph(const AnfNodePtrList &lst, const AnfNodePtrList auto graph_id = graph_sum_; auto graph = ConstructKernelGraph(lst, outputs); MS_EXCEPTION_IF_NULL(graph); + // Optimize + Optimize(graph); // Select kernel build info SelectKernel(graph); // Convert kernel Graph to model predictmodel::StepConvertGraph(graph); // Start gpu kernel runtime StartKernelRT(); - // AllReduce Optimize - Optimize(graph); + // HardwareOptimize + HardwareOptimize(graph); // Assign CUDA streams AssignStream(graph); // Hide NoOp from execution graph diff --git a/mindspore/ccsrc/session/gpu_session.h b/mindspore/ccsrc/session/gpu_session.h index 0dfb815abe..4e46c2138d 100644 --- a/mindspore/ccsrc/session/gpu_session.h +++ b/mindspore/ccsrc/session/gpu_session.h @@ -51,6 +51,8 @@ class GPUSession : public SessionBasic { void Optimize(const std::shared_ptr &kernel_graph); + void HardwareOptimize(const std::shared_ptr &kernel_graph); + void AssignStream(const std::shared_ptr &kernel_graph); void BuildKernel(const std::shared_ptr &kernel_graph) const; diff --git a/mindspore/ccsrc/utils/utils.h b/mindspore/ccsrc/utils/utils.h index 7380ef501f..f80c13c9a1 100644 --- a/mindspore/ccsrc/utils/utils.h +++ b/mindspore/ccsrc/utils/utils.h @@ -161,6 +161,8 @@ constexpr auto kNMSWithMaskOpName = "NMSWithMask"; constexpr auto kSoftmaxGradExtOpName = "SoftmaxGradExt"; constexpr auto kStridedReadOpName = "StridedRead"; constexpr auto kStridedWriteOpName = "StridedWrite"; +constexpr auto kFusedAdamWeightDecayName = "FusedAdamWeightDecay"; +constexpr auto kFusedAdamName = "FusedAdam"; // attr key name constexpr auto kAttrInputNames = "input_names"; diff --git a/tests/st/ops/gpu/test_adam_fusion.py b/tests/st/ops/gpu/test_adam_fusion.py new file mode 100644 index 0000000000..f0595d12a1 --- /dev/null +++ b/tests/st/ops/gpu/test_adam_fusion.py @@ -0,0 +1,87 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +import numpy as np +import pytest + +import mindspore.context as context +import mindspore.nn as nn +from mindspore import Tensor +from mindspore.common.api import ms_function +from mindspore.ops import operations as P +from mindspore.ops import functional as F +from mindspore.common import dtype as mstype +from mindspore.common.parameter import Parameter + +context.set_context(mode=context.GRAPH_MODE, device_target="GPU", save_graphs=True) + + +class Net(nn.Cell): + def __init__(self, decay_flag=True): + super(Net, self).__init__() + self.decay_flag = decay_flag + self.op_mul = P.Mul() + self.op_square = P.Square() + self.op_sqrt = P.Sqrt() + self.op_cast = P.Cast() + self.op_reshape = P.Reshape() + self.op_shape = P.Shape() + self.param = Parameter(Tensor(np.array([0.1, 0.3, 0.5]).astype(np.float32)), name='param') + self.m = Parameter(Tensor(np.array([0.1, 0.3, 0.5]).astype(np.float32)), name='m') + self.v = Parameter(Tensor(np.array([0.1, 0.3, 0.5]).astype(np.float32)), name='v') + + @ms_function + def construct(self, beta1, beta2, gradient, eps, weight_decay_tensor, lr): + param_fp32 = self.op_cast(self.param, mstype.float32) + m_fp32 = self.op_cast(self.m, mstype.float32) + v_fp32 = self.op_cast(self.v, mstype.float32) + gradient_fp32 = self.op_cast(gradient, mstype.float32) + + next_m = self.op_mul(beta1, m_fp32) + \ + self.op_mul(self.op_cast(F.tuple_to_array((1.0,)), mstype.float32) - beta1, gradient_fp32) + next_v = self.op_mul(beta2, v_fp32) + self.op_mul(self.op_cast(F.tuple_to_array((1.0,)), mstype.float32) - \ + beta2, self.op_square(gradient_fp32)) + update = next_m / (eps + self.op_sqrt(next_v)) + if self.decay_flag: + update = self.op_mul(weight_decay_tensor, param_fp32) + update + update_with_lr = self.op_mul(lr, update) + next_param = param_fp32 - self.op_reshape(update_with_lr, self.op_shape(param_fp32)) + + next_v = F.depend(next_v, F.assign(self.param, next_param)) + next_v = F.depend(next_v, F.assign(self.m, next_m)) + next_v = F.depend(next_v, F.assign(self.v, next_v)) + return next_v + + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard +def test(): + beta1 = Tensor(np.array([0.9]).astype(np.float32)) + beta2 = Tensor(np.array([0.999]).astype(np.float32)) + lr = Tensor(np.array([0.001]).astype(np.float32)) + eps = Tensor(np.array([1e-6]).astype(np.float32)) + weight_decay_tensor = Tensor(np.array([0.001]).astype(np.float32)) + + gradient = Tensor(np.array([0.01, 0.03, 0.05]).astype(np.float32)) + opt = Net(True) + _ = opt(beta1, beta2, gradient, eps, weight_decay_tensor, lr) + + param_expect = np.array([0.09971199, 0.29950103, 0.4993557]).astype(np.float32) + m_expect = np.array([0.091, 0.273, 0.45499998]).astype(np.float32) + v_expect = np.array([0.0999001, 0.29970092, 0.4995025]).astype(np.float32) + assert np.allclose(opt.param.data.asnumpy(), param_expect) + assert np.allclose(opt.m.data.asnumpy(), m_expect) + assert np.allclose(opt.v.data.asnumpy(), v_expect) diff --git a/tests/st/ops/gpu/test_batch_matmul.py b/tests/st/ops/gpu/test_batch_matmul.py index e8450bd81d..7dbab73801 100644 --- a/tests/st/ops/gpu/test_batch_matmul.py +++ b/tests/st/ops/gpu/test_batch_matmul.py @@ -119,6 +119,10 @@ def test_4d_transpose_ab(): [[5612, 5810, 6008, 6206]]]] assert (output.asnumpy() == expect).all() + +@pytest.mark.level0 +@pytest.mark.platform_x86_gpu_training +@pytest.mark.env_onecard def test_4D_fp16(): input_x = Tensor(np.arange(2 * 4 * 1 * 3).reshape(2, 4, 1, 3), mstype.float16) input_y = Tensor(np.arange(2 * 4 * 3 * 4).reshape(2, 4, 3, 4), mstype.float16) @@ -126,13 +130,13 @@ def test_4D_fp16(): context.set_context(mode=context.GRAPH_MODE, device_target="GPU") net = BatchMatMulNet() output = net(input_x, input_y) - expect = [[[[20, 23, 26, 29]], - [[200, 212, 224, 236]], - [[596, 617, 638, 659]], - [[1208, 1238, 1268, 1298]]], - - [[[2036, 2075, 2114, 2153]], - [[3080, 3128, 3176, 3224]], - [[4340, 4397, 4454, 4511]], - [[5816, 5882, 5948, 6014]]]] + expect = np.array([[[[20, 23, 26, 29]], + [[200, 212, 224, 236]], + [[596, 617, 638, 659]], + [[1208, 1238, 1268, 1298]]], + + [[[2036, 2076, 2114, 2152]], + [[3080, 3128, 3176, 3224]], + [[4340, 4396, 4456, 4510]], + [[5816, 5880, 5948, 6016]]]]).astype(np.float16) assert (output.asnumpy() == expect).all() From 1c34c8c970daaa062aa0d34316915d116f06e851 Mon Sep 17 00:00:00 2001 From: lichenever Date: Tue, 23 Jun 2020 15:16:51 +0800 Subject: [PATCH 024/254] fix_hccl_to_support_big_tensor --- mindspore/ccsrc/kernel/hccl/hcom_util.cc | 7 +++---- mindspore/ccsrc/utils/convert_utils_base.h | 10 ++++++++++ 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/mindspore/ccsrc/kernel/hccl/hcom_util.cc b/mindspore/ccsrc/kernel/hccl/hcom_util.cc index 61a4d43eb5..088dbe59d5 100644 --- a/mindspore/ccsrc/kernel/hccl/hcom_util.cc +++ b/mindspore/ccsrc/kernel/hccl/hcom_util.cc @@ -67,18 +67,17 @@ bool HcomUtil::GetHcomDataType(const AnfNodePtr &anf_node, vector &shape, size_t *size) { MS_EXCEPTION_IF_NULL(size); - int tmp_size = 1; + size_t tmp_size = 1; uint32_t type_size = 4; for (size_t i = 0; i < shape.size(); i++) { - IntMulWithOverflowCheck(tmp_size, SizeToInt(shape[i]), &tmp_size); + tmp_size = SizetMulWithOverflowCheck(tmp_size, shape[i]); } if (!GetHcomTypeSize(data_type, &type_size)) { return false; } - IntMulWithOverflowCheck(tmp_size, UintToInt(type_size), &tmp_size); - *size = IntToSize(tmp_size); + *size = SizetMulWithOverflowCheck(tmp_size, type_size); MS_LOG(INFO) << "size[" << *size << "]"; return true; diff --git a/mindspore/ccsrc/utils/convert_utils_base.h b/mindspore/ccsrc/utils/convert_utils_base.h index 76d8930324..3638a43e6a 100644 --- a/mindspore/ccsrc/utils/convert_utils_base.h +++ b/mindspore/ccsrc/utils/convert_utils_base.h @@ -102,6 +102,16 @@ inline void IntMulWithOverflowCheck(int a, int b, int *c) { *c = out; } +inline size_t SizetMulWithOverflowCheck(size_t a, size_t b) { + size_t out = a * b; + if (a != 0) { + if ((out / a) != b) { + MS_LOG(EXCEPTION) << "Mul: a(" << a << ") * b(" << b << ") result is overflow"; + } + } + return out; +} + inline uint8_t *AddressOffset(void *address, size_t offset) { MS_EXCEPTION_IF_NULL(address); return static_cast(address) + offset; From df7fe72928460762a0b2c775efd2af8833611ec4 Mon Sep 17 00:00:00 2001 From: chenfei Date: Tue, 23 Jun 2020 21:45:21 +0800 Subject: [PATCH 025/254] visit stop if tuple getitem and maketuple of function GetCallRealOutputs --- mindspore/ccsrc/session/kernel_graph.cc | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/mindspore/ccsrc/session/kernel_graph.cc b/mindspore/ccsrc/session/kernel_graph.cc index 5c05b3fdcc..8a4982cd4f 100644 --- a/mindspore/ccsrc/session/kernel_graph.cc +++ b/mindspore/ccsrc/session/kernel_graph.cc @@ -43,7 +43,8 @@ void PushNoVisitedNode(const AnfNodePtr &node, std::queue *que, } std::vector GetCallRealOutputs(const AnfNodePtr &call_node) { - auto item_with_index = AnfAlgo::VisitKernelWithReturnType(call_node, 0); + auto item_with_index = + AnfAlgo::VisitKernelWithReturnType(call_node, 0, false, {prim::kPrimTupleGetItem, prim::kPrimMakeTuple}); AnfNodePtr node = item_with_index.first; MS_EXCEPTION_IF_NULL(node); if (AnfAlgo::CheckPrimitiveType(node, prim::kPrimMakeTuple)) { @@ -773,9 +774,7 @@ void KernelGraph::UpdateCallRealInput() { std::vector new_real_inputs; for (auto &real_input : real_inputs) { // if real input is a call node ,find the child graph output act as the new real input - auto item_with_index = AnfAlgo::VisitKernelWithReturnType(real_input, 0); - MS_EXCEPTION_IF_NULL(item_with_index.first); - auto tmp_real_input = GetCallRealOutputs(item_with_index.first); + auto tmp_real_input = GetCallRealOutputs(real_input); std::copy(tmp_real_input.begin(), tmp_real_input.end(), std::back_inserter(new_real_inputs)); } real_inputs_map.emplace_back(parameter, new_real_inputs); From bca037e1becfb4d933901baa9f4840f6bbcc27c6 Mon Sep 17 00:00:00 2001 From: Mahdi Date: Fri, 19 Jun 2020 15:00:34 -0400 Subject: [PATCH 026/254] added cpp test cases for pad and cut out --- tests/ut/cpp/dataset/CMakeLists.txt | 2 ++ tests/ut/cpp/dataset/cut_out_op_test.cc | 44 +++++++++++++++++++++++++ tests/ut/cpp/dataset/pad_op_test.cc | 44 +++++++++++++++++++++++++ 3 files changed, 90 insertions(+) create mode 100644 tests/ut/cpp/dataset/cut_out_op_test.cc create mode 100644 tests/ut/cpp/dataset/pad_op_test.cc diff --git a/tests/ut/cpp/dataset/CMakeLists.txt b/tests/ut/cpp/dataset/CMakeLists.txt index 317f9d67c3..b183cbab95 100644 --- a/tests/ut/cpp/dataset/CMakeLists.txt +++ b/tests/ut/cpp/dataset/CMakeLists.txt @@ -17,6 +17,7 @@ SET(DE_UT_SRCS circular_pool_test.cc client_config_test.cc connector_test.cc + cut_out_op_test.cc datatype_test.cc decode_op_test.cc execution_tree_test.cc @@ -28,6 +29,7 @@ SET(DE_UT_SRCS normalize_op_test.cc one_hot_op_test.cc pad_end_op_test.cc + pad_op_test.cc path_test.cc project_op_test.cc queue_test.cc diff --git a/tests/ut/cpp/dataset/cut_out_op_test.cc b/tests/ut/cpp/dataset/cut_out_op_test.cc new file mode 100644 index 0000000000..462fb3a875 --- /dev/null +++ b/tests/ut/cpp/dataset/cut_out_op_test.cc @@ -0,0 +1,44 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "common/common.h" +#include "common/cvop_common.h" +#include "dataset/kernels/image/cut_out_op.h" +#include "utils/log_adapter.h" + +using namespace mindspore::dataset; +using mindspore::MsLogLevel::INFO; +using mindspore::ExceptionType::NoExceptionType; +using mindspore::LogStream; + +class MindDataTestCutOutOp : public UT::CVOP::CVOpCommon { + protected: + MindDataTestCutOutOp() : CVOpCommon() {} + + std::shared_ptr output_tensor_; +}; + +TEST_F(MindDataTestCutOutOp, TestOp) { + MS_LOG(INFO) << "Doing testCutOut."; + std::unique_ptr op(new CutOutOp(50, 50, 5, false, 0, 0, 0)); + + EXPECT_TRUE(op->OneToOne()); + Status s = op->Compute(input_tensor_, &output_tensor_); + EXPECT_EQ(input_tensor_->shape()[0], output_tensor_->shape()[0]); + EXPECT_EQ(input_tensor_->shape()[1], output_tensor_->shape()[1]); + EXPECT_EQ(input_tensor_->shape()[2], output_tensor_->shape()[2]); + EXPECT_EQ(s, Status::OK()); +} + diff --git a/tests/ut/cpp/dataset/pad_op_test.cc b/tests/ut/cpp/dataset/pad_op_test.cc new file mode 100644 index 0000000000..b659d009f3 --- /dev/null +++ b/tests/ut/cpp/dataset/pad_op_test.cc @@ -0,0 +1,44 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "common/common.h" +#include "common/cvop_common.h" +#include "dataset/kernels/image/pad_op.h" +#include "utils/log_adapter.h" + +using namespace mindspore::dataset; +using mindspore::MsLogLevel::INFO; +using mindspore::ExceptionType::NoExceptionType; +using mindspore::LogStream; + +class MindDataTestPadOp : public UT::CVOP::CVOpCommon { + protected: + MindDataTestPadOp() : CVOpCommon() {} + + std::shared_ptr output_tensor_; +}; + +TEST_F(MindDataTestPadOp, TestOp) { + MS_LOG(INFO) << "Doing testPad."; + std::unique_ptr op(new PadOp(10, 20, 30, 40, BorderType::kConstant)); + EXPECT_TRUE(op->OneToOne()); + Status s = op->Compute(input_tensor_, &output_tensor_); + size_t actual = 0; + if (s == Status::OK()) { + actual = output_tensor_->shape()[0] * output_tensor_->shape()[1] * output_tensor_->shape()[2]; + } + EXPECT_EQ(actual, (input_tensor_->shape()[0] + 30) * (input_tensor_->shape()[1] + 70) * 3); + EXPECT_EQ(s, Status::OK()); +} From 0d52888fc5a7abe04ff9bb9a8024e19e9832cc4a Mon Sep 17 00:00:00 2001 From: heleiwang Date: Tue, 23 Jun 2020 17:50:05 +0800 Subject: [PATCH 027/254] fix misspell and check parameters --- mindspore/ccsrc/dataset/engine/gnn/graph.cc | 29 ++++++++++++++ mindspore/ccsrc/dataset/engine/gnn/graph.h | 2 + mindspore/dataset/engine/validators.py | 42 ++++++++++----------- tests/ut/cpp/dataset/gnn_graph_test.cc | 22 ++++++++++- 4 files changed, 73 insertions(+), 22 deletions(-) diff --git a/mindspore/ccsrc/dataset/engine/gnn/graph.cc b/mindspore/ccsrc/dataset/engine/gnn/graph.cc index 1017657397..a143bd4e38 100644 --- a/mindspore/ccsrc/dataset/engine/gnn/graph.cc +++ b/mindspore/ccsrc/dataset/engine/gnn/graph.cc @@ -149,14 +149,37 @@ Status Graph::GetAllNeighbors(const std::vector &node_list, NodeType return Status::OK(); } +Status Graph::CheckSamplesNum(NodeIdType samples_num) { + NodeIdType all_nodes_number = + std::accumulate(node_type_map_.begin(), node_type_map_.end(), 0, + [](NodeIdType t1, const auto &t2) -> NodeIdType { return t1 + t2.second.size(); }); + if ((samples_num < 1) || (samples_num > all_nodes_number)) { + std::string err_msg = "Wrong samples number, should be between 1 and " + std::to_string(all_nodes_number) + + ", got " + std::to_string(samples_num); + RETURN_STATUS_UNEXPECTED(err_msg); + } + return Status::OK(); +} + Status Graph::GetSampledNeighbors(const std::vector &node_list, const std::vector &neighbor_nums, const std::vector &neighbor_types, std::shared_ptr *out) { CHECK_FAIL_RETURN_UNEXPECTED(!node_list.empty(), "Input node_list is empty."); CHECK_FAIL_RETURN_UNEXPECTED(neighbor_nums.size() == neighbor_types.size(), "The sizes of neighbor_nums and neighbor_types are inconsistent."); + for (const auto &num : neighbor_nums) { + RETURN_IF_NOT_OK(CheckSamplesNum(num)); + } + for (const auto &type : neighbor_types) { + if (node_type_map_.find(type) == node_type_map_.end()) { + std::string err_msg = "Invalid neighbor type:" + std::to_string(type); + RETURN_STATUS_UNEXPECTED(err_msg); + } + } std::vector> neighbors_vec(node_list.size()); for (size_t node_idx = 0; node_idx < node_list.size(); ++node_idx) { + std::shared_ptr input_node; + RETURN_IF_NOT_OK(GetNodeByNodeId(node_list[node_idx], &input_node)); neighbors_vec[node_idx].emplace_back(node_list[node_idx]); std::vector input_list = {node_list[node_idx]}; for (size_t i = 0; i < neighbor_nums.size(); ++i) { @@ -204,6 +227,12 @@ Status Graph::NegativeSample(const std::vector &data, const std::uno Status Graph::GetNegSampledNeighbors(const std::vector &node_list, NodeIdType samples_num, NodeType neg_neighbor_type, std::shared_ptr *out) { CHECK_FAIL_RETURN_UNEXPECTED(!node_list.empty(), "Input node_list is empty."); + RETURN_IF_NOT_OK(CheckSamplesNum(samples_num)); + if (node_type_map_.find(neg_neighbor_type) == node_type_map_.end()) { + std::string err_msg = "Invalid neighbor type:" + std::to_string(neg_neighbor_type); + RETURN_STATUS_UNEXPECTED(err_msg); + } + std::vector> neighbors_vec; neighbors_vec.resize(node_list.size()); for (size_t node_idx = 0; node_idx < node_list.size(); ++node_idx) { diff --git a/mindspore/ccsrc/dataset/engine/gnn/graph.h b/mindspore/ccsrc/dataset/engine/gnn/graph.h index ea10363053..344a6c6bf2 100644 --- a/mindspore/ccsrc/dataset/engine/gnn/graph.h +++ b/mindspore/ccsrc/dataset/engine/gnn/graph.h @@ -226,6 +226,8 @@ class Graph { Status NegativeSample(const std::vector &input_data, const std::unordered_set &exclude_data, int32_t samples_num, std::vector *out_samples); + Status CheckSamplesNum(NodeIdType samples_num); + std::string dataset_file_; int32_t num_workers_; // The number of worker threads std::mt19937 rnd_; diff --git a/mindspore/dataset/engine/validators.py b/mindspore/dataset/engine/validators.py index 005f7072aa..5bfd7656d3 100644 --- a/mindspore/dataset/engine/validators.py +++ b/mindspore/dataset/engine/validators.py @@ -1110,10 +1110,10 @@ def check_gnn_list_or_ndarray(param, param_name): for m in param: if not isinstance(m, int): raise TypeError( - "Each membor in {0} should be of type int. Got {1}.".format(param_name, type(m))) + "Each member in {0} should be of type int. Got {1}.".format(param_name, type(m))) elif isinstance(param, np.ndarray): if not param.dtype == np.int32: - raise TypeError("Each membor in {0} should be of type int32. Got {1}.".format( + raise TypeError("Each member in {0} should be of type int32. Got {1}.".format( param_name, param.dtype)) else: raise TypeError("Wrong input type for {0}, should be list or numpy.ndarray, got {1}".format( @@ -1196,15 +1196,15 @@ def check_gnn_get_sampled_neighbors(method): # check neighbor_nums; required argument neighbor_nums = param_dict.get("neighbor_nums") check_gnn_list_or_ndarray(neighbor_nums, 'neighbor_nums') - if len(neighbor_nums) > 6: - raise ValueError("Wrong number of input members for {0}, should be less than or equal to 6, got {1}".format( + if not neighbor_nums or len(neighbor_nums) > 6: + raise ValueError("Wrong number of input members for {0}, should be between 1 and 6, got {1}".format( 'neighbor_nums', len(neighbor_nums))) # check neighbor_types; required argument neighbor_types = param_dict.get("neighbor_types") check_gnn_list_or_ndarray(neighbor_types, 'neighbor_types') - if len(neighbor_nums) > 6: - raise ValueError("Wrong number of input members for {0}, should be less than or equal to 6, got {1}".format( + if not neighbor_types or len(neighbor_types) > 6: + raise ValueError("Wrong number of input members for {0}, should be between 1 and 6, got {1}".format( 'neighbor_types', len(neighbor_types))) if len(neighbor_nums) != len(neighbor_types): @@ -1256,7 +1256,7 @@ def check_gnn_random_walk(method): return new_method -def check_aligned_list(param, param_name, membor_type): +def check_aligned_list(param, param_name, member_type): """Check whether the structure of each member of the list is the same.""" if not isinstance(param, list): @@ -1264,27 +1264,27 @@ def check_aligned_list(param, param_name, membor_type): if not param: raise TypeError( "Parameter {0} or its members are empty".format(param_name)) - membor_have_list = None + member_have_list = None list_len = None - for membor in param: - if isinstance(membor, list): - check_aligned_list(membor, param_name, membor_type) - if membor_have_list not in (None, True): + for member in param: + if isinstance(member, list): + check_aligned_list(member, param_name, member_type) + if member_have_list not in (None, True): raise TypeError("The type of each member of the parameter {0} is inconsistent".format( param_name)) - if list_len is not None and len(membor) != list_len: + if list_len is not None and len(member) != list_len: raise TypeError("The size of each member of parameter {0} is inconsistent".format( param_name)) - membor_have_list = True - list_len = len(membor) + member_have_list = True + list_len = len(member) else: - if not isinstance(membor, membor_type): - raise TypeError("Each membor in {0} should be of type int. Got {1}.".format( - param_name, type(membor))) - if membor_have_list not in (None, False): + if not isinstance(member, member_type): + raise TypeError("Each member in {0} should be of type int. Got {1}.".format( + param_name, type(member))) + if member_have_list not in (None, False): raise TypeError("The type of each member of the parameter {0} is inconsistent".format( param_name)) - membor_have_list = False + member_have_list = False def check_gnn_get_node_feature(method): @@ -1300,7 +1300,7 @@ def check_gnn_get_node_feature(method): check_aligned_list(node_list, 'node_list', int) elif isinstance(node_list, np.ndarray): if not node_list.dtype == np.int32: - raise TypeError("Each membor in {0} should be of type int32. Got {1}.".format( + raise TypeError("Each member in {0} should be of type int32. Got {1}.".format( node_list, node_list.dtype)) else: raise TypeError("Wrong input type for {0}, should be list or numpy.ndarray, got {1}".format( diff --git a/tests/ut/cpp/dataset/gnn_graph_test.cc b/tests/ut/cpp/dataset/gnn_graph_test.cc index ce2aca4ffd..dc74e66b0c 100644 --- a/tests/ut/cpp/dataset/gnn_graph_test.cc +++ b/tests/ut/cpp/dataset/gnn_graph_test.cc @@ -158,6 +158,18 @@ TEST_F(MindDataTestGNNGraph, TestGetSampledNeighbors) { s = graph.GetSampledNeighbors({}, {10}, {meta_info.node_type[1]}, &neighbors); EXPECT_TRUE(s.ToString().find("Input node_list is empty.") != std::string::npos); + neighbors.reset(); + s = graph.GetSampledNeighbors({-1, 1}, {10}, {meta_info.node_type[1]}, &neighbors); + EXPECT_TRUE(s.ToString().find("Invalid node id") != std::string::npos); + + neighbors.reset(); + s = graph.GetSampledNeighbors(node_list, {2, 50}, {meta_info.node_type[0], meta_info.node_type[1]}, &neighbors); + EXPECT_TRUE(s.ToString().find("Wrong samples number") != std::string::npos); + + neighbors.reset(); + s = graph.GetSampledNeighbors(node_list, {2}, {5}, &neighbors); + EXPECT_TRUE(s.ToString().find("Invalid neighbor type") != std::string::npos); + neighbors.reset(); s = graph.GetSampledNeighbors(node_list, {2, 3, 4}, {meta_info.node_type[1], meta_info.node_type[0]}, &neighbors); EXPECT_TRUE(s.ToString().find("The sizes of neighbor_nums and neighbor_types are inconsistent.") != @@ -198,9 +210,17 @@ TEST_F(MindDataTestGNNGraph, TestGetNegSampledNeighbors) { s = graph.GetNegSampledNeighbors({}, 3, meta_info.node_type[1], &neg_neighbors); EXPECT_TRUE(s.ToString().find("Input node_list is empty.") != std::string::npos); + neg_neighbors.reset(); + s = graph.GetNegSampledNeighbors({-1, 1}, 3, meta_info.node_type[1], &neg_neighbors); + EXPECT_TRUE(s.ToString().find("Invalid node id") != std::string::npos); + + neg_neighbors.reset(); + s = graph.GetNegSampledNeighbors(node_list, 50, meta_info.node_type[1], &neg_neighbors); + EXPECT_TRUE(s.ToString().find("Wrong samples number") != std::string::npos); + neg_neighbors.reset(); s = graph.GetNegSampledNeighbors(node_list, 3, 3, &neg_neighbors); - EXPECT_TRUE(s.ToString().find("Invalid node type:3") != std::string::npos); + EXPECT_TRUE(s.ToString().find("Invalid neighbor type") != std::string::npos); } TEST_F(MindDataTestGNNGraph, TestRandomWalk) { From 343889cdb75d4b720ba0c7523506642abfa197ca Mon Sep 17 00:00:00 2001 From: chenjianping Date: Tue, 23 Jun 2020 22:16:18 +0800 Subject: [PATCH 028/254] building _ms_mpi with mpi_interface --- cmake/package.cmake | 5 + mindspore/ccsrc/CMakeLists.txt | 5 +- mindspore/ccsrc/device/CMakeLists.txt | 17 ++- mindspore/ccsrc/device/cpu/mpi/mpi_adapter.cc | 123 ++++++++++-------- mindspore/ccsrc/device/cpu/mpi/mpi_adapter.h | 40 ++++-- .../ccsrc/device/cpu/mpi/mpi_interface.cc | 33 +++++ .../ccsrc/device/gpu/mpi/mpi_initializer.cc | 7 - .../ccsrc/kernel/cpu/allgather_cpu_kernel.cc | 2 +- .../embedding_look_up_comm_grad_cpu_kernel.cc | 4 +- .../cpu/embedding_look_up_cpu_kernel.cc | 6 +- .../kernel/cpu/reduce_scatter_cpu_kernel.cc | 4 +- mindspore/context.py | 38 ------ mindspore/parallel/mpi/_mpi_config.py | 2 +- 13 files changed, 161 insertions(+), 125 deletions(-) create mode 100644 mindspore/ccsrc/device/cpu/mpi/mpi_interface.cc diff --git a/cmake/package.cmake b/cmake/package.cmake index 1cff396ef1..42821cf41d 100644 --- a/cmake/package.cmake +++ b/cmake/package.cmake @@ -128,6 +128,11 @@ if (ENABLE_MPI) DESTINATION ${INSTALL_BASE_DIR} COMPONENT mindspore ) + install( + TARGETS mpi_adapter + DESTINATION ${INSTALL_LIB_DIR} + COMPONENT mindspore + ) endif () if (ENABLE_GPU) diff --git a/mindspore/ccsrc/CMakeLists.txt b/mindspore/ccsrc/CMakeLists.txt index 80f82fd7ea..b6330e67b8 100644 --- a/mindspore/ccsrc/CMakeLists.txt +++ b/mindspore/ccsrc/CMakeLists.txt @@ -126,11 +126,12 @@ endforeach () set_property(SOURCE ${SUB_OBJECTS_SRC} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_ME) add_library(mindspore STATIC ${SUB_OBJECTS_SRC}) target_link_libraries(mindspore proto_input) -if (ENABLE_CPU AND ENABLE_MPI) - target_link_libraries(mindspore securec mindspore::flatbuffers mindspore::ompi) +if (ENABLE_MPI) + target_link_libraries(mindspore securec mindspore::flatbuffers mpi_adapter) else () target_link_libraries(mindspore securec mindspore::flatbuffers) endif () + if (NOT WIN32) target_link_libraries(mindspore dl) endif() diff --git a/mindspore/ccsrc/device/CMakeLists.txt b/mindspore/ccsrc/device/CMakeLists.txt index 7178a01ce6..951b6a1f9c 100644 --- a/mindspore/ccsrc/device/CMakeLists.txt +++ b/mindspore/ccsrc/device/CMakeLists.txt @@ -14,17 +14,22 @@ endif () if (ENABLE_CPU) file(GLOB_RECURSE CPU_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "cpu/*.cc") - if (NOT ENABLE_MPI) - list(REMOVE_ITEM CPU_SRC_LIST "cpu/mpi/mpi_adapter.cc") - endif () + list(REMOVE_ITEM CPU_SRC_LIST "cpu/mpi/mpi_adapter.cc") + list(REMOVE_ITEM CPU_SRC_LIST "cpu/mpi/mpi_interface.cc") endif () if (ENABLE_MPI) # _ms_mpi - set_property(SOURCE "gpu/mpi/mpi_initializer.cc" + file(GLOB_RECURSE MPI_SRC_LIST "cpu/mpi/mpi_adapter.cc") + set_property(SOURCE ${MPI_SRC_LIST} + PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_DEVICE) + add_library(mpi_adapter SHARED ${MPI_SRC_LIST}) + target_link_libraries(mpi_adapter PRIVATE mindspore::ompi) + + set_property(SOURCE "cpu/mpi/mpi_interface.cc" PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_DEVICE) - pybind11_add_module(_ms_mpi "gpu/mpi/mpi_initializer.cc") - target_link_libraries(_ms_mpi PRIVATE mindspore::pybind11_module mindspore::ompi) + pybind11_add_module(_ms_mpi "cpu/mpi/mpi_interface.cc") + target_link_libraries(_ms_mpi PRIVATE mindspore::pybind11_module mpi_adapter) endif () # gpu diff --git a/mindspore/ccsrc/device/cpu/mpi/mpi_adapter.cc b/mindspore/ccsrc/device/cpu/mpi/mpi_adapter.cc index 0d49846bf7..1307e49644 100644 --- a/mindspore/ccsrc/device/cpu/mpi/mpi_adapter.cc +++ b/mindspore/ccsrc/device/cpu/mpi/mpi_adapter.cc @@ -15,13 +15,41 @@ */ #include "device/cpu/mpi/mpi_adapter.h" +#ifdef ENABLE_MPI #include -#include "utils/mpi/mpi_config.h" +#include +#include "pybind11/pybind11.h" +#endif // ENABLE_MPI #include "utils/log_adapter.h" namespace mindspore { namespace device { namespace cpu { +std::shared_ptr MPIAdapter::instance_ = nullptr; +std::shared_ptr MPIAdapter::Instance() { + if (instance_ == nullptr) { + MS_LOG(DEBUG) << "Create new mpi adapter instance."; + instance_.reset(new (std::nothrow) MPIAdapter()); + } + return instance_; +} + +#ifdef ENABLE_MPI + +#define RAISE_EXCEPTION(message) \ + { \ + std::ostringstream oss; \ + oss << "[" << __FILE__ << "] [" << __LINE__ << "] " << message; \ + pybind11::pybind11_fail(oss.str()); \ + } + +#define RAISE_EXCEPTION_WITH_PARAM(message, param) \ + { \ + std::ostringstream oss; \ + oss << "[" << __FILE__ << "] [" << __LINE__ << "] " << message << param; \ + pybind11::pybind11_fail(oss.str()); \ + } + namespace { MPI_Op GetMpiOp(const std::string &op_type) { if (op_type == "sum") { @@ -33,7 +61,8 @@ MPI_Op GetMpiOp(const std::string &op_type) { } else if (op_type == "prod") { return MPI_PROD; } - MS_LOG(EXCEPTION) << "unsupport op_type:" << op_type; + + RAISE_EXCEPTION_WITH_PARAM("unsupport op_type: ", op_type); return MPI_SUM; } @@ -46,80 +75,72 @@ int GetScatterIndex(int rankid, const std::vector &ranks_group) { } } if (scatter_index == -1) { - MS_LOG(EXCEPTION) << "process rankid " << rankid << " does not in the input rank group!"; + RAISE_EXCEPTION_WITH_PARAM("local rankid does not in the input rank group!local rank id:", rankid); } return scatter_index; } } // namespace -MPIAdapter::MPIAdapter() : rank_id_(0), rank_size_(0), comm_group_world_(MPI_GROUP_NULL) { Init(); } +MPIAdapter::MPIAdapter() : comm_group_world_(MPI_GROUP_NULL) { Init(); } MPIAdapter::~MPIAdapter() { + int finalized; + MPI_Finalized(&finalized); + if (finalized != 0) { + return; + } + for (auto iter = ranks_group_.begin(); iter != ranks_group_.end(); ++iter) { MPI_Group_free(&iter->second); } + ranks_group_.clear(); if (comm_group_world_ != MPI_GROUP_NULL) { MPI_Group_free(&comm_group_world_); + comm_group_world_ = MPI_GROUP_NULL; } - int finalized; - MPI_Finalized(&finalized); - if (finalized == 0) { - MPI_Finalize(); - } + MPI_Finalize(); } -MPIAdapter &MPIAdapter::Instance() { - static MPIAdapter instance; - return instance; -} - -int MPIAdapter::GetRankId() const { return rank_id_; } - void MPIAdapter::Init() { static bool init = false; if (init) { return; } - auto mpi_config_ptr = MpiConfig::GetInstance(); - MS_EXCEPTION_IF_NULL(mpi_config_ptr); - if (!mpi_config_ptr->enable_mpi()) { - MS_LOG(EXCEPTION) << "MPI is disabled now!Please enable mpi with mpi config first."; - } + int init_flag = 0; if (MPI_Initialized(&init_flag) != MPI_SUCCESS) { - MS_LOG(EXCEPTION) << "Check mpi initialized fail!"; + RAISE_EXCEPTION("Check mpi initialized fail!"); } if (init_flag == 0) { auto ret = MPI_Init(nullptr, nullptr); if (ret != MPI_SUCCESS) { - MS_LOG(EXCEPTION) << "Failed to init mpi!"; + RAISE_EXCEPTION("Failed to init mpi!"); } } MPI_Comm_group(MPI_COMM_WORLD, &comm_group_world_); if (comm_group_world_ == MPI_GROUP_NULL) { - MS_LOG(EXCEPTION) << "comm_group_world_ init fail!"; + RAISE_EXCEPTION("comm_group_world_ init fail!"); } auto ret = MPI_Comm_rank(MPI_COMM_WORLD, &rank_id_); if (ret != MPI_SUCCESS) { - MS_LOG(EXCEPTION) << "Failed to init mpi rank id!"; + RAISE_EXCEPTION("Failed to init mpi rank id!"); } ret = MPI_Comm_size(MPI_COMM_WORLD, &rank_size_); if (ret != MPI_SUCCESS) { - MS_LOG(EXCEPTION) << "Failed to init mpi rank size!rankid:" << rank_id_; + RAISE_EXCEPTION_WITH_PARAM("Failed to init mpi rank size!rankid:", rank_id_) } init = true; } MPI_Group MPIAdapter::AddGroup(const std::vector &ranks) { if (ranks.size() > static_cast(rank_size_) || ranks.empty()) { - MS_LOG(EXCEPTION) << "input rank size: " << ranks.size() << ", max rank size: " << rank_size_; + RAISE_EXCEPTION_WITH_PARAM("input rank size:", ranks.size()); } if (std::find(ranks.begin(), ranks.end(), rank_id_) == ranks.end()) { - MS_LOG(ERROR) << "rankid:" << rank_id_ << " is not in the input group."; - return MPI_GROUP_NULL; + RAISE_EXCEPTION_WITH_PARAM("local rankid does not in the input rank group!local rank id:", rank_id_); } std::lock_guard lock(group_mutex_); auto iter = ranks_group_.find(ranks); @@ -135,29 +156,28 @@ MPI_Group MPIAdapter::AddGroup(const std::vector &ranks) { MPI_Group group = MPI_GROUP_NULL; MPI_Group_incl(comm_group_world_, ranks.size(), ranks_input.data(), &group); if (group == MPI_GROUP_NULL) { - MS_LOG(EXCEPTION) << "create mpi group fail!rankid:" << rank_id_; + RAISE_EXCEPTION_WITH_PARAM("create mpi group fail!rankid:", rank_id_) } ranks_group_[ranks] = group; - MS_LOG(INFO) << "rank:" << rank_id_ << " add group:" << group; return group; } bool MPIAdapter::ReduceScatter(const float *input, float *output, const std::vector &ranks_group, size_t data_num, const std::string &op_type) { if (ranks_group.empty()) { - MS_LOG(ERROR) << "input rank group is empty!"; + RAISE_EXCEPTION("input rank group is empty!"); return false; } auto group = AddGroup(ranks_group); if (group == MPI_GROUP_NULL) { - MS_LOG(EXCEPTION) << "Get mpi group fail!rankid:" << rank_id_; + RAISE_EXCEPTION_WITH_PARAM("Get mpi group fail!rankid:", rank_id_) } MPI_Comm comm; MPI_Comm_create_group(MPI_COMM_WORLD, group, 0, &comm); if (comm == MPI_COMM_NULL) { - MS_LOG(EXCEPTION) << "create mpi comm fail!rankid:" << rank_id_; + RAISE_EXCEPTION_WITH_PARAM("create mpi comm fail!rankid:", rank_id_); } std::vector receive_count(ranks_group.size(), 0); for (size_t i = 0; i < ranks_group.size(); ++i) { @@ -168,13 +188,13 @@ bool MPIAdapter::ReduceScatter(const float *input, float *output, const std::vec auto ret = MPI_Reduce_scatter(input, output, receive_count.data(), MPI_FLOAT, op, comm); bool result = true; if (ret != MPI_SUCCESS) { - MS_LOG(ERROR) << "mpi reduce_scatter fail!ret = " << ret << ", rankid:" << rank_id_; + RAISE_EXCEPTION_WITH_PARAM("mpi reduce_scatter fail!ret = ", ret); result = false; } ret = MPI_Comm_free(&comm); if (ret != MPI_SUCCESS) { - MS_LOG(WARNING) << "mpi comm free fail! ret = " << ret << ", rankid:" << rank_id_; + RAISE_EXCEPTION_WITH_PARAM("mpi comm free fail! ret = ", ret); } return result; } @@ -184,19 +204,18 @@ bool MPIAdapter::ReduceScatterOverwriteInput(float *input, const std::vector &ranks_group, size_t data_num, - const std::string &op_type = kOpTypeSum); - bool ReduceScatterOverwriteInput(float *input, const std::vector &ranks_group, size_t input_data_num, - size_t output_size, const std::string &op_type = kOpTypeSum, - float *output = nullptr); - bool AllGather(const float *input, float *output, const std::vector &ranks_group, size_t data_num); + FUNC_EXPORT static std::shared_ptr Instance(); + FUNC_EXPORT int GetRankId() const { return rank_id_; } + FUNC_EXPORT int GetRankSize() const { return rank_size_; } +#ifdef ENABLE_MPI + FUNC_EXPORT ~MPIAdapter(); + FUNC_EXPORT bool ReduceScatter(const float *input, float *output, const std::vector &ranks_group, + size_t data_num, const std::string &op_type = kOpTypeSum); + FUNC_EXPORT bool ReduceScatterOverwriteInput(float *input, const std::vector &ranks_group, size_t in_data_num, + size_t output_size, const std::string &op_type = kOpTypeSum, + float *output = nullptr); + FUNC_EXPORT bool AllGather(const float *input, float *output, const std::vector &ranks_group, size_t data_num); +#else + FUNC_EXPORT ~MPIAdapter() = default; +#endif // ENABLE_MPI private: +#ifdef ENABLE_MPI MPIAdapter(); void Init(); MPI_Group AddGroup(const std::vector &ranks); - int rank_id_; - int rank_size_; MPI_Group comm_group_world_; // key:ranks group, value: mpi group std::map, MPI_Group> ranks_group_; std::mutex group_mutex_; +#else + MPIAdapter() = default; +#endif // ENABLE_MPI + int rank_id_{-1}; + int rank_size_{0}; + + static std::shared_ptr instance_; }; } // namespace cpu } // namespace device } // namespace mindspore -#endif // ENABLE_MPI #endif // MINDSPORE_CCSRC_DEVICE_CPU_MPI_MPI_ADAPTER_H_ diff --git a/mindspore/ccsrc/device/cpu/mpi/mpi_interface.cc b/mindspore/ccsrc/device/cpu/mpi/mpi_interface.cc new file mode 100644 index 0000000000..c047c9ae4e --- /dev/null +++ b/mindspore/ccsrc/device/cpu/mpi/mpi_interface.cc @@ -0,0 +1,33 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include "device/cpu/mpi/mpi_adapter.h" + +namespace mindspore { +namespace device { +namespace cpu { +int get_rank_id() { return MPIAdapter::Instance()->GetRankId(); } + +int get_rank_size() { return MPIAdapter::Instance()->GetRankSize(); } + +PYBIND11_MODULE(_ms_mpi, mpi_interface) { + mpi_interface.doc() = "mindspore mpi python wrapper"; + mpi_interface.def("get_rank_id", &get_rank_id, "get rank id"); + mpi_interface.def("get_rank_size", &get_rank_size, "get rank size"); +} +} // namespace cpu +} // namespace device +} // namespace mindspore diff --git a/mindspore/ccsrc/device/gpu/mpi/mpi_initializer.cc b/mindspore/ccsrc/device/gpu/mpi/mpi_initializer.cc index bcad74e5b5..9baec161e5 100644 --- a/mindspore/ccsrc/device/gpu/mpi/mpi_initializer.cc +++ b/mindspore/ccsrc/device/gpu/mpi/mpi_initializer.cc @@ -17,7 +17,6 @@ #include "device/gpu/mpi/mpi_initializer.h" #include -#include #include namespace mindspore { @@ -54,12 +53,6 @@ MPIInitializer &MPIInitializer::GetInstance() { int MPIInitializer::get_rank_id() { return MPIInitializer::GetInstance().rank_id_; } int MPIInitializer::get_rank_size() { return MPIInitializer::GetInstance().rank_size_; } - -PYBIND11_MODULE(_ms_mpi, mpi_initializer) { - mpi_initializer.doc() = "mindspore mpi python wrapper"; - mpi_initializer.def("get_rank_id", &MPIInitializer::get_rank_id, "get rank id"); - mpi_initializer.def("get_rank_size", &MPIInitializer::get_rank_size, "get rank size"); -} } // namespace gpu } // namespace device } // namespace mindspore diff --git a/mindspore/ccsrc/kernel/cpu/allgather_cpu_kernel.cc b/mindspore/ccsrc/kernel/cpu/allgather_cpu_kernel.cc index abb0c65d27..276d03625b 100644 --- a/mindspore/ccsrc/kernel/cpu/allgather_cpu_kernel.cc +++ b/mindspore/ccsrc/kernel/cpu/allgather_cpu_kernel.cc @@ -47,7 +47,7 @@ bool AllGatherCPUKernel::Launch(const std::vector &inputs, auto output_addr = reinterpret_cast(outputs[0]->addr); auto input_data_num = inputs[0]->size / sizeof(float); - return device::cpu::MPIAdapter::Instance().AllGather(input_addr, output_addr, ranks_group_, input_data_num); + return device::cpu::MPIAdapter::Instance()->AllGather(input_addr, output_addr, ranks_group_, input_data_num); } } // namespace kernel } // namespace mindspore diff --git a/mindspore/ccsrc/kernel/cpu/embedding_look_up_comm_grad_cpu_kernel.cc b/mindspore/ccsrc/kernel/cpu/embedding_look_up_comm_grad_cpu_kernel.cc index 837cb647e3..8a061f63c1 100644 --- a/mindspore/ccsrc/kernel/cpu/embedding_look_up_comm_grad_cpu_kernel.cc +++ b/mindspore/ccsrc/kernel/cpu/embedding_look_up_comm_grad_cpu_kernel.cc @@ -51,8 +51,8 @@ bool EmbeddingLookUpCommGradCPUKernel::Launch(const std::vectorAllGather(input_addr + i * input_split_lens, + output_addr + i * output_split_lens, rank_group, input_split_lens); } #if defined(_WIN32) || defined(_WIN64) auto end_time = std::chrono::steady_clock::now(); diff --git a/mindspore/ccsrc/kernel/cpu/embedding_look_up_cpu_kernel.cc b/mindspore/ccsrc/kernel/cpu/embedding_look_up_cpu_kernel.cc index e91b5d8109..a63f825ec0 100644 --- a/mindspore/ccsrc/kernel/cpu/embedding_look_up_cpu_kernel.cc +++ b/mindspore/ccsrc/kernel/cpu/embedding_look_up_cpu_kernel.cc @@ -105,9 +105,9 @@ bool EmbeddingLookUpCPUKernel::Launch(const std::vector &inp size_t reduce_scatter_out_lens = one_split_lens / 8; const std::vector &group = {0, 1, 2, 3, 4, 5, 6, 7}; for (int i = 0; i < split_num_; i++) { - device::cpu::MPIAdapter::Instance().ReduceScatter(reinterpret_cast(gather_v2_out_) + i * one_split_lens, - output_addr + i * reduce_scatter_out_lens, group, - one_split_lens / 8, "sum"); + device::cpu::MPIAdapter::Instance()->ReduceScatter(reinterpret_cast(gather_v2_out_) + i * one_split_lens, + output_addr + i * reduce_scatter_out_lens, group, + one_split_lens / 8, "sum"); } } #endif diff --git a/mindspore/ccsrc/kernel/cpu/reduce_scatter_cpu_kernel.cc b/mindspore/ccsrc/kernel/cpu/reduce_scatter_cpu_kernel.cc index fd8a74eb6b..b0c25387e2 100644 --- a/mindspore/ccsrc/kernel/cpu/reduce_scatter_cpu_kernel.cc +++ b/mindspore/ccsrc/kernel/cpu/reduce_scatter_cpu_kernel.cc @@ -47,8 +47,8 @@ bool ReduceScatterCPUKernel::Launch(const std::vector &input auto output_addr = reinterpret_cast(outputs[0]->addr); auto output_data_num = outputs[0]->size / sizeof(float); - return device::cpu::MPIAdapter::Instance().ReduceScatter(input_addr, output_addr, ranks_group_, output_data_num, - op_type_); + return device::cpu::MPIAdapter::Instance()->ReduceScatter(input_addr, output_addr, ranks_group_, output_data_num, + op_type_); } } // namespace kernel } // namespace mindspore diff --git a/mindspore/context.py b/mindspore/context.py index ad601f8fab..aaf663882d 100644 --- a/mindspore/context.py +++ b/mindspore/context.py @@ -25,7 +25,6 @@ from mindspore._c_expression import MSContext from mindspore._checkparam import args_type_check from mindspore.parallel._auto_parallel_context import _set_auto_parallel_context, _get_auto_parallel_context, \ _reset_auto_parallel_context -from mindspore.parallel.mpi._mpi_config import _set_mpi_config, _get_mpi_config __all__ = ['GRAPH_MODE', 'PYNATIVE_MODE', 'set_context', 'get_context', 'set_auto_parallel_context', 'get_auto_parallel_context', 'reset_auto_parallel_context'] @@ -608,40 +607,3 @@ def get_context(attr_key): raise ValueError( "Get context keyword %s is not recognized!" % attr_key) return getattr(_context(), attr_key) - -@args_type_check(enable_mpi=bool) -def set_mpi_config(**kwargs): - """ - Sets mpi config for running environment. - - mpi config should be configured before running your program. If there is no configuration, - mpi moudle will be disabled by default. - - Note: - Attribute name is required for setting attributes. - - Args: - enable_mpi (bool): Whether to enable mpi. Default: False. - - Raises: - ValueError: If input key is not an attribute in mpi config. - - Examples: - >>> mpiconfig.set_mpi_config(enable_mpi=True) - """ - _set_mpi_config(**kwargs) - -def get_mpi_config(attr_key): - """ - Gets mpi config attribute value according to the input key. - - Args: - attr_key (str): The key of the attribute. - - Returns: - Object, The value of given attribute key. - - Raises: - ValueError: If input key is not an attribute in context. - """ - return _get_mpi_config(attr_key) diff --git a/mindspore/parallel/mpi/_mpi_config.py b/mindspore/parallel/mpi/_mpi_config.py index e43305fb76..811066ca5d 100644 --- a/mindspore/parallel/mpi/_mpi_config.py +++ b/mindspore/parallel/mpi/_mpi_config.py @@ -104,7 +104,7 @@ def _get_mpi_config(attr_key): Object, The value of given attribute key. Raises: - ValueError: If input key is not an attribute in context. + ValueError: If input key is not an attribute in config. """ if not hasattr(_mpi_config(), attr_key): raise ValueError("Get context keyword %s is not recognized!" % attr_key) From 830af41c1114e977000a038eac31f40a3b094de0 Mon Sep 17 00:00:00 2001 From: Alexey Shevlyakov Date: Tue, 23 Jun 2020 10:28:34 -0400 Subject: [PATCH 029/254] fix memory leak --- mindspore/ccsrc/dataset/engine/execution_tree.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mindspore/ccsrc/dataset/engine/execution_tree.cc b/mindspore/ccsrc/dataset/engine/execution_tree.cc index 5c921bba84..80a11aca02 100644 --- a/mindspore/ccsrc/dataset/engine/execution_tree.cc +++ b/mindspore/ccsrc/dataset/engine/execution_tree.cc @@ -209,10 +209,10 @@ Status ExecutionTree::Prepare() { Status ExecutionTree::PrepareTreePreAction() { bool modified = false; - std::vector pre_actions; + std::vector> pre_actions; // Construct pre actions - pre_actions.push_back(new MapColumnReorder()); - pre_actions.push_back(new GlobalShufflePass()); + pre_actions.push_back(std::make_unique()); + pre_actions.push_back(std::make_unique()); // Apply pre action passes for (auto &pass : pre_actions) { RETURN_IF_NOT_OK(pass->Run(this, &modified)); From b7a50b24d63cdec34126648ecab6dcd71843cc6a Mon Sep 17 00:00:00 2001 From: zhoufeng Date: Tue, 23 Jun 2020 22:12:37 +0800 Subject: [PATCH 030/254] Disable cuda9.2, use cuda10.1 as default Signed-off-by: zhoufeng --- build.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/build.sh b/build.sh index 608b723425..c6f80b1269 100755 --- a/build.sh +++ b/build.sh @@ -50,7 +50,7 @@ usage() echo " -D Enable dumping of function graph ir, default on" echo " -z Compile dataset & mindrecord, default on" echo " -M Enable MPI and NCCL for GPU training, gpu default on" - echo " -V Specify the minimum required cuda version, default CUDA 9.2" + echo " -V Specify the minimum required cuda version, default CUDA 10.1" echo " -I Compile predict, default off" echo " -K Compile with AKG, default off" echo " -s Enable serving module, default off" @@ -89,7 +89,7 @@ checkopts() ENABLE_DUMP_IR="on" COMPILE_MINDDATA="on" ENABLE_MPI="off" - CUDA_VERSION="9.2" + CUDA_VERSION="10.1" COMPILE_PREDICT="off" USE_GLOG="on" PREDICT_PLATFORM="" @@ -194,6 +194,10 @@ checkopts() usage exit 1 fi + if [[ "X$OPTARG" == "X9.2" ]]; then + echo "Unsupported CUDA version 9.2" + exit 1 + fi CUDA_VERSION="$OPTARG" ;; P) From 2a22d9dcc9b4556befbe595c009c7c9641b4e2b4 Mon Sep 17 00:00:00 2001 From: jonyguo Date: Tue, 23 Jun 2020 22:16:10 +0800 Subject: [PATCH 031/254] fix: change field name from 'data' to 'image' in mindrecord which similar with ImageFolderDatasetV2 --- .../ImageNet_Similar_Perf/imagenet/mr_api.py | 6 +++--- mindspore/mindrecord/tools/imagenet_to_mr.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/example/cv_to_mindrecord/ImageNet_Similar_Perf/imagenet/mr_api.py b/example/cv_to_mindrecord/ImageNet_Similar_Perf/imagenet/mr_api.py index c8129ec9ff..12272b4faf 100644 --- a/example/cv_to_mindrecord/ImageNet_Similar_Perf/imagenet/mr_api.py +++ b/example/cv_to_mindrecord/ImageNet_Similar_Perf/imagenet/mr_api.py @@ -26,8 +26,8 @@ import os import pickle ######## mindrecord_schema begin ########## -mindrecord_schema = {"label": {"type": "int64"}, - "data": {"type": "bytes"}, +mindrecord_schema = {"label": {"type": "int32"}, + "image": {"type": "bytes"}, "file_name": {"type": "string"}} ######## mindrecord_schema end ########## @@ -121,5 +121,5 @@ def mindrecord_dict_data(task_id): if not image_bytes: print("The image file: {} is invalid.".format(file_name)) continue - data["data"] = image_bytes + data["image"] = image_bytes yield data diff --git a/mindspore/mindrecord/tools/imagenet_to_mr.py b/mindspore/mindrecord/tools/imagenet_to_mr.py index e941e76477..0aa870384e 100644 --- a/mindspore/mindrecord/tools/imagenet_to_mr.py +++ b/mindspore/mindrecord/tools/imagenet_to_mr.py @@ -114,7 +114,7 @@ class ImageNetToMR: if not image_bytes: logger.warning("The image file: {} is invalid.".format(file_name)) continue - data["data"] = image_bytes + data["image"] = image_bytes yield data def transform(self): @@ -126,8 +126,8 @@ class ImageNetToMR: """ t0_total = time.time() - imagenet_schema_json = {"label": {"type": "int64"}, - "data": {"type": "bytes"}, + imagenet_schema_json = {"label": {"type": "int32"}, + "image": {"type": "bytes"}, "file_name": {"type": "string"}} logger.info("transformed MindRecord schema is: {}".format(imagenet_schema_json)) From c6c6fbed385650974b4c00980a7ec292423f2253 Mon Sep 17 00:00:00 2001 From: hesham Date: Tue, 23 Jun 2020 20:33:28 -0400 Subject: [PATCH 032/254] Slice docstring changes --- mindspore/dataset/transforms/c_transforms.py | 38 ++++++++++---------- 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/mindspore/dataset/transforms/c_transforms.py b/mindspore/dataset/transforms/c_transforms.py index ffe711b106..48e986202c 100644 --- a/mindspore/dataset/transforms/c_transforms.py +++ b/mindspore/dataset/transforms/c_transforms.py @@ -75,28 +75,30 @@ class Slice(cde.SliceOp): Slice operation to extract a tensor out using the given n slices. The functionality of Slice is similar to NumPy indexing feature. - (Currently only rank 1 Tensors are supported) + (Currently only rank-1 tensors are supported). Args: - *slices(Variable length argument list): Maximum `n` number of arguments to slice a tensor of rank `n`. + *slices(Variable length argument list, supported types are, int, list(int), slice, None or Ellipses): + Maximum `n` number of arguments to slice a tensor of rank `n`. One object in slices can be one of: - 1. int: slice this index only. Negative index is supported. - 2. slice object: slice the generated indices from the slice object. Similar to `start:stop:step`. - 3. None: slice the whole dimension. Similar to `:` in python indexing. - 4. Ellipses ...: slice all dimensions between the two slices. + 1. :py:obj:`int`: Slice this index only. Negative index is supported. + 2. :py:obj:`list(int)`: Slice these indices ion the list only. Negative indices are supdeported. + 3. :py:obj:`slice`: Slice the generated indices from the slice object. Similar to `start:stop:step`. + 4. :py:obj:`None`: Slice the whole dimension. Similar to `:` in python indexing. + 5. :py:obj:`Ellipses`: Slice all dimensions between the two slices. Similar to `...` in python indexing. Examples: - >>> # Data before - >>> # | col | - >>> # +---------+ - >>> # | [1,2,3] | - >>> # +---------| - >>> data = data.map(operations=Slice(slice(1,3))) # slice indices 1 and 2 only - >>> # Data after - >>> # | col | - >>> # +------------+ - >>> # | [1,2] | - >>> # +------------| + >>> # Data before + >>> # | col | + >>> # +---------+ + >>> # | [1,2,3] | + >>> # +---------| + >>> data = data.map(operations=Slice(slice(1,3))) # slice indices 1 and 2 only + >>> # Data after + >>> # | col | + >>> # +---------+ + >>> # | [2,3] | + >>> # +---------| """ @check_slice_op @@ -167,7 +169,7 @@ class PadEnd(cde.PadEndOp): Pad input tensor according to `pad_shape`, need to have same rank. Args: - pad_shape (list of `int`): list on integers representing the shape needed. Dimensions that set to `None` will + pad_shape (list(int)): list on integers representing the shape needed. Dimensions that set to `None` will not be padded (i.e., original dim will be used). Shorter dimensions will truncate the values. pad_value (python types (str, bytes, int, float, or bool), optional): value used to pad. Default to 0 or empty string in case of Tensors of strings. From 108dd7a4a21617cc7157e7881b673161df072aac Mon Sep 17 00:00:00 2001 From: ougongchang Date: Tue, 23 Jun 2020 10:51:13 +0800 Subject: [PATCH 033/254] Make sure record the first step data in SummaryCollector, and catch the ValueError when the loss is not a Scalar. --- .../train/callback/_summary_collector.py | 49 ++++++++++++------- 1 file changed, 32 insertions(+), 17 deletions(-) diff --git a/mindspore/train/callback/_summary_collector.py b/mindspore/train/callback/_summary_collector.py index f86c1a2d32..6d5ec45d5b 100644 --- a/mindspore/train/callback/_summary_collector.py +++ b/mindspore/train/callback/_summary_collector.py @@ -73,7 +73,7 @@ class SummaryCollector(Callback): summary_dir (str): The collected data will be persisted to this directory. If the directory does not exist, it will be created automatically. collect_freq (int): Set the frequency of data collection, it should be greater then zero, - and the unit is `step`. Default: 10. + and the unit is `step`. Default: 10. The first step will be recorded at any time. It is important to note that if the data sink mode is used, the unit will become the `epoch`. It is not recommended to collect data too frequently, which can affect performance. collect_specified_data (Union[None, dict]): Perform custom operations on the collected data. Default: None. @@ -142,9 +142,6 @@ class SummaryCollector(Callback): 'histogram_regular': None } - # _OPTIMIZER_FAILED means find optimizer failed, so we will not collect data about optimizer. - _OPTIMIZER_FAILED = 'Failed' - def __init__(self, summary_dir, collect_freq=10, collect_specified_data=None, keep_default_action=True, custom_lineage_data=None): super(SummaryCollector, self).__init__() @@ -158,7 +155,8 @@ class SummaryCollector(Callback): self._check_action(keep_default_action) self._collect_specified_data = self._process_specified_data(collect_specified_data, keep_default_action) - logger.info(f"For `collect_specified_data` the value after processing is: {self._collect_specified_data}.") + msg = f"For 'collect_specified_data' the value after processing is: {self._collect_specified_data}." + logger.info(msg) self._check_custom_lineage_data(custom_lineage_data) self._custom_lineage_data = custom_lineage_data @@ -167,6 +165,7 @@ class SummaryCollector(Callback): self._has_saved_train_network = False self._has_saved_custom_data = False self._is_parse_loss_success = True + self._first_step = True def __enter__(self): self._record = SummaryRecord(log_dir=self._summary_dir) @@ -228,7 +227,7 @@ class SummaryCollector(Callback): if specified_data is None: if action: return self._DEFAULT_SPECIFIED_DATA - return None + return dict() check_value_type('collect_specified_data', specified_data, [dict, type(None)]) @@ -282,9 +281,13 @@ class SummaryCollector(Callback): cb_params = run_context.original_args() if cb_params.mode == ModeEnum.TRAIN.value: - if cb_params.cur_step_num % self._collect_freq: + + # Make sure the first step data is recorded + if not self._first_step and cb_params.cur_step_num % self._collect_freq: return + self._first_step = False + if not self._has_saved_train_network: self._collect_graphs(cb_params) @@ -357,7 +360,8 @@ class SummaryCollector(Callback): input_data = input_data[0] try: self._record.add_value(PluginEnum.IMAGE.value, 'input_data/auto', input_data) - except ValueError: + except ValueError as ex: + logger.warning(str(ex)) self._collect_specified_data['collect_input_data'] = False return @@ -395,7 +399,12 @@ class SummaryCollector(Callback): loss = self._get_loss(cb_params) if loss is None: return - self._record.add_value(PluginEnum.SCALAR.value, 'loss/auto', loss) + + try: + self._record.add_value(PluginEnum.SCALAR.value, 'loss/auto', loss) + except ValueError as exc: + logger.warning(str(exc)) + self._collect_specified_data['collect_metric'] = False def _get_loss(self, cb_params): """ @@ -446,7 +455,9 @@ class SummaryCollector(Callback): Returns: Union[Optimizer, None], if parse optimizer success, will return a optimizer, else return None. """ - if self._optimizer == self._OPTIMIZER_FAILED: + # 'optimizer_failed' means find optimizer failed, so we will not collect data about optimizer. + optimizer_failed = 'Failed' + if self._optimizer == optimizer_failed: return None if self._optimizer is not None: @@ -458,9 +469,11 @@ class SummaryCollector(Callback): optimizer = self._parse_optimizer_by_network(network) if optimizer is None or not isinstance(optimizer, Optimizer): - logger.warning("Can not find optimizer in network, or the optimizer does not inherit Mindpore's optimizer, " - "so we will not collect data about optimizer in SummaryCollector.") - optimizer = self._OPTIMIZER_FAILED + logger.warning("Can not find optimizer in network, or the optimizer does not inherit MindSpore's " + "optimizer, so we will not collect data about optimizer in SummaryCollector.") + optimizer = None + + self._optimizer = optimizer if optimizer is not None else optimizer_failed return optimizer @@ -469,6 +482,8 @@ class SummaryCollector(Callback): """Parse optimizer from network, if parse success will return a optimizer, else return None.""" optimizer = None for _, cell in network.cells_and_names(): + if isinstance(cell, Optimizer): + return cell try: optimizer = getattr(cell, 'optimizer') except AttributeError: @@ -489,11 +504,11 @@ class SummaryCollector(Callback): if 'histogram_regular' not in self._collect_specified_data: return - self._optimizer = self._get_optimizer(cb_params) - if self._optimizer is None: + optimizer = self._get_optimizer(cb_params) + if optimizer is None: return - parameters = self._optimizer.parameters + parameters = optimizer.parameters regular = self._collect_specified_data.get('histogram_regular') if regular is not None: for parameter in parameters: @@ -538,7 +553,7 @@ class SummaryCollector(Callback): train_lineage[LineageMetadata.loss] = None optimizer = self._get_optimizer(cb_params) - learning_rate = self._get_learning_rate(optimizer) + learning_rate = self._get_learning_rate(optimizer) if optimizer is not None else None if learning_rate is not None: train_lineage[LineageMetadata.learning_rate] = list(np.atleast_1d(learning_rate.asnumpy()))[0] From 7029bc5dd30ef8e848e594eef8b1413648e53e0e Mon Sep 17 00:00:00 2001 From: hongxing Date: Tue, 23 Jun 2020 16:30:32 +0200 Subject: [PATCH 034/254] fix onehot axis --- .../rec_core/rec_generate_strategy.cc | 149 ++++++++++++++---- .../rec_core/rec_generate_strategy.h | 30 ++-- .../auto_parallel/rec_core/rec_graph.h | 3 +- .../auto_parallel/rec_core/rec_parse_graph.cc | 22 +-- .../auto_parallel/rec_core/rec_parse_graph.h | 13 +- 5 files changed, 157 insertions(+), 60 deletions(-) diff --git a/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_generate_strategy.cc b/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_generate_strategy.cc index 19e07aae02..630833f4a6 100644 --- a/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_generate_strategy.cc +++ b/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_generate_strategy.cc @@ -28,10 +28,10 @@ namespace mindspore { namespace parallel { -void GenerateStrategy(std::shared_ptr graph, const std::vector> &ops, - const std::shared_ptr>> eli_list, +void GenerateStrategy(const std::shared_ptr &graph, const std::vector> &ops, + const std::shared_ptr>> &eli_list, const std::vector> &input_tensor_names, - const std::shared_ptr> index_list) { + const std::shared_ptr> &index_list) { MS_EXCEPTION_IF_NULL(graph); MS_EXCEPTION_IF_NULL(eli_list); MS_EXCEPTION_IF_NULL(index_list); @@ -140,10 +140,24 @@ std::vector> PrepareOneHot(const std::shared_ptr &gr const std::vector> &ops, const size_t iter_graph, const size_t iter_ops) { std::vector> strategies = MakeRecSearchStrategy(graph, ops, iter_graph, iter_ops); - strategies[0][0] = strategies[0][1]; - strategies[0][1] = 1; - graph->nodes[iter_graph].tensor_parm.tensor_str.str_h = graph->nodes[iter_graph].tensor_parm.tensor_str.str_w; - graph->nodes[iter_graph].tensor_parm.tensor_str.str_w = 1.0; + + int32_t axis = -1; + auto iter = ops[iter_ops]->attrs().find(AXIS); + if (iter != ops[iter_ops]->attrs().end()) { + MS_EXCEPTION_IF_NULL(iter->second); + if (iter->second->isa()) { + axis = iter->second->cast()->value(); + } else { + MS_LOG(EXCEPTION) << ops[iter_ops]->name() << ": The value of axis is not int."; + } + } + if (axis == -1) { + strategies[0][0] = strategies[0][1]; + strategies[0][1] = 1; + graph->nodes[iter_graph].tensor_parm.tensor_str.str_h = graph->nodes[iter_graph].tensor_parm.tensor_str.str_w; + graph->nodes[iter_graph].tensor_parm.tensor_str.str_w = 1.0; + } + std::vector s_empty = {}; strategies.push_back(s_empty); strategies.push_back(s_empty); @@ -221,7 +235,7 @@ std::vector> MakeRecSearchStrategy(const std::shared_ptrname() << ": Tensor's output size is unexcepted."; } strategies.push_back(s); } @@ -241,7 +255,7 @@ std::vector> MakeDataParallelStrategy(const std::shared_ptr StrategyPtr origin_strategy = ops[iter_ops]->strategy(); std::vector> strategies; size_t max_device_num = g_device_manager->DeviceNum(); - size_t target_tensor_batch = ops[iter_ops]->outputs_tensor_info()[0].shape()[0]; + size_t target_tensor_batch = ops[iter_ops]->inputs_tensor_info()[0].shape()[0]; for (size_t iter_op_inputs = 0; iter_op_inputs < ops[iter_ops]->inputs_tensor_info().size(); iter_op_inputs++) { if (iter_op_inputs >= origin_strategy->GetInputDim().size()) { MS_LOG(EXCEPTION) << "Failure: Strategy's InputDim out of range."; @@ -256,8 +270,10 @@ std::vector> MakeDataParallelStrategy(const std::shared_ptr } else { s.push_back(1); } + } else if (input_size == 0) { + s = {}; } else { - MS_LOG(ERROR) << "Tensor's shape is unknown."; + MS_LOG(EXCEPTION) << ops[iter_ops]->name() << ": Tensor's shape is unknown."; } } strategies.push_back(s); @@ -304,13 +320,13 @@ std::vector> PrepareStrategy(const std::shared_ptr & } } -void GeneratePartitionedOperatorStrategy(const std::shared_ptr graph, +void GeneratePartitionedOperatorStrategy(const std::shared_ptr &graph, const std::vector> &ops, - const std::shared_ptr> index_list) { + const std::shared_ptr> &index_list) { for (size_t iter_ops = 0; iter_ops < (size_t)index_list->size(); iter_ops++) { std::vector> strategies; size_t iter_graph = index_list->at(iter_ops); - if (iter_graph != SIZE_MAX) { + if (iter_graph != SIZE_MAX && ops[iter_ops]->type() != GET_NEXT) { strategies = PrepareStrategy(graph, ops, iter_graph, iter_ops); } StrategyPtr sp = std::make_shared(0, strategies); @@ -335,7 +351,7 @@ size_t FindIndexOfOperatorIncoming(const std::vector> & return incoming_op_index; } -std::vector CopyIncomingOperatorOutputStrategy(const std::shared_ptr graph, +std::vector CopyIncomingOperatorOutputStrategy(const std::shared_ptr &graph, const std::vector> &ops, const size_t iter_ops, const size_t iter_graph) { std::vector s; @@ -354,8 +370,10 @@ std::vector CopyIncomingOperatorOutputStrategy(const std::shared_ptrnodes[iter_graph].tensor_parm.tensor_str.str_c); s.push_back(1 / graph->nodes[iter_graph].tensor_parm.tensor_str.str_h); s.push_back(1 / graph->nodes[iter_graph].tensor_parm.tensor_str.str_w); + } else if (input_stra_dim == 0) { + s = {}; } else { - MS_LOG(ERROR) << "Tensor's shape is unknown."; + MS_LOG(EXCEPTION) << ops[iter_ops]->name() << ": Tensor's shape is unknown."; } break; } @@ -365,7 +383,8 @@ std::vector CopyIncomingOperatorOutputStrategy(const std::shared_ptr PrepareIncomingOperatorInputStrategy(const std::vector> &ops, const size_t incoming_op_index) { std::vector s; - if (ops[incoming_op_index]->type() == RESHAPE || ops[incoming_op_index]->type() == GATHERV2) { + if (ops[incoming_op_index]->type() == RESHAPE || ops[incoming_op_index]->type() == GATHERV2 || + ops[incoming_op_index]->type() == TRANSPOSE) { return s; } auto strategy = ops[incoming_op_index]->selected_strategy(); @@ -433,13 +452,23 @@ std::vector ModifyStrategyIfSqueezeIncoming(const std::vector> &ops, const size_t iter_ops) { + bool keepdims = false; + auto keep_dims_iter = ops[iter_ops]->attrs().find(KEEP_DIMS); + if (keep_dims_iter == ops[iter_ops]->attrs().end()) { + MS_LOG(EXCEPTION) << ops[iter_ops]->name() << ": Don't have attr keep_dims."; + } + MS_EXCEPTION_IF_NULL(keep_dims_iter->second); + if (!keep_dims_iter->second->isa()) { + MS_LOG(EXCEPTION) << ops[iter_ops]->name() << ": Keep_dims is not a bool."; + } + keepdims = keep_dims_iter->second->cast()->value(); + return keepdims; +} + std::vector GetDimList(const std::vector> &ops, const size_t iter_ops) { std::vector dim_list; - bool keep_dims; - if (!ops[iter_ops]->attrs().find(KEEP_DIMS)->second->isa()) { - MS_LOG(EXCEPTION) << "Failure: Parameter keep_dims is not a boolean value." << std::endl; - } - keep_dims = ops[iter_ops]->attrs().find(KEEP_DIMS)->second->cast()->value(); + bool keep_dims = GetKeepDims(ops, iter_ops); if (keep_dims != false) { return dim_list; } @@ -485,6 +514,62 @@ std::vector ModifyStrategyIfReduceIncoming(const std::vector GetDimListFromAttrs(const std::vector> &ops, const size_t iter_ops) { + std::vector dim_list; + auto iter = ops[iter_ops]->attrs().find(AXIS); + if (iter == ops[iter_ops]->attrs().end()) { + MS_LOG(EXCEPTION) << ops[iter_ops]->name() << ": Don't have attr axis."; + } + auto input_dim = ops[iter_ops]->inputs_tensor_info()[0].shape().size(); + MS_EXCEPTION_IF_NULL(iter->second); + if (iter->second->isa()) { + auto attr_axis = GetValue>(iter->second); + if (attr_axis.empty()) { + for (size_t i = 0; i < input_dim; ++i) { + dim_list.push_back(SizeToInt(i)); + } + } else { + for (auto &axis : attr_axis) { + axis < 0 ? dim_list.push_back(axis + SizeToInt(input_dim)) : dim_list.push_back(axis); + } + } + } else if (iter->second->isa()) { + int axis = GetValue(iter->second); + axis < 0 ? dim_list.push_back(axis + SizeToInt(input_dim)) : dim_list.push_back(axis); + } else { + MS_LOG(EXCEPTION) << "Axis type is invalid."; + } + return dim_list; +} + +std::vector ModifyStrategyIfArgIncoming(const std::vector> &ops, + const size_t incoming_op_index, std::vector s) { + bool keepdims = GetKeepDims(ops, incoming_op_index); + if (keepdims) { + return s; + } + + std::vector s_Arg; + std::vector axis_list; + for (size_t i = 0; i < s.size(); i++) { + axis_list.push_back(i); + } + + auto dim_list = GetDimListFromAttrs(ops, incoming_op_index); + for (auto axis : dim_list) { + auto it = find(axis_list.begin(), axis_list.end(), axis); + if (it == axis_list.end()) { + MS_LOG(EXCEPTION) << "Failure: Can not find dimension indexes in Axis." << std::endl; + } + axis_list.erase(it); + } + + for (size_t i = 0; i < (size_t)axis_list.size(); i++) { + s_Arg.push_back(s[axis_list[i]]); + } + return s_Arg; +} + std::vector CopyIncomingOperatorInputStrategy(const std::vector> &ops, const size_t iter_ops, const size_t incoming_op_index) { std::vector s; @@ -497,6 +582,9 @@ std::vector CopyIncomingOperatorInputStrategy(const std::vectortype() == REDUCE_MIN || ops[incoming_op_index]->type() == REDUCE_MEAN) { s = ModifyStrategyIfReduceIncoming(ops, incoming_op_index, s); } + if (ops[incoming_op_index]->type() == ARGMAXWITHVALUE || ops[incoming_op_index]->type() == ARGMINWITHVALUE) { + s = ModifyStrategyIfArgIncoming(ops, incoming_op_index, s); + } } return s; } @@ -551,11 +639,11 @@ std::vector> GenerateStrategiesFromStrategy(const std::vect return stra; } -void GenerateEliminatedOperatorStrategyForward(const std::shared_ptr graph, +void GenerateEliminatedOperatorStrategyForward(const std::shared_ptr &graph, const std::vector> &ops, const std::vector> &input_tensor_names, - const std::shared_ptr> index_list, - const std::shared_ptr> no_stra_op_list) { + const std::shared_ptr> &index_list, + const std::shared_ptr> &no_stra_op_list) { if (no_stra_op_list->size() == 0) { return; } @@ -624,7 +712,8 @@ std::vector CopyOutgoingOperatorInputStrategy(const std::vector s; if (ops[iter_ops]->type() == REDUCE_MAX || ops[iter_ops]->type() == REDUCE_MIN || ops[iter_ops]->type() == REDUCE_SUM || ops[iter_ops]->type() == REDUCE_MEAN || ops[iter_ops]->type() == RESHAPE || - ops[iter_ops]->type() == GATHERV2) { + ops[iter_ops]->type() == GATHERV2 || ops[iter_ops]->type() == TRANSPOSE || + ops[iter_ops]->type() == ARGMAXWITHVALUE || ops[iter_ops]->type() == ARGMINWITHVALUE) { return s; } @@ -656,7 +745,7 @@ std::vector CopyOutgoingOperatorInputStrategy(const std::vector> &ops, const std::vector> &input_tensor_names, - const std::shared_ptr> no_stra_op_list) { + const std::shared_ptr> &no_stra_op_list) { if (no_stra_op_list->size() == 0) { return; } @@ -686,16 +775,16 @@ void GenerateEliminatedOperatorStrategyBackward(const std::vector graph, +void GenerateRemainingOperatorStrategy(const std::shared_ptr &graph, const std::vector> &ops, const std::vector> &input_tensor_names, - const std::shared_ptr> index_list, - const std::shared_ptr> no_stra_op_list) { + const std::shared_ptr> &index_list, + const std::shared_ptr> &no_stra_op_list) { if (no_stra_op_list->size() == 0) { return; } - size_t no_stra_op_list_size; + size_t no_stra_op_list_size = no_stra_op_list->size(); do { no_stra_op_list_size = no_stra_op_list->size(); GenerateEliminatedOperatorStrategyForward(graph, ops, input_tensor_names, index_list, no_stra_op_list); diff --git a/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_generate_strategy.h b/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_generate_strategy.h index c9604b449f..1e8080f2b7 100644 --- a/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_generate_strategy.h +++ b/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_generate_strategy.h @@ -27,10 +27,10 @@ namespace mindspore { namespace parallel { -void GenerateStrategy(std::shared_ptr graph, const std::vector> &ops, - const std::shared_ptr>> eli_list, +void GenerateStrategy(const std::shared_ptr &graph, const std::vector> &ops, + const std::shared_ptr>> &eli_list, const std::vector> &input_tensor_names, - const std::shared_ptr> index_list); + const std::shared_ptr> &index_list); std::vector> PrepareMatMul(const std::shared_ptr &graph, const std::vector> &ops, const size_t iter_graph, const size_t iter_ops); @@ -50,12 +50,12 @@ std::vector> MakeDataParallelStrategy(const std::shared_ptr std::vector> PrepareStrategy(const std::shared_ptr &graph, const std::vector> &ops, const size_t iter_graph, const size_t iter_ops); -void GeneratePartitionedOperatorStrategy(const std::shared_ptr graph, +void GeneratePartitionedOperatorStrategy(const std::shared_ptr &graph, const std::vector> &ops, - const std::shared_ptr> index_list); + const std::shared_ptr> &index_list); size_t FindIndexOfOperatorIncoming(const std::vector> &input_tensor_names, const size_t iter_ops); -std::vector CopyIncomingOperatorOutputStrategy(const std::shared_ptr graph, +std::vector CopyIncomingOperatorOutputStrategy(const std::shared_ptr &graph, const std::vector> &ops, const size_t iter_ops, const size_t iter_graph); std::vector PrepareIncomingOperatorInputStrategy(const std::vector> &ops, @@ -63,19 +63,23 @@ std::vector PrepareIncomingOperatorInputStrategy(const std::vector GetAxisList(const std::vector> &ops, const int iter_ops); std::vector ModifyStrategyIfSqueezeIncoming(const std::vector> &ops, const size_t incoming_op_index, std::vector s); +bool GetKeepDims(const std::vector> &ops, const size_t iter_ops); std::vector GetDimList(const std::vector> &ops, const size_t iter_ops); std::vector ModifyStrategyIfReduceIncoming(const std::vector> &ops, const size_t incoming_op_index, std::vector s); +std::vector GetDimListFromAttrs(const std::vector> &ops, const size_t iter_ops); +std::vector ModifyStrategyIfArgIncoming(const std::vector> &ops, + const size_t incoming_op_index, std::vector s); std::vector CopyIncomingOperatorInputStrategy(const std::vector> &ops, const size_t iter_ops, const size_t incoming_op_index); std::vector> GenerateStrategiesFromStrategy(const std::vector> &ops, const size_t iter_ops, std::vector basic_stra); -void GenerateEliminatedOperatorStrategyForward(std::shared_ptr graph, +void GenerateEliminatedOperatorStrategyForward(const std::shared_ptr &graph, const std::vector> &ops, const std::vector> &input_tensor_names, - const std::shared_ptr> index_list, - const std::shared_ptr> no_stra_op_list); + const std::shared_ptr> &index_list, + const std::shared_ptr> &no_stra_op_list); std::vector ModifyStrategyIfSqueezeOutgoing(const std::vector> &ops, const size_t iter_ops, std::vector s); std::vector CopyOutgoingOperatorInputStrategy(const std::vector> &ops, @@ -83,12 +87,12 @@ std::vector CopyOutgoingOperatorInputStrategy(const std::vector> &ops, const std::vector> &input_tensor_names, - const std::shared_ptr> no_stra_op_list); -void GenerateRemainingOperatorStrategy(const std::shared_ptr graph, + const std::shared_ptr> &no_stra_op_list); +void GenerateRemainingOperatorStrategy(const std::shared_ptr &graph, const std::vector> &ops, const std::vector> &input_tensor_names, - const std::shared_ptr> index_list, - const std::shared_ptr> no_stra_op_list); + const std::shared_ptr> &index_list, + const std::shared_ptr> &no_stra_op_list); } // namespace parallel } // namespace mindspore #endif // PARALLEL_AUTO_PARALLEL_REC_GENERATE_STRATEGY_H_ diff --git a/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_graph.h b/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_graph.h index 647b857e16..9007218d15 100644 --- a/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_graph.h +++ b/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_graph.h @@ -50,7 +50,8 @@ enum OperatorType { kRecCast, kRecReduce, kRecPReLU, - kRecGatherV2 + kRecGatherV2, + kRecArgWithValue }; enum InfoType { kApplication, kConstant }; diff --git a/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_parse_graph.cc b/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_parse_graph.cc index 3e4eafe0a4..58884be9db 100644 --- a/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_parse_graph.cc +++ b/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_parse_graph.cc @@ -163,8 +163,8 @@ size_t GetIndexInInputTensorNames(const std::vector> &i return SIZE_MAX; } -void Eliminate_Aux(const size_t node_index, const std::shared_ptr graph, - const std::shared_ptr>> eli_list) { +void Eliminate_Aux(const size_t node_index, const std::shared_ptr &graph, + const std::shared_ptr>> &eli_list) { std::vector eli; eli.push_back(node_index); for (size_t i = 0; i < (size_t)graph->nodes[node_index].node_out.size(); i++) { @@ -211,18 +211,18 @@ void Eliminate_Aux(const size_t node_index, const std::shared_ptr graph, } } -std::shared_ptr EliminateGraph(const std::shared_ptr graph, - const std::shared_ptr>> eli_list, - const std::shared_ptr> index_list) { +std::shared_ptr EliminateGraph(const std::shared_ptr &graph, + const std::shared_ptr>> &eli_list, + const std::shared_ptr> &index_list) { MS_EXCEPTION_IF_NULL(graph); - const std::set type_list = { - OperatorType::kRecReLU, OperatorType::kRecLog, OperatorType::kRecExp, OperatorType::kRecAdd, - OperatorType::kRecElmWiseOp, OperatorType::kRecBiasAdd, OperatorType::kRecSub, OperatorType::kRecMul, - OperatorType::kRecDiv, OperatorType::kRecSqueeze, OperatorType::kRecReduce, OperatorType::kRecCast, - OperatorType::kRecReshape, OperatorType::kRecGatherV2}; + const std::set elementwise_type = { + OperatorType::kRecReLU, OperatorType::kRecLog, OperatorType::kRecExp, OperatorType::kRecAdd, + OperatorType::kRecElmWiseOp, OperatorType::kRecBiasAdd, OperatorType::kRecSub, OperatorType::kRecMul, + OperatorType::kRecDiv, OperatorType::kRecSqueeze, OperatorType::kRecReduce, OperatorType::kRecCast, + OperatorType::kRecReshape, OperatorType::kRecGatherV2, OperatorType::kRecArgWithValue}; for (size_t node_index = 0; node_index < (size_t)graph->nodes.size(); node_index++) { auto type = graph->nodes[node_index].apply.op_type; - if (type_list.find(type) != type_list.end()) { + if (elementwise_type.find(type) != elementwise_type.end()) { Eliminate_Aux(node_index, graph, eli_list); } } diff --git a/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_parse_graph.h b/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_parse_graph.h index 1be8e4c796..a696e88332 100644 --- a/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_parse_graph.h +++ b/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_parse_graph.h @@ -47,6 +47,8 @@ const std::map DictOpType{ {REDUCE_MIN, OperatorType::kRecReduce}, {REDUCE_MEAN, OperatorType::kRecReduce}, {GATHERV2, OperatorType::kRecGatherV2}, + {ARGMAXWITHVALUE, OperatorType::kRecArgWithValue}, + {ARGMINWITHVALUE, OperatorType::kRecArgWithValue}, {RELU, OperatorType::kRecReLU}, {"ReLU6", OperatorType::kRecReLU}, @@ -59,6 +61,7 @@ const std::map DictOpType{ {PRELU, OperatorType::kRecPReLU}, + {TRANSPOSE, OperatorType::kRecElmWiseOp}, {L2_NORMALIZE, OperatorType::kRecElmWiseOp}, {TENSOR_ADD, OperatorType::kRecElmWiseOp}, {SUB, OperatorType::kRecElmWiseOp}, @@ -123,12 +126,12 @@ void MakeEdge(const std::vector> &input_tensor_names, s size_t GetIndexInInputTensorNames(const std::vector> &input_tensor_names, const std::string &input_name); -void Eliminate_Aux(const size_t node_index, const std::shared_ptr graph, - const std::shared_ptr>> eli_list); +void Eliminate_Aux(const size_t node_index, const std::shared_ptr &graph, + const std::shared_ptr>> &eli_list); -std::shared_ptr EliminateGraph(const std::shared_ptr graph, - const std::shared_ptr>> eli_list, - const std::shared_ptr> index_list); +std::shared_ptr EliminateGraph(const std::shared_ptr &graph, + const std::shared_ptr>> &eli_list, + const std::shared_ptr> &index_list); } // namespace parallel } // namespace mindspore #endif // PARALLEL_AUTO_PARALLEL_REC_PARSE_GRAPH_H_ From c703c4d7c8925fb5cf57a7520c398d047d2ed636 Mon Sep 17 00:00:00 2001 From: gukecai Date: Sat, 20 Jun 2020 15:22:36 +0800 Subject: [PATCH 035/254] hcom one stream --- .../device/ascend/ascend_kernel_runtime.cc | 12 +- .../device/ascend/ascend_stream_assign.cc | 948 ++++++++++-------- .../device/ascend/ascend_stream_assign.h | 111 +- mindspore/ccsrc/device/kernel_adjust.cc | 18 +- mindspore/ccsrc/device/kernel_adjust.h | 6 +- mindspore/ccsrc/session/ascend_session.cc | 6 +- mindspore/ccsrc/session/ascend_session.h | 2 +- .../tasksink/ascend_stream_assign_stub.cc | 2 +- 8 files changed, 619 insertions(+), 486 deletions(-) diff --git a/mindspore/ccsrc/device/ascend/ascend_kernel_runtime.cc b/mindspore/ccsrc/device/ascend/ascend_kernel_runtime.cc index fb2a3f350b..bf949d16e7 100644 --- a/mindspore/ccsrc/device/ascend/ascend_kernel_runtime.cc +++ b/mindspore/ccsrc/device/ascend/ascend_kernel_runtime.cc @@ -338,23 +338,25 @@ bool AscendKernelRuntime::GenTask(const session::KernelGraph *graph) { return true; } AscendStreamAssign &assign_instance = AscendStreamAssign::GetInstance(); - AscendStreamMng &stream_manager = AscendStreamMng::GetInstance(); + AscendResourceMng &resource_manager = AscendResourceMng::GetInstance(); AscendLabelAssign &label_assign_instance = AscendLabelAssign::GetInstance(); // the streams' flag not HEAD_STREAM std::vector wait_active_stream_list; assign_instance.GetWaitStreams(&wait_active_stream_list); std::vector force_copy_stream_list; assign_instance.GetHcomStreams(&force_copy_stream_list); - MS_LOG(INFO) << "call DavinciModel total stream num:" << stream_manager.GetCurAllocStreamNum() - << ", total event num:" << assign_instance.total_event_num() + + MS_LOG(INFO) << "call DavinciModel total stream num:" << resource_manager.get_cur_stream_num() + << ", total event num:" << resource_manager.get_cur_event_num() << ", total label num:" << label_assign_instance.GetLabelNum(NOT_NULL(graph)) << ", wait_active_stream_list size:" << wait_active_stream_list.size() << ", force_copy_stream_list size:" << force_copy_stream_list.size(); std::vector> empty_list; std::shared_ptr model = std::make_shared( task_info_list, empty_list, empty_list, empty_list, empty_list, wait_active_stream_list, force_copy_stream_list, 0, - 0, 0, 0, 0, 0, stream_manager.GetCurAllocStreamNum(), label_assign_instance.GetLabelNum(NOT_NULL(graph)), - assign_instance.total_event_num(), 0); + 0, 0, 0, 0, 0, resource_manager.get_cur_stream_num(), label_assign_instance.GetLabelNum(NOT_NULL(graph)), + resource_manager.get_cur_event_num(), 0); + auto ret = graph_model_map_.insert(std::make_pair(graph->graph_id(), model)); if (!ret.second) { MS_LOG(EXCEPTION) << "Duplicate GraphId! Please check in ascend_session."; diff --git a/mindspore/ccsrc/device/ascend/ascend_stream_assign.cc b/mindspore/ccsrc/device/ascend/ascend_stream_assign.cc index f0bad6b492..8f8f022bdb 100644 --- a/mindspore/ccsrc/device/ascend/ascend_stream_assign.cc +++ b/mindspore/ccsrc/device/ascend/ascend_stream_assign.cc @@ -34,110 +34,136 @@ namespace ascend { const uint32_t kHcomMaxTask = 5; const uint32_t kCommonMaxTask = 350; -void AscendStreamAssign::AssignStream(const shared_ptr &graph_ptr) { +void AscendStreamAssign::AssignStream(const NotNull &graph_ptr) { if (IsTaskSink()) { Reset(); ReorderIndependentOrders(graph_ptr); AssignAllNodesStream(graph_ptr); UpdateAtomicAddrCleanStreamId(graph_ptr); - FindHcomParallelStreams(graph_ptr); InsertStreamActive(graph_ptr); - InsertSendRecvForHcomParallel(graph_ptr); - InsertSendRecvForIndependent(graph_ptr); - UpdateEventId(graph_ptr); + InsertEventForHcomParallel(graph_ptr); + InsertEventForIndependentParallel(graph_ptr); GetNeedActiveStreams(graph_ptr); graph_ptr->PrintGraphExecuteOrder(); - CheckStreamAssign(graph_ptr); + CheckResourceAssign(graph_ptr); MS_LOG(INFO) << "after finish stream assign"; // Get info for D Model - AscendStreamMng &stream_manager = AscendStreamMng::GetInstance(); - generator::IRModelUtil::GetInstance().set_event_num(total_event_num()); - generator::IRModelUtil::GetInstance().set_stream_num(stream_manager.GetCurAllocStreamNum()); + AscendResourceMng &resource_manager = AscendResourceMng::GetInstance(); + generator::IRModelUtil::GetInstance().set_event_num(resource_manager.get_cur_event_num()); + generator::IRModelUtil::GetInstance().set_stream_num(resource_manager.get_cur_stream_num()); // Init to 1,temporarily generator::IRModelUtil::GetInstance().set_batch_num(1); } } -// section 0 -void AscendStreamAssign::CheckStreamAssign(const shared_ptr &graph_ptr) { - MS_EXCEPTION_IF_NULL(graph_ptr); - std::set streams; - uint32_t max_stream = 0; - uint32_t min_stream = kInvalidStreamId; - const std::vector &cnode_ptr_list = graph_ptr->execution_order(); +// section 1 +void AscendStreamAssign::ReorderIndependentOrders(const NotNull &graph_ptr) { + std::vector exe_orders; + std::vector independents; + std::vector others; + + auto cnode_ptr_list = graph_ptr->execution_order(); + MS_LOG(INFO) << "before reorder, graph orders size:" << cnode_ptr_list.size(); for (size_t i = 0; i < cnode_ptr_list.size(); ++i) { - CNodePtr cur_cnode_ptr = cnode_ptr_list[i]; + auto cur_cnode_ptr = cnode_ptr_list[i]; MS_EXCEPTION_IF_NULL(cur_cnode_ptr); - uint32_t stream_id = AnfAlgo::GetStreamId(cur_cnode_ptr); - if (stream_id == kInvalidStreamId) { - MS_LOG(EXCEPTION) << "node [" << AnfAlgo::GetCNodeName(cur_cnode_ptr) << "] had not been assigned streams"; + if (IsIndependentNode(cur_cnode_ptr)) { + independents.emplace_back(cur_cnode_ptr); + } else { + others.emplace_back(cur_cnode_ptr); } + } - streams.emplace(stream_id); - if (stream_id > max_stream) { - max_stream = stream_id; - } - if (stream_id < min_stream) { - min_stream = stream_id; - } + if (others.empty() || independents.empty()) { + MS_LOG(INFO) << "independent or others is empty, no need reorder"; + return; } - if (!streams.empty()) { - if (min_stream != 0) { - MS_LOG(EXCEPTION) << "before stream assign, assigned stream should start from 0, now is from " << min_stream; + std::set processed; + for (size_t i = 0; i < others.size(); i++) { + auto begin = others.begin() + i; + auto end = begin + 1; + bool flag = false; + for (size_t j = 0; j < independents.size(); j++) { + auto cur_independent = independents[j]; + auto it = std::find(processed.begin(), processed.end(), cur_independent.get()); + if (it != processed.end()) { + continue; + } + + auto res = FindTargetOp(begin, end, cur_independent); + if (res != end) { + flag = true; + exe_orders.emplace_back(cur_independent); + exe_orders.emplace_back(*begin); + processed.emplace(cur_independent.get()); + break; + } } - if (max_stream != (streams.size() - 1)) { - MS_LOG(EXCEPTION) << "before stream assign, assigned stream should be consecutive"; + + if (!flag) { + exe_orders.emplace_back(*begin); } } + + MS_LOG(INFO) << "after reorder, graph orders size:" << exe_orders.size(); + if (processed.size() != independents.size()) { + MS_LOG(WARNING) << "processed independent nodes size is not equal to exiting independent nodes size"; + return; + } + + graph_ptr->set_execution_order(exe_orders); } -// section 1 -void AscendStreamAssign::AssignAllNodesStream(const shared_ptr &graph_ptr) { - MS_EXCEPTION_IF_NULL(graph_ptr); +// section 2 +void AscendStreamAssign::AssignAllNodesStream(const NotNull &graph_ptr) { auto cnode_ptr_list = graph_ptr->execution_order(); - CNodePtr pre_cnode_ptr = nullptr; - uint32_t cur_index = 0; - uint32_t cur_stream_id = 0; - bool exit_independent = false; - AscendStreamMng &stream_manager = AscendStreamMng::GetInstance(); + bool exit_hcom = false; + AscendResourceMng &resource_manager = AscendResourceMng::GetInstance(); for (size_t i = 0; i < cnode_ptr_list.size(); ++i) { CNodePtr cur_cnode_ptr = cnode_ptr_list[i]; MS_EXCEPTION_IF_NULL(cur_cnode_ptr); + // node has been assigned stream before if (AnfAlgo::GetStreamId(cur_cnode_ptr) != kInvalidStreamId) { continue; } + + if (IsHcom(cur_cnode_ptr)) { + exit_hcom = true; + continue; + } + if (IsIndependentNode(cur_cnode_ptr)) { exit_independent = true; continue; } - // first common node, only exe one time - if (pre_cnode_ptr == nullptr) { - uint32_t cur_stream_num = stream_manager.GetCurAllocStreamNum(); - if (cur_stream_num == 0) { - cur_stream_id = stream_manager.ApplyNewStream(); - } else { - cur_stream_id = stream_manager.GetCurAllocStream(); + + AssignCommonStreamId(cur_cnode_ptr); + } + MS_LOG(INFO) << "common start from 0, common stream nums:" << resource_manager.get_cur_stream_num(); + + if (exit_hcom) { + uint32_t first_hcom_stream_id = resource_manager.ApplyNewStream(); + for (size_t i = 0; i < cnode_ptr_list.size(); ++i) { + CNodePtr cur_cnode_ptr = cnode_ptr_list[i]; + // node has been assigned stream before + if (AnfAlgo::GetStreamId(cur_cnode_ptr) != kInvalidStreamId) { + continue; } - ++cur_index; - pre_cnode_ptr = cur_cnode_ptr; - AnfAlgo::SetStreamId(cur_stream_id, cur_cnode_ptr.get()); + if (IsHcom(cur_cnode_ptr)) { - hcom_stream_list_.emplace(cur_stream_id); + AssignHcomStreamId(cur_cnode_ptr); } - continue; } - - AssignCommonStreamId(cur_cnode_ptr, &pre_cnode_ptr, &cur_index, &cur_stream_id); + MS_LOG(INFO) << "hcom start from :" << first_hcom_stream_id << ", hcom stream nums:" << hcom_stream_map_.size(); } if (exit_independent) { - uint32_t first_independent_stream_id = stream_manager.ApplyNewStream(); + uint32_t first_independ = resource_manager.ApplyNewStream(); for (size_t i = 0; i < cnode_ptr_list.size(); ++i) { CNodePtr cur_cnode_ptr = cnode_ptr_list[i]; - MS_EXCEPTION_IF_NULL(cur_cnode_ptr); if (AnfAlgo::GetStreamId(cur_cnode_ptr) != kInvalidStreamId) { continue; } @@ -145,28 +171,75 @@ void AscendStreamAssign::AssignAllNodesStream(const shared_ptrsecond < kCommonMaxTask) { + AnfAlgo::SetStreamId(it->first, cur_cnode_ptr.get()); + it->second++; + } else { + cur_common_stream_id = resource_manager.ApplyNewStream(); + AnfAlgo::SetStreamId(cur_common_stream_id, cur_cnode_ptr.get()); + common_stream_map_.insert(std::make_pair(cur_common_stream_id, 1)); + } } +} - MS_LOG(INFO) << "total stream nums:" << stream_manager.GetCurAllocStreamNum(); +void AscendStreamAssign::AssignHcomStreamId(const CNodePtr &cur_cnode_ptr) { + MS_EXCEPTION_IF_NULL(cur_cnode_ptr); + AscendResourceMng &resource_manager = AscendResourceMng::GetInstance(); + uint32_t cur_hcom_stream_id = resource_manager.GetCurAllocStreamId(); + auto it = hcom_stream_map_.find(cur_hcom_stream_id); + if (it == hcom_stream_map_.end()) { + AnfAlgo::SetStreamId(cur_hcom_stream_id, cur_cnode_ptr.get()); + hcom_stream_map_.insert(std::make_pair(cur_hcom_stream_id, 1)); + } else { + if (it->second < kHcomMaxTask) { + AnfAlgo::SetStreamId(it->first, cur_cnode_ptr.get()); + it->second++; + } else { + cur_hcom_stream_id = resource_manager.ApplyNewStream(); + AnfAlgo::SetStreamId(cur_hcom_stream_id, cur_cnode_ptr.get()); + hcom_stream_map_.insert(std::make_pair(cur_hcom_stream_id, 1)); + } + } } void AscendStreamAssign::AssignIndependentStreamId(const CNodePtr &cur_cnode_ptr) { MS_EXCEPTION_IF_NULL(cur_cnode_ptr); - AscendStreamMng &stream_manager = AscendStreamMng::GetInstance(); - uint32_t cur_independent_id = stream_manager.GetCurAllocStream(); + AscendResourceMng &resource_manager = AscendResourceMng::GetInstance(); + uint32_t cur_independent_id = resource_manager.GetCurAllocStreamId(); auto it = independent_stream_map_.find(cur_independent_id); if (it == independent_stream_map_.end()) { AnfAlgo::SetStreamId(cur_independent_id, cur_cnode_ptr.get()); - independent_stream_map_.emplace(cur_independent_id, 1); + independent_stream_map_.insert(std::make_pair(cur_independent_id, 1)); } else { if (it->second < kCommonMaxTask) { AnfAlgo::SetStreamId(it->first, cur_cnode_ptr.get()); it->second++; } else { - cur_independent_id = stream_manager.ApplyNewStream(); + cur_independent_id = resource_manager.ApplyNewStream(); AnfAlgo::SetStreamId(cur_independent_id, cur_cnode_ptr.get()); - independent_stream_map_.emplace(cur_independent_id, 1); + independent_stream_map_.insert(std::make_pair(cur_independent_id, 1)); } } } @@ -188,7 +261,7 @@ bool AscendStreamAssign::IsIndependentNode(const CNodePtr &node_ptr) { return true; } - const std::vector &inputs = node_ptr->inputs(); + auto inputs = node_ptr->inputs(); for (size_t i = 1; i < inputs.size(); i++) { if (!inputs[i]->isa()) { return false; @@ -198,86 +271,105 @@ bool AscendStreamAssign::IsIndependentNode(const CNodePtr &node_ptr) { return true; } -void AscendStreamAssign::AssignCommonStreamId(const CNodePtr &cur_cnode_ptr, CNodePtr *pre_cnode_ptr, - uint32_t *cur_index, uint32_t *cur_stream_id) { - MS_EXCEPTION_IF_NULL(cur_cnode_ptr); - MS_EXCEPTION_IF_NULL(pre_cnode_ptr); - MS_EXCEPTION_IF_NULL(*pre_cnode_ptr); - AscendStreamMng &stream_manager = AscendStreamMng::GetInstance(); - bool over_max_hcom_task = (IsHcom(cur_cnode_ptr) && (*cur_index) % kHcomMaxTask == 0); - bool over_max_common_task = (!IsHcom(cur_cnode_ptr) && (*cur_index) % kCommonMaxTask == 0); - bool pre_common_cur_hcom = (IsHcom(cur_cnode_ptr) && !IsHcom(*pre_cnode_ptr)); - bool pre_hcom_cur_common = (!IsHcom(cur_cnode_ptr) && IsHcom(*pre_cnode_ptr)); - if (over_max_hcom_task || over_max_common_task || pre_common_cur_hcom || pre_hcom_cur_common) { - *cur_index = 0; - *cur_stream_id = stream_manager.ApplyNewStream(); - } - - ++(*cur_index); - AnfAlgo::SetStreamId(*cur_stream_id, cur_cnode_ptr.get()); - *pre_cnode_ptr = cur_cnode_ptr; - - // record ll hcom streams as hcom stream has different stream flag - if (IsHcom(cur_cnode_ptr)) { - auto it = std::find(hcom_stream_list_.begin(), hcom_stream_list_.end(), *cur_stream_id); - if (it == hcom_stream_list_.end()) { - MS_LOG(INFO) << "hcom stream id:" << *cur_stream_id; - hcom_stream_list_.emplace(*cur_stream_id); - } - } -} - -// section 2: -void AscendStreamAssign::UpdateAtomicAddrCleanStreamId(const shared_ptr &graph_ptr) { +// section 3: +void AscendStreamAssign::UpdateAtomicAddrCleanStreamId(const NotNull &graph_ptr) { MS_LOG(INFO) << "start"; - MS_EXCEPTION_IF_NULL(graph_ptr); - const std::vector &cnode_ptr_list = graph_ptr->execution_order(); + auto cnode_ptr_list = graph_ptr->execution_order(); for (size_t i = 0; i < cnode_ptr_list.size(); ++i) { CNodePtr cur_cnode_ptr = cnode_ptr_list[i]; MS_EXCEPTION_IF_NULL(cur_cnode_ptr); // update AtomicAddrClean stream same witch the next node if (i > 0 && AnfAlgo::GetCNodeName(cnode_ptr_list[i - 1]) == kAtomicAddrCleanOpName) { - MS_LOG(INFO) << "update AtomicAddrClean stream id from[" << AnfAlgo::GetStreamId(cnode_ptr_list[i - 1]) - << "] to [" << AnfAlgo::GetStreamId(cur_cnode_ptr) << "]"; AnfAlgo::SetStreamId(AnfAlgo::GetStreamId(cur_cnode_ptr), cnode_ptr_list[i - 1].get()); } } MS_LOG(INFO) << "end"; } -// section 3 -void AscendStreamAssign::FindHcomParallelStreams(const shared_ptr &graph_ptr) { - MS_EXCEPTION_IF_NULL(graph_ptr); +// section 4 +void AscendStreamAssign::InsertStreamActive(const NotNull &graph_ptr) { + MS_LOG(INFO) << "start"; + GetProcessedStream(graph_ptr); + std::vector update_cnode_list; CNodePtr cur_cnode_ptr = nullptr; CNodePtr pre_cnode_ptr = nullptr; uint32_t pre_stream_id = UINT32_MAX; + + bool independent_flag = !(independent_stream_map_.empty()); + bool hcom_flag = !(hcom_stream_map_.empty()); auto cnode_ptr_list = graph_ptr->execution_order(); - for (uint32_t i = 0; i < cnode_ptr_list.size(); ++i) { + for (size_t i = 0; i < cnode_ptr_list.size(); ++i) { cur_cnode_ptr = cnode_ptr_list[i]; MS_EXCEPTION_IF_NULL(cur_cnode_ptr); - uint32_t cur_stream_id = AnfAlgo::GetStreamId(cur_cnode_ptr); - if (i == 0) { - pre_cnode_ptr = cur_cnode_ptr; - pre_stream_id = cur_stream_id; + if (IsIndependentNode(cur_cnode_ptr)) { + update_cnode_list.emplace_back(cur_cnode_ptr); continue; } - bool pre_fusion_hcom = IsFusionHcom(pre_cnode_ptr); - bool diff_stream = (pre_stream_id != cur_stream_id); - if (diff_stream && pre_fusion_hcom) { - inner_parallel_streams_.emplace_back(std::vector{pre_stream_id, cur_stream_id}); + if (IsHcom(cur_cnode_ptr)) { + update_cnode_list.emplace_back(cur_cnode_ptr); + continue; + } + uint32_t cur_stream_id = AnfAlgo::GetStreamId(cur_cnode_ptr); + bool processed = IsProcessedStream(cur_stream_id); + // 1)inner stream assign, need insert active op + if (!processed) { + MS_LOG(INFO) << "common stream active info:" << pre_stream_id << "->active" << cur_stream_id; + CNodePtr active_ptr = KernelAdjust::GetInstance().CreateStreamActiveOp(graph_ptr); + // 1.set stream id + AnfAlgo::SetStreamId(pre_stream_id, active_ptr.get()); + // 2.set active stream ids + std::vector active_index_list{cur_stream_id}; + AnfAlgo::SetNodeAttr(kAttrActiveStreamList, MakeValue>(active_index_list), active_ptr); + update_cnode_list.emplace_back(active_ptr); } - pre_cnode_ptr = cur_cnode_ptr; + if ((independent_flag || hcom_flag) && (AnfAlgo::GetCNodeName(cur_cnode_ptr) == kStreamSwitchOpName)) { + MS_LOG(INFO) << "Insert StreamActive op after FP StreamSwitch for stream parallel"; + UpdateStreamSwitch(graph_ptr, cur_cnode_ptr, &update_cnode_list); + } else { + update_cnode_list.emplace_back(cur_cnode_ptr); + } + + processed_streams_.emplace(cur_stream_id); pre_stream_id = cur_stream_id; + pre_cnode_ptr = cur_cnode_ptr; } + graph_ptr->set_execution_order(update_cnode_list); + MS_LOG(INFO) << "end"; } -// section 4 -void AscendStreamAssign::UpdateStreamSwitch(const std::shared_ptr &graph_ptr, - const CNodePtr &switch_ptr, const vector &independent_stream, +void AscendStreamAssign::GetProcessedStream(const NotNull &graph_ptr) { + // 0 stream is activated at first + processed_streams_.emplace(0); + auto cnode_ptr_list = graph_ptr->execution_order(); + for (size_t i = 0; i < cnode_ptr_list.size(); ++i) { + auto cur_cnode_ptr = cnode_ptr_list[i]; + uint32_t cur_stream_id = AnfAlgo::GetStreamId(cur_cnode_ptr); + + if (AnfAlgo::GetCNodeName(cur_cnode_ptr) == kStreamSwitchOpName) { + auto primitive = AnfAlgo::GetCNodePrimitive(cur_cnode_ptr); + MS_EXCEPTION_IF_NULL(primitive); + auto true_stream_id = GetValue(primitive->GetAttr(kAttrTrueBranchStream)); + processed_streams_.emplace(true_stream_id); + + auto value_ptr = primitive->GetAttr(kStreamNeedActivedFirst); + if (value_ptr == nullptr) { + continue; + } + auto need_active = GetValue(value_ptr); + if (need_active) { + processed_streams_.emplace(cur_stream_id); + } + } + } + for (const auto &item : processed_streams_) { + MS_LOG(INFO) << "before active:" << item << " is been processed"; + } +} + +void AscendStreamAssign::UpdateStreamSwitch(const NotNull &graph_ptr, const CNodePtr &switch_ptr, vector *orders) { - MS_EXCEPTION_IF_NULL(orders); orders->emplace_back(switch_ptr); auto primitive = AnfAlgo::GetCNodePrimitive(switch_ptr); MS_EXCEPTION_IF_NULL(primitive); @@ -291,203 +383,270 @@ void AscendStreamAssign::UpdateStreamSwitch(const std::shared_ptrDebugString() << "]"; MS_EXCEPTION_IF_NULL(switch_ptr); auto true_stream_id = GetValue(primitive->GetAttr(kAttrTrueBranchStream)); - MS_LOG(INFO) << "streamswtich stream id[" << AnfAlgo::GetStreamId(switch_ptr) << "], true_logic_id[" << true_stream_id - << "]"; + MS_LOG(INFO) << "streamswtich stream id:" << AnfAlgo::GetStreamId(switch_ptr) + << "; active stream id:" << true_stream_id; CNodePtr active_ptr = KernelAdjust::GetInstance().CreateStreamActiveOp(graph_ptr); - MS_LOG(INFO) << "start update StreamActive op[" << active_ptr->DebugString() << "]"; AnfAlgo::SetStreamId(true_stream_id, active_ptr.get()); - AnfAlgo::SetNodeAttr(kAttrActiveStreamList, MakeValue>(independent_stream), active_ptr); - independent_stream_activated_ = true; + vector active_ids; + // active indepdent stream + for (const auto &item : independent_stream_map_) { + active_ids.emplace_back(item.first); + } + // active hcom stream + for (const auto &item : hcom_stream_map_) { + active_ids.emplace_back(item.first); + } + AnfAlgo::SetNodeAttr(kAttrActiveStreamList, MakeValue>(active_ids), active_ptr); // update processed stream - for (auto &item : independent_stream) { - processed_streams_.emplace(item); + independent_stream_activated_ = true; + for (const auto &item : independent_stream_map_) { + processed_streams_.emplace(item.first); + } + + hcom_stream_activated_ = true; + for (const auto &item : hcom_stream_map_) { + processed_streams_.emplace(item.first); } orders->emplace_back(active_ptr); -} // namespace ascend +} -void AscendStreamAssign::InsertStreamActive(const std::shared_ptr &graph_ptr) { - MS_LOG(INFO) << "start"; - MS_EXCEPTION_IF_NULL(graph_ptr); - std::vector update_cnode_list; - CNodePtr cur_cnode_ptr = nullptr; - CNodePtr pre_cnode_ptr = nullptr; - uint32_t pre_stream_id = UINT32_MAX; - std::vector independent_stream; - MS_LOG(INFO) << "independent stream size:" << independent_stream_map_.size(); - for (auto item : independent_stream_map_) { - independent_stream.emplace_back(item.first); +bool AscendStreamAssign::IsProcessedStream(uint32_t stream_id) { + auto it = std::find(processed_streams_.begin(), processed_streams_.end(), stream_id); + if (it != processed_streams_.end()) { + return true; } + return false; +} + +// section5 +void AscendStreamAssign::InsertEventForHcomParallel(const NotNull &graph_ptr) { + MS_LOG(INFO) << "start"; + InsertEventCommonDependHcom(graph_ptr); + InsertEventHcomDependCommon(graph_ptr); + InsertEventHcomDependHcom(graph_ptr); + MS_LOG(INFO) << "end"; +} - bool independent_flag = !(independent_stream.empty()); +void AscendStreamAssign::InsertEventCommonDependHcom(const NotNull &graph_ptr) { + AscendResourceMng &resource_manager = AscendResourceMng::GetInstance(); + auto cnode_ptr_list = graph_ptr->execution_order(); + vector cnodes = cnode_ptr_list; + uint32_t cur_event_id = resource_manager.ApplyNewEvent(); + auto it = cnodes.begin(); + while (it != cnodes.end() && (it + 1) != cnodes.end()) { + MS_EXCEPTION_IF_NULL(*it); + MS_EXCEPTION_IF_NULL(*(it + 1)); + if (IsHcom(*it) && !IsHcom(*(it + 1))) { + CNodePtr send_cnode_ptr = CreateSendApplyKernel(graph_ptr, cur_event_id, AnfAlgo::GetStreamId(*it)); + it = cnodes.insert(it + 1, send_cnode_ptr); + + auto target = FindTargetOp(it, cnodes.end(), *(it - 1)); + if (target == cnodes.end()) { + MS_LOG(WARNING) << "hcom node:" << (*(it - 1))->fullname_with_scope() + << ", can't find target for insert recv op, no insert send/recv"; + it = cnodes.erase(it); + continue; + } + + if (IsHcom(*target)) { + it = cnodes.erase(it); + continue; + } + + // deal recv op + uint32_t stream_id = AnfAlgo::GetStreamId(*target); + CNodePtr recv_cnode_ptr = CreateRecvApplyKernel(graph_ptr, cur_event_id, stream_id); + (void)cnodes.insert(target, recv_cnode_ptr); + cur_event_id = resource_manager.ApplyNewEvent(); + } + ++it; + } + // one event allocated additional, should delete + resource_manager.DeleteEvent(); + graph_ptr->set_execution_order(cnodes); + MS_LOG(INFO) << "after common depend hcom, total event nums:" << resource_manager.get_cur_event_num(); +} - const std::vector &cnode_ptr_list = graph_ptr->execution_order(); +void AscendStreamAssign::InsertEventHcomDependCommon(const NotNull &graph_ptr) { + AscendResourceMng &resource_manager = AscendResourceMng::GetInstance(); + auto cnode_ptr_list = graph_ptr->execution_order(); + vector cnodes; + CNodePtr cur_cnode_ptr = nullptr; + uint32_t pre_stream_id = UINT32_MAX; for (size_t i = 0; i < cnode_ptr_list.size(); ++i) { cur_cnode_ptr = cnode_ptr_list[i]; - MS_EXCEPTION_IF_NULL(cur_cnode_ptr); uint32_t cur_stream_id = AnfAlgo::GetStreamId(cur_cnode_ptr); - if (IsIndependentNode(cur_cnode_ptr)) { - update_cnode_list.emplace_back(cur_cnode_ptr); + MS_EXCEPTION_IF_NULL(cur_cnode_ptr); + if (i == 0) { + cnodes.emplace_back(cur_cnode_ptr); + pre_stream_id = cur_stream_id; continue; } - bool inner_active = false; - if (pre_cnode_ptr != nullptr) { - inner_active = pre_stream_id != cur_stream_id && AnfAlgo::GetCNodeName(pre_cnode_ptr) != kStreamSwitchOpName && - AnfAlgo::GetCNodeName(pre_cnode_ptr) != kSendOpName; + if (!IsHcom(cur_cnode_ptr)) { + cnodes.emplace_back(cur_cnode_ptr); + pre_stream_id = cur_stream_id; + continue; } - bool processed = IsProcessedStream(cur_stream_id); - // 1)inner stream assign, need insert active op - if (inner_active && !processed) { - MS_LOG(INFO) << "Inner insert active op, self stream id[" << pre_stream_id << "]"; - CNodePtr active_ptr = KernelAdjust::GetInstance().CreateStreamActiveOp(graph_ptr); - // 1.set stream id - AnfAlgo::SetStreamId(pre_stream_id, active_ptr.get()); - // 2.set active stream ids - std::vector active_index_list; - GetParallelStream(cur_stream_id, pre_stream_id, &active_index_list); - AnfAlgo::SetNodeAttr(kAttrActiveStreamList, MakeValue>(active_index_list), active_ptr); - update_cnode_list.emplace_back(active_ptr); + if (cur_stream_id == pre_stream_id) { + cnodes.emplace_back(cur_cnode_ptr); + pre_stream_id = cur_stream_id; + continue; } - if (independent_flag && (AnfAlgo::GetCNodeName(cur_cnode_ptr) == kStreamSwitchOpName)) { - MS_LOG(INFO) << "Insert StreamActive op after FP StreamSwitch for stream parallel"; - UpdateStreamSwitch(graph_ptr, cur_cnode_ptr, independent_stream, &update_cnode_list); - } else { - update_cnode_list.emplace_back(cur_cnode_ptr); + if (!IsHcom(cnode_ptr_list[i - 1])) { + uint32_t cur_event_id = resource_manager.ApplyNewEvent(); + auto send = CreateSendApplyKernel(graph_ptr, cur_event_id, pre_stream_id); + cnodes.emplace_back(send); + auto recv = CreateRecvApplyKernel(graph_ptr, cur_event_id, cur_stream_id); + cnodes.emplace_back(recv); + cnodes.emplace_back(cur_cnode_ptr); } - - processed_streams_.emplace(cur_stream_id); pre_stream_id = cur_stream_id; - pre_cnode_ptr = cur_cnode_ptr; } - graph_ptr->set_execution_order(update_cnode_list); - MS_LOG(INFO) << "end"; -} -bool AscendStreamAssign::IsProcessedStream(uint32_t stream_id) { - auto it = std::find(processed_streams_.begin(), processed_streams_.end(), stream_id); - if (it != processed_streams_.end()) { - return true; - } - return false; -} - -void AscendStreamAssign::GetParallelStream(uint32_t cur_stream_id, uint32_t stream_acitve_id, - vector *parallel_streams) { - MS_EXCEPTION_IF_NULL(parallel_streams); - for (size_t i = 0; i < inner_parallel_streams_.size(); i++) { - const auto &cur_parallel_streams = inner_parallel_streams_[i]; - auto it = std::find(cur_parallel_streams.begin(), cur_parallel_streams.end(), cur_stream_id); - if (it != cur_parallel_streams.end()) { - MS_LOG(INFO) << "stream id:" << cur_stream_id << " is parallel stream"; - for (size_t j = 0; j < cur_parallel_streams.size(); j++) { - if (cur_parallel_streams[j] == stream_acitve_id) { - MS_LOG(INFO) << "one of parallel stream id" << cur_parallel_streams[j] - << "is same with streamacvite stream id" << stream_acitve_id; - continue; - } - (*parallel_streams).emplace_back(cur_parallel_streams[j]); - processed_streams_.emplace(cur_parallel_streams[j]); - } - return; - } - } - - processed_streams_.emplace(cur_stream_id); - (*parallel_streams).push_back(cur_stream_id); + graph_ptr->set_execution_order(cnodes); + MS_LOG(INFO) << "after hcom depend common, total event nums:" << resource_manager.get_cur_event_num(); } -// section5 -void AscendStreamAssign::InsertSendRecvForDiffHcom(const shared_ptr &graph_ptr) { - MS_LOG(INFO) << "start"; - MS_EXCEPTION_IF_NULL(graph_ptr); +void AscendStreamAssign::InsertEventHcomDependHcom(const NotNull &graph_ptr) { + AscendResourceMng &resource_manager = AscendResourceMng::GetInstance(); auto cnode_ptr_list = graph_ptr->execution_order(); - vector fusion_hcom_index; - vector orders; + uint32_t first_hcom_stream = kInvalidStreamId; + uint32_t last_hcom_stream = kInvalidStreamId; + // key: stream id, value:hcom index + std::map> hcom_index; for (size_t i = 0; i < cnode_ptr_list.size(); i++) { auto cur_cnode = cnode_ptr_list[i]; - if (IsFusionHcom(cur_cnode)) { - fusion_hcom_index.emplace_back(i); + if (!IsHcom(cur_cnode)) { + continue; + } + uint32_t cur_stream_id = AnfAlgo::GetStreamId(cur_cnode); + auto it = hcom_index.find(cur_stream_id); + if (it != hcom_index.end()) { + hcom_index[cur_stream_id].emplace_back(i); + } else { + hcom_index[cur_stream_id] = {i}; + } + + // record first hcom stream id + if (first_hcom_stream == kInvalidStreamId) { + first_hcom_stream = cur_stream_id; + } + + // record last hcom stream id + if (cur_stream_id != last_hcom_stream) { + last_hcom_stream = cur_stream_id; } } - if (fusion_hcom_index.size() < 2) { - MS_LOG(INFO) << "fusion hcom size is less than 2, no need insert event between them"; + + if (hcom_index.size() < 2) { + MS_LOG(INFO) << "different stream hcom size is less than 2, no need insert event between them"; return; } - uint32_t first_index = fusion_hcom_index[0]; - uint32_t last_index = fusion_hcom_index[fusion_hcom_index.size() - 1]; - uint32_t cur_event_id = total_event_num_; - uint32_t pre_hcom_stream_id = kInvalidStreamId; - std::copy(cnode_ptr_list.begin(), cnode_ptr_list.begin() + first_index, std::back_inserter(orders)); - for (size_t i = first_index; i <= last_index; i++) { + InsertEventBetweenHcom(graph_ptr, hcom_index, first_hcom_stream, last_hcom_stream); + MS_LOG(INFO) << "after hcom depend hcom, total event nums:" << resource_manager.get_cur_event_num(); +} + +void AscendStreamAssign::InsertEventBetweenHcom(const NotNull &graph_ptr, + const map> &hcom_index, + uint32_t first_hcom_stream, uint32_t last_hcom_stream) { + vector orders; + AscendResourceMng &resource_manager = AscendResourceMng::GetInstance(); + auto cnode_ptr_list = graph_ptr->execution_order(); + uint32_t cur_event_id = resource_manager.ApplyNewEvent(); + size_t first_stream_last_index = hcom_index.at(first_hcom_stream).back(); + size_t last_stream_first_index = hcom_index.at(last_hcom_stream).front(); + std::copy(cnode_ptr_list.begin(), cnode_ptr_list.begin() + first_stream_last_index, std::back_inserter(orders)); + for (size_t i = first_stream_last_index; i <= last_stream_first_index; i++) { auto cur_cnode = cnode_ptr_list[i]; - auto it = std::find(fusion_hcom_index.begin(), fusion_hcom_index.end(), i); - if (it == fusion_hcom_index.end()) { + if (!IsSatisfiedHcom(hcom_index, cur_cnode, i)) { orders.emplace_back(cur_cnode); continue; } auto cur_hcom_stream_id = AnfAlgo::GetStreamId(cur_cnode); - if (cur_hcom_stream_id == pre_hcom_stream_id) { - orders.emplace_back(cur_cnode); - continue; - } - if (i == first_index) { + if (i == first_stream_last_index) { // first fusion hcom orders.emplace_back(cur_cnode); auto send = CreateSendApplyKernel(graph_ptr, cur_event_id, cur_hcom_stream_id); orders.emplace_back(send); - } else if (i == last_index) { + } else if (i == last_stream_first_index) { // last fusion hcom auto recv = CreateRecvApplyKernel(graph_ptr, cur_event_id, cur_hcom_stream_id); orders.emplace_back(recv); orders.emplace_back(cur_cnode); - cur_event_id++; } else { - auto recv = CreateRecvApplyKernel(graph_ptr, cur_event_id, cur_hcom_stream_id); - orders.emplace_back(recv); - cur_event_id++; - orders.emplace_back(cur_cnode); - auto send = CreateSendApplyKernel(graph_ptr, cur_event_id, cur_hcom_stream_id); - orders.emplace_back(send); + auto cur_stream_hcom_size = hcom_index.at(cur_hcom_stream_id).size(); + if (cur_stream_hcom_size == 1) { + auto recv = CreateRecvApplyKernel(graph_ptr, cur_event_id, cur_hcom_stream_id); + orders.emplace_back(recv); + cur_event_id = resource_manager.ApplyNewEvent(); + orders.emplace_back(cur_cnode); + auto send = CreateSendApplyKernel(graph_ptr, cur_event_id, cur_hcom_stream_id); + orders.emplace_back(send); + } else { + // current stream, first hcom:add recv op + if (i == hcom_index.at(cur_hcom_stream_id).front()) { + auto recv = CreateRecvApplyKernel(graph_ptr, cur_event_id, cur_hcom_stream_id); + orders.emplace_back(recv); + cur_event_id = resource_manager.ApplyNewEvent(); + orders.emplace_back(cur_cnode); + } else if (i == hcom_index.at(cur_hcom_stream_id).back()) { + // current stream, last hcom:add send op + orders.emplace_back(cur_cnode); + auto send = CreateSendApplyKernel(graph_ptr, cur_event_id, cur_hcom_stream_id); + orders.emplace_back(send); + } else { + // current stream, not first and last op + orders.emplace_back(cur_cnode); + } + } } - pre_hcom_stream_id = cur_hcom_stream_id; } - std::copy(cnode_ptr_list.begin() + last_index + 1, cnode_ptr_list.end(), std::back_inserter(orders)); + std::copy(cnode_ptr_list.begin() + last_stream_first_index + 1, cnode_ptr_list.end(), std::back_inserter(orders)); graph_ptr->set_execution_order(orders); - total_event_num_ = cur_event_id; - MS_LOG(INFO) << "after indsert between allreduce, total event nums[" << total_event_num_ << "]\n end"; } -void AscendStreamAssign::InsertSendRecvForHcomParallel(const shared_ptr &graph_ptr) { +bool AscendStreamAssign::IsSatisfiedHcom(const std::map> &hcom_index, const CNodePtr &node_ptr, + size_t index) { + MS_EXCEPTION_IF_NULL(node_ptr); + auto cur_hcom_stream_id = AnfAlgo::GetStreamId(node_ptr); + auto it = hcom_index.find(cur_hcom_stream_id); + if (it == hcom_index.end()) { + return false; + } + auto iter = std::find(hcom_index.at(cur_hcom_stream_id).begin(), hcom_index.at(cur_hcom_stream_id).end(), index); + if (iter == hcom_index.at(cur_hcom_stream_id).end()) { + return false; + } + return true; +} + +// section6 +void AscendStreamAssign::InsertEventForIndependentParallel(const NotNull &graph_ptr) { MS_LOG(INFO) << "start"; - MS_EXCEPTION_IF_NULL(graph_ptr); + AscendResourceMng &resource_manager = AscendResourceMng::GetInstance(); auto cnode_ptr_list = graph_ptr->execution_order(); vector cnodes = cnode_ptr_list; - uint32_t cur_event_id = 0; + uint32_t cur_event_id = resource_manager.ApplyNewEvent(); auto it = cnodes.begin(); - while (it != cnodes.end() && (it + 1) != cnodes.end()) { + while (it != cnodes.end()) { MS_EXCEPTION_IF_NULL(*it); - MS_EXCEPTION_IF_NULL(*(it + 1)); - if (IsHcom(*it) && !IsHcom(*(it + 1))) { - bool is_fusion = IsFusionHcom(*it); - if (!is_fusion) { - ++it; - continue; - } + if (IsIndependentNode(*it)) { + MS_LOG(INFO) << "deal independent op[" << (*it)->DebugString() << "]"; CNodePtr send_cnode_ptr = CreateSendApplyKernel(graph_ptr, cur_event_id, AnfAlgo::GetStreamId(*it)); it = cnodes.insert(it + 1, send_cnode_ptr); auto target = FindTargetOp(it, cnodes.end(), *(it - 1)); if (target == cnodes.end()) { - MS_LOG(WARNING) << "hcom node[" << (*(it - 1))->fullname_with_scope() - << "] can't find target for insert recv op, no insert send/recv"; + MS_LOG(DEBUG) << "independ node[" << (*(it - 1))->fullname_with_scope() + << "] can't find target for insert recv op, no insert send/recv"; it = cnodes.erase(it); continue; } @@ -496,67 +655,31 @@ void AscendStreamAssign::InsertSendRecvForHcomParallel(const shared_ptrset_execution_order(cnodes); - total_event_num_ = cur_event_id; - MS_LOG(INFO) << "after insert send/recv for hcom parallel, total event nums[" << total_event_num_ << "]"; - - // Insert Send/Recv between Hcom(such as:AllReduce1 Send1 Common Recv1 AllReduce2) - InsertSendRecvForDiffHcom(graph_ptr); + MS_LOG(INFO) << "after independent parallel, total event nums:" << resource_manager.get_cur_event_num(); MS_LOG(INFO) << "end"; } -void AscendStreamAssign::UpdateEventId(const shared_ptr &graph_ptr) { - MS_LOG(INFO) << "start"; - MS_EXCEPTION_IF_NULL(graph_ptr); +// section7 +void AscendStreamAssign::GetNeedActiveStreams(const NotNull &graph_ptr) { CNodePtr cur_cnode_ptr = nullptr; - // key:virutal event id, value:real event id - std::unordered_map event_id_map; - uint32_t event_id; auto cnode_ptr_list = graph_ptr->execution_order(); - for (size_t i = 0; i < cnode_ptr_list.size(); ++i) { - cur_cnode_ptr = cnode_ptr_list[i]; - MS_EXCEPTION_IF_NULL(cur_cnode_ptr); - if (AnfAlgo::GetCNodeName(cur_cnode_ptr) == kSendOpName || AnfAlgo::GetCNodeName(cur_cnode_ptr) == kRecvOpName) { - auto primitive = AnfAlgo::GetCNodePrimitive(cur_cnode_ptr); - MS_EXCEPTION_IF_NULL(primitive); - event_id = GetValue(primitive->GetAttr(kAttrEventId)); - // before stream assign, send/recv event_id assign from kFirstEventId - if (event_id < kFirstEventId) { - continue; - } - auto it = event_id_map.find(event_id); - if (it == event_id_map.end()) { - event_id_map.insert(std::make_pair(event_id, total_event_num_)); - AnfAlgo::SetNodeAttr(kAttrEventId, MakeValue(total_event_num_), cur_cnode_ptr); - total_event_num_++; - } else { - AnfAlgo::SetNodeAttr(kAttrEventId, MakeValue(it->second), cur_cnode_ptr); - } - } - } -} + // 1)first stream 0 should be actived first; + need_first_active_streams_.emplace_back(0); -void AscendStreamAssign::GetNeedActiveStreams(const shared_ptr &graph_ptr) { - MS_EXCEPTION_IF_NULL(graph_ptr); - CNodePtr cur_cnode_ptr = nullptr; - auto cnode_ptr_list = graph_ptr->execution_order(); - // 1)stream witch kStreamNeedActivedFirst attr should be actived; + // 2)stream witch kStreamNeedActivedFirst attr should be actived; for (size_t i = 0; i < cnode_ptr_list.size(); ++i) { cur_cnode_ptr = cnode_ptr_list[i]; MS_EXCEPTION_IF_NULL(cur_cnode_ptr); - ValuePtr value_ptr = nullptr; auto primitive = AnfAlgo::GetCNodePrimitive(cur_cnode_ptr); - if (primitive != nullptr) { - value_ptr = primitive->GetAttr(kStreamNeedActivedFirst); - } else { - auto func_graph = AnfAlgo::GetCNodeFuncGraphPtr(cur_cnode_ptr); - MS_EXCEPTION_IF_NULL(func_graph); - value_ptr = func_graph->get_attr(kStreamNeedActivedFirst); - } + MS_EXCEPTION_IF_NULL(primitive); + auto value_ptr = primitive->GetAttr(kStreamNeedActivedFirst); if (value_ptr == nullptr) { continue; } @@ -569,20 +692,115 @@ void AscendStreamAssign::GetNeedActiveStreams(const shared_ptr &graph_ptr) { + CheckStreamAssign(graph_ptr); + CheckEventAssign(graph_ptr); +} + +void AscendStreamAssign::CheckStreamAssign(const NotNull &graph_ptr) { + AscendResourceMng &resource_manager = AscendResourceMng::GetInstance(); + std::set streams; + uint32_t max_stream = 0; + uint32_t min_stream = kInvalidStreamId; + auto cnode_ptr_list = graph_ptr->execution_order(); + for (size_t i = 0; i < cnode_ptr_list.size(); ++i) { + CNodePtr cur_cnode_ptr = cnode_ptr_list[i]; + MS_EXCEPTION_IF_NULL(cur_cnode_ptr); + uint32_t stream_id = AnfAlgo::GetStreamId(cur_cnode_ptr); + if (stream_id == kInvalidStreamId) { + MS_LOG(EXCEPTION) << "node:" << AnfAlgo::GetCNodeName(cur_cnode_ptr) << "had not been assigned stream"; + } + + (void)streams.emplace(stream_id); + if (stream_id > max_stream) { + max_stream = stream_id; + } + if (stream_id < min_stream) { + min_stream = stream_id; + } + } + + // check stream assign + if (!streams.empty()) { + if (min_stream != 0) { + MS_LOG(EXCEPTION) << "stream should start from 0, now is from " << min_stream; + } + uint32_t assigned_stream_num = resource_manager.get_cur_stream_num(); + if ((max_stream != assigned_stream_num - 1) || (streams.size() != assigned_stream_num)) { + MS_LOG(EXCEPTION) << "stream should be consecutive, max stream id:" << max_stream + << "; alloc stream nums:" << assigned_stream_num << "; streams size:" << streams.size(); + } + } } -CNodePtr AscendStreamAssign::CreateSendApplyKernel(const std::shared_ptr &graph_ptr, - uint32_t event_id, uint32_t stream_id) { - MS_EXCEPTION_IF_NULL(graph_ptr); +void AscendStreamAssign::CheckEventAssign(const NotNull &graph_ptr) { + AscendResourceMng &resource_manager = AscendResourceMng::GetInstance(); + std::map> event_map; + uint32_t max_event_id = 0; + uint32_t min_event_id = kInvalidEventId; + auto cnode_ptr_list = graph_ptr->execution_order(); + for (size_t i = 0; i < cnode_ptr_list.size(); ++i) { + CNodePtr cur_cnode_ptr = cnode_ptr_list[i]; + MS_EXCEPTION_IF_NULL(cur_cnode_ptr); + auto name = AnfAlgo::GetCNodeName(cur_cnode_ptr); + if (name == kSendOpName || name == kRecvOpName) { + uint32_t event_id = AnfAlgo::GetNodeAttr(cur_cnode_ptr, kAttrEventId); + if (event_id > max_event_id) { + max_event_id = event_id; + } + + if (event_id < min_event_id) { + min_event_id = event_id; + } + auto it = event_map.find(event_id); + if (it == event_map.end()) { + event_map[event_id] = {cur_cnode_ptr}; + } else { + event_map[event_id].emplace_back(cur_cnode_ptr); + } + } + } + // check event assign + if (!event_map.empty()) { + if (min_event_id != 0) { + MS_LOG(EXCEPTION) << "event should start from 0, now is from " << min_event_id; + } + uint32_t assigned_event_num = resource_manager.get_cur_event_num(); + if ((max_event_id != assigned_event_num - 1) || (event_map.size() != assigned_event_num)) { + MS_LOG(EXCEPTION) << "event should be consecutive"; + } + for (const auto &item : event_map) { + if (item.second.size() != 2) { + MS_LOG(EXCEPTION) << "send/recv should be in pair and share one event id"; + } + auto first_name = AnfAlgo::GetCNodeName(item.second[0]); + auto second_name = AnfAlgo::GetCNodeName(item.second[1]); + if (!(first_name == kSendOpName && second_name == kRecvOpName)) { + MS_LOG(EXCEPTION) << "send should be before recv"; + } + } + } +} + +// section9 +CNodePtr AscendStreamAssign::CreateSendApplyKernel(const NotNull &graph_ptr, uint32_t event_id, + uint32_t stream_id) { auto send_op = std::make_shared(kSendOpName); MS_EXCEPTION_IF_NULL(send_op); auto send_apply = std::make_shared(send_op); @@ -601,9 +819,8 @@ CNodePtr AscendStreamAssign::CreateSendApplyKernel(const std::shared_ptr &graph_ptr, - uint32_t event_id, uint32_t stream_id) { - MS_EXCEPTION_IF_NULL(graph_ptr); +CNodePtr AscendStreamAssign::CreateRecvApplyKernel(const NotNull &graph_ptr, uint32_t event_id, + uint32_t stream_id) { auto recv_op = std::make_shared(kRecvOpName); MS_EXCEPTION_IF_NULL(recv_op); auto recv_apply = std::make_shared(recv_op); @@ -649,42 +866,6 @@ vector::iterator AscendStreamAssign::FindTargetOp(vector::it ++begin; } return end; -} // namespace ascend - -void AscendStreamAssign::InsertSendRecvForIndependent(const shared_ptr &graph_ptr) { - MS_LOG(INFO) << "start"; - MS_EXCEPTION_IF_NULL(graph_ptr); - auto cnode_ptr_list = graph_ptr->execution_order(); - vector cnodes = cnode_ptr_list; - uint32_t cur_event_id = total_event_num_; - auto it = cnodes.begin(); - while (it != cnodes.end()) { - MS_EXCEPTION_IF_NULL(*it); - if (IsIndependentNode(*it)) { - MS_LOG(INFO) << "deal independent op[" << (*it)->DebugString() << "]"; - CNodePtr send_cnode_ptr = CreateSendApplyKernel(graph_ptr, cur_event_id, AnfAlgo::GetStreamId(*it)); - it = cnodes.insert(it + 1, send_cnode_ptr); - - auto target = FindTargetOp(it, cnodes.end(), *(it - 1)); - if (target == cnodes.end()) { - MS_LOG(DEBUG) << "independ node[" << (*(it - 1))->fullname_with_scope() - << "] can't find target for insert recv op, no insert send/recv"; - it = cnodes.erase(it); - continue; - } - - // deal recv op - uint32_t stream_id = AnfAlgo::GetStreamId(*target); - CNodePtr recv_cnode_ptr = CreateRecvApplyKernel(graph_ptr, cur_event_id, stream_id); - (void)cnodes.insert(target, recv_cnode_ptr); - ++cur_event_id; - } - ++it; - } - graph_ptr->set_execution_order(cnodes); - total_event_num_ = cur_event_id; - MS_LOG(INFO) << "total event nums[" << total_event_num_ << "]"; - MS_LOG(INFO) << "end"; } bool AscendStreamAssign::IsTaskSink() { @@ -701,8 +882,8 @@ bool AscendStreamAssign::IsTaskSink() { void AscendStreamAssign::GetWaitStreams(vector *wait_active_stream_list) { MS_EXCEPTION_IF_NULL(wait_active_stream_list); - AscendStreamMng &stream_manager = AscendStreamMng::GetInstance(); - uint32_t total_stream_num = stream_manager.GetCurAllocStreamNum(); + AscendResourceMng &resource_manager = AscendResourceMng::GetInstance(); + uint32_t total_stream_num = resource_manager.get_cur_stream_num(); if (total_stream_num == 0) { MS_LOG(INFO) << "total_common_stream_num is zero"; return; @@ -713,7 +894,7 @@ void AscendStreamAssign::GetWaitStreams(vector *wait_active_stream_lis auto it = std::find(need_first_active_streams_.begin(), need_first_active_streams_.end(), i); if (it == need_first_active_streams_.end()) { MS_LOG(INFO) << "wait common stream id = " << i; - (*wait_active_stream_list).push_back(i); + wait_active_stream_list->push_back(i); } } } @@ -723,94 +904,21 @@ bool AscendStreamAssign::IsHcom(const CNodePtr &apply_kernel) { return AnfAlgo::GetKernelType(apply_kernel) == HCCL_KERNEL; } -bool AscendStreamAssign::IsFusionHcom(const CNodePtr &cur_cnode_ptr) { - MS_EXCEPTION_IF_NULL(cur_cnode_ptr); - bool is_hcom = IsHcom(cur_cnode_ptr); - if (!is_hcom) { - return false; - } - - if (!AnfAlgo::HasNodeAttr(kAttrFusion, cur_cnode_ptr)) { - return false; - } - - if (AnfAlgo::GetNodeAttr(cur_cnode_ptr, kAttrFusion) == 0) { - return false; - } - - return true; -} - void AscendStreamAssign::GetHcomStreams(std::vector *streams) { MS_EXCEPTION_IF_NULL(streams); - for (const auto &stream : hcom_stream_list_) { - (*streams).emplace_back(stream); + for (const auto &item : hcom_stream_map_) { + streams->emplace_back(item.first); } } -void AscendStreamAssign::ReorderIndependentOrders(const shared_ptr &graph_ptr) { - MS_EXCEPTION_IF_NULL(graph_ptr); - CNodePtr cur_cnode_ptr = nullptr; - std::vector exe_orders; - std::vector independents; - std::vector others; - auto cnode_ptr_list = graph_ptr->execution_order(); - MS_LOG(INFO) << "before reorder, graph orders size:" << cnode_ptr_list.size(); - for (size_t i = 0; i < cnode_ptr_list.size(); ++i) { - cur_cnode_ptr = cnode_ptr_list[i]; - MS_EXCEPTION_IF_NULL(cur_cnode_ptr); - if (IsIndependentNode(cur_cnode_ptr)) { - independents.emplace_back(cur_cnode_ptr); - } else { - others.emplace_back(cur_cnode_ptr); - } - } - if (others.empty() || independents.empty()) { - MS_LOG(INFO) << "independent or others is empty, no need reorder"; - return; - } - - std::set processed; - for (size_t i = 0; i < others.size(); i++) { - auto begin = others.begin() + i; - auto end = begin + 1; - bool flag = false; - for (size_t j = 0; j < independents.size(); j++) { - auto cur_independent = independents[j]; - auto it = std::find(processed.begin(), processed.end(), cur_independent.get()); - if (it != processed.end()) { - continue; - } - auto res = FindTargetOp(begin, end, cur_independent); - if (res != end) { - flag = true; - exe_orders.emplace_back(cur_independent); - exe_orders.emplace_back(*begin); - processed.emplace(cur_independent.get()); - break; - } - } - if (!flag) { - exe_orders.emplace_back(*begin); - } - } - MS_LOG(INFO) << "after reorder, graph orders size:" << exe_orders.size(); - if (processed.size() != independents.size()) { - MS_LOG(WARNING) << "processed independent nodes size is not equal to exiting independent nodes size"; - return; - } - - graph_ptr->set_execution_order(exe_orders); -} - void AscendStreamAssign::Reset() { - total_event_num_ = 0; independent_stream_activated_ = false; + hcom_stream_activated_ = false; independent_stream_map_.clear(); + hcom_stream_map_.clear(); + common_stream_map_.clear(); processed_streams_.clear(); - hcom_stream_list_.clear(); need_first_active_streams_.clear(); - inner_parallel_streams_.clear(); } } // namespace ascend } // namespace device diff --git a/mindspore/ccsrc/device/ascend/ascend_stream_assign.h b/mindspore/ccsrc/device/ascend/ascend_stream_assign.h index bb918cfc79..625ab6ad6e 100644 --- a/mindspore/ccsrc/device/ascend/ascend_stream_assign.h +++ b/mindspore/ccsrc/device/ascend/ascend_stream_assign.h @@ -29,6 +29,7 @@ #include "runtime/rt_model.h" #include "runtime/stream.h" #include "session/kernel_graph.h" +#include "utils/contract.h" namespace mindspore { namespace device { @@ -38,35 +39,59 @@ using std::shared_ptr; using std::unordered_map; using std::unordered_set; using std::vector; -using CnodeKey = void *; const uint32_t kInvalidStreamId = UINT32_MAX; -class AscendStreamMng { +const uint32_t kInvalidEventId = UINT32_MAX; +class AscendResourceMng { public: - static AscendStreamMng &GetInstance() { - static AscendStreamMng instance; + static AscendResourceMng &GetInstance() { + static AscendResourceMng instance; return instance; } - void Reset() { - cur_stream_id = 0; - cur_stream_num = 0; + void ResetResource() { + cur_stream_num_ = 0; + cur_event_num_ = 0; } uint32_t ApplyNewStream() { - if (!cur_stream_num) { - cur_stream_num++; + if (!cur_stream_num_) { + uint32_t cur_stream_id = cur_stream_num_; + cur_stream_num_++; return cur_stream_id; } - cur_stream_num++; - cur_stream_id++; + uint32_t cur_stream_id = cur_stream_num_; + cur_stream_num_++; return cur_stream_id; } + uint32_t ApplyNewEvent() { + if (!cur_event_num_) { + uint32_t cur_event_id = cur_event_num_; + cur_event_num_++; + return cur_event_id; + } + uint32_t cur_event_id = cur_event_num_; + cur_event_num_++; + return cur_event_id; + } - uint32_t GetCurAllocStream() { return cur_stream_id; } - uint32_t GetCurAllocStreamNum() { return cur_stream_num; } + void DeleteEvent() { + if (!cur_event_num_) { + MS_LOG(WARNING) << "total event num is 0, no event to delete"; + } else { + --cur_event_num_; + } + } + uint32_t get_cur_stream_num() { return cur_stream_num_; } + uint32_t GetCurAllocStreamId() { + if (!cur_stream_num_) { + MS_LOG(EXCEPTION) << "stream nums is 0, no stream id should be get"; + } + return cur_stream_num_ - 1; + } + uint32_t get_cur_event_num() { return cur_event_num_; } private: - uint32_t cur_stream_num{0}; - uint32_t cur_stream_id{0}; + uint32_t cur_stream_num_{0}; + uint32_t cur_event_num_{0}; }; class AscendStreamAssign { @@ -79,39 +104,42 @@ class AscendStreamAssign { AscendStreamAssign(const AscendStreamAssign &) = delete; AscendStreamAssign &operator=(const AscendStreamAssign &) = delete; - uint32_t total_event_num() const { return total_event_num_; } + void AssignStream(const NotNull &graph_ptr); void GetHcomStreams(std::vector *streams); - - void AssignStream(const std::shared_ptr &graph_ptr); void GetWaitStreams(vector *wait_active_stream_list); - CNodePtr CreateSendApplyKernel(const std::shared_ptr &graph_ptr, uint32_t event_id, - uint32_t stream_id); - CNodePtr CreateRecvApplyKernel(const std::shared_ptr &graph_ptr, uint32_t event_id, - uint32_t stream_id); + CNodePtr CreateSendApplyKernel(const NotNull &graph_ptr, uint32_t event_id, uint32_t stream_id); + CNodePtr CreateRecvApplyKernel(const NotNull &graph_ptr, uint32_t event_id, uint32_t stream_id); private: AscendStreamAssign() = default; ~AscendStreamAssign() = default; void Reset(); - void CheckStreamAssign(const std::shared_ptr &graph_ptr); - void AssignAllNodesStream(const std::shared_ptr &graph_ptr); - void AssignCommonStreamId(const CNodePtr &cur_cnode_ptr, CNodePtr *pre_cnode_ptr, uint32_t *cur_index, - uint32_t *cur_stream_id); + void CheckResourceAssign(const NotNull &graph_ptr); + void CheckStreamAssign(const NotNull &graph_ptr); + void CheckEventAssign(const NotNull &graph_ptr); + void AssignAllNodesStream(const NotNull &graph_ptr); + void AssignCommonStreamId(const CNodePtr &cur_cnode_ptr); + void AssignHcomStreamId(const CNodePtr &cur_cnode_ptr); void AssignIndependentStreamId(const CNodePtr &cur_cnode_ptr); - void UpdateAtomicAddrCleanStreamId(const std::shared_ptr &graph_ptr); - void FindHcomParallelStreams(const std::shared_ptr &graph_ptr); - void InsertStreamActive(const std::shared_ptr &graph_ptr); - void UpdateStreamSwitch(const std::shared_ptr &graph_ptr, const CNodePtr &switch_ptr, - const vector &independent_stream, vector *orders); - void InsertSendRecvForIndependent(const std::shared_ptr &graph_ptr); - void InsertSendRecvForHcomParallel(const std::shared_ptr &graph_ptr); - void InsertSendRecvForDiffHcom(const shared_ptr &graph_ptr); - void UpdateEventId(const std::shared_ptr &graph_ptr); - void GetNeedActiveStreams(const std::shared_ptr &graph_ptr); - void ReorderIndependentOrders(const std::shared_ptr &graph_ptr); + void UpdateAtomicAddrCleanStreamId(const NotNull &graph_ptr); + void FindHcomParallelStreams(const NotNull &graph_ptr); + void InsertStreamActive(const NotNull &graph_ptr); + void UpdateStreamSwitch(const NotNull &graph_ptr, const CNodePtr &switch_ptr, + vector *orders); + void InsertEventForIndependentParallel(const NotNull &graph_ptr); + void InsertEventForHcomParallel(const NotNull &graph_ptr); + void InsertEventCommonDependHcom(const NotNull &graph_ptr); + void InsertEventHcomDependCommon(const NotNull &graph_ptr); + void InsertEventHcomDependHcom(const NotNull &graph_ptr); + void InsertEventBetweenHcom(const NotNull &graph_ptr, const map> &hcom_index, + uint32_t first_hcom_stream, uint32_t last_hcom_stream); + bool IsSatisfiedHcom(const std::map> &hcom_index, const CNodePtr &node_ptr, size_t index); + + void GetProcessedStream(const NotNull &graph_ptr); + void GetNeedActiveStreams(const NotNull &graph_ptr); + void ReorderIndependentOrders(const NotNull &graph_ptr); bool IsTaskSink(); - bool IsFusionHcom(const CNodePtr &cur_cnode_ptr); bool IsHcom(const CNodePtr &cur_cnode_ptr); bool IsIndependentNode(const CNodePtr &node_ptr); bool IsProcessedStream(uint32_t stream_id); @@ -119,14 +147,13 @@ class AscendStreamAssign { const CNodePtr &node); void GetParallelStream(uint32_t cur_stream_id, uint32_t stream_acitve_id, std::vector *parallel_streams); - uint32_t total_event_num_{0}; bool independent_stream_activated_{false}; + bool hcom_stream_activated_{false}; std::map independent_stream_map_{}; + std::map hcom_stream_map_{}; + std::map common_stream_map_{}; std::set processed_streams_{}; - std::set hcom_stream_list_{}; std::vector need_first_active_streams_{}; - std::vector> inner_parallel_streams_{}; - // new policy end }; } // namespace ascend diff --git a/mindspore/ccsrc/device/kernel_adjust.cc b/mindspore/ccsrc/device/kernel_adjust.cc index 93007764af..9fd3132829 100644 --- a/mindspore/ccsrc/device/kernel_adjust.cc +++ b/mindspore/ccsrc/device/kernel_adjust.cc @@ -103,8 +103,8 @@ CNodePtr KernelAdjust::CreateRecvApplyKernel(const std::shared_ptr &kernel_graph_ptr) { - device::ascend::AscendStreamMng &stream_manager = device::ascend::AscendStreamMng::GetInstance(); - stream_manager.Reset(); + device::ascend::AscendResourceMng &resource_manager = device::ascend::AscendResourceMng::GetInstance(); + resource_manager.ResetResource(); if (!NeedInsertSwitch()) { return; } @@ -135,17 +135,16 @@ void KernelAdjust::InsertSwitchLoop(const std::shared_ptr } std::vector exec_order; - // getnext loop process // getnext loop stream switch op CNodePtr getnext_switch_app = CreateStreamSwitchOp(kernel_graph_ptr, switch_loop_input); MS_EXCEPTION_IF_NULL(getnext_switch_app); - uint32_t getnext_switch_stream_id = stream_manager.ApplyNewStream(); + uint32_t getnext_switch_stream_id = resource_manager.ApplyNewStream(); AnfAlgo::SetStreamId(getnext_switch_stream_id, getnext_switch_app.get()); exec_order.push_back(getnext_switch_app); // getnext op - uint32_t getnext_stream_id = stream_manager.ApplyNewStream(); + uint32_t getnext_stream_id = resource_manager.ApplyNewStream(); size_t i = 0; for (; i < orders.size(); i++) { auto node = orders[i]; @@ -160,7 +159,8 @@ void KernelAdjust::InsertSwitchLoop(const std::shared_ptr AnfAlgo::SetNodeAttr(kAttrTrueBranchStream, MakeValue(getnext_stream_id), getnext_switch_app); // getnext loop send - CNodePtr send = CreateSendApplyKernel(kernel_graph_ptr, kFirstEventId); + uint32_t getnext_event_id = resource_manager.ApplyNewEvent(); + CNodePtr send = CreateSendApplyKernel(kernel_graph_ptr, getnext_event_id); AnfAlgo::SetStreamId(getnext_stream_id, send.get()); exec_order.push_back(send); @@ -168,14 +168,14 @@ void KernelAdjust::InsertSwitchLoop(const std::shared_ptr // fpbp loop stream switch CNodePtr fpbp_switch_app = CreateStreamSwitchOp(kernel_graph_ptr, switch_loop_input); MS_EXCEPTION_IF_NULL(fpbp_switch_app); - uint32_t fpbp_switch_stream_id = stream_manager.ApplyNewStream(); + uint32_t fpbp_switch_stream_id = resource_manager.ApplyNewStream(); AnfAlgo::SetStreamId(fpbp_switch_stream_id, fpbp_switch_app.get()); AnfAlgo::SetNodeAttr(kStreamNeedActivedFirst, MakeValue(true), fpbp_switch_app); exec_order.push_back(fpbp_switch_app); // fpbp loop recv - CNodePtr recv = CreateRecvApplyKernel(kernel_graph_ptr, kFirstEventId); - uint32_t fpbp_stream_id = stream_manager.ApplyNewStream(); + CNodePtr recv = CreateRecvApplyKernel(kernel_graph_ptr, getnext_event_id); + uint32_t fpbp_stream_id = resource_manager.ApplyNewStream(); AnfAlgo::SetStreamId(fpbp_stream_id, recv.get()); exec_order.push_back(recv); diff --git a/mindspore/ccsrc/device/kernel_adjust.h b/mindspore/ccsrc/device/kernel_adjust.h index 1a7436b396..5dc559408a 100644 --- a/mindspore/ccsrc/device/kernel_adjust.h +++ b/mindspore/ccsrc/device/kernel_adjust.h @@ -38,12 +38,8 @@ constexpr auto kIterLoopParamName = "iter_loop"; constexpr auto kZeroParamName = "zero"; constexpr auto kOneParamName = "one"; constexpr auto kStreamNeedActivedFirst = "stream_need_active_first"; +constexpr uint32_t kSecondStreamSwitchLabel = 2; -const uint32_t kFirstStreamSwitchLabel = 0; -const uint32_t kGetNextLabel = 1; -const uint32_t kSecondStreamSwitchLabel = 2; -const uint32_t kInvalidEventId = UINT32_MAX; -const uint32_t kFirstEventId = kInvalidEventId / 2; namespace device { class KernelAdjust { public: diff --git a/mindspore/ccsrc/session/ascend_session.cc b/mindspore/ccsrc/session/ascend_session.cc index bae10ed943..ce0883597e 100644 --- a/mindspore/ccsrc/session/ascend_session.cc +++ b/mindspore/ccsrc/session/ascend_session.cc @@ -303,7 +303,7 @@ GraphId AscendSession::CompileGraph(NotNull func_graph) { // adjust kernel AdjustKernel(root_graph); // assign stream - AssignStream(root_graph); + AssignStream(NOT_NULL(root_graph)); // insert profiling point device::KernelAdjust::GetInstance().Profiling(NOT_NULL(root_graph.get())); // build kernel @@ -375,7 +375,7 @@ void AscendSession::BuildGraph(GraphId graph_id) { // adjust execution order because merge child graph and other special operations AdjustKernel(graph); // Assign streams for control sink and hccl and so on - AssignStream(graph); + AssignStream(NOT_NULL(graph)); device::KernelAdjust::GetInstance().Profiling(NOT_NULL(graph.get())); // build kernel if node is cnode @@ -627,7 +627,7 @@ void AscendSession::RunOpAdjustKernel(const std::shared_ptr &kernel MS_LOG(INFO) << "Finish!"; } -void AscendSession::AssignStream(const std::shared_ptr &kernel_graph) const { +void AscendSession::AssignStream(NotNull kernel_graph) const { MS_LOG(INFO) << "Start!"; device::ascend::AscendStreamAssign::GetInstance().AssignStream(kernel_graph); MS_LOG(INFO) << "Finish!"; diff --git a/mindspore/ccsrc/session/ascend_session.h b/mindspore/ccsrc/session/ascend_session.h index 7857330115..ce7327b91e 100755 --- a/mindspore/ccsrc/session/ascend_session.h +++ b/mindspore/ccsrc/session/ascend_session.h @@ -76,7 +76,7 @@ class AscendSession : public SessionBasic { void HardwareOptimize(const std::shared_ptr &kernel_graph) const; void AdjustKernel(const std::shared_ptr &kernel_graph) const; void RunOpAdjustKernel(const std::shared_ptr &kernel_graph) const; - void AssignStream(const std::shared_ptr &kernel_graph) const; + void AssignStream(NotNull kernel_graph) const; void AssignLabel(NotNull kernel_graph) const; void BuildKernel(const std::shared_ptr &kernel_graph) const; void MemoryAlloc(KernelGraph *kernel_graph) const; diff --git a/tests/ut/cpp/stub/tasksink/ascend_stream_assign_stub.cc b/tests/ut/cpp/stub/tasksink/ascend_stream_assign_stub.cc index fba52323cf..a6ec3a50b5 100755 --- a/tests/ut/cpp/stub/tasksink/ascend_stream_assign_stub.cc +++ b/tests/ut/cpp/stub/tasksink/ascend_stream_assign_stub.cc @@ -26,7 +26,7 @@ void AscendLabelAssign::AssignLabel(NotNull graph) { return 1; } uint32_t AscendLabelAssign::GetLabelNum(NotNull> graph) { return 1; } -void AscendStreamAssign::AssignStream(const KernelGraphPtr &graph) { return; } +void AscendStreamAssign::AssignStream(const NotNull &graph_ptr) { return; } void AscendStreamAssign::GetWaitStreams(vector *wait_active_stream_list) { return; } From 947a93c8644a5b323b60d298a998f468b31c065f Mon Sep 17 00:00:00 2001 From: geekun Date: Tue, 23 Jun 2020 16:29:22 +0800 Subject: [PATCH 036/254] fix infer value bug --- mindspore/common/tensor.py | 8 ++++++++ mindspore/ops/operations/math_ops.py | 6 ++++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/mindspore/common/tensor.py b/mindspore/common/tensor.py index 0a631b954f..ddd7cbfabc 100644 --- a/mindspore/common/tensor.py +++ b/mindspore/common/tensor.py @@ -22,6 +22,10 @@ from . import dtype as mstype from ._register_for_tensor import tensor_operator_registry __all__ = ['Tensor', 'MetaTensor'] +np_types = (np.int8, np.int16, np.int32, np.int64, + np.uint8, np.uint16, np.uint32, np.uint64, np.float16, + np.float32, np.float64, np.bool_) + class Tensor(Tensor_): @@ -54,6 +58,10 @@ class Tensor(Tensor_): """ def __init__(self, input_data, dtype=None): + # If input data is numpy number, convert it to np array + if isinstance(input_data, np_types): + input_data = np.array(input_data) + # If input_data is tuple/list/numpy.ndarray, it's support in check_type method. check_type('tensor input_data', input_data, (Tensor_, float, int)) if dtype is not None: diff --git a/mindspore/ops/operations/math_ops.py b/mindspore/ops/operations/math_ops.py index 08cd481582..5044783ea7 100644 --- a/mindspore/ops/operations/math_ops.py +++ b/mindspore/ops/operations/math_ops.py @@ -888,7 +888,8 @@ class Neg(PrimitiveWithInfer): def infer_value(self, input_x): if input_x is not None: input_x = input_x.asnumpy() - return Tensor(-input_x) + out = np.array(-input_x, input_x.dtype) + return Tensor(out) return None @@ -1667,7 +1668,8 @@ class Div(_MathBinaryOp): if x is not None and y is not None: x = x.asnumpy() y = y.asnumpy() - return Tensor(x / y) + out = np.array(x / y, x.dtype) + return Tensor(out) return None From f9c3023e9471467508ee09e52e706402e35c4210 Mon Sep 17 00:00:00 2001 From: leopz Date: Wed, 24 Jun 2020 10:21:19 +0800 Subject: [PATCH 037/254] fix changefilemod print --- mindspore/ccsrc/utils/utils.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mindspore/ccsrc/utils/utils.h b/mindspore/ccsrc/utils/utils.h index 7380ef501f..f27e646348 100644 --- a/mindspore/ccsrc/utils/utils.h +++ b/mindspore/ccsrc/utils/utils.h @@ -287,7 +287,7 @@ const std::set kFloatDataTypeSet = {kNumberTypeFloat16, kNumberTypeFloat static inline void ChangeFileMode(const std::string &file_name, mode_t mode) { try { if (chmod(file_name.c_str(), mode) != 0) { - MS_LOG(WARNING) << "Change file `" << file_name << "` to mode " << std::oct << mode << " fail."; + MS_LOG(DEBUG) << "Change file `" << file_name << "` to mode " << std::oct << mode << " fail."; } } catch (std::exception &e) { MS_LOG(DEBUG) << "File `" << file_name << "` change mode failed! May be not exist."; From 091d88849d0712a90d07b4e4811c1c000851cf6f Mon Sep 17 00:00:00 2001 From: jojobugfree Date: Tue, 23 Jun 2020 23:00:32 +0800 Subject: [PATCH 038/254] codex && code review --- .../ascend/profiling/profiling_engine_impl.cc | 1 + .../ascend/profiling/profiling_manager.cc | 30 +++++---- .../ascend/profiling/profiling_manager.h | 10 ++- .../ascend/profiling/profiling_utils.cc | 4 ++ .../device/ascend/tasksink/runtime_utils.cc | 20 +++--- .../device/ascend/tasksink/task_generator.cc | 67 ++++++++++--------- .../device/ascend/tasksink/task_generator.h | 1 + .../ccsrc/device/cpu/cpu_kernel_runtime.cc | 1 + 8 files changed, 73 insertions(+), 61 deletions(-) diff --git a/mindspore/ccsrc/device/ascend/profiling/profiling_engine_impl.cc b/mindspore/ccsrc/device/ascend/profiling/profiling_engine_impl.cc index cbecb3030d..a393409334 100644 --- a/mindspore/ccsrc/device/ascend/profiling/profiling_engine_impl.cc +++ b/mindspore/ccsrc/device/ascend/profiling/profiling_engine_impl.cc @@ -28,6 +28,7 @@ PluginIntf *ProfilingEngineImpl::CreatePlugin() { int ProfilingEngineImpl::ReleasePlugin(PluginIntf *plugin) { if (plugin != nullptr) { delete plugin; + plugin = nullptr; } return 0; } diff --git a/mindspore/ccsrc/device/ascend/profiling/profiling_manager.cc b/mindspore/ccsrc/device/ascend/profiling/profiling_manager.cc index fec1aac685..a2fe5b852d 100644 --- a/mindspore/ccsrc/device/ascend/profiling/profiling_manager.cc +++ b/mindspore/ccsrc/device/ascend/profiling/profiling_manager.cc @@ -15,11 +15,8 @@ */ #include "device/ascend/profiling/profiling_manager.h" - #include #include - -#include #include "securec/include/securec.h" #include "./prof_mgr_core.h" #include "device/ascend/profiling/plugin_impl.h" @@ -30,9 +27,6 @@ #include "utils/convert_utils.h" #include "runtime/base.h" -using std::vector; -using Json = nlohmann::json; - namespace mindspore { namespace device { namespace ascend { @@ -124,35 +118,43 @@ bool ProfilingManager::StartupProfiling(uint32_t device_id) { auto context = MsContext::GetInstance(); MS_EXCEPTION_IF_NULL(context); const string prof_options_str = context->profiling_options(); - vector opts = Split(prof_options_str, ':'); + std::vector opts = Split(prof_options_str, ':'); if (opts.empty()) { MS_LOG(WARNING) << "Profiling is enabled, but profiling option is not set!"; return true; } // current one docker only use one device` - Json p_device; + nlohmann::json p_device; // JOBID auto job_id = GetJobId(); p_device["jobID"] = std::to_string(job_id); // device_id p_device["deviceID"] = std::to_string(device_id); // features:'training_trace', 'task_trace' etc - Json features; - for (vector::size_type i = 0; i < opts.size(); i++) { - Json f; + nlohmann::json features; + for (std::vector::size_type i = 0; i < opts.size(); i++) { + nlohmann::json f; f["name"] = opts[i]; features[i] = f; } p_device["features"] = features; // only one device, but sProfMgrStartUp API require for device list - Json devices; + nlohmann::json devices; devices[0] = p_device; - Json startCfg; + nlohmann::json startCfg; startCfg["startCfg"] = devices; + if (!ProfStartUp(NOT_NULL(&startCfg))) { + MS_LOG(ERROR) << "ProfMgrStartUp failed."; + return false; + } + return true; +} + +bool ProfilingManager::ProfStartUp(NotNull startCfg) { // convert json to string std::stringstream ss; - ss << startCfg; + ss << *startCfg; std::string cfg = ss.str(); MS_LOG(INFO) << "profiling config " << cfg; auto ret = rtProfilerStart(); diff --git a/mindspore/ccsrc/device/ascend/profiling/profiling_manager.h b/mindspore/ccsrc/device/ascend/profiling/profiling_manager.h index c30c6898ea..05b5248996 100644 --- a/mindspore/ccsrc/device/ascend/profiling/profiling_manager.h +++ b/mindspore/ccsrc/device/ascend/profiling/profiling_manager.h @@ -20,18 +20,15 @@ #include #include #include +#include +#include "utils/contract.h" #include "utils/context/ms_context.h" + using std::map; using std::string; - namespace mindspore { namespace device { namespace ascend { -// PROFILING_CUSTOM_LOGID_START 3 -const uint64_t kProfilingFpStartLogId = 1; -const uint64_t kProfilingBpEndLogId = 2; -const uint64_t kProfilingIterEndLogId = 255; - class ProfilingEngineImpl; class ProfilingManager { public: @@ -52,6 +49,7 @@ class ProfilingManager { ~ProfilingManager() { prof_handle_ = nullptr; } private: + bool ProfStartUp(NotNull json); std::shared_ptr engine_0_; uint32_t device_id_; void *prof_handle_; diff --git a/mindspore/ccsrc/device/ascend/profiling/profiling_utils.cc b/mindspore/ccsrc/device/ascend/profiling/profiling_utils.cc index 131a22805d..17ac4c4530 100644 --- a/mindspore/ccsrc/device/ascend/profiling/profiling_utils.cc +++ b/mindspore/ccsrc/device/ascend/profiling/profiling_utils.cc @@ -33,6 +33,10 @@ constexpr char kCustomNode[] = "PROFILING_CUSTOM_"; constexpr char kFpStartNode[] = "PROFILING_FP_START"; constexpr char kBpEndNode[] = "PROFILING_BP_END"; constexpr char kIterEndNode[] = "PROFILING_ITER_END"; +// PROFILING_CUSTOM_LOGID_START 3 +constexpr uint64_t kProfilingFpStartLogId = 1; +constexpr uint64_t kProfilingBpEndLogId = 2; +constexpr uint64_t kProfilingIterEndLogId = 255; std::map> ProfilingUtils::graph_profiling_cnode_; std::map> ProfilingUtils::graph_kernel_name_; std::map>> ProfilingUtils::graph_point_; diff --git a/mindspore/ccsrc/device/ascend/tasksink/runtime_utils.cc b/mindspore/ccsrc/device/ascend/tasksink/runtime_utils.cc index 603dd989e5..3faeefb820 100644 --- a/mindspore/ccsrc/device/ascend/tasksink/runtime_utils.cc +++ b/mindspore/ccsrc/device/ascend/tasksink/runtime_utils.cc @@ -58,9 +58,9 @@ bool RuntimeUtils::HcomDistribute(const std::shared_ptr &task_info if (task_info->hccl_type() == kBroadcastOpName) { // call hcom broadcast interface to run op const string tag_broadcast = kHcomBroadcast + std::to_string(task_counter++) + kUnderline + std::to_string(0); - ret = hcom_broadcast(tag_broadcast.c_str(), reinterpret_cast(task_info->input_data_addr()), - static_cast(task_info->count()), static_cast(task_info->data_type()), - static_cast(task_info->root_id()), hccl_group.c_str(), stream); + ret = hcom_broadcast(tag_broadcast.c_str(), task_info->input_data_addr(), static_cast(task_info->count()), + static_cast(task_info->data_type()), static_cast(task_info->root_id()), + hccl_group.c_str(), stream); if (ret != HCCL_SUCCESS) { MS_LOG(ERROR) << "hcom_broadcast fail, return ret: " << static_cast(ret); return false; @@ -68,9 +68,9 @@ bool RuntimeUtils::HcomDistribute(const std::shared_ptr &task_info } else if (task_info->hccl_type() == kAllGatherOpName) { // call hcom allgather interface to run op const string tag_all_gather = kHcomAllGather + std::to_string(task_counter++) + kUnderline + std::to_string(0); - ret = hcom_all_gather(tag_all_gather.c_str(), reinterpret_cast(task_info->input_data_addr()), - reinterpret_cast(task_info->output_data_addr()), static_cast(task_info->count()), - static_cast(task_info->data_type()), hccl_group.c_str(), stream); + ret = hcom_all_gather(tag_all_gather.c_str(), task_info->input_data_addr(), task_info->output_data_addr(), + static_cast(task_info->count()), static_cast(task_info->data_type()), + hccl_group.c_str(), stream); if (ret != HCCL_SUCCESS) { MS_LOG(ERROR) << "hcom_all_gather fail, return ret: " << ret; return false; @@ -78,9 +78,8 @@ bool RuntimeUtils::HcomDistribute(const std::shared_ptr &task_info } else if (task_info->hccl_type() == kAllReduceOpName) { // call hcom allreduce interface to run op const string tag_all_reduce = kHcomAllReduce + std::to_string(task_counter++) + kUnderline + std::to_string(0); - ret = hcom_all_reduce(tag_all_reduce.c_str(), reinterpret_cast(task_info->input_data_addr()), - reinterpret_cast(task_info->output_data_addr()), static_cast(task_info->count()), - static_cast(task_info->data_type()), + ret = hcom_all_reduce(tag_all_reduce.c_str(), task_info->input_data_addr(), task_info->output_data_addr(), + static_cast(task_info->count()), static_cast(task_info->data_type()), static_cast(task_info->op_type()), hccl_group.c_str(), stream); if (ret != HCCL_SUCCESS) { MS_LOG(ERROR) << "hcom_all_reduce fail, return ret: " << ret; @@ -90,8 +89,7 @@ bool RuntimeUtils::HcomDistribute(const std::shared_ptr &task_info // call hcom reducescatter interface to run op const string tag_reduce_scatter = kHcomReduceScatter + std::to_string(task_counter++) + kUnderline + std::to_string(0); - ret = hcom_reduce_scatter(tag_reduce_scatter.c_str(), reinterpret_cast(task_info->input_data_addr()), - reinterpret_cast(task_info->output_data_addr()), + ret = hcom_reduce_scatter(tag_reduce_scatter.c_str(), task_info->input_data_addr(), task_info->output_data_addr(), static_cast(task_info->count()), static_cast(task_info->data_type()), static_cast(task_info->op_type()), hccl_group.c_str(), stream); if (ret != HCCL_SUCCESS) { diff --git a/mindspore/ccsrc/device/ascend/tasksink/task_generator.cc b/mindspore/ccsrc/device/ascend/tasksink/task_generator.cc index 0cdf751801..e026459ae9 100644 --- a/mindspore/ccsrc/device/ascend/tasksink/task_generator.cc +++ b/mindspore/ccsrc/device/ascend/tasksink/task_generator.cc @@ -40,39 +40,46 @@ bool TaskGenerator::GenTasks(const std::vector &anf_node_list, std::ve return true; } +void TaskGenerator::LaunchAddrCleanAkgKernel(const CNodePtr &anf_node_ptr, AddressPtrList *kernel_inputs) { + MS_EXCEPTION_IF_NULL(anf_node_ptr); + MS_EXCEPTION_IF_NULL(kernel_inputs); + // akg process + // set atomic clean addr + if (AnfAlgo::HasNodeAttr(kAttrAtomicOutputIndexs, anf_node_ptr)) { + auto clean_output_indexs = AnfAlgo::GetNodeAttr>(anf_node_ptr, kAttrAtomicOutputIndexs); + auto graph = anf_node_ptr->func_graph(); + MS_EXCEPTION_IF_NULL(graph); + auto manager = graph->manager(); + MS_EXCEPTION_IF_NULL(manager); + auto node_users = manager->node_users(); + if (node_users[anf_node_ptr].empty()) { + MS_LOG(EXCEPTION) << "Node users of " << anf_node_ptr->ToString() << " is empty."; + } + auto depend_node = node_users[anf_node_ptr].pop().first; + if (!IsPrimitiveCNode(depend_node, prim::kPrimDepend)) { + MS_LOG(EXCEPTION) << "Checking Depend node failed"; + } + if (node_users[depend_node].empty()) { + MS_LOG(EXCEPTION) << "Node users of " << depend_node->ToString() << " is empty."; + } + auto post_node = node_users[depend_node].pop().first; + for (auto index : clean_output_indexs) { + auto device_address = AnfAlgo::GetOutputAddr(post_node, index); + kernel::AddressPtr input = std::make_shared(); + MS_EXCEPTION_IF_NULL(input); + input->addr = device_address->ptr_; + input->size = device_address->size_; + kernel_inputs->push_back(input); + } + MS_LOG(DEBUG) << "AtomicAddClean clean output size: " << clean_output_indexs.size(); + } +} + void TaskGenerator::LaunchAddrCleanKernel(const CNodePtr &anf_node_ptr, AddressPtrList *kernel_inputs) { MS_EXCEPTION_IF_NULL(anf_node_ptr); + MS_EXCEPTION_IF_NULL(kernel_inputs); if (anf_node_ptr->inputs().size() != 2) { - // akg process - // set atomic clean addr - if (AnfAlgo::HasNodeAttr(kAttrAtomicOutputIndexs, anf_node_ptr)) { - auto clean_output_indexs = AnfAlgo::GetNodeAttr>(anf_node_ptr, kAttrAtomicOutputIndexs); - auto graph = anf_node_ptr->func_graph(); - MS_EXCEPTION_IF_NULL(graph); - auto manager = graph->manager(); - MS_EXCEPTION_IF_NULL(manager); - auto node_users = manager->node_users(); - if (node_users[anf_node_ptr].empty()) { - MS_LOG(EXCEPTION) << "Node users of " << anf_node_ptr->ToString() << " is empty."; - } - auto depend_node = node_users[anf_node_ptr].pop().first; - if (!IsPrimitiveCNode(depend_node, prim::kPrimDepend)) { - MS_LOG(EXCEPTION) << "Checking Depend node failed"; - } - if (node_users[depend_node].empty()) { - MS_LOG(EXCEPTION) << "Node users of " << depend_node->ToString() << " is empty."; - } - auto post_node = node_users[depend_node].pop().first; - for (auto index : clean_output_indexs) { - auto device_address = AnfAlgo::GetOutputAddr(post_node, index); - kernel::AddressPtr input = std::make_shared(); - input->addr = device_address->ptr_; - MS_EXCEPTION_IF_NULL(input->addr); - input->size = device_address->size_; - kernel_inputs->push_back(input); - } - MS_LOG(DEBUG) << "AtomicAddClean clean output size: " << clean_output_indexs.size(); - } + LaunchAddrCleanAkgKernel(anf_node_ptr, kernel_inputs); return; } MS_EXCEPTION_IF_NULL(anf_node_ptr->inputs()[1]); diff --git a/mindspore/ccsrc/device/ascend/tasksink/task_generator.h b/mindspore/ccsrc/device/ascend/tasksink/task_generator.h index ffedcd7930..ecd5889b04 100644 --- a/mindspore/ccsrc/device/ascend/tasksink/task_generator.h +++ b/mindspore/ccsrc/device/ascend/tasksink/task_generator.h @@ -48,6 +48,7 @@ class TaskGenerator { private: static void LaunchAddrCleanKernel(const CNodePtr &anf_node_ptr, AddressPtrList *kernel_inputs); + static void LaunchAddrCleanAkgKernel(const CNodePtr &anf_node_ptr, AddressPtrList *kernel_inputs); static bool LaunchKernel(const CNodePtr &anf_node_ptr, uint32_t stream_id, std::vector *task_info_list); static bool LaunchAllKernel(const std::vector &anf_node_list, std::vector *task_info_list, uint32_t graph_id); diff --git a/mindspore/ccsrc/device/cpu/cpu_kernel_runtime.cc b/mindspore/ccsrc/device/cpu/cpu_kernel_runtime.cc index cfcc1b7c79..cdb848add2 100644 --- a/mindspore/ccsrc/device/cpu/cpu_kernel_runtime.cc +++ b/mindspore/ccsrc/device/cpu/cpu_kernel_runtime.cc @@ -79,6 +79,7 @@ void CPUKernelRuntime::AssignValueNodeAddress(session::KernelGraph *kernel_graph std::vector data_shape = tensor->shape(); size_t tensor_size = std::accumulate(data_shape.begin(), data_shape.end(), type_size, std::multiplies()); DeviceAddressPtr address = CreateDeviceAddress(nullptr, tensor_size, kOpFormat_DEFAULT, kNumberTypeFloat32); + MS_EXCEPTION_IF_NULL(address); if (tensor->data_type() == kNumberTypeFloat32 || tensor->data_type() == kNumberTypeInt32) { address->ptr_ = tensor->data_c(false); } else { From 43e0967024269fea1796329faab7785675114715 Mon Sep 17 00:00:00 2001 From: He Wei Date: Thu, 11 Jun 2020 14:53:35 +0800 Subject: [PATCH 039/254] Decouple ir::Tensor class from python --- mindspore/ccsrc/debug/anf_ir_utils.cc | 7 +- mindspore/ccsrc/debug/debug_services.cc | 6 +- .../device/ascend/ascend_device_address.cc | 8 +- .../ccsrc/device/cpu/cpu_kernel_runtime.cc | 10 +- mindspore/ccsrc/device/kernel_adjust.cc | 10 +- mindspore/ccsrc/device/kernel_runtime.cc | 2 +- mindspore/ccsrc/ir/meta_tensor.h | 2 +- mindspore/ccsrc/ir/tensor.cc | 602 ++++++------------ mindspore/ccsrc/ir/tensor.h | 249 +++----- mindspore/ccsrc/ir/tensor_py.cc | 377 +++++++++++ mindspore/ccsrc/ir/tensor_py.h | 114 ++++ mindspore/ccsrc/onnx/ir_exporter.cc | 3 +- mindspore/ccsrc/onnx/onnx_exporter.cc | 3 +- mindspore/ccsrc/operator/prim_structures.cc | 5 +- .../optimizer/irpass/arithmetic_simplify.h | 6 +- .../ccsrc/optimizer/irpass/branch_culling.cc | 4 +- .../optimizer/irpass/special_op_eliminate.h | 2 +- .../optimizer/irpass/specialize_transform.h | 2 +- .../ccsrc/parallel/ops_info/gather_v2_info.cc | 4 +- .../parallel/ops_info/reduce_method_info.cc | 4 +- mindspore/ccsrc/pipeline/pipeline_ge.cc | 6 +- .../ascend/ir_fission/topk_split.cc | 2 +- mindspore/ccsrc/pre_activate/common/helper.cc | 2 +- .../ccsrc/predict/converter/kernel2ms.cc | 2 +- mindspore/ccsrc/pynative/pynative_execute.cc | 13 +- mindspore/ccsrc/pynative/pynative_execute.h | 1 + .../ccsrc/pynative/pynative_execute_ge.cc | 13 +- .../ccsrc/session/ascend_inference_session.cc | 7 +- mindspore/ccsrc/session/ascend_session.cc | 4 +- mindspore/ccsrc/session/gpu_session.cc | 2 +- mindspore/ccsrc/session/session_basic.cc | 9 +- mindspore/ccsrc/transform/util.cc | 2 +- mindspore/ccsrc/utils/convert_utils.cc | 7 +- .../ccsrc/utils/load_onnx/anf_model_parser.cc | 8 +- mindspore/ccsrc/utils/tensorprint_utils.cc | 2 +- mindspore/common/tensor.py | 4 +- tests/st/ops/gpu/test_rmsprop.py | 7 +- tests/ut/cpp/ir/meta_tensor_test.cc | 53 +- tests/ut/cpp/parallel/step_parallel_test.cc | 32 +- .../cpp/pipeline/static_analysis/data_test.cc | 8 +- ...onvert_const_input_to_tensor_input_test.cc | 2 +- tests/ut/cpp/transform/convert_test.cc | 2 +- tests/ut/cpp/transform/graph_runner_test.cc | 23 +- tests/ut/cpp/transform/transform_base_test.cc | 9 +- 44 files changed, 958 insertions(+), 682 deletions(-) create mode 100644 mindspore/ccsrc/ir/tensor_py.cc create mode 100644 mindspore/ccsrc/ir/tensor_py.h diff --git a/mindspore/ccsrc/debug/anf_ir_utils.cc b/mindspore/ccsrc/debug/anf_ir_utils.cc index 2b8e61ab15..e4d21a74a4 100644 --- a/mindspore/ccsrc/debug/anf_ir_utils.cc +++ b/mindspore/ccsrc/debug/anf_ir_utils.cc @@ -27,6 +27,7 @@ #include "utils/symbolic.h" #include "ir/meta_func_graph.h" #include "ir/param_value_py.h" +#include "ir/tensor_py.h" #include "pipeline/parse/python_adapter.h" #include "pipeline/parse/resolve.h" #include "operator/composite/composite.h" @@ -39,6 +40,8 @@ #include "utils/context/ms_context.h" #include "operator/ops.h" +using mindspore::tensor::TensorPy; + namespace mindspore { // max number of elements in sequence const int NUM_MAX_SEQUENCE_ELEMS = 0x00FFFFFF; @@ -399,7 +402,7 @@ std::string AnfExporter::GetValueText(const FuncGraphPtr &func_graph, const Valu oss << value->DumpText(); } else if (value->isa()) { auto tensor_ptr = dyn_cast(value); - oss << value->DumpText() << "@" << DumpObject(tensor_ptr->data(), "T"); + oss << value->DumpText() << "@" << DumpObject(TensorPy::AsNumpy(*tensor_ptr), "T"); } else if (value->isa() || value->isa() || value->isa()) { oss << value->DumpText(); } else if (value->isa()) { @@ -1813,7 +1816,7 @@ class IrParser { if (tensor_data == nullptr) { return TOK_ERROR; } - *val_ptr = std::make_shared(tensor_data, TypeIdToType(type)); + *val_ptr = TensorPy::MakeTensor(tensor_data, TypeIdToType(type)); return lexer_.GetNextToken(); } diff --git a/mindspore/ccsrc/debug/debug_services.cc b/mindspore/ccsrc/debug/debug_services.cc index 8d46e00f19..cb883eef51 100644 --- a/mindspore/ccsrc/debug/debug_services.cc +++ b/mindspore/ccsrc/debug/debug_services.cc @@ -117,7 +117,7 @@ void DebugServices::check_watchpoints(std::vector *name, std::vecto continue; } - float *start_addr = reinterpret_cast(tensor_ptr->data_c(false)); + float *start_addr = reinterpret_cast(tensor_ptr->data_c()); unsigned int num_elements = (tensor_ptr->data().nbytes()) / sizeof(float); std::unordered_map::iterator it_w_table_check; @@ -144,7 +144,7 @@ void DebugServices::check_watchpoints(std::vector *name, std::vecto name->push_back(name_no_slot); slot->push_back(std::to_string(tensor_list[i]->GetSlot())); - data_ptr->push_back(reinterpret_cast(tensor_ptr->data_c(false))); + data_ptr->push_back(reinterpret_cast(tensor_ptr->data_c())); data_size->push_back(tensor_ptr->data().nbytes()); int condition_item = -1; @@ -182,7 +182,7 @@ void DebugServices::read_nodes_tensors(std::vector name, std::vecto continue; } ret_name->push_back(std::get<0>(result)); - data_ptr->push_back(reinterpret_cast(std::get<1>(result)->GetTensor()->data_c(false))); + data_ptr->push_back(reinterpret_cast(std::get<1>(result)->GetTensor()->data_c())); data_size->push_back(std::get<1>(result)->GetTensor()->data().nbytes()); dtype->push_back(std::get<1>(result)->GetTensor()->Dtype()); shape->push_back(std::get<1>(result)->GetTensor()->shape()); diff --git a/mindspore/ccsrc/device/ascend/ascend_device_address.cc b/mindspore/ccsrc/device/ascend/ascend_device_address.cc index 71a16607ef..ccf9dfe405 100644 --- a/mindspore/ccsrc/device/ascend/ascend_device_address.cc +++ b/mindspore/ccsrc/device/ascend/ascend_device_address.cc @@ -329,12 +329,12 @@ bool AscendDeviceAddress::DumpMemToFile(bool trans_flag, const std::string &file MS_LOG(INFO) << "E2E Dump path is " << path; mindspore::tensor::TensorPtr out_tensor = std::make_shared(host_type, host_shape); size_t host_size = out_tensor->data().nbytes(); - ret = SyncDeviceToHost(host_shape, host_size, host_type, out_tensor->data_c(true)); + ret = SyncDeviceToHost(host_shape, host_size, host_type, out_tensor->data_c()); if (!ret) { MS_LOG(ERROR) << "Copy device mem to host failed"; return ret; } - ret = mindspore::Dump::DumpToFile(path, out_tensor->data_c(false), host_size); + ret = mindspore::Dump::DumpToFile(path, out_tensor->data_c(), host_size); } else { auto host_tmp = std::vector(size_); auto ret_rt_memcpy = rtMemcpy(host_tmp.data(), size_, ptr_, size_, RT_MEMCPY_DEVICE_TO_HOST); @@ -364,7 +364,7 @@ bool AscendDeviceAddress::LoadMemToHost(bool trans_flag, const std::string &tens MS_LOG(INFO) << "E2E tensor name is " << tensor_name; mindspore::tensor::TensorPtr out_tensor = std::make_shared(host_type, host_shape); size_t host_size = out_tensor->data().nbytes(); - ret = SyncDeviceToHost(host_shape, host_size, host_type, out_tensor->data_c(true)); + ret = SyncDeviceToHost(host_shape, host_size, host_type, out_tensor->data_c()); if (!ret) { MS_LOG(ERROR) << "Copy device mem to host failed"; return ret; @@ -379,7 +379,7 @@ bool AscendDeviceAddress::LoadMemToHost(bool trans_flag, const std::string &tens } else { mindspore::tensor::TensorPtr out_tensor = std::make_shared(type_id_, host_shape); size_t host_size = out_tensor->data().nbytes(); - auto ret_rt_memcpy = rtMemcpy(out_tensor->data_c(true), host_size, ptr_, host_size, RT_MEMCPY_DEVICE_TO_HOST); + auto ret_rt_memcpy = rtMemcpy(out_tensor->data_c(), host_size, ptr_, host_size, RT_MEMCPY_DEVICE_TO_HOST); auto tensor_data = std::make_shared(); tensor_data->SetName(tensor_name); diff --git a/mindspore/ccsrc/device/cpu/cpu_kernel_runtime.cc b/mindspore/ccsrc/device/cpu/cpu_kernel_runtime.cc index cfcc1b7c79..c18c053858 100644 --- a/mindspore/ccsrc/device/cpu/cpu_kernel_runtime.cc +++ b/mindspore/ccsrc/device/cpu/cpu_kernel_runtime.cc @@ -80,11 +80,11 @@ void CPUKernelRuntime::AssignValueNodeAddress(session::KernelGraph *kernel_graph size_t tensor_size = std::accumulate(data_shape.begin(), data_shape.end(), type_size, std::multiplies()); DeviceAddressPtr address = CreateDeviceAddress(nullptr, tensor_size, kOpFormat_DEFAULT, kNumberTypeFloat32); if (tensor->data_type() == kNumberTypeFloat32 || tensor->data_type() == kNumberTypeInt32) { - address->ptr_ = tensor->data_c(false); + address->ptr_ = tensor->data_c(); } else { address->ptr_ = resource_manager_.MemMalloc(tensor_size); if (!address->SyncHostToDevice(data_shape, LongToSize(tensor->data().nbytes()), tensor->data_type(), - tensor->data_c(false))) { + tensor->data_c())) { MS_LOG(EXCEPTION) << "Value node sync host to device failed!"; } } @@ -177,7 +177,7 @@ BaseRef CPUKernelRuntime::CreatTensorForOutput(const session::KernelWithIndex &k tensor->set_device_address(address); need_sync_outputs->emplace_back(tensor); } else { - address->ptr_ = tensor->data_c(true); + address->ptr_ = tensor->data_c(); address->ref_count_ = INIT_NODE_REF; (void)bound_addresses->insert(address); } @@ -220,11 +220,11 @@ void CPUKernelRuntime::BindInputOutput(const session::KernelGraph *kernel_graph, size_t tensor_size = std::accumulate(data_shape.begin(), data_shape.end(), sizeof(float), std::multiplies()); if (tensor->data_type() == kNumberTypeFloat32 || tensor->data_type() == kNumberTypeInt32) { - address->ptr_ = tensor->data_c(false); + address->ptr_ = tensor->data_c(); } else { address->ptr_ = resource_manager_.MemMalloc(tensor_size); if (!address->SyncHostToDevice(data_shape, LongToSize(tensor->data().nbytes()), tensor->data_type(), - tensor->data_c(false))) { + tensor->data_c())) { MS_LOG(EXCEPTION) << "Parameter node sync host to device failed!"; } tensor->set_dirty(true); diff --git a/mindspore/ccsrc/device/kernel_adjust.cc b/mindspore/ccsrc/device/kernel_adjust.cc index 93007764af..cfccfb3506 100644 --- a/mindspore/ccsrc/device/kernel_adjust.cc +++ b/mindspore/ccsrc/device/kernel_adjust.cc @@ -390,7 +390,7 @@ bool KernelAdjust::StepLoadCtrlInputs(const std::shared_ptrset_device_address(device_address); if (!device_address->SyncHostToDevice(trans::GetRuntimePaddingShape(pk_node, 0), LongToSize(tensor->data().nbytes()), tensor->data_type(), - tensor->data_c(false))) { + tensor->data_c())) { MS_LOG(INFO) << "SyncHostToDevice failed."; return false; } @@ -407,14 +407,14 @@ void KernelAdjust::LoadSwitchInputs(std::vector *inputs) { tensor::TensorPtr loop_count_tensor = std::make_shared(kInt32->type_id(), shp); MS_EXCEPTION_IF_NULL(loop_count_tensor); int32_t *val = nullptr; - val = static_cast(loop_count_tensor->data_c(true)); + val = static_cast(loop_count_tensor->data_c()); MS_EXCEPTION_IF_NULL(val); *val = 0; inputs->push_back(loop_count_tensor); tensor::TensorPtr iter_loop_tensor = std::make_shared(kInt32->type_id(), shp); MS_EXCEPTION_IF_NULL(iter_loop_tensor); - val = static_cast(iter_loop_tensor->data_c(true)); + val = static_cast(iter_loop_tensor->data_c()); MS_EXCEPTION_IF_NULL(val); *val = SizeToInt(LongToSize(ConfigManager::GetInstance().iter_num())); MS_LOG(INFO) << "iter_loop_tensor = " << *val; @@ -422,14 +422,14 @@ void KernelAdjust::LoadSwitchInputs(std::vector *inputs) { tensor::TensorPtr zero_tensor = std::make_shared(kInt32->type_id(), shp); MS_EXCEPTION_IF_NULL(zero_tensor); - val = static_cast(zero_tensor->data_c(true)); + val = static_cast(zero_tensor->data_c()); MS_EXCEPTION_IF_NULL(val); *val = 0; inputs->push_back(zero_tensor); tensor::TensorPtr one_tensor = std::make_shared(kInt32->type_id(), shp); MS_EXCEPTION_IF_NULL(one_tensor); - val = static_cast(one_tensor->data_c(true)); + val = static_cast(one_tensor->data_c()); MS_EXCEPTION_IF_NULL(val); *val = 1; inputs->push_back(one_tensor); diff --git a/mindspore/ccsrc/device/kernel_runtime.cc b/mindspore/ccsrc/device/kernel_runtime.cc index 07b9257bb2..4581141790 100644 --- a/mindspore/ccsrc/device/kernel_runtime.cc +++ b/mindspore/ccsrc/device/kernel_runtime.cc @@ -543,7 +543,7 @@ void KernelRuntime::AssignValueNodeTensor(const ValueNodePtr &value_node, const } AnfAlgo::SetOutputAddr(address, output_idx, value_node.get()); if (!address->SyncHostToDevice(trans::GetRuntimePaddingShape(value_node, 0), tensor_size, tensor->data_type(), - tensor->data_c(false))) { + tensor->data_c())) { MS_EXCEPTION(NotExistsError) << "ValueNode SyncHostToDevice fail!" << value_node->DebugString() << "node format is" << AnfAlgo::GetOutputFormat(value_node, output_idx) << "node dtype is " << AnfAlgo::GetOutputInferDataType(value_node, output_idx); diff --git a/mindspore/ccsrc/ir/meta_tensor.h b/mindspore/ccsrc/ir/meta_tensor.h index d78caf3b5d..a8c07d6992 100644 --- a/mindspore/ccsrc/ir/meta_tensor.h +++ b/mindspore/ccsrc/ir/meta_tensor.h @@ -115,7 +115,7 @@ class MetaTensor : public Value { // order it represents. // // return A const vector which represents the shape of the tensor. - std::vector shape() const { return shape_; } + const std::vector &shape() const { return shape_; } // brief Sets the shape of a tensor. // diff --git a/mindspore/ccsrc/ir/tensor.cc b/mindspore/ccsrc/ir/tensor.cc index 4b02fdf2a7..c43f5423f1 100644 --- a/mindspore/ccsrc/ir/tensor.cc +++ b/mindspore/ccsrc/ir/tensor.cc @@ -16,319 +16,261 @@ #include "ir/tensor.h" +#include #include #include -#include #include #include #include +#include #include "device/device_address.h" -#include "pybind_api/api_register.h" -#include "pybind_api/export_flags.h" #include "pipeline/static_analysis/abstract_value.h" namespace mindspore { namespace tensor { -static uint64_t count = 0; -void DataBuf2Contiguous(const py::array &src, py::array *const dest) { - if (dest == nullptr) { - MS_LOG(EXCEPTION) << "Failed to copy data to a contiguous buffer as dest is nullptr!"; - } - Py_buffer pybuf_src; - if (PyObject_GetBuffer(src.ptr(), &pybuf_src, PyBUF_ANY_CONTIGUOUS)) { - MS_LOG(EXCEPTION) << "Failed to get buffer info from the src!"; - } +using Bool = unsigned char; - if (!PyBuffer_IsContiguous(&pybuf_src, 'C')) { - if (PyBuffer_ToContiguous(dest->request(true).ptr, &pybuf_src, pybuf_src.len, 'C')) { - MS_LOG(EXCEPTION) << "Can't copy numpy.ndarray to a contiguous buffer."; - } - } else { - *dest = src; - } - - PyBuffer_Release(&pybuf_src); +static std::string MakeId() { + // Use atomic to make id generator thread safe. + static std::atomic last_id{1}; + return std::to_string(last_id.fetch_add(1, std::memory_order_relaxed)); } -Tensor::Tensor(const TypePtr &type_ptr, const py::tuple &shape) { - TypeId data_type = TypeId::kTypeUnknown; - if (type_ptr != nullptr) { - data_type = type_ptr->type_id(); - } - data_type_ = data_type; - shape_.resize(shape.size()); - for (size_t i = 0; i < shape.size(); ++i) { - shape_[i] = py::int_(shape[i]); - } - init(data_type_, shape_, &data_); +static TypeId TypeIdOf(const TypePtr &data_type, TypeId defaultTypeId) { + return data_type ? data_type->type_id() : defaultTypeId; } -Tensor::Tensor(TypeId data_type, const std::vector &shape) { init(data_type, shape, &data_); } - -Tensor::Tensor(const py::array &input, const TypePtr &data_type) { init(input, data_type); } - -Tensor::Tensor(const py::list &input, const TypePtr &data_type) { init(py::array(input), data_type); } - -Tensor::Tensor(const py::tuple &input, const TypePtr &data_type) { init(py::array(input), data_type); } - -Tensor::Tensor(const py::float_ &input, const TypePtr &data_type) { init(py::array(input), data_type); } - -Tensor::Tensor(const py::int_ &input, const TypePtr &data_type) { init(py::array(input), data_type); } - -Tensor::Tensor(const Tensor &tensor, const TypePtr &data_type) - : MetaTensor(tensor), device_address_(tensor.device_address_) { - init(tensor.data_, data_type); - dirty_ = tensor.is_dirty(); - id_ = tensor.id(); +static size_t SizeOf(const std::vector &shape) { + return std::accumulate(shape.begin(), shape.end(), size_t(1), std::multiplies()); } -Tensor &Tensor::operator=(const Tensor &tensor) { - if (this != &tensor) { - MetaTensor::operator=(tensor); - dirty_ = tensor.is_dirty(); - device_address_ = tensor.device_address(); - data_ = tensor.data_; - id_ = tensor.id(); +template +std::vector CopyData(const std::vector &shape, void *data, TypeId data_type) { + const size_t count = SizeOf(shape); + switch (data_type) { + case kNumberTypeBool: { + auto buf = static_cast(data); + return std::vector(buf, buf + count); + } + case kNumberTypeUInt8: { + auto buf = static_cast(data); + return std::vector(buf, buf + count); + } + case kNumberTypeInt8: { + auto buf = static_cast(data); + return std::vector(buf, buf + count); + } + case kNumberTypeInt16: { + auto buf = static_cast(data); + return std::vector(buf, buf + count); + } + case kNumberTypeInt32: { + auto buf = static_cast(data); + return std::vector(buf, buf + count); + } + case kNumberTypeInt64: { + auto buf = static_cast(data); + return std::vector(buf, buf + count); + } + case kNumberTypeUInt16: { + auto buf = static_cast(data); + return std::vector(buf, buf + count); + } + case kNumberTypeUInt32: { + auto buf = static_cast(data); + return std::vector(buf, buf + count); + } + case kNumberTypeUInt64: { + auto buf = static_cast(data); + return std::vector(buf, buf + count); + } + case kNumberTypeFloat16: { + auto buf = static_cast(data); + return std::vector(buf, buf + count); + } + case kNumberTypeFloat32: { + const float *buf = static_cast(data); + return std::vector(buf, buf + count); + } + case kNumberTypeFloat64: { + auto buf = static_cast(data); + return std::vector(buf, buf + count); + } + default: + break; } - return *this; -} -Tensor &Tensor::AssignValue(const Tensor &tensor) { - *this = tensor; - return *this; + MS_LOG(EXCEPTION) << "Cannot construct Tensor because of unsupported data type: " << data_type << "."; } -bool Tensor::operator==(const Tensor &tensor) const { - return (MetaTensor::operator==(tensor) && data_ == tensor.data_); -} - -bool Tensor::ValueEqual(const Tensor &other) const { - auto equal = [&other, this]() -> bool { - auto np = py::module::import("numpy"); - auto equal = np.attr("equal")(data_, other.data_); - auto all_equal = np.attr("all")(equal); - return all_equal.cast(); - }; - return (MetaTensor::operator==(other) && (data_.is(other.data_) || equal())); +// Convert to bool is not allowed. +template <> +std::vector CopyData(const std::vector &shape, void *data, TypeId data_type) { + MS_LOG(EXCEPTION) << "Cannot convert from " << TypeIdLabel(data_type) << " to " << TypeIdLabel(kNumberTypeBool) + << "."; + return {}; } -py::tuple Tensor::GetPyTupleShape() const { - std::vector shape = this->shape(); - py::tuple dims(shape.size()); - for (size_t i = 0; i < dims.size(); ++i) { - dims[i] = py::int_(shape[i]); +template +std::vector CopyData(const std::vector &shape, void *data, size_t data_len) { + size_t size = SizeOf(shape); + if (size * sizeof(T) != data_len) { + MS_LOG(EXCEPTION) << "Incorrect tensor input data length " << data_len << ", expect " << size * sizeof(T) + << " item size " << sizeof(T); } - return dims; + auto buf = static_cast(data); + return {buf, buf + size}; } -int Tensor::DataDim() const { return static_cast(data_.ndim()); } +// Tensor data implementation. +template +class TensorDataImpl : public TensorData { + public: + explicit TensorDataImpl(const std::vector &shape) : shape_(shape), data_(SizeOf(shape)) {} -int Tensor::DataSize() const { return static_cast(data_.size()); } + TensorDataImpl(const std::vector &shape, void *data, size_t data_len) + : shape_(shape), data_(CopyData(shape, data, data_len)) {} -py::array Tensor::data() const { return data_; } + TensorDataImpl(const std::vector &shape, void *data, TypeId data_type) + : shape_(shape), data_(CopyData(shape, data, data_type)) {} -int Tensor::data_type_c() const { return static_cast(data_type_); } + template + TensorDataImpl(const std::vector &shape, InputIt first, InputIt last) : shape_(shape), data_(first, last) {} -std::vector Tensor::shape_c(void) const { return shape(); } + template + TensorDataImpl(const std::vector &shape, Scalar scalar) : shape_(shape), data_({static_cast(scalar)}) {} -void *Tensor::data_c(bool writable) { - // operand of bit operation should be unsigned int. - unsigned int flags = ((unsigned int)data_.flags()) & pybind11::detail::npy_api::NPY_ARRAY_C_CONTIGUOUS_; - bool is_c_contiguous = (flags != 0) ? true : false; - if (!is_c_contiguous) { - py::array data_c; - init(data_type_, shape_, &data_c); - DataBuf2Contiguous(data_, &data_c); - data_ = data_c; - } - return data_.request(writable).ptr; -} + ssize_t size() const override { return data_.size(); } -TypeId Tensor::GetDataType(const py::buffer_info &buf) const { - TypeId data_type = TypeId::kTypeUnknown; - if (buf.format.compare("e") == 0) { - data_type = TypeId::kNumberTypeFloat16; - } else if (buf.format.compare("f") == 0) { - data_type = TypeId::kNumberTypeFloat32; - } else if (buf.format.compare("d") == 0) { - data_type = TypeId::kNumberTypeFloat64; - } else if (buf.format.compare("B") == 0) { - data_type = TypeId::kNumberTypeUInt8; - } else if (buf.format.compare("H") == 0) { - data_type = TypeId::kNumberTypeUInt16; - } else if (buf.format.compare("I") == 0) { - data_type = TypeId::kNumberTypeUInt32; - } else if (buf.format.compare("L") == 0 || buf.format.compare("Q") == 0) { - data_type = TypeId::kNumberTypeUInt64; - } else if (buf.format.compare("b") == 0) { - data_type = TypeId::kNumberTypeInt8; - } else if (buf.format.compare("h") == 0) { - data_type = TypeId::kNumberTypeInt16; - } else if (buf.format.compare("i") == 0) { - data_type = TypeId::kNumberTypeInt32; - } else if (buf.format.compare("l") == 0 || buf.format.compare("q") == 0) { - data_type = TypeId::kNumberTypeInt64; - } else if (buf.format.compare("?") == 0) { - data_type = TypeId::kNumberTypeBool; - } else { - MS_LOG(WARNING) << "Get unsupported DataType " << buf.format << "."; - } - return data_type; -} + ssize_t itemsize() const override { return static_cast(sizeof(T)); } -void Tensor::init(const py::array &input, const TypePtr &type_ptr) { - TypeId data_type = TypeId::kTypeUnknown; - if (type_ptr != nullptr) { - data_type = type_ptr->type_id(); - } - init(input, data_type); -} + ssize_t nbytes() const override { return size() * itemsize(); } -void Tensor::init(const py::array &input, const TypeId &data_type) { - py::buffer_info buf = input.request(); + ssize_t ndim() const override { return static_cast(shape_.size()); } - data_type_ = GetDataType(buf); - if (TypeId::kTypeUnknown == data_type && TypeId::kTypeUnknown == data_type_) { - MS_LOG(EXCEPTION) << "Unsupported tensor type!"; + void *data() override { + static std::vector empty_data(1); + if (data_.empty()) { + // Prevent null pointer for empty data. + return empty_data.data(); + } + return data_.data(); } - std::vector tm = buf.shape; - size_t len = tm.size(); - std::vector dims(len); - for (size_t i = 0; i < len; ++i) { - dims[i] = static_cast(tm[i]); + bool equals(const TensorData &other) const override { + auto ptr = dynamic_cast *>(&other); + if (ptr) { + return (ptr == this) || ((shape_ == ptr->shape_) && (data_ == ptr->data_)); + } + return false; } - (void)set_shape(dims); - - if (TypeId::kTypeUnknown != data_type && TypeId::kTypeUnknown != data_type_ && data_type_ != data_type) { - // If user defined data type is not same as GetDataType from the data - bool success = convert_data(input, data_type_, &data_, data_type); - if (success) { - data_type_ = data_type; - } else { - data_type_ = TypeId::kTypeUnknown; - MS_LOG(EXCEPTION) << "Convert data from " << data_type_ << " to " << data_type << " failed!"; + + std::string ToString() const override { + std::ostringstream ss; + ss << '['; + for (auto value : data_) { + ss << value << ','; } - } else { - data_ = input; + ss << ']'; + return ss.str(); } - dirty_ = true; - id_ = std::to_string((uintptr_t)(this)) + std::to_string(count++); -} -void Tensor::init(TypeId data_type, const std::vector &shape, py::array *const data) { - data_type_ = data_type; - shape_ = shape; + private: + std::vector shape_; + std::vector data_; +}; + +template +TensorDataPtr MakeTensorData(TypeId data_type, const std::vector &shape, Args... args) { switch (data_type) { case kNumberTypeBool: - *data = py::array_t(shape); - break; + // std::vector is a specialization of std::vector, + // it may use single bit instead of sizeof(bool) bytes, + // so we use std::vector for bool tensors. + return std::make_shared>(shape, args...); + case kNumberTypeUInt8: + return std::make_shared>(shape, args...); case kNumberTypeInt8: - *data = py::array_t(shape); - break; + return std::make_shared>(shape, args...); case kNumberTypeInt16: - *data = py::array_t(shape); - break; + return std::make_shared>(shape, args...); case kNumberTypeInt32: - *data = py::array_t(shape); - break; + return std::make_shared>(shape, args...); case kNumberTypeInt64: - *data = py::array_t(shape); - break; - case kNumberTypeUInt8: - *data = py::array_t(shape); - break; + return std::make_shared>(shape, args...); case kNumberTypeUInt16: - *data = py::array_t(shape); - break; + return std::make_shared>(shape, args...); case kNumberTypeUInt32: - *data = py::array_t(shape); - break; + return std::make_shared>(shape, args...); case kNumberTypeUInt64: - *data = py::array_t(shape); - break; + return std::make_shared>(shape, args...); case kNumberTypeFloat16: - *data = py::array_t(shape); - break; + return std::make_shared>(shape, args...); case kNumberTypeFloat32: - *data = py::array_t(shape); - break; + return std::make_shared>(shape, args...); case kNumberTypeFloat64: - *data = py::array_t(shape); - break; + return std::make_shared>(shape, args...); default: - MS_LOG(EXCEPTION) << "Cannot construct Tensor because of unsupported data type: " << data_type << "."; break; } - id_ = std::to_string((uintptr_t)(this)) + std::to_string(count++); + MS_LOG(EXCEPTION) << "Cannot construct Tensor because of unsupported data type: " << data_type << "."; } -TypePtr Tensor::SetDtype(const TypePtr type_ptr) { - MS_EXCEPTION_IF_NULL(type_ptr); - (void)set_data_type(type_ptr->type_id()); - return type_ptr; -} +Tensor::Tensor(const Tensor &tensor) + : MetaTensor(tensor), + init_flag_(tensor.init_flag_), + data_(tensor.data_), + dirty_(tensor.dirty_), + id_(tensor.id_), + device_address_(tensor.device_address_) {} -TypeId Tensor::set_data_type(const TypeId data_type) { - if (data_.size() > 0 && data_type_ != data_type) { - bool success = convert_data(data_, data_type_, &data_, data_type); - if (success) { - data_type_ = data_type; - } else { - MS_LOG(EXCEPTION) << "Convert data from " << data_type_ << " to " << data_type << " failed!"; - } - } else if (data_.size() == 0) { - data_type_ = data_type; - } +Tensor::Tensor(const Tensor &tensor, TypeId data_type) + : MetaTensor(data_type, tensor.shape_), + init_flag_(tensor.init_flag_), + data_(MakeTensorData(data_type, tensor.shape_, tensor.data_->data(), tensor.data_type_)), + dirty_(tensor.dirty_), + id_(tensor.id_), + device_address_(tensor.device_address_) {} - return data_type_; -} +Tensor::Tensor(TypeId data_type, const std::vector &shape, TensorDataPtr data) + : MetaTensor(data_type, shape), data_(std::move(data)), id_(MakeId()) {} -bool Tensor::is_init() { return init_flag_; } +Tensor::Tensor(TypeId data_type, const std::vector &shape) + : Tensor(data_type, shape, MakeTensorData(data_type, shape)) {} -void Tensor::set_init_flag(bool flag) { init_flag_ = flag; } +Tensor::Tensor(TypeId data_type, const std::vector &shape, void *data, size_t data_len) + : Tensor(data_type, shape, MakeTensorData(data_type, shape, data, data_len)) {} -bool Tensor::convert_data(const py::array &in, const TypeId in_data_type, py::array *const out, - const TypeId out_data_type) { - if (out == nullptr) { - return false; - } +Tensor::Tensor(TypeId data_type, const std::vector &shape, void *data, TypeId src_data_type) + : Tensor(data_type, shape, MakeTensorData(data_type, shape, data, src_data_type)) {} - bool result = true; - if (TypeId::kTypeUnknown == in_data_type || TypeId::kTypeUnknown == out_data_type) { - result = false; - } else if (in_data_type == out_data_type) { - *out = in; - } else if (TypeId::kNumberTypeFloat64 == out_data_type) { - *out = in.attr("astype").cast()("float64").cast(); - } else if (TypeId::kNumberTypeFloat32 == out_data_type) { - *out = in.attr("astype").cast()("float32").cast(); - } else if (TypeId::kNumberTypeFloat16 == out_data_type) { - *out = in.attr("astype").cast()("float16").cast(); - } else if (TypeId::kNumberTypeInt64 == out_data_type) { - *out = in.attr("astype").cast()("int64").cast(); - } else if (TypeId::kNumberTypeInt32 == out_data_type) { - *out = in.attr("astype").cast()("int32").cast(); - } else if (TypeId::kNumberTypeInt16 == out_data_type) { - *out = in.attr("astype").cast()("int16").cast(); - } else if (TypeId::kNumberTypeInt8 == out_data_type) { - *out = in.attr("astype").cast()("int8").cast(); - } else if (TypeId::kNumberTypeUInt8 == out_data_type) { - *out = in.attr("astype").cast()("uint8").cast(); - } else if (TypeId::kNumberTypeUInt16 == out_data_type) { - *out = in.attr("astype").cast()("uint16").cast(); - } else if (TypeId::kNumberTypeUInt32 == out_data_type) { - *out = in.attr("astype").cast()("uint32").cast(); - } else if (TypeId::kNumberTypeUInt64 == out_data_type) { - *out = in.attr("astype").cast()("uint64").cast(); - } else { - data_type_ = TypeId::kTypeUnknown; - MS_LOG(EXCEPTION) << "Cannot convert from " << TypeIdLabel(in_data_type) << " to " << TypeIdLabel(out_data_type) - << "."; - } +Tensor::Tensor(const std::vector &input, const TypePtr &data_type) + : MetaTensor(TypeIdOf(data_type, kNumberTypeInt32), {static_cast(input.size())}), + data_(MakeTensorData(data_type_, shape_, input.begin(), input.end())), + id_(MakeId()) {} + +Tensor::Tensor(const std::vector &input, const TypePtr &data_type) + : MetaTensor(TypeIdOf(data_type, kNumberTypeFloat32), {static_cast(input.size())}), + data_(MakeTensorData(data_type_, shape_, input.begin(), input.end())), + id_(MakeId()) {} + +Tensor::Tensor(int64_t input, const TypePtr &data_type) + : MetaTensor(TypeIdOf(data_type, kNumberTypeInt32), {}), + data_(MakeTensorData(data_type_, {}, input)), + id_(MakeId()) {} - return result; +Tensor::Tensor(double input, const TypePtr &data_type) + : MetaTensor(TypeIdOf(data_type, kNumberTypeFloat32), {}), + data_(MakeTensorData(data_type_, {}, input)), + id_(MakeId()) {} + +bool Tensor::operator==(const Tensor &tensor) const { + return (&tensor == this || (MetaTensor::operator==(tensor) && data_ == tensor.data_)); +} + +bool Tensor::ValueEqual(const Tensor &tensor) const { + return (&tensor == this || (MetaTensor::operator==(tensor) && data_->equals(*tensor.data_))); } abstract::AbstractBasePtr Tensor::ToAbstract() { @@ -355,7 +297,7 @@ std::string Tensor::ToString() const { buf << "Tensor shape:[" << shape() << "]" << this->Dtype()->ToString(); // only print small tensor if (DataSize() < small_tensor_size) { - buf << "val:" << std::string(py::str(data())); + buf << "val:" << data().ToString(); } return buf.str(); } @@ -365,164 +307,25 @@ std::string Tensor::ToStringRepr() const { auto type_ptr = this->Dtype(); MS_EXCEPTION_IF_NULL(type_ptr); buf << "Tensor shape:[" << shape() << "]" << type_ptr->ToString(); - buf << "\nval:" << std::string(py::str(data())); + buf << "\nval:" << data().ToString(); return buf.str(); } -py::array Tensor::data_sync() { +void Tensor::data_sync() const { if (device_address_ != nullptr) { - if (!device_address_->SyncDeviceToHost(this->shape(), static_cast(this->data().nbytes()), this->data_type(), - this->data_c(true))) { + if (!device_address_->SyncDeviceToHost(shape(), static_cast(data().nbytes()), data_type(), data_c())) { MS_LOG(EXCEPTION) << "SyncDeviceToHost when asnumpy."; } } - return data_; } -REGISTER_PYBIND_DEFINE(Tensor, ([](const py::module *m) { - // dtype should define before Tensor, because Tensor init depend dtype - (void)py::class_>(*m, "Tensor") - .def(py::init(), py::arg("dtype"), py::arg("shape")) - .def(py::init(), py::arg("input"), py::arg("dtype") = nullptr) - .def(py::init(), py::arg("input"), py::arg("dtype") = nullptr) - .def(py::init(), py::arg("input"), py::arg("dtype") = nullptr) - .def(py::init(), py::arg("input"), py::arg("dtype") = nullptr) - .def(py::init(), py::arg("input"), py::arg("dtype") = nullptr) - .def(py::init(), py::arg("input"), py::arg("dtype") = nullptr) - .def_readonly(PYTHON_TENSOR_FLAG, &Tensor::parse_info_) - .def_property_readonly("dtype", &Tensor::Dtype, R"mydelimiter( - Get the tensor's data type. - - Returns: - type, the data type of tensor. - - Examples: - >>> data = mindspore.Tensor(np.ones((2, 1), np.int32)) - >>> data.dtype - Int32 - )mydelimiter") - .def_property_readonly("shape", &Tensor::GetPyTupleShape, R"mydelimiter( - Get the tensor's shape. - - Returns: - tuple[int], the shape of tensor. - - Examples: - >>> data = mindspore.Tensor(np.ones((3, 3))) - >>> data.shape() - (3, 3) - )mydelimiter") - .def("asnumpy", &Tensor::data_sync, R"mydelimiter( - Convert tensor to numpy.ndarray. - - Returns: - numpy.ndarray. - - Examples: - >>> data = mindspore.Tensor(np.ones((2, 3))) - >>> array = data.asnumpy() - >>> array - array([[1., 1., 1.], - [1., 1., 1.]]) - )mydelimiter") - .def("size", &Tensor::DataSize, R"mydelimiter( - Get tensor's data size. - - Returns: - int, the size of tensor. - - Examples: - >>> data = mindspore.Tensor(np.ones((2, 3))) - >>> data.size() - 6 - )mydelimiter") - .def("is_init", &Tensor::is_init, R"mydelimiter( - Get tensor init_flag. - - Returns: - bool, whether the tensor init. - - Examples: - >>> data = mindspore.Tensor(np.ones((2, 3))) - >>> data.is_init() - False - )mydelimiter") - .def("set_init_flag", &Tensor::set_init_flag, R"mydelimiter( - Set tensor init_flag. - - Examples: - >>> data = mindspore.Tensor(np.ones((2, 3))) - >>> data.set_init_flag(True) - )mydelimiter") - .def("dim", &Tensor::DataDim, R"mydelimiter( - Get tensor's data dimension. - - Returns: - int, the dimension of tensor. - - Examples: - >>> data = mindspore.Tensor(np.ones((2, 3))) - >>> data.dim() - 2 - )mydelimiter") - .def("set_dtype", &Tensor::SetDtype, R"mydelimiter( - Set the tensor's data type. - - Arg: - dtype (:class:`mindspore.dtype`): The type of output tensor. - - Examples: - >>> data = mindspore.Tensor(np.ones((1, 2), np.float32)) - >>> data.set_dtype(mindspore.int32) - mindspore.int32 - )mydelimiter") - .def("assign_value", &Tensor::AssignValue, R"mydelimiter( - Assign another tensor value to this. - - Arg: - value (:class:`mindspore.tensor`): The value tensor. - - Examples: - >>> data = mindspore.Tensor(np.ones((1, 2), np.float32)) - >>> data2 = mindspore.Tensor(np.ones((2, 2), np.float32)) - >>> data.assign_value(data2) - >>> data.shape - (2, 2) - )mydelimiter") - .def("__str__", &Tensor::ToString) - .def("__repr__", &Tensor::ToStringRepr) - .def(py::pickle( - [](const Tensor &t) { // __getstate__ - /* Return a tuple that fully encodes the state of the object */ - return py::make_tuple(t.data()); - }, - [](const py::tuple &t) { // __setstate__ - if (t.size() != 1) { - throw std::runtime_error("Invalid state!"); - } - /* Create a new C++ instance */ - Tensor tensor(t[0].cast()); - return tensor; - })); - (void)py::class_>(*m, "MetaTensor") - .def(py::init>(), py::arg("dtype"), py::arg("shape")) - .def(py::pickle( - [](const MetaTensor &t) { // __getstate__ - /* Return a tuple that fully encodes the state of the object */ - return py::make_tuple(static_cast(t.data_type()), t.shape()); - }, - [](const py::tuple &t) { // __setstate__ - if (t.size() != 2) { - throw std::runtime_error("Invalid state!"); - } - /* Create a new C++ instance */ - MetaTensor tensor(TypeId(t[0].cast()), t[1].cast>()); - return tensor; - })) - .def_readonly(PYTHON_META_TENSOR_FLAG, &MetaTensor::parse_info_) - .def_property_readonly("dtype", &MetaTensor::Dtype, "Get the MetaTensor's dtype.") - .def_property_readonly("shape", &MetaTensor::shape, "Get the MetaTensor's shape."); - })); +TypeId Tensor::set_data_type(const TypeId data_type) { + if (data_type != data_type_) { + data_ = MakeTensorData(data_type, shape_, data_->data(), data_type_); + return MetaTensor::set_data_type(data_type); + } + return data_type; +} } // namespace tensor namespace inference { @@ -530,8 +333,6 @@ MSTensor *MSTensor::CreateTensor(TypeId data_type, const std::vector &shape return new Tensor(data_type, shape); } -Tensor::Tensor() { this->tensor_impl_ = std::make_shared(); } - Tensor::Tensor(TypeId data_type, const std::vector &shape) { this->tensor_impl_ = std::make_shared(data_type, shape); } @@ -585,7 +386,8 @@ size_t Tensor::Size() const { void *Tensor::MutableData() const { MS_ASSERT(this->tensor_impl_ != nullptr); - return this->tensor_impl_->data_c(true); + return this->tensor_impl_->data_c(); } + } // namespace inference } // namespace mindspore diff --git a/mindspore/ccsrc/ir/tensor.h b/mindspore/ccsrc/ir/tensor.h index 1ce657143b..cd30827129 100644 --- a/mindspore/ccsrc/ir/tensor.h +++ b/mindspore/ccsrc/ir/tensor.h @@ -20,9 +20,7 @@ #include #include #include - -#include "pybind11/numpy.h" -#include "pybind11/pybind11.h" +#include #include "Eigen/Core" #include "device/device_address.h" @@ -30,63 +28,8 @@ #include "include/ms_tensor.h" #include "utils/log_adapter.h" -namespace py = pybind11; - using float16 = Eigen::half; -namespace pybind11 { -namespace detail { -// Similar to enums in `pybind11/numpy.h`. Determined by doing: -// python3 -c 'import numpy as np; print(np.dtype(np.float16).num)' -constexpr int NPY_FLOAT16 = 23; - -template -struct npy_scalar_caster { - PYBIND11_TYPE_CASTER(T, _("PleaseOverride")); - using Array = array_t; - - bool load(handle src, bool convert) { - // Taken from Eigen casters. Permits either scalar dtype or scalar array. - handle type = dtype::of().attr("type"); - if (!convert && !isinstance(src) && !isinstance(src, type)) return false; - - Array tmp = Array::ensure(src); - if (tmp && tmp.size() == 1 && tmp.ndim() == 0) { - this->value = *tmp.data(); - return true; - } - - return false; - } - - static handle cast(T src, return_value_policy, handle) { - Array tmp({1}); - tmp.mutable_at(0) = src; - tmp.resize({}); - - // You could also just return the array if you want a scalar array. - object scalar = tmp[tuple()]; - return scalar.release(); - } -}; - -template <> -struct npy_format_descriptor { - static constexpr auto name = "float16"; - static pybind11::dtype dtype() { - handle ptr = npy_api::get().PyArray_DescrFromType_(NPY_FLOAT16); - return reinterpret_borrow(ptr); - } - virtual ~npy_format_descriptor() {} -}; - -template <> -struct type_caster : public npy_scalar_caster { - static constexpr auto name = "float16"; -}; -} // namespace detail -} // namespace pybind11 - using mindspore::device::DeviceAddress; using DeviceAddressPtr = std::shared_ptr; // brief mindspore namespace. @@ -98,179 +41,195 @@ namespace mindspore { // // A sub namespace in ME to support tensor related definition. namespace tensor { +// Tensor data interface. +class TensorData { + public: + /// Total number of elements. + virtual ssize_t size() const = 0; + /// Byte size of a single element. + virtual ssize_t itemsize() const = 0; + /// Total number of bytes. + virtual ssize_t nbytes() const = 0; + /// Number of dimensions. + virtual ssize_t ndim() const = 0; + /// Data pointer. + virtual void *data() = 0; + /// Is data equals. + virtual bool equals(const TensorData &other) const = 0; + /// To string. + virtual std::string ToString() const = 0; +}; + +using TensorDataPtr = std::shared_ptr; + // Tensor entity class class Tensor : public MetaTensor { public: - Tensor() = default; abstract::AbstractBasePtr ToAbstract() override; - // brief Constructor for Python. + + // brief Create tensor from another tensor, data is shared. + // + // param tensor [Tensor] The input tensor. + explicit Tensor(const Tensor &tensor); + + // brief Create tensor with given data type from another tensor. // - // param type_ptr [TypePty] Data type of the tensor. - // param py_shape [py::tuple] The shape represented by py::tuple of the tensor. - Tensor(const TypePtr &type_ptr, const py::tuple &shape); + // param tensor [Tensor] The input tensor. + // param data_type [TypeId] The new tensor data type. + Tensor(const Tensor &tensor, TypeId data_type); - // brief Constructor for C++. + // brief Create tensor with the given shared tensor data. + // + // param data_type [TypeId] Data type of the tensor. + // param shape The shape represented by std::vector of the tensor. + // param data The shared tensor data. + Tensor(TypeId data_type, const std::vector &shape, TensorDataPtr data); + + // brief Create an all zero tensor. // // param data_type [TypeId] Data type of the tensor. // param shape The shape represented by std::vector of the tensor. Tensor(TypeId data_type, const std::vector &shape); - // brief Constructor for Python. + // brief Create a tensor with input data buffer. // - // param input [py::array] Data value of the tensor. // param data_type [TypeId] Data type of the tensor. - explicit Tensor(const py::array &input, const TypePtr &data_type = nullptr); + // param shape The shape represented by std::vector of the tensor. + // param data The input data to be copied into tensor. + // param data_len The length of data in bytes. + Tensor(TypeId data_type, const std::vector &shape, void *data, size_t data_len); - // brief Constructor + // brief Create a tensor with input data buffer and given source data type. // - // param input [py::list] the data for tensor - // param data_type [TypeId] data type - explicit Tensor(const py::list &input, const TypePtr &data_type = nullptr); + // param data_type [TypeId] Data type of the tensor. + // param shape The shape represented by std::vector of the tensor. + // param data The input data to be copied into tensor. + // param src_data_type The source data type. + Tensor(TypeId data_type, const std::vector &shape, void *data, TypeId src_data_type); - // brief Constructor + // brief Create 1 dimension tensor from an int vector. // - // param input [py::tuple] the data for tensor + // param input [std::vector] the data for tensor // param data_type [TypeId] data type - explicit Tensor(const py::tuple &input, const TypePtr &data_type = nullptr); + explicit Tensor(const std::vector &input, const TypePtr &data_type = nullptr); - // brief Constructor + // brief Create 1 dimension tensor from a float vector. // - // param input [py::float_] the data for tensor + // param input [std::vector] the data for tensor // param data_type [TypeId] data type - explicit Tensor(const py::float_ &input, const TypePtr &data_type = nullptr); + explicit Tensor(const std::vector &input, const TypePtr &data_type = nullptr); - // brief Constructor + // brief Create 0 dimension tensor from an int scalar. // - // param input [py::int_] the data for tensor + // param input [int64] the data for tensor // param data_type [TypeId] data type - explicit Tensor(const py::int_ &input, const TypePtr &data_type = nullptr); + explicit Tensor(int64_t input, const TypePtr &data_type = nullptr); - // brief Constructor + // brief Create 0 dimension tensor from a float scalar. // - // param input [Tensor] the data for tensor + // param input [double] the data for tensor // param data_type [TypeId] data type - Tensor(const Tensor &tensor, const TypePtr &data_type = nullptr); + explicit Tensor(double input, const TypePtr &data_type = nullptr); ~Tensor() override = default; MS_DECLARE_PARENT(Tensor, MetaTensor); - // brief Overloads operator = for Tensor. - // - // The constructed Tensor object has the same type and shape with tensor. - // - // param tensor An existing Tensor object. - Tensor &operator=(const Tensor &tensor); - // brief Compares two Tensor objects. // - // Compare two tensor objects to see if they have same data type, shape and - // data value. + // Compare two tensor objects to see if they have same data type, shape and data address. // // param tensor The Tensor object to be compared. - // return true: If having same type, shape and data, return true, or return false. + // return true: If having same type, shape and data address, return true, or return false. bool operator==(const Tensor &tensor) const; - // It is different from 'operator==' which just compare shape/type/address, it do real value comparison. - bool ValueEqual(const Tensor &other) const; - - // assgin value to this tensor - Tensor &AssignValue(const Tensor &tensor); + // It is different from 'operator==' which just compare shape/type/address, + // it do real value comparison. + bool ValueEqual(const Tensor &tensor) const; bool operator==(const Value &other) const override { if (other.isa()) { - auto other_ = static_cast(other); + auto &other_ = static_cast(other); return *this == other_; - } else { - return false; } + return false; } - py::tuple GetPyTupleShape() const; - // brief Gets tensor's dimension // // return The number of dimensions of the tensor data. - int DataDim() const; + int DataDim() const { return static_cast(data().ndim()); } // brief Getting tensor data size // // return The total number of elements of the tensor data. - int DataSize() const; - - // brief Tensor's data value. - // - // return [py::array] The tensor's data in py::array. - py::array data() const; + int DataSize() const { return static_cast(data().size()); } // brief Get the data type fo the tensor for C++ // // return [int] The tensor's data type will be cast to int to return. - int data_type_c() const; + int data_type_c() const { return static_cast(data_type_); } // brief Get the tensor's shape for C++ // // return [std::vector] - std::vector shape_c(void) const; + std::vector shape_c(void) const { return shape(); } // brief Get Tensor data pointer for c++ type // // param writable true if writable, false if read only // return The pointer to the object - void *data_c(bool writable = false); + void *data_c() { return data().data(); } // brief Get Tensor data byte-size for c++ type // // return byte size of Tensor data - size_t Size() const { return this->data().nbytes(); } + size_t Size() const { return data().nbytes(); } - // brief Get data type from tensor data. + void *data_c() const { return data_->data(); } + + // brief Sync data with device. + void data_sync() const; + + // brief Get the internal data object. // - // param buf The buffer info of the py::array data. - // return The [TypeId] of the tensor data. - TypeId GetDataType(const py::buffer_info &buf) const; + // return The reference to internal data object. + TensorData &data() { return *data_; } - // brief Sets the data type of a tensor. + // brief Get the internal data shared pointer. // - // param data_type The data type of the tensor to be set. + // return The reference to internal data object. + const TensorDataPtr &data_ptr() const { return data_; } + + // brief Get the internal data object. // + // return The reference to internal data object. + const TensorData &data() const { return *data_; } + TypeId set_data_type(const TypeId data_type) override; - TypePtr SetDtype(const TypePtr type_ptr) override; + std::string GetShapeAndDataTypeInfo() const; + std::string ToString() const override; - std::string ToStringRepr() const; - py::array data_; // < Tensor's data value - const bool parse_info_ = true; - bool is_init(); - void set_init_flag(bool flag); - private: - // brief init tensor - // - // param input [py::array] the data for tensor - // param data_type [TypeId] data type - // return true if succeed, false if failed. - void init(const py::array &input, const TypeId &data_type); - void init(const py::array &input, const TypePtr &type_ptr); - bool init_flag_{false}; - // brief init tensor attribute - // - // param data_type [TypeId] Data type of the tensor. - // param shape [py::array] The shape of the tensor. - // return true if succeed, false if failed. - void init(TypeId data_type, const std::vector &shape, py::array *data); + std::string ToStringRepr() const; - bool convert_data(const py::array &in, const TypeId in_data_type, py::array *out, const TypeId out_data_type); + bool is_init() { return init_flag_; } + void set_init_flag(bool flag) { init_flag_ = flag; } - public: bool is_dirty() const { return dirty_; } void set_dirty(const bool dirty) { dirty_ = dirty; } + DeviceAddressPtr device_address() const { return device_address_; } void set_device_address(const DeviceAddressPtr &device_address) { device_address_ = device_address; } - py::array data_sync(); + std::string id() const { return id_; } + const bool parse_info_ = true; + private: + bool init_flag_{false}; + TensorDataPtr data_{nullptr}; bool dirty_{true}; std::string id_{""}; DeviceAddressPtr device_address_{nullptr}; @@ -282,8 +241,6 @@ using TensorPtrList = std::vector>; namespace inference { class Tensor : public MSTensor { public: - Tensor(); - Tensor(TypeId data_type, const std::vector &shape); explicit Tensor(std::shared_ptr tensor_ptr); diff --git a/mindspore/ccsrc/ir/tensor_py.cc b/mindspore/ccsrc/ir/tensor_py.cc new file mode 100644 index 0000000000..1c763d48f4 --- /dev/null +++ b/mindspore/ccsrc/ir/tensor_py.cc @@ -0,0 +1,377 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "ir/tensor_py.h" + +#include +#include +#include +#include +#include + +#include "device/device_address.h" +#include "pybind_api/api_register.h" +#include "pybind_api/export_flags.h" +#include "pipeline/static_analysis/abstract_value.h" + +namespace mindspore { +namespace tensor { + +static TypeId GetDataType(const py::buffer_info &buf) { + if (buf.format.size() == 1) { + switch (buf.format.front()) { + case 'e': + case 'f': + case 'd': + switch (buf.itemsize) { + case 2: + return TypeId::kNumberTypeFloat16; + case 4: + return TypeId::kNumberTypeFloat32; + case 8: + return TypeId::kNumberTypeFloat64; + } + break; + case 'b': + case 'h': + case 'i': + case 'l': + case 'q': + switch (buf.itemsize) { + case 1: + return TypeId::kNumberTypeInt8; + case 2: + return TypeId::kNumberTypeInt16; + case 4: + return TypeId::kNumberTypeInt32; + case 8: + return TypeId::kNumberTypeInt64; + } + break; + case 'B': + case 'H': + case 'I': + case 'L': + case 'Q': + switch (buf.itemsize) { + case 1: + return TypeId::kNumberTypeUInt8; + case 2: + return TypeId::kNumberTypeUInt16; + case 4: + return TypeId::kNumberTypeUInt32; + case 8: + return TypeId::kNumberTypeUInt64; + } + break; + case '?': + return TypeId::kNumberTypeBool; + } + } + MS_LOG(WARNING) << "Unsupported DataType format " << buf.format << " item size " << buf.itemsize; + return TypeId::kTypeUnknown; +} + +static std::string GetPyTypeFormat(TypeId data_type) { + switch (data_type) { + case TypeId::kNumberTypeFloat16: + return "e"; + case TypeId::kNumberTypeFloat32: + return py::format_descriptor::format(); + case TypeId::kNumberTypeFloat64: + return py::format_descriptor::format(); + case TypeId::kNumberTypeUInt8: + return py::format_descriptor::format(); + case TypeId::kNumberTypeUInt16: + return py::format_descriptor::format(); + case TypeId::kNumberTypeUInt32: + return py::format_descriptor::format(); + case TypeId::kNumberTypeUInt64: + return py::format_descriptor::format(); + case TypeId::kNumberTypeInt8: + return py::format_descriptor::format(); + case TypeId::kNumberTypeInt16: + return py::format_descriptor::format(); + case TypeId::kNumberTypeInt32: + return py::format_descriptor::format(); + case TypeId::kNumberTypeInt64: + return py::format_descriptor::format(); + case TypeId::kNumberTypeBool: + return py::format_descriptor::format(); + default: + MS_LOG(WARNING) << "Unsupported DataType " << data_type << "."; + return ""; + } +} + +static bool IsCContiguous(const py::array &input) { + auto flags = static_cast(input.flags()); + return (flags & pybind11::detail::npy_api::NPY_ARRAY_C_CONTIGUOUS_) != 0; +} + +TensorPtr TensorPy::MakeTensor(const py::array &input, const TypePtr &type_ptr) { + // Get input buffer info. + py::buffer_info buf = input.request(); + // Check data types. + auto data_type = type_ptr ? type_ptr->type_id() : TypeId::kTypeUnknown; + auto buf_type = GetDataType(buf); + if (buf_type == TypeId::kTypeUnknown && data_type == TypeId::kTypeUnknown) { + MS_LOG(EXCEPTION) << "Unsupported tensor type!"; + } + // Use buf type as data type if type_ptr not set. + if (data_type == TypeId::kTypeUnknown) { + data_type = buf_type; + } + // Convert input array to C contiguous if need. + std::unique_ptr tmp_buf; + if (!IsCContiguous(input)) { + Py_buffer pybuf; + if (PyObject_GetBuffer(input.ptr(), &pybuf, PyBUF_ANY_CONTIGUOUS)) { + MS_LOG(EXCEPTION) << "Failed to get buffer from the input!"; + } + tmp_buf = std::make_unique(pybuf.len); + if (PyBuffer_ToContiguous(tmp_buf.get(), &pybuf, pybuf.len, 'C')) { + MS_LOG(EXCEPTION) << "Can't copy numpy.ndarray to a contiguous buffer."; + } + PyBuffer_Release(&pybuf); + buf.ptr = tmp_buf.get(); + } + // Get tensor shape. + std::vector shape(buf.shape.begin(), buf.shape.end()); + if (data_type == buf_type) { + // Use memory copy if input data type is same as the required type. + return std::make_shared(data_type, shape, buf.ptr, buf.size * buf.itemsize); + } + // Create tensor with data type converted. + return std::make_shared(data_type, shape, buf.ptr, buf_type); +} + +static std::vector GetStrides(const std::vector &shape, ssize_t item_size) { + std::vector strides; + strides.reserve(shape.size()); + const auto ndim = shape.size(); + for (size_t i = 0; i < ndim; ++i) { + auto stride = item_size; + for (size_t j = i + 1; j < ndim; ++j) { + stride *= shape[j]; + } + strides.push_back(stride); + } + return strides; +} + +static py::buffer_info GetPyBufferInfo(const Tensor &tensor) { + std::vector shape(tensor.shape().begin(), tensor.shape().end()); + std::vector strides = GetStrides(shape, tensor.data().itemsize()); + return py::buffer_info{ + tensor.data_c(), tensor.data().itemsize(), GetPyTypeFormat(tensor.data_type()), tensor.DataDim(), shape, strides}; +} + +py::tuple TensorPy::GetPyTupleShape(const Tensor &tensor) { + auto &shape = tensor.shape(); + py::tuple dims(shape.size()); + for (size_t i = 0; i < dims.size(); ++i) { + dims[i] = py::int_(shape[i]); + } + return dims; +} + +py::array TensorPy::SyncAsNumpy(const Tensor &tensor) { + tensor.data_sync(); + auto info = GetPyBufferInfo(tensor); + py::object self = py::cast(&tensor); + return py::array(py::dtype(info), info.shape, info.strides, info.ptr, self); +} + +py::array TensorPy::AsNumpy(const Tensor &tensor) { + auto info = GetPyBufferInfo(tensor); + py::object self = py::cast(&tensor); + return py::array(py::dtype(info), info.shape, info.strides, info.ptr, self); +} + +static std::vector GetShapeFromTuple(const py::tuple &tuple) { + std::vector shape; + const size_t size = tuple.size(); + shape.reserve(tuple.size()); + for (size_t i = 0; i < size; ++i) { + shape.push_back(py::int_(tuple[i])); + } + return shape; +} + +REGISTER_PYBIND_DEFINE(Tensor, ([](const py::module *m) { + // Define python Tensor class. + // dtype should define before Tensor, because Tensor init depend dtype + (void)py::class_>(*m, "Tensor") + .def(py::init([](const Tensor &tensor) { return std::make_shared(tensor); }), + py::arg("input")) + .def(py::init([](const Tensor &tensor, const TypePtr &type_ptr) { + TypeId data_type = type_ptr ? type_ptr->type_id() : kTypeUnknown; + if (data_type == kTypeUnknown || tensor.data_type() == data_type) { + return std::make_shared(tensor); + } + return std::make_shared(tensor, data_type); + }), + py::arg("input"), py::arg("dtype")) + .def(py::init([](const TypePtr &type_ptr, const py::tuple &shape) { + auto data_type = type_ptr ? type_ptr->type_id() : TypeId::kNumberTypeFloat64; + return std::make_shared(data_type, GetShapeFromTuple(shape)); + }), + py::arg("dtype"), py::arg("shape")) + .def(py::init([](const py::array &input, const TypePtr &type_ptr) { + return TensorPy::MakeTensor(input, type_ptr); + }), + py::arg("input"), py::arg("dtype") = nullptr) + .def(py::init([](py::float_ input, const TypePtr &type_ptr) { + return TensorPy::MakeTensor(py::array(input), type_ptr); + }), + py::arg("input"), py::arg("dtype") = nullptr) + .def(py::init([](py::int_ input, const TypePtr &type_ptr) { + return TensorPy::MakeTensor(py::array(input), type_ptr); + }), + py::arg("input"), py::arg("dtype") = nullptr) + .def(py::init([](py::list input, const TypePtr &type_ptr) { + return TensorPy::MakeTensor(py::array(input), type_ptr); + }), + py::arg("input"), py::arg("dtype") = nullptr) + .def(py::init([](py::tuple input, const TypePtr &type_ptr) { + return TensorPy::MakeTensor(py::array(input), type_ptr); + }), + py::arg("input"), py::arg("dtype") = nullptr) + .def_readonly(PYTHON_TENSOR_FLAG, &Tensor::parse_info_) + .def_property_readonly("dtype", &Tensor::Dtype, R"mydelimiter( + Get the tensor's data type. + + Returns: + type, the data type of tensor. + + Examples: + >>> data = mindspore.Tensor(np.ones((2, 1), np.int32)) + >>> data.dtype + Int32 + )mydelimiter") + .def_property_readonly("shape", TensorPy::GetPyTupleShape, R"mydelimiter( + Get the tensor's shape. + + Returns: + tuple[int], the shape of tensor. + + Examples: + >>> data = mindspore.Tensor(np.ones((3, 3))) + >>> data.shape() + (3, 3) + )mydelimiter") + .def("asnumpy", TensorPy::SyncAsNumpy, R"mydelimiter( + Convert tensor to numpy.ndarray. + + Returns: + numpy.ndarray. + + Examples: + >>> data = mindspore.Tensor(np.ones((2, 3))) + >>> array = data.asnumpy() + >>> array + array([[1., 1., 1.], + [1., 1., 1.]]) + )mydelimiter") + .def("size", &Tensor::DataSize, R"mydelimiter( + Get tensor's data size. + + Returns: + int, the size of tensor. + + Examples: + >>> data = mindspore.Tensor(np.ones((2, 3))) + >>> data.size() + 6 + )mydelimiter") + .def("is_init", &Tensor::is_init, R"mydelimiter( + Get tensor init_flag. + + Returns: + bool, whether the tensor init. + + Examples: + >>> data = mindspore.Tensor(np.ones((2, 3))) + >>> data.is_init() + False + )mydelimiter") + .def("set_init_flag", &Tensor::set_init_flag, R"mydelimiter( + Set tensor init_flag. + + Examples: + >>> data = mindspore.Tensor(np.ones((2, 3))) + >>> data.set_init_flag(True) + )mydelimiter") + .def("dim", &Tensor::DataDim, R"mydelimiter( + Get tensor's data dimension. + + Returns: + int, the dimension of tensor. + + Examples: + >>> data = mindspore.Tensor(np.ones((2, 3))) + >>> data.dim() + 2 + )mydelimiter") + .def("set_dtype", &Tensor::SetDtype, R"mydelimiter( + Set the tensor's data type. + + Arg: + dtype (:class:`mindspore.dtype`): The type of output tensor. + + Examples: + >>> data = mindspore.Tensor(np.ones((1, 2), np.float32)) + >>> data.set_dtype(mindspore.int32) + mindspore.int32 + )mydelimiter") + .def("__str__", &Tensor::ToString) + .def("__repr__", &Tensor::ToStringRepr) + .def(py::pickle( + [](const Tensor &t) { // __getstate__ + /* Return a tuple that fully encodes the state of the object */ + return py::make_tuple(TensorPy::AsNumpy(t)); + }, + [](const py::tuple &t) { // __setstate__ + if (t.size() != 1) { + throw std::runtime_error("Invalid state!"); + } + /* Create a new C++ instance */ + return TensorPy::MakeTensor(t[0].cast()); + })); + // Define python MetaTensor class. + (void)py::class_>(*m, "MetaTensor") + .def(py::init>(), py::arg("dtype"), py::arg("shape")) + .def_readonly(PYTHON_META_TENSOR_FLAG, &MetaTensor::parse_info_) + .def_property_readonly("dtype", &MetaTensor::Dtype, "Get the MetaTensor's dtype.") + .def_property_readonly("shape", &MetaTensor::shape, "Get the MetaTensor's shape.") + .def(py::pickle( + [](const MetaTensor &t) { // __getstate__ + /* Return a tuple that fully encodes the state of the object */ + return py::make_tuple(static_cast(t.data_type()), t.shape()); + }, + [](const py::tuple &t) { // __setstate__ + if (t.size() != 2) { + throw std::runtime_error("Invalid state!"); + } + /* Create a new C++ instance */ + MetaTensor tensor(TypeId(t[0].cast()), t[1].cast>()); + return tensor; + })); + })); + +} // namespace tensor +} // namespace mindspore diff --git a/mindspore/ccsrc/ir/tensor_py.h b/mindspore/ccsrc/ir/tensor_py.h new file mode 100644 index 0000000000..18ee547071 --- /dev/null +++ b/mindspore/ccsrc/ir/tensor_py.h @@ -0,0 +1,114 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_IR_TENSOR_PY_H_ +#define MINDSPORE_CCSRC_IR_TENSOR_PY_H_ + +#include +#include +#include + +#include "pybind11/pybind11.h" +#include "pybind11/numpy.h" + +#include "ir/tensor.h" + +namespace py = pybind11; + +namespace pybind11 { +namespace detail { +// Similar to enums in `pybind11/numpy.h`. Determined by doing: +// python3 -c 'import numpy as np; print(np.dtype(np.float16).num)' +constexpr int NPY_FLOAT16 = 23; + +template +struct npy_scalar_caster { + PYBIND11_TYPE_CASTER(T, _("PleaseOverride")); + using Array = array_t; + + bool load(handle src, bool convert) { + // Taken from Eigen casters. Permits either scalar dtype or scalar array. + handle type = dtype::of().attr("type"); + if (!convert && !isinstance(src) && !isinstance(src, type)) return false; + + Array tmp = Array::ensure(src); + if (tmp && tmp.size() == 1 && tmp.ndim() == 0) { + this->value = *tmp.data(); + return true; + } + + return false; + } + + static handle cast(T src, return_value_policy, handle) { + Array tmp({1}); + tmp.mutable_at(0) = src; + tmp.resize({}); + + // You could also just return the array if you want a scalar array. + object scalar = tmp[tuple()]; + return scalar.release(); + } +}; + +template <> +struct npy_format_descriptor { + static constexpr auto name = "float16"; + static pybind11::dtype dtype() { + handle ptr = npy_api::get().PyArray_DescrFromType_(NPY_FLOAT16); + return reinterpret_borrow(ptr); + } + virtual ~npy_format_descriptor() {} +}; + +template <> +struct type_caster : public npy_scalar_caster { + static constexpr auto name = "float16"; +}; +} // namespace detail +} // namespace pybind11 + +using mindspore::device::DeviceAddress; +using DeviceAddressPtr = std::shared_ptr; +// brief mindspore namespace. +// +// mindspore namespace is the top level namespace of Mindsporeession project. +// Other namespace should be a sub namespace of mindspore namespace in the ME project. +namespace mindspore { +// brief mindspore::tensor namespace +// +// A sub namespace in ME to support tensor related definition. +namespace tensor { +// Tensor python wrapper and adapter class. +class TensorPy { + public: + // brief Create Tensor from a numpy array object. + // + // param input [py::array] Data value of the tensor. + // param data_type [TypeId] Data type of the tensor. + static TensorPtr MakeTensor(const py::array &input, const TypePtr &data_type = nullptr); + + static py::array SyncAsNumpy(const Tensor &tensor); + + static py::array AsNumpy(const Tensor &tensor); + + static py::tuple GetPyTupleShape(const Tensor &tensor); +}; + +} // namespace tensor +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_IR_TENSOR_PY_H_ diff --git a/mindspore/ccsrc/onnx/ir_exporter.cc b/mindspore/ccsrc/onnx/ir_exporter.cc index d74233d79a..2f02f483f5 100644 --- a/mindspore/ccsrc/onnx/ir_exporter.cc +++ b/mindspore/ccsrc/onnx/ir_exporter.cc @@ -23,6 +23,7 @@ #include #include +#include "ir/tensor_py.h" #include "ir/param_value_py.h" #include "debug/anf_ir_utils.h" #include "operator/ops.h" @@ -257,7 +258,7 @@ void IrExportBuilder::SetTensorToAttributeProto(const ValuePtr &value, onnx::Att attr_proto->set_type(onnx::AttributeProto_AttributeType_TENSOR); onnx::TensorProto *tensor_proto = attr_proto->mutable_t(); auto data = value->cast(); - tensor_proto->set_raw_data(data->data().request(true).ptr, static_cast(data->data().nbytes())); + tensor_proto->set_raw_data(data->data_c(), static_cast(data->data().nbytes())); auto dtype = data->data_type(); auto shape = data->shape_c(); tensor_proto->set_data_type(GetOnnxDataType(dtype)); diff --git a/mindspore/ccsrc/onnx/onnx_exporter.cc b/mindspore/ccsrc/onnx/onnx_exporter.cc index 2a038bbf1a..65a841246b 100644 --- a/mindspore/ccsrc/onnx/onnx_exporter.cc +++ b/mindspore/ccsrc/onnx/onnx_exporter.cc @@ -27,6 +27,7 @@ #include "proto/onnx.pb.h" #include "operator/ops.h" #include "ir/param_value_py.h" +#include "ir/tensor_py.h" namespace mindspore { enum OpMergeMode { @@ -1190,7 +1191,7 @@ void OnnxExporter::SetNodeAttribute(const ValuePtr &value, onnx::NodeProto *cons attr_proto->set_type(onnx::AttributeProto_AttributeType_TENSOR); onnx::TensorProto *tensor_proto = attr_proto->mutable_t(); auto data = dyn_cast(value); - tensor_proto->set_raw_data(data->data().request(true).ptr, static_cast(data->data().nbytes())); + tensor_proto->set_raw_data(data->data_c(), static_cast(data->data().nbytes())); auto dtype = data->data_type(); auto shape = data->shape_c(); diff --git a/mindspore/ccsrc/operator/prim_structures.cc b/mindspore/ccsrc/operator/prim_structures.cc index 33c7a1e209..03c432483a 100644 --- a/mindspore/ccsrc/operator/prim_structures.cc +++ b/mindspore/ccsrc/operator/prim_structures.cc @@ -21,6 +21,9 @@ #include "pipeline/static_analysis/param_validator.h" #include "operator/ops.h" #include "utils/convert_utils.h" +#include "ir/tensor_py.h" + +using mindspore::tensor::TensorPy; namespace mindspore { namespace abstract { @@ -554,7 +557,7 @@ AbstractBasePtr InferImplTuple2Array(const AnalysisEnginePtr &, const PrimitiveP py::tuple data_tuple = ValuePtrToPyData(input->BuildValue()); py::array data = py::array(data_tuple); - auto tensor = std::make_shared(data); + auto tensor = TensorPy::MakeTensor(data); auto ret = tensor->ToAbstract(); ret->set_value(tensor); MS_LOG(DEBUG) << "Tuple2arry result AbstractTensor: " << ret->ToString(); diff --git a/mindspore/ccsrc/optimizer/irpass/arithmetic_simplify.h b/mindspore/ccsrc/optimizer/irpass/arithmetic_simplify.h index ae44ec1f7d..270db8305f 100644 --- a/mindspore/ccsrc/optimizer/irpass/arithmetic_simplify.h +++ b/mindspore/ccsrc/optimizer/irpass/arithmetic_simplify.h @@ -153,7 +153,7 @@ class TensorMultiplyBase : public AnfVisitor { } tensor::TensorPtr tensor_ptr = dyn_cast(value); - return tensor_ptr->data_c(writable); + return tensor_ptr->data_c(); } // Make a new tensor (when possible) with the same shape as of `node` @@ -171,7 +171,7 @@ class TensorMultiplyBase : public AnfVisitor { auto new_tensor_ptr = std::make_shared(tensor_type_ptr->type_id(), tensor_shape); size_t mem_size = GetTypeByte(tensor_type_ptr) * IntToSize(new_tensor_ptr->ElementsNum()); - char *data = reinterpret_cast(new_tensor_ptr->data_c(true)); + char *data = reinterpret_cast(new_tensor_ptr->data_c()); if (x == nullptr) { std::memset(data, 0, mem_size); @@ -546,7 +546,7 @@ class ConstantDuplicateMul : public AnfVisitor { auto new_tensor_ptr = std::make_shared(tensor_3_type_ptr->type_id(), tensor_out_shape); size_t mem_size = GetTypeByte(tensor_3_type_ptr) * IntToSize(new_tensor_ptr->ElementsNum()); - char *data = reinterpret_cast(new_tensor_ptr->data_c(true)); + char *data = reinterpret_cast(new_tensor_ptr->data_c()); memcpy(data, data_out, mem_size); auto new_vnode = NewValueNode(new_tensor_ptr); diff --git a/mindspore/ccsrc/optimizer/irpass/branch_culling.cc b/mindspore/ccsrc/optimizer/irpass/branch_culling.cc index 0253cd2b39..837588c37d 100644 --- a/mindspore/ccsrc/optimizer/irpass/branch_culling.cc +++ b/mindspore/ccsrc/optimizer/irpass/branch_culling.cc @@ -191,7 +191,7 @@ inline void ResetSharedOp() { tensor::TensorPtr ConstData() { std::vector shp = {1}; tensor::TensorPtr const_data = std::make_shared(kInt32->type_id(), shp); - auto *val = static_cast(const_data->data_c(true)); + auto *val = static_cast(const_data->data_c()); *val = 0; return const_data; } @@ -267,7 +267,7 @@ CNodePtr GenerateSwitchControlDependNode(const FuncGraphPtr &graph, const AnfNod auto PrimSquare = prim::GetPythonOps("square", "mindspore.ops.functional")->cast(); std::vector shp = {1}; tensor::TensorPtr const_data = std::make_shared(kInt32->type_id(), shp); - auto *val = static_cast(const_data->data_c(true)); + auto *val = static_cast(const_data->data_c()); *val = 0; // for the control_depend netoutput node , add two const data to merge the flow ,one for depended node with same // switch the other use the opposite diff --git a/mindspore/ccsrc/optimizer/irpass/special_op_eliminate.h b/mindspore/ccsrc/optimizer/irpass/special_op_eliminate.h index 1dc8fbb344..dcba80431a 100644 --- a/mindspore/ccsrc/optimizer/irpass/special_op_eliminate.h +++ b/mindspore/ccsrc/optimizer/irpass/special_op_eliminate.h @@ -178,7 +178,7 @@ class ZeroLikeFillZero : public AnfVisitor { tensor::TensorPtr new_tensor_ptr = std::make_shared(tensor_type_ptr->type_id(), tensor_shape); size_t mem_size = GetTypeByte(tensor_type_ptr) * IntToSize(new_tensor_ptr->ElementsNum()); - char *data = reinterpret_cast(new_tensor_ptr->data_c(true)); + char *data = reinterpret_cast(new_tensor_ptr->data_c()); (void)memset_s(data, mem_size, 0, mem_size); auto new_cnode = NewValueNode(new_tensor_ptr); diff --git a/mindspore/ccsrc/optimizer/irpass/specialize_transform.h b/mindspore/ccsrc/optimizer/irpass/specialize_transform.h index 6ac4e40f5e..3db9e7bd51 100644 --- a/mindspore/ccsrc/optimizer/irpass/specialize_transform.h +++ b/mindspore/ccsrc/optimizer/irpass/specialize_transform.h @@ -71,7 +71,7 @@ class SpecializeTransform { continue; } if (value_args[i] != nullptr) { - auto const_tensor = *value_args[i]; + auto &const_tensor = *value_args[i]; auto const_tensor_ptr = std::make_shared(const_tensor); AnfNodePtr arg = NewValueNode(const_tensor_ptr); (void)mng->Replace(params[i], arg); diff --git a/mindspore/ccsrc/parallel/ops_info/gather_v2_info.cc b/mindspore/ccsrc/parallel/ops_info/gather_v2_info.cc index 2ca5a408b1..078be08128 100644 --- a/mindspore/ccsrc/parallel/ops_info/gather_v2_info.cc +++ b/mindspore/ccsrc/parallel/ops_info/gather_v2_info.cc @@ -210,8 +210,8 @@ OperatorVector CreateSubOp(int32_t sub_value) { OperatorName operator_name = SUB; OperatorAttrs operator_attrs; - py::tuple tuple = py::make_tuple(sub_value); - mindspore::tensor::TensorPtr tensor_ptr = std::make_shared(tuple, kInt32); + std::vector tensor_data = {sub_value}; + mindspore::tensor::TensorPtr tensor_ptr = std::make_shared(tensor_data, kInt32); ValuePtr op_param_value = MakeValue(tensor_ptr); Attr op1_param = std::make_pair("", op_param_value); diff --git a/mindspore/ccsrc/parallel/ops_info/reduce_method_info.cc b/mindspore/ccsrc/parallel/ops_info/reduce_method_info.cc index 44eab20588..7304666a77 100644 --- a/mindspore/ccsrc/parallel/ops_info/reduce_method_info.cc +++ b/mindspore/ccsrc/parallel/ops_info/reduce_method_info.cc @@ -204,8 +204,8 @@ ForwardOp CreatReduceMeanForwardOp(const std::vector &forward_group, cons OperatorName operator1_name = REAL_DIV; std::vector device_list = forward_group[0].GetDevicesList(); auto divisor = static_cast(device_list.size()); - py::tuple tuple = py::make_tuple(divisor); - mindspore::tensor::TensorPtr tensor_ptr = std::make_shared(tuple, dtype); + std::vector tensor_data = {divisor}; + mindspore::tensor::TensorPtr tensor_ptr = std::make_shared(tensor_data, dtype); ValuePtr op1_param_value = MakeValue(tensor_ptr); Attr op1_param = std::make_pair("divisor", op1_param_value); OperatorParams operator1_params = {std::make_pair(op1_param, 2)}; diff --git a/mindspore/ccsrc/pipeline/pipeline_ge.cc b/mindspore/ccsrc/pipeline/pipeline_ge.cc index ea0ca14c7a..8ec1602315 100644 --- a/mindspore/ccsrc/pipeline/pipeline_ge.cc +++ b/mindspore/ccsrc/pipeline/pipeline_ge.cc @@ -156,11 +156,11 @@ void ConvertObjectToTensors(const py::dict &dict, TensorOrderMap *const tensors) if (py::isinstance(item.second.attr("default_input"))) { // convert float to tensor with shape([1]) tensor = std::make_shared(kNumberTypeFloat32, std::vector({1})); - *(static_cast(tensor->data_c(true))) = py::cast(item.second.attr("default_input")); + *(static_cast(tensor->data_c())) = py::cast(item.second.attr("default_input")); } else if (py::isinstance(item.second.attr("default_input"))) { // convert int to tensor with shape([1]) tensor = std::make_shared(kNumberTypeInt32, std::vector({1})); - *(static_cast(tensor->data_c(true))) = py::cast(item.second.attr("default_input")); + *(static_cast(tensor->data_c())) = py::cast(item.second.attr("default_input")); } else if (py::hasattr(item.second.attr("default_input"), PYTHON_TENSOR_FLAG)) { // cast tensor tensor = py::cast>(item.second.attr("default_input")); @@ -330,7 +330,7 @@ py::object ExtractGeneralCnodeRet(const AbstractBasePtr &cnode_data, const py::t MS_LOG(EXCEPTION) << "The shape of the tensor derived is not Shape, is " << shape->ToString(); } auto shape_me = shape->cast()->shape(); - auto shape_ge = py::cast(data[*count]).shape(); + auto shape_ge = py::cast(data[*count]).shape(); if (shape_ge != shape_me) { MS_LOG(EXCEPTION) << "The shape of the " << *count << "th tensor returned: " << shape_ge << " is not the same as the shape of the tensor derived: " << shape_me; diff --git a/mindspore/ccsrc/pre_activate/ascend/ir_fission/topk_split.cc b/mindspore/ccsrc/pre_activate/ascend/ir_fission/topk_split.cc index 1cace41fc4..c8477353f9 100644 --- a/mindspore/ccsrc/pre_activate/ascend/ir_fission/topk_split.cc +++ b/mindspore/ccsrc/pre_activate/ascend/ir_fission/topk_split.cc @@ -44,7 +44,7 @@ tensor::TensorPtr CreateTensor(const AnfNodePtr &node) { indices_tensor->set_device_info(device_info); // 2 set value of tensor - auto data_ptr = indices_tensor->data_c(true); + auto data_ptr = indices_tensor->data_c(); MS_EXCEPTION_IF_NULL(data_ptr); std::vector half_data; for (size_t i = 0; i < last_dim; ++i) { diff --git a/mindspore/ccsrc/pre_activate/common/helper.cc b/mindspore/ccsrc/pre_activate/common/helper.cc index 1c2ade201c..e1db0ed6ed 100644 --- a/mindspore/ccsrc/pre_activate/common/helper.cc +++ b/mindspore/ccsrc/pre_activate/common/helper.cc @@ -348,7 +348,7 @@ tensor::TensorPtr CreateTensorWithValueTuple(const ValueTuplePtr &value_tuple_pt MS_EXCEPTION_IF_NULL(tensor); tensor::DeviceInfo device_info{kOpFormat_DEFAULT, type_ptr}; tensor->set_device_info(device_info); - auto data_ptr = tensor->data_c(true); + auto data_ptr = tensor->data_c(); MS_EXCEPTION_IF_NULL(data_ptr); auto elem_num = values.size() * data_length; auto ret_code = memcpy_s(data_ptr, static_cast(tensor->data().nbytes()), values.data(), elem_num); diff --git a/mindspore/ccsrc/predict/converter/kernel2ms.cc b/mindspore/ccsrc/predict/converter/kernel2ms.cc index 902efac720..1b1277aade 100644 --- a/mindspore/ccsrc/predict/converter/kernel2ms.cc +++ b/mindspore/ccsrc/predict/converter/kernel2ms.cc @@ -538,7 +538,7 @@ bool Kernel2Ms::KernelInput2MS(const std::vector &input_tensors) { auto match_idx = match_to_rel_idxs[j]; auto real_tensor = input_tensors[match_idx]; auto real_size = LongToSize(real_tensor->data().nbytes()); - auto real_data = real_tensor->data_c(false); + auto real_data = real_tensor->data_c(); MS_EXCEPTION_IF_NULL(real_data); if (sub_ms_graph_->allTensors[cache_idx] != nullptr) { sub_ms_graph_->allTensors[cache_idx]->data.resize(real_size); diff --git a/mindspore/ccsrc/pynative/pynative_execute.cc b/mindspore/ccsrc/pynative/pynative_execute.cc index 75653ff5d2..2b8d504f1a 100644 --- a/mindspore/ccsrc/pynative/pynative_execute.cc +++ b/mindspore/ccsrc/pynative/pynative_execute.cc @@ -22,6 +22,7 @@ #include #include +#include "ir/tensor_py.h" #include "ir/param_value_py.h" #include "utils/any.h" #include "utils/utils.h" @@ -51,6 +52,8 @@ #include "pynative/pynative_execute_ge.h" #endif +using mindspore::tensor::TensorPy; + const char SINGLE_OP_GRAPH[] = "single_op_graph"; // primitive unable to infer value for constant input in PyNative mode const std::set vm_operators = {"make_ref", "HookBackward", "stop_gradient"}; @@ -171,7 +174,8 @@ py::tuple ConvertInputs(const PrimitivePyPtr &prim, const py::list &args, py::tu py_args[i] = std::make_shared(py::cast(py_args[i]), tensor_ptr->Dtype()); (*out_args_list)[i] = py_args[i]; } else { - py_args[i] = std::make_shared(py::cast(py_args[i]), tensor_ptr->Dtype()); + double arg_value = py::cast(py_args[i]); + py_args[i] = std::make_shared(arg_value, tensor_ptr->Dtype()); (*out_args_list)[i] = py_args[i]; } continue; @@ -262,7 +266,7 @@ py::object RunOpInVM(const OpExecInfoPtr &op_exec_info, PynativeStatusCode *stat result[i] = py::getattr(input, "data"); } else { auto tensor = py::cast(op_inputs[i]); - auto new_tensor = std::make_shared(tensor->data()); + auto new_tensor = std::make_shared(tensor->data_type(), tensor->shape(), tensor->data_ptr()); result[i] = new_tensor; } } @@ -366,13 +370,14 @@ void ConvertPyObjectToTensor(const py::object &input_object, const PrimitivePtr if (py::isinstance(input_object)) { tensor_ptr = py::cast(input_object); } else if (py::isinstance(input_object)) { - tensor_ptr = std::make_shared(py::cast(input_object), kFloat32); + double input_value = py::cast(input_object); + tensor_ptr = std::make_shared(input_value, kFloat32); *tensor_mask = kValueNodeTensorMask; } else if (py::isinstance(input_object)) { tensor_ptr = std::make_shared(py::cast(input_object), kInt32); *tensor_mask = kValueNodeTensorMask; } else if (py::isinstance(input_object)) { - tensor_ptr = std::make_shared(py::cast(input_object), nullptr); + tensor_ptr = TensorPy::MakeTensor(py::cast(input_object), nullptr); } else if (py::isinstance(input_object)) { auto list_inputs = py::cast(input_object); py::tuple tuple_inputs(list_inputs.size()); diff --git a/mindspore/ccsrc/pynative/pynative_execute.h b/mindspore/ccsrc/pynative/pynative_execute.h index 310cf0cb1e..3b030f7035 100644 --- a/mindspore/ccsrc/pynative/pynative_execute.h +++ b/mindspore/ccsrc/pynative/pynative_execute.h @@ -26,6 +26,7 @@ #include #include "pybind11/pybind11.h" +#include "pybind11/numpy.h" #include "pynative/base.h" #include "utils/context/ms_context.h" diff --git a/mindspore/ccsrc/pynative/pynative_execute_ge.cc b/mindspore/ccsrc/pynative/pynative_execute_ge.cc index 7357bdd710..8e10468236 100644 --- a/mindspore/ccsrc/pynative/pynative_execute_ge.cc +++ b/mindspore/ccsrc/pynative/pynative_execute_ge.cc @@ -28,9 +28,12 @@ #include "pipeline/parse/data_converter.h" #include "pipeline/static_analysis/prim.h" #include "session/session_factory.h" +#include "ir/tensor_py.h" const char SINGLE_OP_GRAPH[] = "single_op_graph"; +using mindspore::tensor::TensorPy; + namespace mindspore { namespace pynative { using MeTensor = mindspore::tensor::Tensor; @@ -56,15 +59,15 @@ MeTensorPtr ConvertPyObjToTensor(const py::object &obj) { if (py::isinstance(obj)) { me_tensor_ptr = py::cast(obj); } else if (py::isinstance(obj)) { - me_tensor_ptr = std::make_shared(py::cast(obj), nullptr); + me_tensor_ptr = TensorPy::MakeTensor(py::array(py::cast(obj)), nullptr); } else if (py::isinstance(obj)) { - me_tensor_ptr = std::make_shared(py::cast(obj), nullptr); + me_tensor_ptr = TensorPy::MakeTensor(py::array(py::cast(obj)), nullptr); } else if (py::isinstance(obj)) { - me_tensor_ptr = std::make_shared(py::cast(obj), nullptr); + me_tensor_ptr = TensorPy::MakeTensor(py::array(py::cast(obj)), nullptr); } else if (py::isinstance(obj)) { - me_tensor_ptr = std::make_shared(py::cast(obj), nullptr); + me_tensor_ptr = TensorPy::MakeTensor(py::array(py::cast(obj)), nullptr); } else if (py::isinstance(obj)) { - me_tensor_ptr = std::make_shared(py::cast(obj), nullptr); + me_tensor_ptr = TensorPy::MakeTensor(py::cast(obj), nullptr); } else { MS_LOG(EXCEPTION) << "Run op inputs type is invalid!"; } diff --git a/mindspore/ccsrc/session/ascend_inference_session.cc b/mindspore/ccsrc/session/ascend_inference_session.cc index ff53874502..6295fca1c3 100644 --- a/mindspore/ccsrc/session/ascend_inference_session.cc +++ b/mindspore/ccsrc/session/ascend_inference_session.cc @@ -16,6 +16,7 @@ #include "session/ascend_inference_session.h" #include "operator/ops.h" #include "ir/tensor.h" +#include "ir/tensor_py.h" #include "ir/anf.h" #include "ir/param_value_py.h" #include "device/kernel_runtime.h" @@ -26,6 +27,8 @@ #include "utils/config_manager.h" #include "utils/base_ref_extends.h" +using mindspore::tensor::TensorPy; + namespace mindspore { namespace session { void AscendInferenceSession::LoadInputData(const std::shared_ptr &kernel_graph, @@ -51,7 +54,7 @@ void AscendInferenceSession::LoadInputData(const std::shared_ptr &k auto py_param = param_value->value(); MS_EXCEPTION_IF_NULL(py_param); py::array py_array = py_param.cast(); - tensor = std::make_shared(py_array); + tensor = TensorPy::MakeTensor(py_array); } else { tensor = inputs[no_weight_input++]; } @@ -78,7 +81,7 @@ void AscendInferenceSession::LoadInputData(const std::shared_ptr &k MS_EXCEPTION_IF_NULL(device_address); if (!device_address->SyncHostToDevice(trans::GetRuntimePaddingShape(pk_node, 0), LongToSize(tensor->data().nbytes()), tensor->data_type(), - tensor->data_c(false))) { + tensor->data_c())) { MS_LOG(EXCEPTION) << "SyncHostToDevice failed."; } } diff --git a/mindspore/ccsrc/session/ascend_session.cc b/mindspore/ccsrc/session/ascend_session.cc index 7ef6551f2b..9ae16a1dbd 100644 --- a/mindspore/ccsrc/session/ascend_session.cc +++ b/mindspore/ccsrc/session/ascend_session.cc @@ -989,7 +989,7 @@ void AscendSession::InsertSwitchToGraph(GraphId condition_graph_id, GraphId true MS_EXCEPTION_IF_NULL(condition_graph); tensor::TensorPtr tensor = std::make_shared(kNumberTypeInt32, std::vector{1}); int32_t *val = nullptr; - val = static_cast(tensor->data_c(true)); + val = static_cast(tensor->data_c()); MS_EXCEPTION_IF_NULL(val); *val = 0; auto value_node = std::make_shared(tensor); @@ -1523,7 +1523,7 @@ void AscendSession::SyncInitialTenosrToDevice() { auto addr = AnfAlgo::GetOutputAddr(backend_parameter, 0); MS_EXCEPTION_IF_NULL(addr); if (!addr->SyncHostToDevice(trans::GetRuntimePaddingShape(backend_parameter, 0), tensor_size, - front_tensor->data_type(), front_tensor->data_c(false))) { + front_tensor->data_type(), front_tensor->data_c())) { MS_LOG(EXCEPTION) << "Tensor SyncHostToDevice fail!"; } } diff --git a/mindspore/ccsrc/session/gpu_session.cc b/mindspore/ccsrc/session/gpu_session.cc index a0a43f2edd..cda71961d6 100644 --- a/mindspore/ccsrc/session/gpu_session.cc +++ b/mindspore/ccsrc/session/gpu_session.cc @@ -129,7 +129,7 @@ void GPUSession::LoadInputData(const std::shared_ptr &kernel_graph, MS_EXCEPTION_IF_NULL(device_address); if (!device_address->SyncHostToDevice(trans::GetRuntimePaddingShape(pk_node, 0), LongToSize(tensor->data().nbytes()), tensor->data_type(), - tensor->data_c(false))) { + tensor->data_c())) { MS_LOG(EXCEPTION) << "SyncHostToDevice failed."; } } diff --git a/mindspore/ccsrc/session/session_basic.cc b/mindspore/ccsrc/session/session_basic.cc index 730c20d699..e080c86285 100644 --- a/mindspore/ccsrc/session/session_basic.cc +++ b/mindspore/ccsrc/session/session_basic.cc @@ -96,8 +96,7 @@ BaseRef CreateOneTensor(const AnfNodePtr &node, size_t output_index, const Kerne tensor->set_device_address(AnfAlgo::GetMutableOutputAddr(node, output_index)); tensor->set_dirty(false); } else if (!address->SyncDeviceToHost(trans::GetRuntimePaddingShape(node, output_index), - LongToSize(tensor->data().nbytes()), tensor->data_type(), - tensor->data_c(true))) { + LongToSize(tensor->data().nbytes()), tensor->data_type(), tensor->data_c())) { MS_LOG(INFO) << "output sync device to host error!!!"; tensor->set_dirty(false); } @@ -218,7 +217,7 @@ size_t LoadCtrlInputTensor(const std::shared_ptr &graph, std::vecto } auto tensor = (*inputs_params)[0]; MS_EXCEPTION_IF_NULL(tensor); - auto *val = static_cast(tensor->data_c(true)); + auto *val = static_cast(tensor->data_c()); MS_EXCEPTION_IF_NULL(val); *val = 0; tensor->set_dirty(true); @@ -720,7 +719,7 @@ void SessionBasic::LoadInputData(const std::shared_ptr &kernel_grap MS_EXCEPTION_IF_NULL(device_address); if (!device_address->SyncHostToDevice(trans::GetRuntimePaddingShape(pk_node, 0), LongToSize(tensor->data().nbytes()), tensor->data_type(), - tensor->data_c(false))) { + tensor->data_c())) { MS_LOG(EXCEPTION) << "SyncHostToDevice failed."; } } @@ -815,7 +814,7 @@ void SessionBasic::Summary(KernelGraph *graph) { continue; } if (!address->SyncDeviceToHost(trans::GetRuntimePaddingShape(node, index), LongToSize(tensor->data().nbytes()), - tensor->data_type(), tensor->data_c(true))) { + tensor->data_type(), tensor->data_c())) { MS_LOG(ERROR) << "Failed to sync output from device to host."; } tensor->set_dirty(false); diff --git a/mindspore/ccsrc/transform/util.cc b/mindspore/ccsrc/transform/util.cc index 3f856fe564..b848ec117b 100644 --- a/mindspore/ccsrc/transform/util.cc +++ b/mindspore/ccsrc/transform/util.cc @@ -342,7 +342,7 @@ MeTensorPtr TransformUtil::GenerateMeTensor(const GeTensorPtr &ge_tensor, const MeTensor me_tensor(me_type, me_dims); // Get the writable data pointer of the tensor and cast it to its data type - auto me_data_ptr = reinterpret_cast(me_tensor.data_c(true)); + auto me_data_ptr = reinterpret_cast(me_tensor.data_c()); size_t me_data_size = static_cast(me_tensor.data().nbytes()); MS_EXCEPTION_IF_NULL(me_data_ptr); MS_EXCEPTION_IF_NULL(ge_tensor); diff --git a/mindspore/ccsrc/utils/convert_utils.cc b/mindspore/ccsrc/utils/convert_utils.cc index 6e28e38ed1..15e08f7794 100644 --- a/mindspore/ccsrc/utils/convert_utils.cc +++ b/mindspore/ccsrc/utils/convert_utils.cc @@ -579,11 +579,12 @@ tensor::TensorPtr ScalarToTensor(const ScalarPtr &scalar) { } tensor::TensorPtr tensor = nullptr; if (scalar->isa()) { - tensor = std::make_shared(py::float_(GetValue(scalar)), kFloat32); + tensor = std::make_shared(static_cast(GetValue(scalar)), kFloat32); } else if (scalar->isa()) { - tensor = std::make_shared(py::int_(GetValue(scalar)), kInt32); + tensor = std::make_shared(static_cast(GetValue(scalar)), kInt32); } else if (scalar->isa()) { - tensor = std::make_shared(py::array(py::bool_(GetValue(scalar))), kBool); + const int64_t bool_value = GetValue(scalar) ? 1 : 0; + tensor = std::make_shared(bool_value, kBool); } else { auto type = scalar->type(); auto type_str = (type == nullptr) ? "nullptr" : type->ToString(); diff --git a/mindspore/ccsrc/utils/load_onnx/anf_model_parser.cc b/mindspore/ccsrc/utils/load_onnx/anf_model_parser.cc index e44eb23001..c3dfa5194f 100644 --- a/mindspore/ccsrc/utils/load_onnx/anf_model_parser.cc +++ b/mindspore/ccsrc/utils/load_onnx/anf_model_parser.cc @@ -22,12 +22,14 @@ #include #include "google/protobuf/io/zero_copy_stream_impl.h" #include "ir/tensor.h" +#include "ir/tensor_py.h" #include "ir/param_value_py.h" #include "operator/ops.h" #include "pipeline/static_analysis/abstract_value.h" #include "proto/onnx.pb.h" #include "utils/log_adapter.h" +using mindspore::tensor::TensorPy; using std::string; namespace mindspore { @@ -117,11 +119,11 @@ bool MSANFModelParser::BuildParameterForFuncGraph(const ParameterPtr &node, cons if (default_para_map_.find(value_proto.name()) != default_para_map_.end()) { const onnx::TensorProto initialize_proto = default_para_map_[value_proto.name()]; std::string initial_data = initialize_proto.raw_data(); - auto *tensor_data_buf = reinterpret_cast(tensor_info->data_c(true)); + auto *tensor_data_buf = reinterpret_cast(tensor_info->data_c()); MS_EXCEPTION_IF_NULL(tensor_data_buf); memcpy_s(tensor_data_buf, tensor_info->data().nbytes(), initial_data.data(), initial_data.size()); - py::array array_data = tensor_info->data(); + py::array array_data = TensorPy::AsNumpy(*tensor_info); ParamValuePyPtr para_value_ptr = std::make_shared(); MS_EXCEPTION_IF_NULL(para_value_ptr); para_value_ptr->set_value(array_data); @@ -249,7 +251,7 @@ bool MSANFModelParser::ObtainValueNodeInTensorForm(const std::string &value_node } tensor::TensorPtr tensor_info = std::make_shared(kDefaultValueSwitchMap[attr_tensor_type], shape); const std::string &tensor_buf = attr_tensor.raw_data(); - auto *tensor_data_buf = reinterpret_cast(tensor_info->data_c(true)); + auto *tensor_data_buf = reinterpret_cast(tensor_info->data_c()); memcpy_s(tensor_data_buf, tensor_info->data().nbytes(), tensor_buf.data(), tensor_buf.size()); auto new_value_node = NewValueNode(MakeValue(tensor_info)); MS_EXCEPTION_IF_NULL(new_value_node); diff --git a/mindspore/ccsrc/utils/tensorprint_utils.cc b/mindspore/ccsrc/utils/tensorprint_utils.cc index 0d464e88a8..cf7b339edd 100644 --- a/mindspore/ccsrc/utils/tensorprint_utils.cc +++ b/mindspore/ccsrc/utils/tensorprint_utils.cc @@ -87,7 +87,7 @@ bool PrintTensorToString(const char *str_data_ptr, mindspore::tensor::Tensor *co const size_t &memory_size) { MS_EXCEPTION_IF_NULL(str_data_ptr); MS_EXCEPTION_IF_NULL(print_tensor); - auto *tensor_data_ptr = static_cast(print_tensor->data_c(true)); + auto *tensor_data_ptr = static_cast(print_tensor->data_c()); MS_EXCEPTION_IF_NULL(tensor_data_ptr); auto cp_ret = memcpy_s(tensor_data_ptr, static_cast(print_tensor->data().nbytes()), str_data_ptr, memory_size); diff --git a/mindspore/common/tensor.py b/mindspore/common/tensor.py index 0a631b954f..0c29c1bf54 100644 --- a/mindspore/common/tensor.py +++ b/mindspore/common/tensor.py @@ -61,9 +61,9 @@ class Tensor(Tensor_): if isinstance(input_data, np.ndarray) and (not input_data.flags['FORC']): input_data = np.ascontiguousarray(input_data) if dtype is None: - super(Tensor, self).__init__(input_data) + Tensor_.__init__(self, input_data) else: - super(Tensor, self).__init__(input_data, dtype) + Tensor_.__init__(self, input_data, dtype) self._virtual_flag = False self._init_flag = False diff --git a/tests/st/ops/gpu/test_rmsprop.py b/tests/st/ops/gpu/test_rmsprop.py index 24d1003475..f578fb82c8 100644 --- a/tests/st/ops/gpu/test_rmsprop.py +++ b/tests/st/ops/gpu/test_rmsprop.py @@ -55,6 +55,7 @@ def rmsprop_numpy(variable, gradients, mean_square, moment, mean_square = mean_square * decay + (1.0 - decay) * gradients * gradients moment = momentum * moment + learning_rate / np.sqrt(mean_square + epsilon) * gradients variable = variable - moment + return variable, gradients, mean_square, moment def rmspropcented_numpy(variable, gradients, mean_gradients, mean_square, moment, @@ -64,7 +65,7 @@ def rmspropcented_numpy(variable, gradients, mean_gradients, mean_square, moment moment = momentum * moment + learning_rate / np.sqrt( mean_square - mean_gradients * mean_gradients + epsilon) * gradients variable = variable - moment - + return variable, gradients, mean_gradients, mean_square, moment @pytest.mark.level0 @pytest.mark.platform_x86_gpu_training @@ -85,12 +86,14 @@ def test_rmsprop(): moment_ms = Tensor(moment_np) if centered: + variable_np, gradients_np, mean_gradients_np, mean_square_np, moment_np = \ rmspropcented_numpy(variable_np, gradients_np, mean_gradients_np, mean_square_np, moment_np, learning_rate, decay, momentum, epsilon) net = NetCenteredRMSProp(learning_rate, decay, momentum, epsilon) _ = net(variable_ms, gradients_ms, mean_gradients_ms, mean_square_ms, moment_ms) else: + variable_np, gradients_np, mean_square_np, moment_np = \ rmsprop_numpy(variable_np, gradients_np, mean_square_np, moment_np, learning_rate, decay, momentum, epsilon) net = NetRMSProp(learning_rate, decay, momentum, epsilon) @@ -136,11 +139,13 @@ def test_rmspropcenter(): moment_ms = Tensor(moment_np) if centered: + variable_np, gradients_np, mean_gradients_np, mean_square_np, moment_np = \ rmspropcented_numpy(variable_np, gradients_np, mean_gradients_np, mean_square_np, moment_np, learning_rate, decay, momentum, epsilon) net = NetCenteredRMSProp(learning_rate, decay, momentum, epsilon) _ = net(variable_ms, gradients_ms, mean_gradients_ms, mean_square_ms, moment_ms) else: + variable_np, gradients_np, mean_square_np, moment_np = \ rmsprop_numpy(variable_np, gradients_np, mean_square_np, moment_np, learning_rate, decay, momentum, epsilon) net = NetRMSProp(learning_rate, decay, momentum, epsilon) diff --git a/tests/ut/cpp/ir/meta_tensor_test.cc b/tests/ut/cpp/ir/meta_tensor_test.cc index 720b292027..537d4c460e 100644 --- a/tests/ut/cpp/ir/meta_tensor_test.cc +++ b/tests/ut/cpp/ir/meta_tensor_test.cc @@ -22,6 +22,9 @@ #include "securec/include/securec.h" #include "ir/tensor.h" +#include "ir/tensor_py.h" + +using mindspore::tensor::TensorPy; namespace mindspore { namespace tensor { @@ -90,9 +93,7 @@ TEST_F(TestMetaTensor, EqualTest) { class TestTensor : public UT::Common { public: TestTensor() {} - virtual void SetUp() { - UT::InitPythonPath(); - } + virtual void SetUp() { UT::InitPythonPath(); } }; py::array_t BuildInputTensor() { @@ -124,7 +125,7 @@ TEST_F(TestTensor, PyArrayScalarTest) { TEST_F(TestTensor, InitScalarTest) { std::vector dimensions; Tensor tensor(TypeId::kNumberTypeInt64, dimensions); - uint8_t *data_buf = reinterpret_cast(tensor.data_c(true)); + uint8_t *data_buf = reinterpret_cast(tensor.data_c()); int64_t num = 1; errno_t ret = memcpy_s(data_buf, sizeof(int64_t), &num, sizeof(int64_t)); @@ -172,9 +173,9 @@ TEST_F(TestTensor, InitTensorPtrTest) { } TEST_F(TestTensor, InitByTupleTest) { - py::tuple dimensions = py::make_tuple(2, 3, 4); + const std::vector shape = {2, 3, 4}; TypePtr data_type = kFloat32; - Tensor tuple_tensor = Tensor(data_type, dimensions); + Tensor tuple_tensor(data_type->type_id(), shape); ASSERT_EQ(2, tuple_tensor.DimensionSize(0)); ASSERT_EQ(3, tuple_tensor.DimensionSize(1)); ASSERT_EQ(4, tuple_tensor.DimensionSize(2)); @@ -184,8 +185,8 @@ TEST_F(TestTensor, InitByTupleTest) { ASSERT_EQ(TypeId::kNumberTypeFloat32, tuple_tensor.data_type()); py::tuple tuple = py::make_tuple(1.0, 2.0, 3, 4, 5, 6); - TensorPtr tensor = std::make_shared(tuple, kFloat64); - py::array array = tensor->data(); + TensorPtr tensor = TensorPy::MakeTensor(py::array(tuple), kFloat64); + py::array array = TensorPy::AsNumpy(*tensor); std::cout << "Dim: " << array.ndim() << std::endl; ASSERT_EQ(1, array.ndim()); @@ -203,24 +204,24 @@ TEST_F(TestTensor, InitByTupleTest) { TEST_F(TestTensor, EqualTest) { py::tuple tuple = py::make_tuple(1, 2, 3, 4, 5, 6); - TensorPtr tensor_int8 = std::make_shared(tuple, kInt8); + TensorPtr tensor_int8 = TensorPy::MakeTensor(py::array(tuple), kInt8); ASSERT_TRUE(*tensor_int8 == *tensor_int8); ASSERT_EQ(TypeId::kNumberTypeInt8, tensor_int8->data_type_c()); - TensorPtr tensor_int16 = std::make_shared(tuple, kInt16); + TensorPtr tensor_int16 = TensorPy::MakeTensor(py::array(tuple), kInt16); ASSERT_EQ(TypeId::kNumberTypeInt16, tensor_int16->data_type_c()); - TensorPtr tensor_int32 = std::make_shared(tuple, kInt32); + TensorPtr tensor_int32 = TensorPy::MakeTensor(py::array(tuple), kInt32); ASSERT_EQ(TypeId::kNumberTypeInt32, tensor_int32->data_type_c()); - TensorPtr tensor_float16 = std::make_shared(tuple, kFloat16); + TensorPtr tensor_float16 = TensorPy::MakeTensor(py::array(tuple), kFloat16); ASSERT_EQ(TypeId::kNumberTypeFloat16, tensor_float16->data_type_c()); - TensorPtr tensor_float32 = std::make_shared(tuple, kFloat32); + TensorPtr tensor_float32 = TensorPy::MakeTensor(py::array(tuple), kFloat32); ASSERT_EQ(TypeId::kNumberTypeFloat32, tensor_float32->data_type_c()); - TensorPtr tensor_float64 = std::make_shared(tuple, kFloat64); + TensorPtr tensor_float64 = TensorPy::MakeTensor(py::array(tuple), kFloat64); ASSERT_EQ(TypeId::kNumberTypeFloat64, tensor_float64->data_type_c()); } @@ -247,7 +248,7 @@ TEST_F(TestTensor, PyArrayTest) { TEST_F(TestTensor, InitByFloatArrayDataCTest) { // Init tensor data by py::array_t - auto tensor = std::make_shared(BuildInputTensor()); + auto tensor = TensorPy::MakeTensor(BuildInputTensor()); // Print some information of the tensor std::cout << "Datatype: " << tensor->data_type() << std::endl; @@ -269,7 +270,7 @@ TEST_F(TestTensor, InitByFloatArrayDataCTest) { TEST_F(TestTensor, InitByFloatArrayDataTest) { // Init tensor data by py::array_t - TensorPtr tensor = std::make_shared(BuildInputTensor()); + TensorPtr tensor = TensorPy::MakeTensor(BuildInputTensor()); // Print some information of the tensor std::cout << "Datatype: " << tensor->data_type() << std::endl; @@ -291,7 +292,7 @@ TEST_F(TestTensor, InitByFloatArrayDataTest) { // Print each elements std::cout << "Elements: " << std::endl; - py::array_t data = (py::array_t)tensor->data(); + py::array_t data = py::cast>(TensorPy::AsNumpy(*tensor)); auto array = data.unchecked<2>(); for (int i = 0; i < array.shape(0); i++) { for (int j = 0; j < array.shape(1); j++) { @@ -319,17 +320,17 @@ TEST_F(TestTensor, TensorDataTest) { float ge_tensor_data[] = {1.1, 2.2, 3.3, 4.4, 5.5, 6.6}; // Create a Tensor with wanted data type and shape - Tensor tensor = Tensor(TypeId::kNumberTypeFloat32, std::vector({2, 3})); + Tensor tensor(TypeId::kNumberTypeFloat32, std::vector({2, 3})); // Get the writable data pointer from the tensor - float *me_tensor_data = reinterpret_cast(tensor.data_c(true)); + float *me_tensor_data = reinterpret_cast(tensor.data_c()); // Copy data from buffer to tensor's data errno_t ret = memcpy_s(me_tensor_data, tensor.data().nbytes(), ge_tensor_data, sizeof(ge_tensor_data)); ASSERT_EQ(0, ret); // Testify if the data has been copied to the tensor data - py::array_t data = (py::array_t)tensor.data(); + py::array_t data = py::cast>(TensorPy::AsNumpy(tensor)); auto array = data.mutable_unchecked(); for (int i = 0; i < array.shape(0); i++) { for (int j = 0; j < array.shape(1); j++) { @@ -340,5 +341,17 @@ TEST_F(TestTensor, TensorDataTest) { } } +TEST_F(TestTensor, TensorPyCast) { + std::vector shape{2, 3, 4, 5}; + py::tuple py_tuple = py::make_tuple(std::make_shared(kNumberTypeFloat32, shape)); + auto shape1 = py::cast(py_tuple[0]).shape(); + const py::tuple &t = py_tuple; + auto shape2 = py::cast(t[0]).shape(); + auto shape3 = py::cast(t[0]).shape(); + ASSERT_EQ(shape, shape1); + ASSERT_EQ(shape, shape2); + ASSERT_EQ(shape, shape3); +} + } // namespace tensor } // namespace mindspore diff --git a/tests/ut/cpp/parallel/step_parallel_test.cc b/tests/ut/cpp/parallel/step_parallel_test.cc index afc898907b..b526195958 100644 --- a/tests/ut/cpp/parallel/step_parallel_test.cc +++ b/tests/ut/cpp/parallel/step_parallel_test.cc @@ -60,15 +60,9 @@ CNodePtr Make_Node(Shape x, Shape y, Shape out, int condition = 0) { BaseShapePtr shape1 = std::make_shared(x); BaseShapePtr shape2 = std::make_shared(y); BaseShapePtr shape3 = std::make_shared(out); - std::shared_ptr inputs_x = std::make_shared(); - inputs_x->set_data_type(kNumberTypeInt32); - inputs_x->set_shape(x); - std::shared_ptr inputs_y = std::make_shared(); - inputs_y->set_data_type(kNumberTypeInt32); - inputs_y->set_shape(y); - std::shared_ptr inputs_out = std::make_shared(); - inputs_out->set_data_type(kNumberTypeInt32); - inputs_out->set_shape(out); + std::shared_ptr inputs_x = std::make_shared(kNumberTypeInt32, x); + std::shared_ptr inputs_y = std::make_shared(kNumberTypeInt32, y); + std::shared_ptr inputs_out = std::make_shared(kNumberTypeInt32, out); AbstractBasePtr abstract1 = abstract::FromValue(inputs_x, true); AbstractBasePtr abstract2 = abstract::FromValue(inputs_y, true); AbstractBasePtr abstract3 = abstract::FromValue(inputs_out, true); @@ -127,21 +121,11 @@ FuncGraphManagerPtr Make_Manager(int condition = 0) { ParameterPtr param1 = func_graph->add_parameter(); ParameterPtr param2 = func_graph->add_parameter(); ParameterPtr param3 = func_graph->add_parameter(); - std::shared_ptr inputs_x_dim = std::make_shared(); - inputs_x_dim->set_data_type(kNumberTypeInt32); - inputs_x_dim->set_shape(inputs_x); - std::shared_ptr inputs_y_dim = std::make_shared(); - inputs_y_dim->set_data_type(kNumberTypeInt32); - inputs_y_dim->set_shape(inputs_y); - std::shared_ptr inputs_z_dim = std::make_shared(); - inputs_z_dim->set_data_type(kNumberTypeInt32); - inputs_z_dim->set_shape(inputs_z); - std::shared_ptr inputs_out1_dim = std::make_shared(); - inputs_out1_dim->set_data_type(kNumberTypeInt32); - inputs_out1_dim->set_shape(outputs_1); - std::shared_ptr inputs_out2_dim = std::make_shared(); - inputs_out2_dim->set_data_type(kNumberTypeInt32); - inputs_out2_dim->set_shape(outputs_2); + std::shared_ptr inputs_x_dim = std::make_shared(kNumberTypeInt32, inputs_x); + std::shared_ptr inputs_y_dim = std::make_shared(kNumberTypeInt32, inputs_y); + std::shared_ptr inputs_z_dim = std::make_shared(kNumberTypeInt32, inputs_z); + std::shared_ptr inputs_out1_dim = std::make_shared(kNumberTypeInt32, outputs_1); + std::shared_ptr inputs_out2_dim = std::make_shared(kNumberTypeInt32, outputs_2); AbstractBasePtr abstract_x = abstract::FromValue(inputs_x_dim, true); AbstractBasePtr abstract_y = abstract::FromValue(inputs_y_dim, true); AbstractBasePtr abstract_z = abstract::FromValue(inputs_z_dim, true); diff --git a/tests/ut/cpp/pipeline/static_analysis/data_test.cc b/tests/ut/cpp/pipeline/static_analysis/data_test.cc index c0c0bd1452..61a22bbe5f 100644 --- a/tests/ut/cpp/pipeline/static_analysis/data_test.cc +++ b/tests/ut/cpp/pipeline/static_analysis/data_test.cc @@ -113,12 +113,8 @@ TEST_F(TestData, test_build_shape) { std::vector weight1_dims = {2, 20, 5, 5}; std::vector weight2_dims = {2, 2, 5, 5}; - tensor::TensorPtr weight1 = std::make_shared(); - weight1->set_data_type(kNumberTypeInt32); - weight1->set_shape(weight1_dims); - tensor::TensorPtr weight2 = std::make_shared(); - weight2->set_data_type(kNumberTypeInt32); - weight2->set_shape(weight2_dims); + tensor::TensorPtr weight1 = std::make_shared(kNumberTypeInt32, weight1_dims); + tensor::TensorPtr weight2 = std::make_shared(kNumberTypeInt32, weight2_dims); AbstractBasePtr abstract_weight1 = FromValue(weight1, true); AbstractBasePtr abstract_weight2 = FromValue(weight2, true); diff --git a/tests/ut/cpp/pre_activate/pass/convert_const_input_to_tensor_input_test.cc b/tests/ut/cpp/pre_activate/pass/convert_const_input_to_tensor_input_test.cc index 8dfb0b178f..1749e54d94 100644 --- a/tests/ut/cpp/pre_activate/pass/convert_const_input_to_tensor_input_test.cc +++ b/tests/ut/cpp/pre_activate/pass/convert_const_input_to_tensor_input_test.cc @@ -104,7 +104,7 @@ TEST_F(TestHWConstInputToTensorInput, test_value_tuple_tensor_input) { EXPECT_TRUE(IsValueNode(input1)); auto tensor = input1->cast()->value()->cast(); ASSERT_TRUE(tensor != nullptr); - auto data = tensor->data_c(false); + auto data = tensor->data_c(); EXPECT_EQ(std::vector((int *)data, (int *)data + 4), std::vector({2, 4, 2, 2})); } } // namespace opt diff --git a/tests/ut/cpp/transform/convert_test.cc b/tests/ut/cpp/transform/convert_test.cc index 7d18663f38..f8f48920e0 100644 --- a/tests/ut/cpp/transform/convert_test.cc +++ b/tests/ut/cpp/transform/convert_test.cc @@ -706,7 +706,7 @@ TEST_F(TestConvert, TestConvertTensor) { auto type_id = kNumberTypeFloat32; MeTensor me_tensor(type_id, dims); // Get the writable data pointer of the tensor and cast it to its data type - uint8_t* me_data_ptr = reinterpret_cast(me_tensor.data_c(true)); + uint8_t* me_data_ptr = reinterpret_cast(me_tensor.data_c()); // Copy or use the writable data pointer of the ME tensor memcpy_s(me_data_ptr, me_tensor.data().nbytes(), data, 12 * sizeof(float)); auto me_tensor_ptr = std::make_shared(me_tensor); diff --git a/tests/ut/cpp/transform/graph_runner_test.cc b/tests/ut/cpp/transform/graph_runner_test.cc index ab73005453..1b87cea464 100644 --- a/tests/ut/cpp/transform/graph_runner_test.cc +++ b/tests/ut/cpp/transform/graph_runner_test.cc @@ -18,6 +18,7 @@ #include #include "common/common_test.h" #include "ir/dtype.h" +#include "ir/tensor_py.h" #include "transform/transform_base_test.h" #include "common/py_func_graph_fetcher.h" #include "pipeline/static_analysis/static_analysis.h" @@ -35,6 +36,8 @@ #define private public #include "transform/graph_runner.h" +using mindspore::tensor::TensorPy; + namespace mindspore { namespace transform { class TestGraphRunner : public UT::Common { @@ -70,7 +73,7 @@ std::shared_ptr MakeGeGraph() { return std::make_shared(anf_graph); } namespace { -std::shared_ptr> DoExecGraph(const std::vector& inputs) { +std::shared_ptr> DoExecGraph(const std::vector &inputs) { std::vector ge_tensor_ptrs = TransformUtil::ConvertInputTensors(inputs, kOpFormat_NCHW); std::vector ge_outputs; @@ -109,7 +112,7 @@ TEST_F(TestGraphRunner, TestGeTensorConstructor) { MeTensor tensor = MeTensor(TypeId::kNumberTypeFloat32, std::vector({1, 2, 3})); // Get the writable data pointer from the tensor - float* me_tensor_data = reinterpret_cast(tensor.data_c(true)); + float *me_tensor_data = reinterpret_cast(tensor.data_c()); // Copy data from buffer to tensor's data memcpy_s(me_tensor_data, static_cast(tensor.data().nbytes()), ge_tensor_data, sizeof(ge_tensor_data)); @@ -119,11 +122,11 @@ TEST_F(TestGraphRunner, TestGeTensorConstructor) { py::tuple py_tuple = py::make_tuple(py::make_tuple(py::make_tuple(1.1f, 2.2f, 3.3f), py::make_tuple(4.4f, 5.5f, 6.6f))); py::array my_arry = py::array(py_tuple).attr("astype").cast()("float32").cast(); - MeTensor tensor_tuple = MeTensor(my_arry, kFloat32); - PrintMeTensor(&tensor_tuple); + auto tensor_tuple = TensorPy::MakeTensor(my_arry, kFloat32); + PrintMeTensor(tensor_tuple.get()); - py::array tensor_array = tensor.data(); - py::array tensor_tuple_array = tensor_tuple.data(); + py::array tensor_array = TensorPy::AsNumpy(tensor); + py::array tensor_tuple_array = TensorPy::AsNumpy(*tensor_tuple); assert(memcmp(ge_tensor_data, tensor_array.data(), sizeof(ge_tensor_data)) == 0); assert(memcmp(ge_tensor_data, tensor_tuple_array.data(), sizeof(ge_tensor_data)) == 0); } @@ -131,7 +134,7 @@ TEST_F(TestGraphRunner, TestGeTensorConstructor) { #if (!defined ENABLE_GE) TEST_F(TestGraphRunner, TestRunGraphException) { - DfGraphManager& graph_manager = DfGraphManager::GetInstance(); + DfGraphManager &graph_manager = DfGraphManager::GetInstance(); graph_manager.ClearGraph(); std::map dict; @@ -167,7 +170,7 @@ TEST_F(TestGraphRunner, TestRunGraphException) { } TEST_F(TestGraphRunner, TestRunGraph) { - DfGraphManager& graph_manager = DfGraphManager::GetInstance(); + DfGraphManager &graph_manager = DfGraphManager::GetInstance(); graph_manager.ClearGraph(); std::shared_ptr convertor = MakeGeGraph(); @@ -183,7 +186,7 @@ TEST_F(TestGraphRunner, TestRunGraph) { py::make_tuple(py::make_tuple(py::make_tuple(1.0, 2.0, 3.0, 4.0), py::make_tuple(4.0, 5.0, 6.0, 7.0))), py::make_tuple(py::make_tuple(py::make_tuple(1.0, 2.0, 3.0, 4.0), py::make_tuple(4.0, 5.0, 6.0, 7.0)))); py::array array = py::array(tuple); - MeTensorPtr me_tensor_ptr = std::make_shared(array, type_id); + MeTensorPtr me_tensor_ptr = TensorPy::MakeTensor(array, type_id); MS_LOG(INFO) << "inputs me tensor data is: "; PrintMeTensor(&(*me_tensor_ptr)); @@ -204,7 +207,7 @@ TEST_F(TestGraphRunner, TestRunGraph) { } TEST_F(TestGraphRunner, TestAPI) { - DfGraphManager& graph_manager = DfGraphManager::GetInstance(); + DfGraphManager &graph_manager = DfGraphManager::GetInstance(); graph_manager.ClearGraph(); std::shared_ptr convertor = MakeGeGraph(); diff --git a/tests/ut/cpp/transform/transform_base_test.cc b/tests/ut/cpp/transform/transform_base_test.cc index eb083e2dd1..50227bc53c 100644 --- a/tests/ut/cpp/transform/transform_base_test.cc +++ b/tests/ut/cpp/transform/transform_base_test.cc @@ -16,6 +16,9 @@ #include #include "common/common_test.h" #include "transform/transform_base_test.h" +#include "ir/tensor_py.h" + +using mindspore::tensor::TensorPy; namespace mindspore { namespace transform { @@ -55,10 +58,10 @@ void PrintMeTensor(MeTensor* tensor) { } std::cout << "the py::str() data is: " << std::endl; - py::array tensor_data = (*tensor).data(); + py::array tensor_data = TensorPy::AsNumpy(*tensor); std::cout << std::string(py::str(tensor_data)) << std::endl; - std::cout << "tensor dtype is: " << std::string(tensor->data().dtype().str()) << std::endl; + std::cout << "tensor dtype is: " << std::string(tensor_data.dtype().str()) << std::endl; } FuncGraphPtr MakeFuncGraph(const PrimitivePtr prim, unsigned int nparam) { @@ -73,7 +76,7 @@ FuncGraphPtr MakeFuncGraph(const PrimitivePtr prim, unsigned int nparam) { std::vector inputs; inputs.push_back(NewValueNode(prim)); for (unsigned int i = 0; i < nparam; i++) { - if ((prim->name() == "ScalarSummary" || prim->name() == "TensorSummary" || + if ((prim->name() == "ScalarSummary" || prim->name() == "TensorSummary" || prim->name() == "ImageSummary" || prim->name() == "HistogramSummary") && i == 0) { auto input = NewValueNode("testSummary"); From 024b52d6e6cad49e5297549a330c200d1790ccca Mon Sep 17 00:00:00 2001 From: huangdongrun Date: Wed, 24 Jun 2020 10:51:52 +0800 Subject: [PATCH 040/254] add inputtoattr prim to white list --- mindspore/ccsrc/operator/ops.cc | 1 + mindspore/ccsrc/operator/ops.h | 1 + .../ccsrc/optimizer/irpass/branch_culling.cc | 36 ++++++++++++------- 3 files changed, 26 insertions(+), 12 deletions(-) diff --git a/mindspore/ccsrc/operator/ops.cc b/mindspore/ccsrc/operator/ops.cc index e6545d311c..88001bf63f 100755 --- a/mindspore/ccsrc/operator/ops.cc +++ b/mindspore/ccsrc/operator/ops.cc @@ -231,6 +231,7 @@ const PrimitivePtr kPrimFakeBprop = std::make_shared("fake_bprop"); const PrimitivePtr kPrimBpropCut = std::make_shared("bprop_cut"); const PrimitivePtr kPrimFakeQuantPerLayer = std::make_shared("FakeQuantPerLayer"); const PrimitivePtr kPrimFakeQuantPerChannel = std::make_shared("FakeQuantPerChannel"); +const PrimitivePtr kPrimApplyRMSProp = std::make_shared("ApplyRMSProp"); // Other miscellaneous const PrimitivePtr kPrimIdentity = std::make_shared("identity"); diff --git a/mindspore/ccsrc/operator/ops.h b/mindspore/ccsrc/operator/ops.h index 01812a5529..ac35a8e2bd 100755 --- a/mindspore/ccsrc/operator/ops.h +++ b/mindspore/ccsrc/operator/ops.h @@ -242,6 +242,7 @@ extern const PrimitivePtr kPrimFakeBprop; extern const PrimitivePtr kPrimBpropCut; extern const PrimitivePtr kPrimFakeQuantPerLayer; extern const PrimitivePtr kPrimFakeQuantPerChannel; +extern const PrimitivePtr kPrimApplyRMSProp; // Other Miscellaneous extern const PrimitivePtr kPrimIdentity; diff --git a/mindspore/ccsrc/optimizer/irpass/branch_culling.cc b/mindspore/ccsrc/optimizer/irpass/branch_culling.cc index 0253cd2b39..a7254c6e32 100644 --- a/mindspore/ccsrc/optimizer/irpass/branch_culling.cc +++ b/mindspore/ccsrc/optimizer/irpass/branch_culling.cc @@ -51,18 +51,30 @@ bool InConvertWhiteList(const AnfNodePtr &node, size_t index) { // node because it is attribute or ge specific reason. // Example : when convert CNode(kPrimReduceSum, x, axis), node of index 2 in CNode->inputs is axis which should not be // converted to switch guarded. - std::vector>> white_list( - {{prim::kPrimApplyMomentum, {1, 2}}, {prim::kPrimMomentum, {2, 3}}, - {prim::kPrimStateSetItem, {1}}, {prim::kPrimTupleGetItem, {2}}, - {prim::kPrimEnvGetItem, {1}}, {prim::kPrimEnvSetItem, {1}}, - {prim::kPrimReduceSum, {2}}, {prim::kPrimReduceMean, {2}}, - {prim::kPrimReduceAll, {2}}, {prim::kPrimCast, {2}}, - {prim::kPrimTranspose, {2}}, {prim::kPrimOneHot, {2}}, - {prim::kPrimGatherV2, {3}}, {prim::kPrimReshape, {2}}, - {prim::kPrimAssign, {1}}, {prim::kPrimAssignAdd, {1}}, - {prim::kPrimAssignSub, {1}}, {prim::kPrimTensorSummary, {1}}, - {prim::kPrimImageSummary, {1}}, {prim::kPrimScalarSummary, {1}}, - {prim::kPrimHistogramSummary, {1}}}); + std::vector>> white_list({{prim::kPrimApplyMomentum, {1, 2}}, + {prim::kPrimMomentum, {2, 3}}, + {prim::kPrimStateSetItem, {1}}, + {prim::kPrimTupleGetItem, {2}}, + {prim::kPrimEnvGetItem, {1}}, + {prim::kPrimEnvSetItem, {1}}, + {prim::kPrimReduceSum, {2}}, + {prim::kPrimReduceMean, {2}}, + {prim::kPrimReduceAll, {2}}, + {prim::kPrimCast, {2}}, + {prim::kPrimTranspose, {2}}, + {prim::kPrimOneHot, {2}}, + {prim::kPrimGatherV2, {3}}, + {prim::kPrimReshape, {2}}, + {prim::kPrimAssign, {1}}, + {prim::kPrimAssignAdd, {1}}, + {prim::kPrimAssignSub, {1}}, + {prim::kPrimTensorSummary, {1}}, + {prim::kPrimImageSummary, {1}}, + {prim::kPrimScalarSummary, {1}}, + {prim::kPrimApplyRMSProp, {6, 7, 8}}, + {prim::kPrimCumSum, {2}}, + {prim::kPrimTile, {2}}, + {prim::kPrimHistogramSummary, {1}}}); for (auto &item : white_list) { auto matched = std::any_of(item.second.begin(), item.second.end(), [&item, &node, &index](size_t idx) { return IsPrimitiveCNode(node, item.first) && idx == index; From 3d116adda43c0b8e1eafddf2757692a828b5719a Mon Sep 17 00:00:00 2001 From: buxue Date: Wed, 24 Jun 2020 11:05:30 +0800 Subject: [PATCH 041/254] Remove the comment for the example --- mindspore/ops/operations/control_ops.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/mindspore/ops/operations/control_ops.py b/mindspore/ops/operations/control_ops.py index e7ac4572ce..3e980ce570 100644 --- a/mindspore/ops/operations/control_ops.py +++ b/mindspore/ops/operations/control_ops.py @@ -47,8 +47,6 @@ class ControlDepend(Primitive): Bool. This operation has no actual data output, it will be used to setup the order of relative operations. Examples: - >>> # In the following example, the data calculation uses original global_step. After the calculation the global - >>> # step should be increased, so the add operation should depend on the data calculation operation. >>> class Net(nn.Cell): >>> def __init__(self): >>> super(Net, self).__init__() From bc0455eaff1b7ae91d410111e0f9ec25e981184c Mon Sep 17 00:00:00 2001 From: huangdongrun Date: Tue, 23 Jun 2020 21:44:20 +0800 Subject: [PATCH 042/254] * fix bool index * change slice setitem to mixed procedure * add testcase for slice assignment --- .../composite/multitype_ops/_compile_utils.py | 18 +++--- .../multitype_ops/_constexpr_utils.py | 7 ++ tests/st/pynative/test_tensor_index.py | 64 ++++++++++++++++--- tests/ut/python/ops/test_tensor_slice.py | 11 ++-- 4 files changed, 78 insertions(+), 22 deletions(-) diff --git a/mindspore/ops/composite/multitype_ops/_compile_utils.py b/mindspore/ops/composite/multitype_ops/_compile_utils.py index 906d74948a..b371b839e1 100644 --- a/mindspore/ops/composite/multitype_ops/_compile_utils.py +++ b/mindspore/ops/composite/multitype_ops/_compile_utils.py @@ -167,12 +167,13 @@ def _tensor_getitem(self, index): return tensor_index_by_tensor(self, index) if isinstance(index, tuple): return tensor_index_by_tuple(self, index) + # bool type should be judged before int + if isinstance(index, bool): + return _tensor_index_by_bool(self, index) if isinstance(index, int): return _tensor_index_by_integer(self, index) if isinstance(index, slice): return tensor_index_by_slice(self, index) - if isinstance(index, bool): - return _tensor_index_by_bool(self, index) if index is None: return F.expand_dims(self, 0) if index is ...: @@ -206,7 +207,8 @@ def tensor_index_by_slice(data, slice_index): """Tensor getitem by a single slice""" shape = F.shape(data) if not shape: - const_utils.raise_index_error("When tensor is indexed by a slice, the dimension of the tensor cannot be 0.") + const_utils.raise_index_error("When tensor is indexed by a slice, the dimension of the tensor\ + cannot be 0.") begin_strides, end_strides, step_strides = const_utils.get_stride_info_from_slice(shape, slice_index) return F.strided_slice(data, begin_strides, end_strides, step_strides) @@ -215,7 +217,11 @@ def _tensor_index_by_integer(data, number): """Tensor getitem by a single integer number""" shape = F.shape(data) if not shape: - const_utils.raise_index_error("When tensor is indexed by an integer, the dimension of the tensor cannot be 0.") + return const_utils.raise_index_error("When tensor is indexed by an integer,\ + the dimension of the tensor cannot be 0.") + if number >= shape[0]: + return const_utils.raise_index_error("index {} is out of bounds for axis 0 with size {}".format( + number, shape[0])) begin_strides, end_strides, step_strides = const_utils.get_stride_info_from_integer(shape, number) shrink_axis_mask = 1 return P.StridedSlice(0, 0, 0, 0, shrink_axis_mask)(data, begin_strides, end_strides, step_strides) @@ -427,8 +433,6 @@ def tensor_setitem_by_tuple_with_number(data, tuple_index, value): indexes_types = hyper_map(F.typeof, tuple_index) index_elements_type = const_utils.tuple_index_elements_type(indexes_types, const_utils.TENSOR_SETITEM) - if index_elements_type == const_utils.NO_TENSOR: - return tensor_setitem_by_slice_with_number(data, tuple_index, value) if index_elements_type == const_utils.ALL_TENSOR: indices = _generate_indices_from_tuple_of_tensor(data, tuple_index, @@ -488,8 +492,6 @@ def tensor_setitem_by_tuple_with_tensor(data, tuple_index, value): indexes_types = hyper_map(F.typeof, tuple_index) index_elements_type = const_utils.tuple_index_elements_type(indexes_types, const_utils.TENSOR_SETITEM) - if index_elements_type == const_utils.NO_TENSOR: - return tensor_setitem_by_slice_with_tensor(data, tuple_index, value) if index_elements_type == const_utils.ALL_TENSOR: indices = _generate_indices_from_tuple_of_tensor(data, tuple_index, diff --git a/mindspore/ops/composite/multitype_ops/_constexpr_utils.py b/mindspore/ops/composite/multitype_ops/_constexpr_utils.py index 02756ffe56..519546af0d 100644 --- a/mindspore/ops/composite/multitype_ops/_constexpr_utils.py +++ b/mindspore/ops/composite/multitype_ops/_constexpr_utils.py @@ -339,6 +339,8 @@ def check_tensors_dtype_same(data_dtype, value_dtype, op_name): @constexpr def generate_broadcast_shape(shapes, op_name): """Generate broadcast shape for a tuple of shape.""" + if not shapes: + return () broadcast_shape = shapes[0] for i, shape in enumerate(shapes): logger.debug(f"Broadcasts the {i}th tensor, the shape is {shape}.") @@ -541,6 +543,11 @@ def generate_index_info_from_tuple_of_mixed_tensors(data_shape, slice_indexes[slice_count].step) # Use list to represent slicing result. indexes_info[pos] = list(range(data_shape[pos]))[slice_obj] + if not indexes_info[pos]: + raise IndexError("An empty slice is not supported, got {}:{}:{}".format( + slice_indexes[slice_count].start, + slice_indexes[slice_count].stop, + slice_indexes[slice_count].step)) slice_count += 1 elif isinstance(ele_type, mstype.ellipsis_type): if ellipsis_num != 0: diff --git a/tests/st/pynative/test_tensor_index.py b/tests/st/pynative/test_tensor_index.py index 77ee7db5d6..3d1c7acdbf 100644 --- a/tests/st/pynative/test_tensor_index.py +++ b/tests/st/pynative/test_tensor_index.py @@ -646,7 +646,7 @@ class TensorAssignWithSlice2(Cell): class TensorAssignWithSlice(Cell): def __init__(self): super(TensorAssignWithSlice, self).__init__() - self.c = 2 + self.c = 2.0 def construct(self, a, b, ck): a[1:3, ::] = b @@ -661,7 +661,47 @@ class TensorAssignWithSlice(Cell): return z -def test_tensor_assign(): +def test_tensor_assign_slice_value_1(): + net = TensorAssignWithSlice() + a = np.arange(60).reshape(3, 4, 5) + ck = np.arange(60).reshape(3, 4, 5) + b = np.array([1]).astype(np.float32) # Tensor([1], dtype=mstype.float32) + tb = Tensor(b, dtype=mstype.float32) + ta = Tensor(a, dtype=mstype.float32) + tck = Tensor(ck, dtype=mstype.float32) + out = net(ta, tb, tck) + a[1:3, ::] = b + a[2:3:, 3:] = b + a[::] = b + a[::] = 2.0 + a[::, ::] = b + a[::, ::] = 2.0 + a[2:3:, 0:, 4:1:-1] = b + a[2:3:, 0:, 4:1:-1] = 2.0 + z = a + ck + assert np.all(z == out.asnumpy()) + + +def test_tensor_assign_slice_value_2(): + net2 = TensorAssignWithSlice2() + a = np.array([1, 2, 3, 4, 5, 6, 7, 8]) + ck = np.array([1, 2, 3, 4, 5, 6, 7, 8]) + b = np.array([1]).astype(np.float32) # Tensor([1], dtype=mstype.float32) + tb = Tensor(b, dtype=mstype.float32) + ta = Tensor(a, dtype=mstype.float32) + tck = Tensor(ck, dtype=mstype.float32) + out = net2(ta, tb, tck) + a[1:5] = b + a[3:4] = 5 + a[-1:1:-1] = b + a[-1:3:-1] = 5 + a[::] = b + a[::] = 9 + z = a + ck + assert np.all(z == out.asnumpy()) + + +def test_tensor_assign_exception(): net = TensorAssignWithSlice() net2 = TensorAssignWithSlice2() net_e1 = TensorAssignWithSliceError1() @@ -677,8 +717,6 @@ def test_tensor_assign(): Tc = Tensor([], dtype=mstype.float32) t = Tensor([1, 2, 3, 4, 5, 6, 7, 8], dtype=mstype.float32) tck = Tensor([1, 2, 3, 4, 5, 6, 7, 8], dtype=mstype.float32) - net(Ta, b, Tck) - net2(t, b, tck) # Error for A[Slice] = Number # 1. A[Slice] = Number, Slice error with pytest.raises(IndexError): @@ -744,9 +782,6 @@ def test_tensor_assign(): # 2. A[::, 1:, ...] = scalar/tensor net = TensorAssignWithTupleEllipsis() net(Ta, b) - Tc = Tensor(1, mstype.float32) - with pytest.raises(ValueError): - net(Ta, Tc) with pytest.raises(ValueError): net(Ta, Tb) @@ -765,7 +800,7 @@ class TensorAssignWithTupleEllipsis(Cell): super(TensorAssignWithTupleEllipsis, self).__init__() def construct(self, a, b): - a[:2, ...] = 1 + a[:2, ...] = 1.0 a[1:, ...] = b return a @@ -955,3 +990,16 @@ def Xtest_tensor_slice_reduce_out_of_bounds_positive(): with pytest.raises(ValueError) as ex: net(input_tensor) assert "For 'StridedSlice' the `begin[0]` should be an int and must less than 6, but got `6`" in str(ex.value) + + +def test_tensor_range(): + a = np.arange(4*5*6).reshape(4, 5, 6).astype(np.float32) + ta = Tensor(a, mstype.float32) + ms_out = [] + for item in ta: + ms_out.append(item) + np_out = [] + for item in a: + np_out.append(item) + for i, elem in enumerate(ms_out): + assert np.all(elem.asnumpy() == np_out[i]) diff --git a/tests/ut/python/ops/test_tensor_slice.py b/tests/ut/python/ops/test_tensor_slice.py index e6b078fd5c..73f705e009 100644 --- a/tests/ut/python/ops/test_tensor_slice.py +++ b/tests/ut/python/ops/test_tensor_slice.py @@ -130,7 +130,7 @@ class TensorAssignWithSlice2(Cell): class TensorAssignWithSlice(Cell): def __init__(self): super(TensorAssignWithSlice, self).__init__() - self.c = 2 + self.c = 2.0 def construct(self, a, b, ck): a[1:3, ::] = b @@ -528,8 +528,7 @@ def test_tensor_assign(): net = TensorAssignWithTupleEllipsis() net(Ta, b) Tc = Tensor(1, mstype.float32) - with pytest.raises(ValueError): - net(Ta, Tc) + net(Ta, Tc) with pytest.raises(ValueError): net(Ta, Tb) @@ -548,7 +547,7 @@ class TensorAssignWithTupleEllipsis(Cell): super(TensorAssignWithTupleEllipsis, self).__init__() def construct(self, a, b): - a[:2, ...] = 1 + a[:2, ...] = 1.0 a[1:, ...] = b return a @@ -579,10 +578,10 @@ class TensorAssignWithTupleInteger(Cell): super(TensorAssignWithTupleInteger, self).__init__() def construct(self, a, b, ck): - a[(1)] = 1 + a[(1)] = 1.0 a[(1)] = b a[(1, 1)] = b - a[(1, 1)] = 1 + a[(1, 1)] = 1.0 z = a + ck return z From d3f9b80066663976d66aaad6cbe43c82a2f0e8dd Mon Sep 17 00:00:00 2001 From: chenzomi Date: Tue, 23 Jun 2020 21:32:35 +0800 Subject: [PATCH 043/254] checkpoint add model_type --- mindspore/_checkparam.py | 11 ++++ mindspore/ccsrc/utils/checkpoint.proto | 1 + mindspore/train/callback/_checkpoint.py | 28 +++++++---- mindspore/train/callback/_loss_monitor.py | 4 +- mindspore/train/serialization.py | 50 ++++++++++++------- model_zoo/lenet/eval.py | 11 ++-- model_zoo/lenet_quant/README.md | 14 +++--- model_zoo/lenet_quant/eval.py | 13 ++--- model_zoo/lenet_quant/eval_quant.py | 14 +++--- model_zoo/lenet_quant/src/lenet.py | 4 +- model_zoo/lenet_quant/src/lenet_fusion.py | 5 +- model_zoo/lenet_quant/train.py | 16 ++++-- model_zoo/lenet_quant/train_quant.py | 21 +++++--- .../python/predict/test_predict_save_model.py | 2 +- tests/ut/python/utils/test_serialize.py | 24 ++++----- 15 files changed, 132 insertions(+), 86 deletions(-) diff --git a/mindspore/_checkparam.py b/mindspore/_checkparam.py index 880d26bfad..d5ac7c3e33 100644 --- a/mindspore/_checkparam.py +++ b/mindspore/_checkparam.py @@ -593,6 +593,17 @@ def check_bool(input_param): raise TypeError("Input type must be bool!") +def check_string(input_param, valid_values): + """String type judgment.""" + if isinstance(input_param, str) and input_param in valid_values: + return input_param + if len(valid_values) == 1: + raise ValueError(f'Input should be str and must be {valid_values[0]},' + f' but got {input_param}.') + raise ValueError(f'Input should be str and must be one of {valid_values},' + f' but got {input_param}.') + + def check_input_format(input_param): """Judge input format.""" if input_param == "NCHW": diff --git a/mindspore/ccsrc/utils/checkpoint.proto b/mindspore/ccsrc/utils/checkpoint.proto index 31c7cd8004..7fca399e2b 100644 --- a/mindspore/ccsrc/utils/checkpoint.proto +++ b/mindspore/ccsrc/utils/checkpoint.proto @@ -22,6 +22,7 @@ message Checkpoint { required TensorProto tensor = 2; } repeated Value value = 1; + required string model_type = 2; } diff --git a/mindspore/train/callback/_checkpoint.py b/mindspore/train/callback/_checkpoint.py index d185377c83..4e686c414f 100644 --- a/mindspore/train/callback/_checkpoint.py +++ b/mindspore/train/callback/_checkpoint.py @@ -21,17 +21,16 @@ import time import mindspore.context as context from mindspore import log as logger -from mindspore._checkparam import check_bool, check_int_non_negative +from mindspore._checkparam import check_bool, check_string, check_int_non_negative from mindspore.train._utils import _make_directory from mindspore.train.serialization import _exec_save_checkpoint, _save_graph - from ._callback import Callback, set_cur_net + _cur_dir = os.getcwd() _save_dir = _cur_dir - def _check_file_name_prefix(file_name_prefix): """ Check file name valid or not. @@ -87,6 +86,7 @@ class CheckpointConfig: Can't be used with keep_checkpoint_max at the same time. integrated_save (bool): Whether to intergrated save in automatic model parallel scene. Default: True. Integrated save function is only supported in automatic parallel scene, not supported in manual parallel. + model_type (str): Model type in `normal`, `fusion` or `quant`. Default: "normal". Raises: ValueError: If the input_param is None or 0. @@ -101,7 +101,8 @@ class CheckpointConfig: save_checkpoint_seconds=0, keep_checkpoint_max=5, keep_checkpoint_per_n_minutes=0, - integrated_save=True): + integrated_save=True, + model_type="normal"): if not save_checkpoint_steps and not save_checkpoint_seconds and \ not keep_checkpoint_max and not keep_checkpoint_per_n_minutes: @@ -115,6 +116,8 @@ class CheckpointConfig: keep_checkpoint_max = check_int_non_negative(keep_checkpoint_max) if keep_checkpoint_per_n_minutes: keep_checkpoint_per_n_minutes = check_int_non_negative(keep_checkpoint_per_n_minutes) + if model_type: + model_type = check_string(model_type, ["normal", "fusion", "quant"]) self._save_checkpoint_steps = save_checkpoint_steps self._save_checkpoint_seconds = save_checkpoint_seconds @@ -129,6 +132,7 @@ class CheckpointConfig: if not self._keep_checkpoint_per_n_minutes or self._keep_checkpoint_per_n_minutes == 0: self._keep_checkpoint_max = 1 + self._model_type = model_type self._integrated_save = check_bool(integrated_save) @property @@ -156,12 +160,18 @@ class CheckpointConfig: """Get the value of _integrated_save.""" return self._integrated_save + @property + def model_type(self): + """Get the value of model_type.""" + return self._model_type + def get_checkpoint_policy(self): """Get the policy of checkpoint.""" checkpoint_policy = {'save_checkpoint_steps': self._save_checkpoint_steps, 'save_checkpoint_seconds': self._save_checkpoint_seconds, 'keep_checkpoint_max': self._keep_checkpoint_max, - 'keep_checkpoint_per_n_minutes': self._keep_checkpoint_per_n_minutes} + 'keep_checkpoint_per_n_minutes': self._keep_checkpoint_per_n_minutes, + 'model_type': self._model_type} return checkpoint_policy @@ -226,7 +236,7 @@ class ModelCheckpoint(Callback): graph_file_name = os.path.join(self._directory, self._prefix + '-graph.meta') _save_graph(cb_params.train_network, graph_file_name) self._graph_saved = True - self._save_ckpt(cb_params) + self._save_ckpt(cb_params, self._config.model_type) def end(self, run_context): """ @@ -237,7 +247,7 @@ class ModelCheckpoint(Callback): """ cb_params = run_context.original_args() _to_save_last_ckpt = True - self._save_ckpt(cb_params, _to_save_last_ckpt) + self._save_ckpt(cb_params, self._config.model_type, _to_save_last_ckpt) from mindspore.parallel._cell_wrapper import destroy_allgather_cell destroy_allgather_cell() @@ -256,7 +266,7 @@ class ModelCheckpoint(Callback): return False - def _save_ckpt(self, cb_params, force_to_save=False): + def _save_ckpt(self, cb_params, model_type, force_to_save=False): """Save checkpoint files.""" if cb_params.cur_step_num == self._last_triggered_step: return @@ -292,7 +302,7 @@ class ModelCheckpoint(Callback): set_cur_net(cb_params.train_network) cb_params.train_network.exec_checkpoint_graph() - _exec_save_checkpoint(cb_params.train_network, gen_file, self._config.integrated_save) + _exec_save_checkpoint(cb_params.train_network, gen_file, model_type, self._config.integrated_save) if os.path.exists(gen_file): shutil.move(gen_file, cur_file) diff --git a/mindspore/train/callback/_loss_monitor.py b/mindspore/train/callback/_loss_monitor.py index 3c1da218c2..3f93c6314d 100644 --- a/mindspore/train/callback/_loss_monitor.py +++ b/mindspore/train/callback/_loss_monitor.py @@ -76,7 +76,7 @@ class LossMonitor(Callback): step_loss = np.mean(step_loss.asnumpy()) self.losses.append(step_loss) - cur_step_in_epoch = int((cb_params.cur_step_num - 1) % cb_params.batch_num) + cur_step_in_epoch = int((cb_params.cur_step_num - 1) % cb_params.batch_num) + 1 if isinstance(step_loss, float) and (np.isnan(step_loss) or np.isinf(step_loss)): raise ValueError("Epoch: [{:3d}/{:3d}], step: [{:5d}/{:5d}]. " @@ -87,7 +87,7 @@ class LossMonitor(Callback): if self._per_print_times != 0 and cb_params.cur_step_num % self._per_print_times == 0: print("Epoch: [{:3d}/{:3d}], step: [{:5d}/{:5d}], " "loss: [{:5.4f}/{:5.4f}], time: [{:5.4f}]".format( - cb_params.cur_epoch_num - 1, cb_params.epoch_num, + cb_params.cur_epoch_num, cb_params.epoch_num, cur_step_in_epoch, int(cb_params.batch_num), step_loss, np.mean(self.losses), step_mseconds), flush=True) diff --git a/mindspore/train/serialization.py b/mindspore/train/serialization.py index c39104c6ff..ce776d6821 100644 --- a/mindspore/train/serialization.py +++ b/mindspore/train/serialization.py @@ -29,6 +29,7 @@ from mindspore.common.api import _executor from mindspore.common import dtype as mstype from mindspore._checkparam import check_input_data + __all__ = ["save_checkpoint", "load_checkpoint", "load_param_into_net", "export"] tensor_to_ms_type = {"Int8": mstype.int8, "Uint8": mstype.uint8, "Int16": mstype.int16, "Uint16": mstype.uint16, @@ -40,6 +41,8 @@ tensor_to_np_type = {"Int8": np.int8, "Uint8": np.uint8, "Int16": np.int16, "Uin "Int32": np.int32, "Uint32": np.uint32, "Int64": np.int64, "Uint64": np.uint64, "Float16": np.float16, "Float32": np.float32, "Float64": np.float64, "Bool": np.bool_} +ModelType = ["normal", "fusion", "quant"] + def _special_process_par(par, new_par): """ @@ -101,20 +104,22 @@ def _update_param(param, new_param): param.set_parameter_data(type(param.data)(new_param.data)) -def save_checkpoint(parameter_list, ckpoint_file_name): +def save_checkpoint(parameter_list, ckpt_file_name, model_type="normal"): """ Saves checkpoint info to a specified file. Args: parameter_list (list): Parameters list, each element is a dict like {"name":xx, "type":xx, "shape":xx, "data":xx}. - ckpoint_file_name (str): Checkpoint file name. + ckpt_file_name (str): Checkpoint file name. + model_type (str): The name of model type. Default: "normal". Raises: RuntimeError: Failed to save the Checkpoint file. """ logger.info("Execute save checkpoint process.") checkpoint_list = Checkpoint() + checkpoint_list.model_type = model_type try: for param in parameter_list: @@ -133,22 +138,23 @@ def save_checkpoint(parameter_list, ckpoint_file_name): for dim in param['data'].shape: param_tensor.dims.append(dim) - with open(ckpoint_file_name, "wb") as f: + with open(ckpt_file_name, "wb") as f: f.write(checkpoint_list.SerializeToString()) - os.chmod(ckpoint_file_name, stat.S_IRUSR) + os.chmod(ckpt_file_name, stat.S_IRUSR) except BaseException as e: - logger.error("Failed to save the checkpoint file %s.", ckpoint_file_name) + logger.error("Failed to save the checkpoint file %s.", ckpt_file_name) raise RuntimeError(e.__str__()) logger.info("Save checkpoint process finish.") -def load_checkpoint(ckpoint_file_name, net=None): +def load_checkpoint(ckpt_file_name, model_type="normal", net=None): """ Loads checkpoint info from a specified file. Args: - ckpoint_file_name (str): Checkpoint file name. + ckpt_file_name (str): Checkpoint file name. + model_type (str): The name of model type in `normal`, `fusion` or `quant`. Default: "normal". net (Cell): Cell network. Default: None Returns: @@ -157,28 +163,33 @@ def load_checkpoint(ckpoint_file_name, net=None): Raises: ValueError: Checkpoint file is incorrect. """ - if not isinstance(ckpoint_file_name, str): - raise ValueError("The ckpoint_file_name must be String.") + if not isinstance(ckpt_file_name, str): + raise ValueError("The ckpt_file_name must be string.") - if not os.path.exists(ckpoint_file_name) or ckpoint_file_name[-5:] != ".ckpt": + if model_type not in ModelType: + raise ValueError(f"The model_type is not in {ModelType}.") + + if not os.path.exists(ckpt_file_name) or ckpt_file_name[-5:] != ".ckpt": raise ValueError("Please input the correct checkpoint file name.") - if os.path.getsize(ckpoint_file_name) == 0: + if os.path.getsize(ckpt_file_name) == 0: raise ValueError("The checkpoint file may be empty, please make sure enter the correct file name.") logger.info("Execute load checkpoint process.") checkpoint_list = Checkpoint() try: - with open(ckpoint_file_name, "rb") as f: + with open(ckpt_file_name, "rb") as f: pb_content = f.read() checkpoint_list.ParseFromString(pb_content) except BaseException as e: - logger.error("Failed to read the checkpoint file %s, please check the correct of the file.", ckpoint_file_name) + logger.error("Failed to read the checkpoint file `%s`, please check the correct of the file.", ckpt_file_name) raise ValueError(e.__str__()) parameter_dict = {} - + if model_type != checkpoint_list.model_type: + raise KeyError("Checkpoint file model type({}) is not equal to input model type({}).".format( + checkpoint_list.model_type, model_type)) try: for element in checkpoint_list.value: data = element.tensor.tensor_content @@ -206,7 +217,7 @@ def load_checkpoint(ckpoint_file_name, net=None): logger.info("Load checkpoint process finish.") except BaseException as e: - logger.error("Failed to load the checkpoint file %s.", ckpoint_file_name) + logger.error("Failed to load the checkpoint file `%s`.", ckpt_file_name) raise RuntimeError(e.__str__()) if net: @@ -303,14 +314,15 @@ def _save_graph(network, file_name): os.chmod(file_name, stat.S_IWUSR | stat.S_IRUSR) -def _exec_save_checkpoint(train_network, ckpoint_file_name, integrated_save=True): +def _exec_save_checkpoint(train_network, ckpt_file_name, model_type="normal", integrated_save=True): """ Saves checkpoint for 'ms' backend. Args: train_network (Network): The train network for training. - ckpoint_file_name (str): The name of checkpoint file. - integrated_save (bool): Whether to intergrated save in automatic model parallel scene. + ckpt_file_name (str): The name of checkpoint file. + model_type (str): The name of model type in `normal`, `fusion` or `quant`. Default: "normal". + integrated_save (bool): Whether to integrated save in automatic model parallel scene. """ param_dict = {} @@ -334,7 +346,7 @@ def _exec_save_checkpoint(train_network, ckpoint_file_name, integrated_save=True each_param["data"] = param_data param_list.append(each_param) - save_checkpoint(param_list, ckpoint_file_name) + save_checkpoint(param_list, ckpt_file_name, model_type) def _get_merged_param_data(net, param_name, param_data): diff --git a/model_zoo/lenet/eval.py b/model_zoo/lenet/eval.py index a9842f4426..bcd5503c39 100644 --- a/model_zoo/lenet/eval.py +++ b/model_zoo/lenet/eval.py @@ -20,16 +20,14 @@ python eval.py --data_path /YourDataPath --ckpt_path Your.ckpt import os import argparse -from src.dataset import create_dataset -from src.config import mnist_cfg as cfg -from src.lenet import LeNet5 import mindspore.nn as nn from mindspore import context from mindspore.train.serialization import load_checkpoint, load_param_into_net -from mindspore.train.callback import ModelCheckpoint, CheckpointConfig from mindspore.train import Model from mindspore.nn.metrics import Accuracy - +from src.dataset import create_dataset +from src.config import mnist_cfg as cfg +from src.lenet import LeNet5 if __name__ == "__main__": parser = argparse.ArgumentParser(description='MindSpore Lenet Example') @@ -49,9 +47,6 @@ if __name__ == "__main__": net_loss = nn.SoftmaxCrossEntropyWithLogits(is_grad=False, sparse=True, reduction="mean") repeat_size = cfg.epoch_size net_opt = nn.Momentum(network.trainable_params(), cfg.lr, cfg.momentum) - config_ck = CheckpointConfig(save_checkpoint_steps=cfg.save_checkpoint_steps, - keep_checkpoint_max=cfg.keep_checkpoint_max) - ckpoint_cb = ModelCheckpoint(prefix="checkpoint_lenet", config=config_ck) model = Model(network, net_loss, net_opt, metrics={"Accuracy": Accuracy()}) print("============== Starting Testing ==============") diff --git a/model_zoo/lenet_quant/README.md b/model_zoo/lenet_quant/README.md index 26cdcc3ecd..2f949f6d76 100644 --- a/model_zoo/lenet_quant/README.md +++ b/model_zoo/lenet_quant/README.md @@ -128,9 +128,9 @@ After all the following we will get the loss value of each step as following: ```bash >>> Epoch: [ 1/ 10] step: [ 1/ 900], loss: [2.3040/2.5234], time: [1.300234] >>> ... ->>> Epoch: [ 10/ 10] step: [887/ 900], loss: [0.0113/0.0223], time: [1.300234] ->>> Epoch: [ 10/ 10] step: [888/ 900], loss: [0.0334/0.0223], time: [1.300234] ->>> Epoch: [ 10/ 10] step: [889/ 900], loss: [0.0233/0.0223], time: [1.300234] +>>> Epoch: [ 9/ 10] step: [887/ 900], loss: [0.0113/0.0223], time: [1.300234] +>>> Epoch: [ 9/ 10] step: [888/ 900], loss: [0.0334/0.0223], time: [1.300234] +>>> Epoch: [ 9/ 10] step: [889/ 900], loss: [0.0233/0.0223], time: [1.300234] ``` Also, you can just run this command instead. @@ -197,9 +197,9 @@ After all the following we will get the loss value of each step as following: ```bash >>> Epoch: [ 1/ 10] step: [ 1/ 900], loss: [2.3040/2.5234], time: [1.300234] >>> ... ->>> Epoch: [ 10/ 10] step: [887/ 900], loss: [0.0113/0.0223], time: [1.300234] ->>> Epoch: [ 10/ 10] step: [888/ 900], loss: [0.0334/0.0223], time: [1.300234] ->>> Epoch: [ 10/ 10] step: [889/ 900], loss: [0.0233/0.0223], time: [1.300234] +>>> Epoch: [ 9/ 10] step: [887/ 900], loss: [0.0113/0.0223], time: [1.300234] +>>> Epoch: [ 9/ 10] step: [888/ 900], loss: [0.0334/0.0223], time: [1.300234] +>>> Epoch: [ 9/ 10] step: [889/ 900], loss: [0.0233/0.0223], time: [1.300234] ``` ### Evaluate quantization aware model @@ -215,7 +215,7 @@ param_dict = load_checkpoint(args.ckpt_path) load_param_into_net(network, param_dict) # convert funsion netwrok to quantization aware network -network = quant.convert_quant_network(network +network = quant.convert_quant_network(network) ``` Also, you can just run this command insread. diff --git a/model_zoo/lenet_quant/eval.py b/model_zoo/lenet_quant/eval.py index c1e3a5fd8c..d94e77279f 100644 --- a/model_zoo/lenet_quant/eval.py +++ b/model_zoo/lenet_quant/eval.py @@ -23,7 +23,6 @@ import argparse import mindspore.nn as nn from mindspore import context from mindspore.train.serialization import load_checkpoint, load_param_into_net -from mindspore.train.callback import ModelCheckpoint, CheckpointConfig from mindspore.train import Model from mindspore.nn.metrics import Accuracy from src.dataset import create_dataset @@ -47,16 +46,18 @@ if __name__ == "__main__": ds_eval = create_dataset(os.path.join(args.data_path, "test"), cfg.batch_size, 1) step_size = ds_eval.get_dataset_size() + # define fusion network network = LeNet5Fusion(cfg.num_classes) + # define loss net_loss = nn.SoftmaxCrossEntropyWithLogits(is_grad=False, sparse=True, reduction="mean") - repeat_size = cfg.epoch_size + # define network optimization net_opt = nn.Momentum(network.trainable_params(), cfg.lr, cfg.momentum) - config_ck = CheckpointConfig(save_checkpoint_steps=cfg.epoch_size * step_size, - keep_checkpoint_max=cfg.keep_checkpoint_max) - ckpoint_cb = ModelCheckpoint(prefix="checkpoint_lenet", config=config_ck) + + # call back and monitor model = Model(network, net_loss, net_opt, metrics={"Accuracy": Accuracy()}) - param_dict = load_checkpoint(args.ckpt_path) + # load check point into network + param_dict = load_checkpoint(args.ckpt_path, network.type) load_param_into_net(network, param_dict) print("============== Starting Testing ==============") diff --git a/model_zoo/lenet_quant/eval_quant.py b/model_zoo/lenet_quant/eval_quant.py index 492f6d36b2..2c2477123f 100644 --- a/model_zoo/lenet_quant/eval_quant.py +++ b/model_zoo/lenet_quant/eval_quant.py @@ -23,7 +23,6 @@ import argparse import mindspore.nn as nn from mindspore import context from mindspore.train.serialization import load_checkpoint, load_param_into_net -from mindspore.train.callback import ModelCheckpoint, CheckpointConfig from mindspore.train import Model from mindspore.nn.metrics import Accuracy from mindspore.train.quant import quant @@ -48,20 +47,21 @@ if __name__ == "__main__": ds_eval = create_dataset(os.path.join(args.data_path, "test"), cfg.batch_size, 1) step_size = ds_eval.get_dataset_size() - # define funsion network + # define fusion network network = LeNet5Fusion(cfg.num_classes) - # convert funsion netwrok to quantization aware network + # convert fusion netwrok to quantization aware network network = quant.convert_quant_network(network, quant_delay=0, bn_fold=False, freeze_bn=10000) + # define loss net_loss = nn.SoftmaxCrossEntropyWithLogits(is_grad=False, sparse=True, reduction="mean") + # define network optimization net_opt = nn.Momentum(network.trainable_params(), cfg.lr, cfg.momentum) - config_ck = CheckpointConfig(save_checkpoint_steps=cfg.epoch_size * step_size, - keep_checkpoint_max=cfg.keep_checkpoint_max) - ckpoint_cb = ModelCheckpoint(prefix="checkpoint_lenet", config=config_ck) + + # call back and monitor model = Model(network, net_loss, net_opt, metrics={"Accuracy": Accuracy()}) # load quantization aware network checkpoint - param_dict = load_checkpoint(args.ckpt_path) + param_dict = load_checkpoint(args.ckpt_path, model_type="quant") load_param_into_net(network, param_dict) print("============== Starting Testing ==============") diff --git a/model_zoo/lenet_quant/src/lenet.py b/model_zoo/lenet_quant/src/lenet.py index 026f1e8df5..1efcf9e7d7 100644 --- a/model_zoo/lenet_quant/src/lenet.py +++ b/model_zoo/lenet_quant/src/lenet.py @@ -34,8 +34,8 @@ class LeNet5(nn.Cell): super(LeNet5, self).__init__() self.num_class = num_class - self.conv1 = nn.Conv2d(channel, 6, 5) - self.conv2 = nn.Conv2d(6, 16, 5) + self.conv1 = nn.Conv2d(channel, 6, 5, pad_mode='valid') + self.conv2 = nn.Conv2d(6, 16, 5, pad_mode='valid') self.fc1 = nn.Dense(16 * 5 * 5, 120) self.fc2 = nn.Dense(120, 84) self.fc3 = nn.Dense(84, self.num_class) diff --git a/model_zoo/lenet_quant/src/lenet_fusion.py b/model_zoo/lenet_quant/src/lenet_fusion.py index 809276a482..88b3593502 100644 --- a/model_zoo/lenet_quant/src/lenet_fusion.py +++ b/model_zoo/lenet_quant/src/lenet_fusion.py @@ -32,11 +32,12 @@ class LeNet5(nn.Cell): def __init__(self, num_class=10, channel=1): super(LeNet5, self).__init__() + self.type = "fusion" self.num_class = num_class # change `nn.Conv2d` to `nn.Conv2dBnAct` - self.conv1 = nn.Conv2dBnAct(channel, 6, 5, activation='relu') - self.conv2 = nn.Conv2dBnAct(6, 16, 5, activation='relu') + self.conv1 = nn.Conv2dBnAct(channel, 6, 5, pad_mode='valid', activation='relu') + self.conv2 = nn.Conv2dBnAct(6, 16, 5, pad_mode='valid', activation='relu') # change `nn.Dense` to `nn.DenseBnAct` self.fc1 = nn.DenseBnAct(16 * 5 * 5, 120, activation='relu') self.fc2 = nn.DenseBnAct(120, 84, activation='relu') diff --git a/model_zoo/lenet_quant/train.py b/model_zoo/lenet_quant/train.py index 6e7a46fb38..b6040776ef 100644 --- a/model_zoo/lenet_quant/train.py +++ b/model_zoo/lenet_quant/train.py @@ -46,16 +46,24 @@ if __name__ == "__main__": ds_train = create_dataset(os.path.join(args.data_path, "train"), cfg.batch_size, cfg.epoch_size) step_size = ds_train.get_dataset_size() + # define fusion network network = LeNet5Fusion(cfg.num_classes) + # define network loss net_loss = nn.SoftmaxCrossEntropyWithLogits(is_grad=False, sparse=True, reduction="mean") + # define network optimization net_opt = nn.Momentum(network.trainable_params(), cfg.lr, cfg.momentum) + + # call back and monitor time_cb = TimeMonitor(data_size=ds_train.get_dataset_size()) - config_ck = CheckpointConfig(save_checkpoint_steps=cfg.epoch_size * step_size, - keep_checkpoint_max=cfg.keep_checkpoint_max) - ckpoint_cb = ModelCheckpoint(prefix="checkpoint_lenet", config=config_ck) + config_ckpt = CheckpointConfig(save_checkpoint_steps=cfg.epoch_size * step_size, + keep_checkpoint_max=cfg.keep_checkpoint_max, + model_type=network.type) + ckpt_callback = ModelCheckpoint(prefix="checkpoint_lenet", config=config_ckpt) + + # define model model = Model(network, net_loss, net_opt, metrics={"Accuracy": Accuracy()}) print("============== Starting Training ==============") - model.train(cfg['epoch_size'], ds_train, callbacks=[time_cb, ckpoint_cb, LossMonitor()], + model.train(cfg['epoch_size'], ds_train, callbacks=[time_cb, ckpt_callback, LossMonitor()], dataset_sink_mode=args.dataset_sink_mode) print("============== End Training ==============") diff --git a/model_zoo/lenet_quant/train_quant.py b/model_zoo/lenet_quant/train_quant.py index 04f595f322..eb1f783a7c 100644 --- a/model_zoo/lenet_quant/train_quant.py +++ b/model_zoo/lenet_quant/train_quant.py @@ -48,23 +48,30 @@ if __name__ == "__main__": ds_train = create_dataset(os.path.join(args.data_path, "train"), cfg.batch_size, cfg.epoch_size) step_size = ds_train.get_dataset_size() - # define funsion network + # define fusion network network = LeNet5Fusion(cfg.num_classes) # load quantization aware network checkpoint - param_dict = load_checkpoint(args.ckpt_path) + param_dict = load_checkpoint(args.ckpt_path, network.type) load_param_into_net(network, param_dict) - # convert funsion netwrok to quantization aware network + # convert fusion network to quantization aware network network = quant.convert_quant_network(network, quant_delay=0, bn_fold=False, freeze_bn=10000) + # define network loss net_loss = nn.SoftmaxCrossEntropyWithLogits(is_grad=False, sparse=True, reduction="mean") + # define network optimization net_opt = nn.Momentum(network.trainable_params(), cfg.lr, cfg.momentum) + + # call back and monitor time_cb = TimeMonitor(data_size=ds_train.get_dataset_size()) - config_ck = CheckpointConfig(save_checkpoint_steps=cfg.epoch_size * step_size, - keep_checkpoint_max=cfg.keep_checkpoint_max) - ckpoint_cb = ModelCheckpoint(prefix="checkpoint_lenet", config=config_ck) + config_ckpt = CheckpointConfig(save_checkpoint_steps=cfg.epoch_size * step_size, + keep_checkpoint_max=cfg.keep_checkpoint_max, + model_type="quant") + ckpt_callback = ModelCheckpoint(prefix="checkpoint_lenet", config=config_ckpt) + + # define model model = Model(network, net_loss, net_opt, metrics={"Accuracy": Accuracy()}) print("============== Starting Training ==============") - model.train(cfg['epoch_size'], ds_train, callbacks=[time_cb, ckpoint_cb, LossMonitor()], + model.train(cfg['epoch_size'], ds_train, callbacks=[time_cb, ckpt_callback, LossMonitor()], dataset_sink_mode=args.dataset_sink_mode) print("============== End Training ==============") diff --git a/tests/ut/python/predict/test_predict_save_model.py b/tests/ut/python/predict/test_predict_save_model.py index 4f5fe16ad4..f57875d073 100644 --- a/tests/ut/python/predict/test_predict_save_model.py +++ b/tests/ut/python/predict/test_predict_save_model.py @@ -85,7 +85,7 @@ if __name__ == '__main__': is_ckpt_exist = os.path.exists(ckpt_file_path) if is_ckpt_exist: - param_dict = load_checkpoint(ckpoint_file_name=ckpt_file_path) + param_dict = load_checkpoint(ckpt_file_name=ckpt_file_path) load_param_into_net(net, param_dict) export(net, input_data, file_name=model_path_name, file_format='LITE') print("test lenet predict success.") diff --git a/tests/ut/python/utils/test_serialize.py b/tests/ut/python/utils/test_serialize.py index 19e9bd72e6..c5b4586566 100644 --- a/tests/ut/python/utils/test_serialize.py +++ b/tests/ut/python/utils/test_serialize.py @@ -111,19 +111,19 @@ def test_save_checkpoint(): os.chmod('./parameters.ckpt', stat.S_IWRITE) os.remove('./parameters.ckpt') - ckpoint_file_name = os.path.join(_cur_dir, './parameters.ckpt') - save_checkpoint(parameter_list, ckpoint_file_name) + ckpt_file_name = os.path.join(_cur_dir, './parameters.ckpt') + save_checkpoint(parameter_list, ckpt_file_name) def test_load_checkpoint_error_filename(): - ckpoint_file_name = 1 + ckpt_file_name = 1 with pytest.raises(ValueError): - load_checkpoint(ckpoint_file_name) + load_checkpoint(ckpt_file_name) def test_load_checkpoint(): - ckpoint_file_name = os.path.join(_cur_dir, './parameters.ckpt') - par_dict = load_checkpoint(ckpoint_file_name) + ckpt_file_name = os.path.join(_cur_dir, './parameters.ckpt') + par_dict = load_checkpoint(ckpt_file_name) assert len(par_dict) == 3 assert par_dict['param_test'].name == 'param_test' @@ -136,17 +136,17 @@ def test_checkpoint_manager(): """ test_checkpoint_manager """ ckp_mgr = _CheckpointManager() - ckpoint_file_name = os.path.join(_cur_dir, './test1.ckpt') - with open(ckpoint_file_name, 'w'): - os.chmod(ckpoint_file_name, stat.S_IWUSR | stat.S_IRUSR) + ckpt_file_name = os.path.join(_cur_dir, './test1.ckpt') + with open(ckpt_file_name, 'w'): + os.chmod(ckpt_file_name, stat.S_IWUSR | stat.S_IRUSR) ckp_mgr.update_ckpoint_filelist(_cur_dir, "test") assert ckp_mgr.ckpoint_num == 1 - ckp_mgr.remove_ckpoint_file(ckpoint_file_name) + ckp_mgr.remove_ckpoint_file(ckpt_file_name) ckp_mgr.update_ckpoint_filelist(_cur_dir, "test") assert ckp_mgr.ckpoint_num == 0 - assert not os.path.exists(ckpoint_file_name) + assert not os.path.exists(ckpt_file_name) another_file_name = os.path.join(_cur_dir, './test2.ckpt') another_file_name = os.path.realpath(another_file_name) @@ -283,7 +283,7 @@ def test_exec_save_checkpoint(): loss_net = WithLossCell(net, loss) train_network = TrainOneStepCell(loss_net, opt) - _exec_save_checkpoint(train_network, ckpoint_file_name="./new_ckpt.ckpt") + _exec_save_checkpoint(train_network, ckpt_file_name="./new_ckpt.ckpt") load_checkpoint("new_ckpt.ckpt") From 8ce081801a912385e9f5b486d7653c9da2649f0e Mon Sep 17 00:00:00 2001 From: liuwenhao4 Date: Tue, 23 Jun 2020 22:53:45 +0800 Subject: [PATCH 044/254] Fix some mistakes of ApplyCenteredRMSProp, aSinh and Sinh vm ops --- mindspore/ops/operations/math_ops.py | 4 ++-- mindspore/ops/operations/nn_ops.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mindspore/ops/operations/math_ops.py b/mindspore/ops/operations/math_ops.py index 08cd481582..af0deacc16 100644 --- a/mindspore/ops/operations/math_ops.py +++ b/mindspore/ops/operations/math_ops.py @@ -1888,7 +1888,7 @@ class Cosh(PrimitiveWithInfer): class Asinh(PrimitiveWithInfer): """ - Compute inverse hyperbolic cosine of x element-wise. + Compute inverse hyperbolic sine of x element-wise. Inputs: - **input_x** (Tensor) - The shape of tensor is :math:`(x_1, x_2, ..., x_R)`. @@ -2645,7 +2645,7 @@ class Sin(PrimitiveWithInfer): class Asin(PrimitiveWithInfer): """ - Computes arccosine of input element-wise. + Computes arcsine of input element-wise. Inputs: - **input_x** (Tensor) - The shape of tensor is :math:`(x_1, x_2, ..., x_R)`. diff --git a/mindspore/ops/operations/nn_ops.py b/mindspore/ops/operations/nn_ops.py index 28944f8b4e..507af3642e 100644 --- a/mindspore/ops/operations/nn_ops.py +++ b/mindspore/ops/operations/nn_ops.py @@ -1905,7 +1905,7 @@ class ApplyCenteredRMSProp(PrimitiveWithInfer): >>> mean_grad = Tensor(np.arange(12).astype(np.float32).reshape(2, 3, 2), mindspore.float32) >>> mean_square = Tensor(np.arange(-8, 4).astype(np.float32).reshape(2, 3, 2), mindspore.float32) >>> moment = Tensor(np.arange(12).astype(np.float32).reshape(2, 3, 2), mindspore.float32) - >>> grad = Tensor(np.arange(12).astype(np.float32).rehspae(2, 3, 2), mindspore.float32) + >>> grad = Tensor(np.arange(12).astype(np.float32).reshape(2, 3, 2), mindspore.float32) >>> learning_rate = Tensor(0.9, mindspore.float32) >>> decay = 0.0 >>> momentum = 1e-10 From e991df44e40e3badea81c578327931df5301c318 Mon Sep 17 00:00:00 2001 From: liuxiao Date: Tue, 23 Jun 2020 19:52:54 +0800 Subject: [PATCH 045/254] fix api and check for some optimizer operators --- mindspore/ops/operations/nn_ops.py | 330 ++++++++++++++++++++--------- 1 file changed, 227 insertions(+), 103 deletions(-) diff --git a/mindspore/ops/operations/nn_ops.py b/mindspore/ops/operations/nn_ops.py index 28944f8b4e..b6556d01f4 100644 --- a/mindspore/ops/operations/nn_ops.py +++ b/mindspore/ops/operations/nn_ops.py @@ -3157,16 +3157,23 @@ class ApplyAdaMax(PrimitiveWithInfer): :math:`\epsilon` represents `epsilon`. Inputs: - - **var** (Parameter) - Variable to be updated. + - **var** (Parameter) - Variable to be updated. With float32 or float16 data type. - **m** (Parameter) - The 1st moment vector in the updating formula. Has the same shape and type as `var`. + With float32 or float16 data type. - **v** (Parameter) - The 2nd moment vector in the updating formula. Mean square gradients, - has the same shape and type as `var`. - - **beta1_power** (float) - :math:`beta_1^t` in the updating formula. - - **lr** (float) - Learning rate, :math:`l` in the updating formula. Has the same type as `var`. - - **beta1** (float) - The exponential decay rate for the 1st moment estimates. - - **beta2** (float) - The exponential decay rate for the 2nd moment estimates. - - **epsilon** (float) - A small value added for numerical stability. + has the same shape and type as `var`. With float32 or float16 data type. + - **beta1_power** (Union[Number, Tensor]) - :math:`beta_1^t` in the updating formula, should be scalar. + With float32 or float16 data type. + - **lr** (Union[Number, Tensor]) - Learning rate, :math:`l` in the updating formula, should be scalar. + With float32 or float16 data type. + - **beta1** (Union[Number, Tensor]) - The exponential decay rate for the 1st moment estimates, + should be scalar. With float32 or float16 data type. + - **beta2** (Union[Number, Tensor]) - The exponential decay rate for the 2nd moment estimates, + should be scalar. With float32 or float16 data type. + - **epsilon** (Union[Number, Tensor]) - A small value added for numerical stability, should be scalar. + With float32 or float16 data type. - **grad** (Tensor) - A tensor for gradient. Has the same shape and type as `var`. + With float32 or float16 data type. Outputs: Tuple of 3 Tensor, the updated parameters. @@ -3176,17 +3183,29 @@ class ApplyAdaMax(PrimitiveWithInfer): - **v** (Tensor) - The same shape and data type as `v`. Examples: - >>> var = Parameter(Tensor(np.random.rand(3, 3).astype(np.float32)), name="var") - >>> m = Parameter(Tensor(np.random.rand(3, 3).astype(np.float32)), name="m") - >>> v = Parameter(Tensor(np.random.rand(3, 3).astype(np.float32)), name="v") + >>> import numpy as np + >>> import mindspore.nn as nn + >>> from mindspore import Tensor, Parameter + >>> from mindspore.ops import operations as P + >>> import mindspore.common.dtype as mstype + >>> class Net(nn.Cell): + >>> def __init__(self): + >>> super(Net, self).__init__() + >>> self.apply_ada_max = P.ApplyAdaMax() + >>> self.var = Parameter(Tensor(np.random.rand(3, 3).astype(np.float32)), name="var") + >>> self.m = Parameter(Tensor(np.random.rand(3, 3).astype(np.float32)), name="m") + >>> self.v = Parameter(Tensor(np.random.rand(3, 3).astype(np.float32)), name="v") + >>> def construct(self, beta1_power, lr, beta1, beta2, epsilon, grad): + >>> out = self.apply_ada_max(self.var, self.m, self.v, beta1_power, lr, beta1, beta2, epsilon, grad) + >>> return out + >>> net = Net() + >>> beta1_power =Tensor(0.9, mstype.float32) + >>> lr = Tensor(0.001, mstype.float32) + >>> beta1 = Tensor(0.9, mstype.float32) + >>> beta2 = Tensor(0.99, mstype.float32) + >>> epsilon = Tensor(1e-10, mstype.float32) >>> grad = Tensor(np.random.rand(3, 3).astype(np.float32)) - >>> beta1_power = 0.9 - >>> lr = 0.001 - >>> beta1 = 0.9 - >>> beta2 = 0.99 - >>> epsilon = 1e-10 - >>> apply_ada_max = P.ApplyAdaMax() - >>> output = apply_ada_max(var, m, v, beta1_power, lr, beta1, beta2, epsilon, grad) + >>> result = net(beta1_power, lr, beta1, beta2, epsilon, grad) """ __mindspore_signature__ = ( @@ -3194,11 +3213,11 @@ class ApplyAdaMax(PrimitiveWithInfer): ('m', sig_rw.RW_WRITE, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), ('v', sig_rw.RW_WRITE, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), ('beta1_power', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, - sig_dtype.T), - ('lr', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), - ('beta1', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), - ('beta2', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), - ('epsilon', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), + sig_dtype.T1), + ('lr', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T2), + ('beta1', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T3), + ('beta2', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T4), + ('epsilon', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T5), ('grad', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T) ) @@ -3208,19 +3227,41 @@ class ApplyAdaMax(PrimitiveWithInfer): def infer_shape(self, var_shape, m_shape, v_shape, beta1_power_shape, lr_shape, beta1_shape, beta2_shape, epsilon_shape, grad_shape): - validator.check("var_shape", var_shape, "m_shape", m_shape, Rel.EQ, self.name) - validator.check("var_shape", var_shape, "v_shape", v_shape, Rel.EQ, self.name) - validator.check("var_shape", var_shape, "grad_shape", grad_shape, Rel.EQ, self.name) + validator.check("m_shape", m_shape, "var_shape", var_shape, Rel.EQ, self.name) + validator.check("v_shape", v_shape, "var_shape", var_shape, Rel.EQ, self.name) + validator.check("grad_shape", grad_shape, "var_shape", var_shape, Rel.EQ, self.name) + beta1_power_shp_len = len(beta1_power_shape) + validator.check_integer("beta1 power's rank", beta1_power_shp_len, 1, Rel.LE, self.name) + if beta1_power_shp_len == 1: + validator.check_integer("beta1_power_shape[0]", beta1_power_shape[0], 1, Rel.EQ, self.name) + lr_shp_len = len(lr_shape) + validator.check_integer("lr's rank", lr_shp_len, 1, Rel.LE, self.name) + if lr_shp_len == 1: + validator.check_integer("lr_shape[0]", lr_shape[0], 1, Rel.EQ, self.name) + beta1_shp_len = len(beta1_shape) + validator.check_integer("beta1's rank", beta1_shp_len, 1, Rel.LE, self.name) + if beta1_shp_len == 1: + validator.check_integer("beta1_shape[0]", beta1_shape[0], 1, Rel.EQ, self.name) + beta2_shp_len = len(beta2_shape) + validator.check_integer("beta2's rank", beta2_shp_len, 1, Rel.LE, self.name) + if beta2_shp_len == 1: + validator.check_integer("beta2_shape[0]", beta2_shape[0], 1, Rel.EQ, self.name) + epsilon_shp_len = len(epsilon_shape) + validator.check_integer("epsilon's rank", epsilon_shp_len, 1, Rel.LE, self.name) + if epsilon_shp_len == 1: + validator.check_integer("epsilon_shape[0]", epsilon_shape[0], 1, Rel.EQ, self.name) return var_shape, m_shape, v_shape def infer_dtype(self, var_dtype, m_dtype, v_dtype, beta1_power_dtype, lr_dtype, beta1_dtype, beta2_dtype, epsilon_dtype, grad_dtype): + valid_types = [mstype.float16, mstype.float32] args = {"var": var_dtype, "m": m_dtype, "v": v_dtype, "grad": grad_dtype} - validator.check_tensor_type_same(args, mstype.number_type, self.name) - - scalar_args = {"beta1_power": beta1_power_dtype, 'lr': lr_dtype, "beta1": beta1_dtype, - "beta2": beta2_dtype, "epsilon": epsilon_dtype} - validator.check_scalar_or_tensor_type_same(scalar_args, [mstype.float16, mstype.float32], self.name, True) + validator.check_tensor_type_same(args, valid_types, self.name) + validator.check_scalar_or_tensor_type_same({"beta1_power": beta1_power_dtype}, valid_types, self.name) + validator.check_scalar_or_tensor_type_same({"lr": lr_dtype}, valid_types, self.name) + validator.check_scalar_or_tensor_type_same({"beta1": beta1_dtype}, valid_types, self.name) + validator.check_scalar_or_tensor_type_same({"beta2": beta2_dtype}, valid_types, self.name) + validator.check_scalar_or_tensor_type_same({"epsilon": epsilon_dtype}, valid_types, self.name) return var_dtype, m_dtype, v_dtype @@ -3238,13 +3279,16 @@ class ApplyAdadelta(PrimitiveWithInfer): var -= lr * update Inputs: - - **var** (Parameter) - Weights to be updated. + - **var** (Parameter) - Weights to be updated. With float32 or float16 data type. - **accum** (Parameter) - Accum to be updated, has the same shape and type as `var`. + With float32 or float16 data type. - **accum_update** (Parameter) - Accum_update to be updated, has the same shape and type as `var`. - - **lr** (float) - Learning rate, has the same type as `var`. - - **rho** (float) - Decay rate. - - **epsilon** (float) - A small value added for numerical stability. - - **grad** (Tensor) - Gradients, has the same shape and type as `var`. + With float32 or float16 data type. + - **lr** (Union[Number, Tensor]) - Learning rate, must be scalar. With float32 or float16 data type. + - **rho** (Union[Number, Tensor]) - Decay rate, must be scalar. With float32 or float16 data type. + - **epsilon** (Union[Number, Tensor]) - A small value added for numerical stability, must be scalar. + With float32 or float16 data type. + - **grad** (Tensor) - Gradients, has the same shape and type as `var`. With float32 or float16 data type. Outputs: Tuple of 3 Tensor, the updated parameters. @@ -3254,15 +3298,27 @@ class ApplyAdadelta(PrimitiveWithInfer): - **accum_update** (Tensor) - The same shape and data type as `accum_update`. Examples: - >>> var = Parameter(Tensor(np.random.rand(3, 3).astype(np.float32)), name="var") - >>> accum = Parameter(Tensor(np.random.rand(3, 3).astype(np.float32)), name="accum") - >>> accum_update = Parameter(Tensor(np.random.rand(3, 3).astype(np.float32)), name="accum_update") + >>> import numpy as np + >>> import mindspore.nn as nn + >>> from mindspore import Tensor, Parameter + >>> from mindspore.ops import operations as P + >>> import mindspore.common.dtype as mstype + >>> class Net(nn.Cell): + >>> def __init__(self): + >>> super(Net, self).__init__() + >>> self.apply_adadelta = P.ApplyAdadelta() + >>> self.var = Parameter(Tensor(np.random.rand(3, 3).astype(np.float32)), name="var") + >>> self.accum = Parameter(Tensor(np.random.rand(3, 3).astype(np.float32)), name="accum") + >>> self.accum_update = Parameter(Tensor(np.random.rand(3, 3).astype(np.float32)), name="accum_update") + >>> def construct(self, lr, rho, epsilon, grad): + >>> out = self.apply_adadelta(self.var, self.accum, self.accum_update, lr, rho, epsilon, grad) + >>> return out + >>> net = Net() + >>> lr = Tensor(0.001, mstype.float32) + >>> rho = Tensor(0.0, mstype.float32) + >>> epsilon = Tensor(1e-6, mstype.float32) >>> grad = Tensor(np.random.rand(3, 3).astype(np.float32)) - >>> lr = 0.001 - >>> rho = 0.0 - >>> epsilon = 1e-6 - >>> apply_adadelta = P.ApplyAdadelta() - >>> output = apply_adadelta(var, accum, accum_update, lr, rho, epsilon, grad) + >>> result = net(lr, rho, epsilon, grad) """ __mindspore_signature__ = ( @@ -3270,9 +3326,9 @@ class ApplyAdadelta(PrimitiveWithInfer): ('accum', sig_rw.RW_WRITE, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), ('accum_update', sig_rw.RW_WRITE, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), - ('lr', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), - ('rho', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), - ('epsilon', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), + ('lr', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T1), + ('rho', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T2), + ('epsilon', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T3), ('grad', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T) ) @@ -3282,18 +3338,31 @@ class ApplyAdadelta(PrimitiveWithInfer): def infer_shape(self, var_shape, accum_shape, accum_update_shape, lr_shape, rho_shape, epsilon_shape, grad_shape): - validator.check("var_shape", var_shape, "accum_shape", accum_shape, Rel.EQ, self.name) - validator.check("var_shape", var_shape, "accum_update_shape", accum_update_shape, Rel.EQ, self.name) - validator.check("var_shape", var_shape, "grad_shape", grad_shape, Rel.EQ, self.name) + validator.check("accum_shape", accum_shape, "var_shape", var_shape, Rel.EQ, self.name) + validator.check("accum_update_shape", accum_update_shape, "var_shape", var_shape, Rel.EQ, self.name) + validator.check("grad_shape", grad_shape, "var_shape", var_shape, Rel.EQ, self.name) + lr_shp_len = len(lr_shape) + validator.check_integer("lr's rank", lr_shp_len, 1, Rel.LE, self.name) + if lr_shp_len == 1: + validator.check_integer("lr_shape[0]", lr_shape[0], 1, Rel.EQ, self.name) + rho_shp_len = len(rho_shape) + validator.check_integer("rho's rank", rho_shp_len, 1, Rel.LE, self.name) + if rho_shp_len == 1: + validator.check_integer("rho_shape[0]", rho_shape[0], 1, Rel.EQ, self.name) + epsilon_shp_len = len(epsilon_shape) + validator.check_integer("lepsilon's rank", epsilon_shp_len, 1, Rel.LE, self.name) + if epsilon_shp_len == 1: + validator.check_integer("epsilon_shape[0]", epsilon_shape[0], 1, Rel.EQ, self.name) return var_shape, accum_shape, accum_update_shape - def infer_dtype(self, var_dtype, accum_dtype, accum_update_dtype, lr_dtype, rho_shape, + def infer_dtype(self, var_dtype, accum_dtype, accum_update_dtype, lr_dtype, rho_dtype, epsilon_dtype, grad_dtype): + valid_types = [mstype.float16, mstype.float32] args = {"var": var_dtype, "accum": accum_dtype, "accum_update": accum_update_dtype, "grad": grad_dtype} - validator.check_tensor_type_same(args, mstype.number_type, self.name) - - scalar_args = {"lr": lr_dtype, "rho": rho_shape, "epsilon": epsilon_dtype} - validator.check_scalar_or_tensor_type_same(scalar_args, [mstype.float16, mstype.float32], self.name, True) + validator.check_tensor_type_same(args, valid_types, self.name) + validator.check_scalar_or_tensor_type_same({"lr": lr_dtype}, valid_types, self.name) + validator.check_scalar_or_tensor_type_same({"rho": rho_dtype}, valid_types, self.name) + validator.check_scalar_or_tensor_type_same({"epsilon": epsilon_dtype}, valid_types, self.name) return var_dtype, accum_dtype, accum_update_dtype @@ -3310,10 +3379,12 @@ class ApplyAdagrad(PrimitiveWithInfer): update_slots (bool): If `True`, `accum` will be updated. Default: True. Inputs: - - **var** (Parameter) - Variable to be updated. + - **var** (Parameter) - Variable to be updated. With float32 or float16 data type. - **accum** (Parameter) - Accum to be updated. The shape and dtype should be the same as `var`. - - **lr** (float): The learning rate value, has the same type as `var`. + With float32 or float16 data type. + - **lr** (Union[Number, Tensor]): The learning rate value, should be scalar. With float32 or float16 data type. - **grad** (Tensor) - A tensor for gradient. The shape and dtype should be the same as `var`. + With float32 or float16 data type. Outputs: Tuple of 2 Tensor, the updated parameters. @@ -3322,18 +3393,30 @@ class ApplyAdagrad(PrimitiveWithInfer): - **accum** (Tensor) - The same shape and data type as `accum`. Examples: - >>> var = Parameter(Tensor(np.random.rand(3, 3).astype(np.float32)), name="var") - >>> accum = Parameter(Tensor(np.random.rand(3, 3).astype(np.float32)), name="accum") + >>> import numpy as np + >>> import mindspore.nn as nn + >>> from mindspore import Tensor, Parameter + >>> from mindspore.ops import operations as P + >>> import mindspore.common.dtype as mstype + >>> class Net(nn.Cell): + >>> def __init__(self): + >>> super(Net, self).__init__() + >>> self.apply_adagrad = P.ApplyAdagrad() + >>> self.var = Parameter(Tensor(np.random.rand(3, 3).astype(np.float32)), name="var") + >>> self.accum = Parameter(Tensor(np.random.rand(3, 3).astype(np.float32)), name="accum") + >>> def construct(self, lr, grad): + >>> out = self.apply_adagrad(self.var, self.accum, lr, grad) + >>> return out + >>> net = Net() + >>> lr = Tensor(0.001, mstype.float32) >>> grad = Tensor(np.random.rand(3, 3).astype(np.float32)) - >>> lr = 0.01 - >>> apply_adagrad = P.ApplyAdagrad() - >>> output = apply_adagrad(var, accum, lr, grad) + >>> result = net(lr, grad) """ __mindspore_signature__ = ( ('var', sig_rw.RW_WRITE, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), ('accum', sig_rw.RW_WRITE, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), - ('lr', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), + ('lr', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T1), ('grad', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T) ) @@ -3342,14 +3425,18 @@ class ApplyAdagrad(PrimitiveWithInfer): validator.check_value_type("update_slots", update_slots, [bool], self.name) def infer_shape(self, var_shape, accum_shape, lr_shape, grad_shape): - validator.check('var shape', var_shape, 'accum shape', accum_shape, Rel.EQ, self.name) - validator.check('var shape', var_shape, 'grad shape', grad_shape, Rel.EQ, self.name) + validator.check('accum shape', accum_shape, 'var shape', var_shape, Rel.EQ, self.name) + validator.check('grad shape', grad_shape, 'var shape', var_shape, Rel.EQ, self.name) + lr_shp_len = len(lr_shape) + validator.check_integer("lr's rank", lr_shp_len, 1, Rel.LE, self.name) + if lr_shp_len == 1: + validator.check_integer("lr_shape[0]", lr_shape[0], 1, Rel.EQ, self.name) return var_shape, accum_shape def infer_dtype(self, var_dtype, accum_dtype, lr_dtype, grad_dtype): args = {'var': var_dtype, 'accum': accum_dtype, 'grad': grad_dtype} - validator.check_tensor_type_same(args, mstype.number_type, self.name) valid_types = [mstype.float16, mstype.float32] + validator.check_tensor_type_same(args, valid_types, self.name) validator.check_scalar_or_tensor_type_same({'lr': lr_dtype}, valid_types, self.name) return var_dtype, accum_dtype @@ -3368,10 +3455,12 @@ class ApplyAdagradV2(PrimitiveWithInfer): update_slots (bool): If `True`, `accum` will be updated. Default: True. Inputs: - - **var** (Parameter) - Variable to be updated. + - **var** (Parameter) - Variable to be updated. With float32 or float16 data type. - **accum** (Parameter) - Accum to be updated. The shape and dtype should be the same as `var`. - - **lr** (float): The learning rate value, has the same type as `var`. + With float32 or float16 data type. + - **lr** (Union[Number, Tensor]): The learning rate value, should be scalar. With float32 or float16 data type. - **grad** (Tensor) - A tensor for gradient. The shape and dtype should be the same as `var`. + With float32 or float16 data type. Outputs: Tuple of 2 Tensor, the updated parameters. @@ -3380,18 +3469,30 @@ class ApplyAdagradV2(PrimitiveWithInfer): - **accum** (Tensor) - The same shape and data type as `m`. Examples: - >>> var = Parameter(Tensor(np.random.rand(3, 3).astype(np.float32)), name="var") - >>> accum = Parameter(Tensor(np.random.rand(3, 3).astype(np.float32)), name="accum") + >>> import numpy as np + >>> import mindspore.nn as nn + >>> from mindspore import Tensor, Parameter + >>> from mindspore.ops import operations as P + >>> import mindspore.common.dtype as mstype + >>> class Net(nn.Cell): + >>> def __init__(self): + >>> super(Net, self).__init__() + >>> self.apply_adagrad_v2 = P.ApplyAdagradV2(epsilon=1e-6) + >>> self.var = Parameter(Tensor(np.random.rand(3, 3).astype(np.float32)), name="var") + >>> self.accum = Parameter(Tensor(np.random.rand(3, 3).astype(np.float32)), name="accum") + >>> def construct(self, lr, grad): + >>> out = self.apply_adagrad_v2(self.var, self.accum, lr, grad) + >>> return out + >>> net = Net() + >>> lr = Tensor(0.001, mstype.float32) >>> grad = Tensor(np.random.rand(3, 3).astype(np.float32)) - >>> lr = 0.01 - >>> apply_adagrad_v2 = P.ApplyAdagradV2(epsilon=1e-6) - >>> output = apply_adagrad_v2(var, accum, lr, grad) + >>> result = net(lr, grad) """ __mindspore_signature__ = ( ('var', sig_rw.RW_WRITE, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), ('accum', sig_rw.RW_WRITE, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), - ('lr', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), + ('lr', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T1), ('grad', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T) ) @@ -3403,12 +3504,16 @@ class ApplyAdagradV2(PrimitiveWithInfer): def infer_shape(self, var_shape, accum_shape, lr_shape, grad_shape): validator.check('var shape', var_shape, 'accum shape', accum_shape, Rel.EQ, self.name) validator.check('var shape', var_shape, 'grad shape', grad_shape, Rel.EQ, self.name) + lr_shp_len = len(lr_shape) + validator.check_integer("lr's rank", lr_shp_len, 1, Rel.LE, self.name) + if lr_shp_len == 1: + validator.check_integer("lr_shape[0]", lr_shape[0], 1, Rel.EQ, self.name) return var_shape, accum_shape def infer_dtype(self, var_dtype, accum_dtype, lr_dtype, grad_dtype): args = {'var': var_dtype, 'accum': accum_dtype, 'grad': grad_dtype} - validator.check_tensor_type_same(args, mstype.number_type, self.name) valid_types = [mstype.float16, mstype.float32] + validator.check_tensor_type_same(args, valid_types, self.name) validator.check_scalar_or_tensor_type_same({'lr': lr_dtype}, valid_types, self.name) return var_dtype, accum_dtype @@ -3508,14 +3613,14 @@ class ApplyProximalAdagrad(PrimitiveWithInfer): use_locking (bool): If True, updating of the var and accum tensors will be protected. Default: False. Inputs: - - **var** (Parameter) - Variable to be updated. The data type should be float. + - **var** (Parameter) - Variable to be updated. The data type should be float16 or float32. - **accum** (Parameter) - Accum to be updated. Must has the same shape and dtype as `var`. - - **lr** (Union[Number, Tensor]): The learning rate value. It should be a scalar tensor or number. - The data type should be float. - - **l1** (Union[Number, Tensor]): l1 regularization strength, must be greater than or equal to zero. - It should be a scalar tensor or number. The data type should be float. - - **l2** (Union[Number, Tensor]): l2 regularization strength, must be greater than or equal to zero. - It should be a scalar tensor or number. The data type should be float. + - **lr** (Union[Number, Tensor]): The learning rate value, should be scalar. The data type should be + float16 or float32. + - **l1** (Union[Number, Tensor]): l1 regularization strength, should be scalar. The data type should be + float16 or float32. + - **l2** (Union[Number, Tensor]): l2 regularization strength, should be scalar. The data type should be + float16 or float32. - **grad** (Tensor) - Gradient. Must has the same shape and dtype as `var`. Outputs: @@ -3549,9 +3654,9 @@ class ApplyProximalAdagrad(PrimitiveWithInfer): __mindspore_signature__ = ( ('var', sig_rw.RW_WRITE, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), ('accum', sig_rw.RW_WRITE, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), - ('lr', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), - ('l1', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), - ('l2', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), + ('lr', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T1), + ('l1', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T2), + ('l2', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T3), ('grad', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T) ) @@ -3561,16 +3666,29 @@ class ApplyProximalAdagrad(PrimitiveWithInfer): self.use_locking = validator.check_value_type("use_locking", use_locking, [bool], self.name) def infer_shape(self, var_shape, accum_shape, lr_shape, l1_shape, l2_shape, grad_shape): - validator.check('var shape', var_shape, 'accum shape', accum_shape, Rel.EQ, self.name) - validator.check('var shape', var_shape, 'grad shape', grad_shape, Rel.EQ, self.name) + validator.check('accum shape', accum_shape, 'var shape', var_shape, Rel.EQ, self.name) + validator.check('grad shape', grad_shape, 'var shape', var_shape, Rel.EQ, self.name) + lr_shp_len = len(lr_shape) + validator.check_integer("lr's rank", lr_shp_len, 1, Rel.LE, self.name) + if lr_shp_len == 1: + validator.check_integer("lr_shape[0]", lr_shape[0], 1, Rel.EQ, self.name) + l1_shp_len = len(l1_shape) + validator.check_integer("l1's rank", l1_shp_len, 1, Rel.LE, self.name) + if l1_shp_len == 1: + validator.check_integer("l1_shape[0]", l1_shape[0], 1, Rel.EQ, self.name) + l2_shp_len = len(l2_shape) + validator.check_integer("l2's rank", l2_shp_len, 1, Rel.LE, self.name) + if l2_shp_len == 1: + validator.check_integer("l2_shape[0]", l2_shape[0], 1, Rel.EQ, self.name) return var_shape, accum_shape def infer_dtype(self, var_dtype, accum_dtype, lr_dtype, l1_dtype, l2_dtype, grad_dtype): valid_types = [mstype.float16, mstype.float32] args = {'var': var_dtype, 'accum': accum_dtype, 'grad': grad_dtype} validator.check_tensor_type_same(args, valid_types, self.name) - scalar_args = {"lr": lr_dtype, "l1": l1_dtype, "l2": l2_dtype} - validator.check_scalar_or_tensor_type_same(scalar_args, valid_types, self.name) + validator.check_scalar_or_tensor_type_same({"lr": lr_dtype}, valid_types, self.name) + validator.check_scalar_or_tensor_type_same({"l1": l1_dtype}, valid_types, self.name) + validator.check_scalar_or_tensor_type_same({"l2": l2_dtype}, valid_types, self.name) return var_dtype, accum_dtype @@ -3592,12 +3710,9 @@ class SparseApplyProximalAdagrad(PrimitiveWithInfer): Inputs: - **var** (Parameter) - Variable tensor to be updated. The data type must be float32. - **accum** (Parameter) - Variable tensor to be updated. Has the same dtype as `var`. - - **lr** (Union[Number, Tensor]): The learning rate value. It should be a scalar tensor or number. - The data type must be float32. - - **l1** (Union[Number, Tensor]): l1 regularization strength, must be greater than or equal to zero. - It should be a scalar tensor or number. The data type must be float32. - - **l2** (Union[Number, Tensor]): l2 regularization strength, must be greater than or equal to zero. - It should be a scalar tensor or number. The data type must be float32. + - **lr** (Union[Number, Tensor]): The learning rate value. The data type must be float32. + - **l1** (Union[Number, Tensor]): l1 regularization strength. The data type must be float32. + - **l2** (Union[Number, Tensor]): l2 regularization strength. The data type must be float32. - **grad** (Tensor) - A tensor of the same type as `var`, for the gradient. The data type must be float32. - **indices** (Tensor) - A vector of indices into the first dimension of `var` and `accum`. @@ -3634,11 +3749,11 @@ class SparseApplyProximalAdagrad(PrimitiveWithInfer): __mindspore_signature__ = ( ('var', sig_rw.RW_WRITE, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), ('accum', sig_rw.RW_WRITE, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), - ('lr', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), - ('l1', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), - ('l2', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), + ('lr', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T1), + ('l1', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T2), + ('l2', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T3), ('grad', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), - ('indices', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T1) + ('indices', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T4) ) @prim_attr_register @@ -3654,8 +3769,9 @@ class SparseApplyProximalAdagrad(PrimitiveWithInfer): def infer_dtype(self, var_dtype, accum_dtype, lr_dtype, l1_dtype, l2_dtype, grad_dtype, indices_dtype): args = {'var': var_dtype, 'accum': accum_dtype, 'grad': grad_dtype} validator.check_tensor_type_same(args, [mstype.float32], self.name) - scalar_args = {"lr": lr_dtype, "l1": l1_dtype, "l2": l2_dtype} - validator.check_scalar_or_tensor_type_same(scalar_args, [mstype.float32], self.name) + validator.check_scalar_or_tensor_type_same({"lr": lr_dtype}, [mstype.float32], self.name) + validator.check_scalar_or_tensor_type_same({"l1": l1_dtype}, [mstype.float32], self.name) + validator.check_scalar_or_tensor_type_same({"l2": l2_dtype}, [mstype.float32], self.name) valid_types = [mstype.int16, mstype.int32, mstype.int64, mstype.uint16, mstype.uint32, mstype.uint64] validator.check_tensor_type_same({'indices': indices_dtype}, valid_types, self.name) @@ -3836,9 +3952,9 @@ class SparseApplyFtrl(PrimitiveWithInfer): use_locking (bool): Use locks for update operation if True . Default: False. Inputs: - - **var** (Tensor): The variable to be updated. - - **accum** (Tensor): The accum to be updated, must be same type and shape as `var`. - - **linear** (Tensor): The linear to be updated, must be same type and shape as `var`. + - **var** (Parameter): The variable to be updated. The data type must be float32. + - **accum** (Parameter): The accum to be updated, must be same type and shape as `var`. + - **linear** (Parameter): The linear to be updated, must be same type and shape as `var`. - **grad** (Tensor): A tensor of the same type as `var`, for the gradient. - **indices** (Tensor): A vector of indices into the first dimension of `var` and `accum`. The shape of `indices` must be the same as `grad` in first dimension. The type must be int32. @@ -3873,6 +3989,14 @@ class SparseApplyFtrl(PrimitiveWithInfer): >>> output = net(grad, indices) """ + __mindspore_signature__ = ( + ('var', sig_rw.RW_WRITE, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), + ('accum', sig_rw.RW_WRITE, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), + ('linear', sig_rw.RW_WRITE, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), + ('grad', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), + ('indices', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T1) + ) + @prim_attr_register def __init__(self, lr, l1, l2, lr_power, use_locking=False): validator.check_value_type("lr", lr, [float], self.name) From 8e2bb7a85cbdceedc1c231ae9547a9cc7b28d68b Mon Sep 17 00:00:00 2001 From: caojian05 Date: Wed, 24 Jun 2020 13:36:04 +0800 Subject: [PATCH 046/254] fix accurancy lower then 92 --- model_zoo/vgg16/src/config.py | 4 +++- model_zoo/vgg16/train.py | 26 ++++++++++++++++---------- 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/model_zoo/vgg16/src/config.py b/model_zoo/vgg16/src/config.py index 8c6ffee98b..a34cf7a1d3 100644 --- a/model_zoo/vgg16/src/config.py +++ b/model_zoo/vgg16/src/config.py @@ -19,7 +19,9 @@ from easydict import EasyDict as edict cifar_cfg = edict({ 'num_classes': 10, - 'lr_init': 0.05, + 'lr_init': 0.01, + 'lr_max': 0.1, + 'warmup_epochs': 5, 'batch_size': 64, 'epoch_size': 70, 'momentum': 0.9, diff --git a/model_zoo/vgg16/train.py b/model_zoo/vgg16/train.py index c582cdd679..33a4f0310c 100644 --- a/model_zoo/vgg16/train.py +++ b/model_zoo/vgg16/train.py @@ -38,20 +38,25 @@ random.seed(1) np.random.seed(1) -def lr_steps(global_step, lr_max=None, total_epochs=None, steps_per_epoch=None): +def lr_steps(global_step, lr_init, lr_max, warmup_epochs, total_epochs, steps_per_epoch): """Set learning rate.""" lr_each_step = [] total_steps = steps_per_epoch * total_epochs - decay_epoch_index = [0.3 * total_steps, 0.6 * total_steps, 0.8 * total_steps] + warmup_steps = steps_per_epoch * warmup_epochs + if warmup_steps != 0: + inc_each_step = (float(lr_max) - float(lr_init)) / float(warmup_steps) + else: + inc_each_step = 0 for i in range(total_steps): - if i < decay_epoch_index[0]: - lr_each_step.append(lr_max) - elif i < decay_epoch_index[1]: - lr_each_step.append(lr_max * 0.1) - elif i < decay_epoch_index[2]: - lr_each_step.append(lr_max * 0.01) + if i < warmup_steps: + lr_value = float(lr_init) + inc_each_step * float(i) else: - lr_each_step.append(lr_max * 0.001) + base = (1.0 - (float(i) - float(warmup_steps)) / (float(total_steps) - float(warmup_steps))) + lr_value = float(lr_max) * base * base + if lr_value < 0.0: + lr_value = 0.0 + lr_each_step.append(lr_value) + current_step = global_step lr_each_step = np.array(lr_each_step).astype(np.float32) learning_rate = lr_each_step[current_step:] @@ -86,7 +91,8 @@ if __name__ == '__main__': if args_opt.pre_trained: load_param_into_net(net, load_checkpoint(args_opt.pre_trained)) - lr = lr_steps(0, lr_max=cfg.lr_init, total_epochs=cfg.epoch_size, steps_per_epoch=batch_num) + lr = lr_steps(0, lr_init=cfg.lr_init, lr_max=cfg.lr_max, warmup_epochs=cfg.warmup_epochs, + total_epochs=cfg.epoch_size, steps_per_epoch=batch_num) opt = Momentum(filter(lambda x: x.requires_grad, net.get_parameters()), Tensor(lr), cfg.momentum, weight_decay=cfg.weight_decay) loss = nn.SoftmaxCrossEntropyWithLogits(sparse=True, reduction='mean', is_grad=False) From 6918943f4d178355e9cae771b143aaf96e2bd00a Mon Sep 17 00:00:00 2001 From: jjfeing Date: Wed, 24 Jun 2020 14:19:36 +0800 Subject: [PATCH 047/254] fix atomic clean when output or workspace empty --- .../ccsrc/device/ascend/kernel_build_ascend.cc | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/mindspore/ccsrc/device/ascend/kernel_build_ascend.cc b/mindspore/ccsrc/device/ascend/kernel_build_ascend.cc index 81d5be6731..bd0b436344 100644 --- a/mindspore/ccsrc/device/ascend/kernel_build_ascend.cc +++ b/mindspore/ccsrc/device/ascend/kernel_build_ascend.cc @@ -207,7 +207,7 @@ static bool IsAtomicNode(const CNodePtr &kernel_node) { } } // process output - std::vector output_indexs; + std::vector output_indexs = {}; for (size_t i = 0; i < output_num; ++i) { auto param_output = parameters_indexs.at(input_num + workspace_num + i); if (param_output == 1) { @@ -215,9 +215,11 @@ static bool IsAtomicNode(const CNodePtr &kernel_node) { MS_LOG(INFO) << "Atomic clear output index: " << i; } } - AnfAlgo::SetNodeAttr(kAttrAtomicOutputIndexs, MakeValue(output_indexs), kernel_node); + if (!output_indexs.empty()) { + AnfAlgo::SetNodeAttr(kAttrAtomicOutputIndexs, MakeValue(output_indexs), kernel_node); + } // process workspace - std::vector workspace_indexs; + std::vector workspace_indexs = {}; for (size_t k = 0; k < workspace_num; ++k) { auto param_workspace = parameters_indexs.at(input_num + k); if (param_workspace == 1) { @@ -225,8 +227,9 @@ static bool IsAtomicNode(const CNodePtr &kernel_node) { MS_LOG(INFO) << "Atomic clear workspace index: " << k; } } - AnfAlgo::SetNodeAttr(kAttrAtomicWorkspaceIndexs, MakeValue(workspace_indexs), kernel_node); - + if (!workspace_indexs.empty()) { + AnfAlgo::SetNodeAttr(kAttrAtomicWorkspaceIndexs, MakeValue(workspace_indexs), kernel_node); + } return !(workspace_indexs.empty() && output_indexs.empty()); } From c9f4889d1b5e95ad0421bb12ff9642225f662104 Mon Sep 17 00:00:00 2001 From: chenjianping Date: Wed, 24 Jun 2020 14:41:20 +0800 Subject: [PATCH 048/254] check mpi_adapter instance --- mindspore/ccsrc/device/ascend/ascend_kernel_runtime.cc | 4 +++- mindspore/ccsrc/kernel/cpu/allgather_cpu_kernel.cc | 5 +++-- .../kernel/cpu/embedding_look_up_comm_grad_cpu_kernel.cc | 6 ++++-- mindspore/ccsrc/kernel/cpu/embedding_look_up_cpu_kernel.cc | 7 ++++--- mindspore/ccsrc/kernel/cpu/reduce_scatter_cpu_kernel.cc | 6 +++--- 5 files changed, 17 insertions(+), 11 deletions(-) diff --git a/mindspore/ccsrc/device/ascend/ascend_kernel_runtime.cc b/mindspore/ccsrc/device/ascend/ascend_kernel_runtime.cc index 5851e885b4..b5ac84a799 100644 --- a/mindspore/ccsrc/device/ascend/ascend_kernel_runtime.cc +++ b/mindspore/ccsrc/device/ascend/ascend_kernel_runtime.cc @@ -59,7 +59,9 @@ std::string GetRankId() { auto mpi_config_ptr = MpiConfig::GetInstance(); MS_EXCEPTION_IF_NULL(mpi_config_ptr); if (mpi_config_ptr->enable_mpi()) { - int rank_id = device::cpu::MPIAdapter::Instance().GetRankId(); + auto mpi_instance = device::cpu::MPIAdapter::Instance(); + MS_EXCEPTION_IF_NULL(mpi_instance); + int rank_id = mpi_instance->GetRankId(); const char *offset = std::getenv("RANK_OFFSET"); if (offset != nullptr) { try { diff --git a/mindspore/ccsrc/kernel/cpu/allgather_cpu_kernel.cc b/mindspore/ccsrc/kernel/cpu/allgather_cpu_kernel.cc index 276d03625b..9cc5126c08 100644 --- a/mindspore/ccsrc/kernel/cpu/allgather_cpu_kernel.cc +++ b/mindspore/ccsrc/kernel/cpu/allgather_cpu_kernel.cc @@ -46,8 +46,9 @@ bool AllGatherCPUKernel::Launch(const std::vector &inputs, auto input_addr = reinterpret_cast(inputs[0]->addr); auto output_addr = reinterpret_cast(outputs[0]->addr); auto input_data_num = inputs[0]->size / sizeof(float); - - return device::cpu::MPIAdapter::Instance()->AllGather(input_addr, output_addr, ranks_group_, input_data_num); + auto mpi_instance = device::cpu::MPIAdapter::Instance(); + MS_EXCEPTION_IF_NULL(mpi_instance); + return mpi_instance->AllGather(input_addr, output_addr, ranks_group_, input_data_num); } } // namespace kernel } // namespace mindspore diff --git a/mindspore/ccsrc/kernel/cpu/embedding_look_up_comm_grad_cpu_kernel.cc b/mindspore/ccsrc/kernel/cpu/embedding_look_up_comm_grad_cpu_kernel.cc index 8a061f63c1..07da3dcc25 100644 --- a/mindspore/ccsrc/kernel/cpu/embedding_look_up_comm_grad_cpu_kernel.cc +++ b/mindspore/ccsrc/kernel/cpu/embedding_look_up_comm_grad_cpu_kernel.cc @@ -50,9 +50,11 @@ bool EmbeddingLookUpCommGradCPUKernel::Launch(const std::vector &rank_group = {0, 1, 2, 3, 4, 5, 6, 7}; size_t input_split_lens = input_size / split_num_ / sizeof(float_t); size_t output_split_lens = output_size / split_num_ / sizeof(float_t); + auto mpi_instance = device::cpu::MPIAdapter::Instance(); + MS_EXCEPTION_IF_NULL(mpi_instance); for (int i = 0; i < split_num_; i++) { - device::cpu::MPIAdapter::Instance()->AllGather(input_addr + i * input_split_lens, - output_addr + i * output_split_lens, rank_group, input_split_lens); + mpi_instance->AllGather(input_addr + i * input_split_lens, output_addr + i * output_split_lens, rank_group, + input_split_lens); } #if defined(_WIN32) || defined(_WIN64) auto end_time = std::chrono::steady_clock::now(); diff --git a/mindspore/ccsrc/kernel/cpu/embedding_look_up_cpu_kernel.cc b/mindspore/ccsrc/kernel/cpu/embedding_look_up_cpu_kernel.cc index a63f825ec0..6c8b1ba28f 100644 --- a/mindspore/ccsrc/kernel/cpu/embedding_look_up_cpu_kernel.cc +++ b/mindspore/ccsrc/kernel/cpu/embedding_look_up_cpu_kernel.cc @@ -104,10 +104,11 @@ bool EmbeddingLookUpCPUKernel::Launch(const std::vector &inp size_t one_split_lens = gatherv2_out_lens_ / split_num_ / sizeof(float); size_t reduce_scatter_out_lens = one_split_lens / 8; const std::vector &group = {0, 1, 2, 3, 4, 5, 6, 7}; + auto mpi_instance = device::cpu::MPIAdapter::Instance(); + MS_EXCEPTION_IF_NULL(mpi_instance); for (int i = 0; i < split_num_; i++) { - device::cpu::MPIAdapter::Instance()->ReduceScatter(reinterpret_cast(gather_v2_out_) + i * one_split_lens, - output_addr + i * reduce_scatter_out_lens, group, - one_split_lens / 8, "sum"); + mpi_instance->ReduceScatter(reinterpret_cast(gather_v2_out_) + i * one_split_lens, + output_addr + i * reduce_scatter_out_lens, group, one_split_lens / 8, "sum"); } } #endif diff --git a/mindspore/ccsrc/kernel/cpu/reduce_scatter_cpu_kernel.cc b/mindspore/ccsrc/kernel/cpu/reduce_scatter_cpu_kernel.cc index b0c25387e2..19a4e907a0 100644 --- a/mindspore/ccsrc/kernel/cpu/reduce_scatter_cpu_kernel.cc +++ b/mindspore/ccsrc/kernel/cpu/reduce_scatter_cpu_kernel.cc @@ -46,9 +46,9 @@ bool ReduceScatterCPUKernel::Launch(const std::vector &input auto input_addr = reinterpret_cast(inputs[0]->addr); auto output_addr = reinterpret_cast(outputs[0]->addr); auto output_data_num = outputs[0]->size / sizeof(float); - - return device::cpu::MPIAdapter::Instance()->ReduceScatter(input_addr, output_addr, ranks_group_, output_data_num, - op_type_); + auto mpi_instance = device::cpu::MPIAdapter::Instance(); + MS_EXCEPTION_IF_NULL(mpi_instance); + return mpi_instance->ReduceScatter(input_addr, output_addr, ranks_group_, output_data_num, op_type_); } } // namespace kernel } // namespace mindspore From 71ffd22a0250ca57257c4597f7910d467c0433a4 Mon Sep 17 00:00:00 2001 From: lizhenyu Date: Wed, 24 Jun 2020 11:49:56 +0800 Subject: [PATCH 049/254] add wide&deep stanalone training script for gpu in model zoo --- model_zoo/wide_and_deep/README.md | 3 +++ model_zoo/wide_and_deep/eval.py | 7 ++--- .../script/run_multigpu_train.sh | 3 ++- .../script/run_standalone_train_for_gpu.sh | 27 +++++++++++++++++++ model_zoo/wide_and_deep/train.py | 9 ++++--- model_zoo/wide_and_deep/train_and_eval.py | 11 +++++--- .../train_and_eval_auto_parallel.py | 3 +++ .../train_and_eval_distribute.py | 11 ++++++-- 8 files changed, 60 insertions(+), 14 deletions(-) create mode 100644 model_zoo/wide_and_deep/script/run_standalone_train_for_gpu.sh diff --git a/model_zoo/wide_and_deep/README.md b/model_zoo/wide_and_deep/README.md index 54367ef173..000e6a5335 100644 --- a/model_zoo/wide_and_deep/README.md +++ b/model_zoo/wide_and_deep/README.md @@ -37,6 +37,7 @@ To train and evaluate the model, command as follows: python train_and_eval.py ``` Arguments: + * `--device_target`: Device where the code will be implemented (Default: Ascend). * `--data_path`: This should be set to the same directory given to the data_download's data_dir argument. * `--epochs`: Total train epochs. * `--batch_size`: Training batch size. @@ -57,6 +58,7 @@ To train the model in one device, command as follows: python train.py ``` Arguments: + * `--device_target`: Device where the code will be implemented (Default: Ascend). * `--data_path`: This should be set to the same directory given to the data_download's data_dir argument. * `--epochs`: Total train epochs. * `--batch_size`: Training batch size. @@ -87,6 +89,7 @@ To evaluate the model, command as follows: python eval.py ``` Arguments: + * `--device_target`: Device where the code will be implemented (Default: Ascend). * `--data_path`: This should be set to the same directory given to the data_download's data_dir argument. * `--epochs`: Total train epochs. * `--batch_size`: Training batch size. diff --git a/model_zoo/wide_and_deep/eval.py b/model_zoo/wide_and_deep/eval.py index 72c30b1f5c..bc3846533f 100644 --- a/model_zoo/wide_and_deep/eval.py +++ b/model_zoo/wide_and_deep/eval.py @@ -26,11 +26,11 @@ from src.datasets import create_dataset from src.metrics import AUCMetric from src.config import WideDeepConfig -context.set_context(mode=context.GRAPH_MODE, device_target="Davinci", - save_graphs=True) - def get_WideDeep_net(config): + """ + Get network of wide&deep model. + """ WideDeep_net = WideDeepModel(config) loss_net = NetWithLossClass(WideDeep_net, config) @@ -91,4 +91,5 @@ if __name__ == "__main__": widedeep_config = WideDeepConfig() widedeep_config.argparse_init() + context.set_context(mode=context.GRAPH_MODE, device_target=widedeep_config.device_target) test_eval(widedeep_config) diff --git a/model_zoo/wide_and_deep/script/run_multigpu_train.sh b/model_zoo/wide_and_deep/script/run_multigpu_train.sh index 987eeaa65e..e0e08ab80e 100644 --- a/model_zoo/wide_and_deep/script/run_multigpu_train.sh +++ b/model_zoo/wide_and_deep/script/run_multigpu_train.sh @@ -14,7 +14,7 @@ # limitations under the License. # ============================================================================ -# bash run_multigpu_train.sh +# bash run_multigpu_train.sh RANK_SIZE EPOCH_SIZE DATASET script_self=$(readlink -f "$0") self_path=$(dirname "${script_self}") RANK_SIZE=$1 @@ -25,4 +25,5 @@ mpirun --allow-run-as-root -n $RANK_SIZE \ python -s ${self_path}/../train_and_eval_distribute.py \ --device_target="GPU" \ --data_path=$DATASET \ + --batch_size=8000 \ --epochs=$EPOCH_SIZE > log.txt 2>&1 & diff --git a/model_zoo/wide_and_deep/script/run_standalone_train_for_gpu.sh b/model_zoo/wide_and_deep/script/run_standalone_train_for_gpu.sh new file mode 100644 index 0000000000..693c62b847 --- /dev/null +++ b/model_zoo/wide_and_deep/script/run_standalone_train_for_gpu.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +# bash run_standalone_train_for_gpu.sh EPOCH_SIZE DATASET +script_self=$(readlink -f "$0") +self_path=$(dirname "${script_self}") +EPOCH_SIZE=$1 +DATASET=$2 + +python -s ${self_path}/../train_and_eval.py \ + --device_target="GPU" \ + --data_path=$DATASET \ + --batch_size=16000 \ + --epochs=$EPOCH_SIZE > log.txt 2>&1 & diff --git a/model_zoo/wide_and_deep/train.py b/model_zoo/wide_and_deep/train.py index ac9750c547..a043be3dc6 100644 --- a/model_zoo/wide_and_deep/train.py +++ b/model_zoo/wide_and_deep/train.py @@ -15,16 +15,16 @@ import os from mindspore import Model, context from mindspore.train.callback import ModelCheckpoint, CheckpointConfig, TimeMonitor - from src.wide_and_deep import PredictWithSigmoid, TrainStepWrap, NetWithLossClass, WideDeepModel from src.callbacks import LossCallBack from src.datasets import create_dataset from src.config import WideDeepConfig -context.set_context(mode=context.GRAPH_MODE, device_target="Ascend", save_graphs=True) - def get_WideDeep_net(configure): + """ + Get network of wide&deep model. + """ WideDeep_net = WideDeepModel(configure) loss_net = NetWithLossClass(WideDeep_net, configure) @@ -72,7 +72,7 @@ def test_train(configure): model = Model(train_net) callback = LossCallBack(config=configure) - ckptconfig = CheckpointConfig(save_checkpoint_steps=1, + ckptconfig = CheckpointConfig(save_checkpoint_steps=ds_train.get_dataset_size(), keep_checkpoint_max=5) ckpoint_cb = ModelCheckpoint(prefix='widedeep_train', directory=configure.ckpt_path, config=ckptconfig) model.train(epochs, ds_train, callbacks=[TimeMonitor(ds_train.get_dataset_size()), callback, ckpoint_cb]) @@ -82,4 +82,5 @@ if __name__ == "__main__": config = WideDeepConfig() config.argparse_init() + context.set_context(mode=context.GRAPH_MODE, device_target=config.device_target) test_train(config) diff --git a/model_zoo/wide_and_deep/train_and_eval.py b/model_zoo/wide_and_deep/train_and_eval.py index 0b37b67a11..e0ab6b2e9e 100644 --- a/model_zoo/wide_and_deep/train_and_eval.py +++ b/model_zoo/wide_and_deep/train_and_eval.py @@ -15,7 +15,7 @@ import os from mindspore import Model, context -from mindspore.train.callback import ModelCheckpoint, CheckpointConfig +from mindspore.train.callback import ModelCheckpoint, CheckpointConfig, TimeMonitor from src.wide_and_deep import PredictWithSigmoid, TrainStepWrap, NetWithLossClass, WideDeepModel from src.callbacks import LossCallBack, EvalCallBack @@ -23,10 +23,11 @@ from src.datasets import create_dataset from src.metrics import AUCMetric from src.config import WideDeepConfig -context.set_context(mode=context.GRAPH_MODE, device_target="Davinci") - def get_WideDeep_net(config): + """ + Get network of wide&deep model. + """ WideDeep_net = WideDeepModel(config) loss_net = NetWithLossClass(WideDeep_net, config) @@ -87,11 +88,13 @@ def test_train_eval(config): out = model.eval(ds_eval) print("=====" * 5 + "model.eval() initialized: {}".format(out)) - model.train(epochs, ds_train, callbacks=[eval_callback, callback, ckpoint_cb]) + model.train(epochs, ds_train, + callbacks=[TimeMonitor(ds_train.get_dataset_size()), eval_callback, callback, ckpoint_cb]) if __name__ == "__main__": wide_deep_config = WideDeepConfig() wide_deep_config.argparse_init() + context.set_context(mode=context.GRAPH_MODE, device_target=wide_deep_config.device_target) test_train_eval(wide_deep_config) diff --git a/model_zoo/wide_and_deep/train_and_eval_auto_parallel.py b/model_zoo/wide_and_deep/train_and_eval_auto_parallel.py index 780c95540c..4c86931b2e 100644 --- a/model_zoo/wide_and_deep/train_and_eval_auto_parallel.py +++ b/model_zoo/wide_and_deep/train_and_eval_auto_parallel.py @@ -40,6 +40,9 @@ init() def get_WideDeep_net(config): + """ + Get network of wide&deep model. + """ WideDeep_net = WideDeepModel(config) loss_net = NetWithLossClass(WideDeep_net, config) loss_net = VirtualDatasetCellTriple(loss_net) diff --git a/model_zoo/wide_and_deep/train_and_eval_distribute.py b/model_zoo/wide_and_deep/train_and_eval_distribute.py index db98bacfec..71f2b11cba 100644 --- a/model_zoo/wide_and_deep/train_and_eval_distribute.py +++ b/model_zoo/wide_and_deep/train_and_eval_distribute.py @@ -33,6 +33,9 @@ sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) def get_WideDeep_net(config): + """ + Get network of wide&deep model. + """ WideDeep_net = WideDeepModel(config) loss_net = NetWithLossClass(WideDeep_net, config) train_net = TrainStepWrap(loss_net) @@ -90,8 +93,12 @@ def train_and_eval(config): callback = LossCallBack(config=config) ckptconfig = CheckpointConfig(save_checkpoint_steps=ds_train.get_dataset_size(), keep_checkpoint_max=5) - ckpoint_cb = ModelCheckpoint(prefix='widedeep_train', - directory=config.ckpt_path, config=ckptconfig) + if config.device_target == "Ascend": + ckpoint_cb = ModelCheckpoint(prefix='widedeep_train', + directory=config.ckpt_path, config=ckptconfig) + elif config.device_target == "GPU": + ckpoint_cb = ModelCheckpoint(prefix='widedeep_train_' + str(get_rank()), + directory=config.ckpt_path, config=ckptconfig) out = model.eval(ds_eval) print("=====" * 5 + "model.eval() initialized: {}".format(out)) model.train(epochs, ds_train, From e829535a867947224cec9870fd17c682f9175a88 Mon Sep 17 00:00:00 2001 From: ms_yan <6576637+ms_yan@user.noreply.gitee.com> Date: Wed, 24 Jun 2020 11:55:17 +0800 Subject: [PATCH 050/254] change GetTensor into GetRow to avoid NullPtr --- .../dataset/engine/datasetops/source/sampler/sampler.cc | 5 ++++- tests/ut/cpp/dataset/zip_op_test.cc | 1 - 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/sampler.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/sampler.cc index 3f737c167c..b3c595870f 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/sampler.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/sampler.cc @@ -91,11 +91,14 @@ void Sampler::Print(std::ostream &out, bool show_all) const { Status Sampler::GetAllIdsThenReset(py::array *data) { std::unique_ptr db; std::shared_ptr sample_ids; + TensorRow sample_row; // A call to derived class to get sample ids wrapped inside a buffer RETURN_IF_NOT_OK(GetNextSample(&db)); // Get the only tensor inside the buffer that contains the actual SampleIds for the entire epoch - RETURN_IF_NOT_OK(db->GetTensor(&sample_ids, 0, 0)); + RETURN_IF_NOT_OK(db->GetRow(0, &sample_row)); + sample_ids = sample_row[0]; + // check this buffer is not a ctrl buffer CHECK_FAIL_RETURN_UNEXPECTED(db->buffer_flags() == DataBuffer::kDeBFlagNone, "ERROR ctrl buffer received"); { diff --git a/tests/ut/cpp/dataset/zip_op_test.cc b/tests/ut/cpp/dataset/zip_op_test.cc index 7885369c07..f8f8fe89db 100644 --- a/tests/ut/cpp/dataset/zip_op_test.cc +++ b/tests/ut/cpp/dataset/zip_op_test.cc @@ -125,7 +125,6 @@ TEST_F(MindDataTestZipOp, MindDataTestZipOpDefault) { EXPECT_TRUE(rc.IsOk()); row_count++; } - MS_LOG(WARNING) <<"row count is: " << row_count; ASSERT_EQ(row_count, 3); // Should be 3 rows fetched } From 9d3c9c69fee3fd969b6f1ca163841f6b6f1d1960 Mon Sep 17 00:00:00 2001 From: huangdongrun Date: Wed, 24 Jun 2020 11:40:46 +0800 Subject: [PATCH 051/254] modify map to C.Map() --- mindspore/_extends/parse/resources.py | 2 +- mindspore/ccsrc/operator/composite/map.cc | 3 +++ .../ut/python/pipeline/parse/test_fix_bug.py | 23 +++++++++++++++++++ 3 files changed, 27 insertions(+), 1 deletion(-) diff --git a/mindspore/_extends/parse/resources.py b/mindspore/_extends/parse/resources.py index 2ae8b7172f..eb89c965df 100644 --- a/mindspore/_extends/parse/resources.py +++ b/mindspore/_extends/parse/resources.py @@ -111,7 +111,7 @@ convert_object_map = { # system function T.len: M.ms_len, T.bool: M.bool_, - T.map: C.HyperMap(), + T.map: C.Map(), T.partial: F.partial, T.zip: C.zip_operation, T.print: F.print_, diff --git a/mindspore/ccsrc/operator/composite/map.cc b/mindspore/ccsrc/operator/composite/map.cc index a054da5f4d..6062f0f5af 100644 --- a/mindspore/ccsrc/operator/composite/map.cc +++ b/mindspore/ccsrc/operator/composite/map.cc @@ -181,6 +181,9 @@ AnfNodePtr Map::FullMakeClass(const std::shared_ptr &type, const FuncGrap } AnfNodePtr Map::Make(const FuncGraphPtr &func_graph, const AnfNodePtr &fn_arg, const ArgsPairList &arg_pairs) { + if (arg_pairs.size() < 1) { + MS_EXCEPTION(TypeError) << "map() must have at least two arguments"; + } bool found = false; TypeId id = kObjectTypeEnd; std::pair pair; diff --git a/tests/ut/python/pipeline/parse/test_fix_bug.py b/tests/ut/python/pipeline/parse/test_fix_bug.py index 5bf7db3798..9b013f95a4 100644 --- a/tests/ut/python/pipeline/parse/test_fix_bug.py +++ b/tests/ut/python/pipeline/parse/test_fix_bug.py @@ -18,6 +18,7 @@ import pytest import mindspore.nn as nn from mindspore import Tensor +from mindspore.ops import composite as C from mindspore.common.api import _executor @@ -93,3 +94,25 @@ def test_compile_unspported(): net = unsupported_method_net() with pytest.raises(RuntimeError): _executor.compile(net, input_me) + + +def test_parser_map_0002(): + class NetMap0002(nn.Cell): + def __init__(self): + super().__init__() + self.relu = nn.ReLU() + self.hypermap = C.Map() + + def mul(self, x=2, y=4): + return x * y + + def construct(self, x): + if map(self.mul) == 8: + x = self.relu(x) + return x + input_np_x = np.random.randn(2, 3, 4, 5).astype(np.float32) + input_me_x = Tensor(input_np_x) + + net = NetMap0002() + with pytest.raises(TypeError): + net(input_me_x) From 8ddf77a47068db24ec5207923b3146371c518688 Mon Sep 17 00:00:00 2001 From: jinyaohui Date: Wed, 24 Jun 2020 14:41:47 +0800 Subject: [PATCH 052/254] fix bug --- mindspore/context.py | 3 +++ mindspore/train/serialization.py | 10 ++++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/mindspore/context.py b/mindspore/context.py index aaf663882d..257ed20766 100644 --- a/mindspore/context.py +++ b/mindspore/context.py @@ -563,6 +563,8 @@ def set_context(**kwargs): check_bprop (bool): Whether to check bprop. Default: False. max_device_memory (str): Sets the maximum memory available for device, currently only supported on GPU. The format is "xxGB". Default: "1024GB". + print_file_path (str): The path of print data to save. If this parameter is set, print data is saved to + a file by default，and turn off printing to the screen. Raises: ValueError: If input key is not an attribute in context. @@ -583,6 +585,7 @@ def set_context(**kwargs): >>> save_graphs_path="/mindspore") >>> context.set_context(enable_profiling=True, profiling_options="training_trace") >>> context.set_context(max_device_memory="3.5GB") + >>> context.set_context(print_file_path="print.pb") """ for key, value in kwargs.items(): if not hasattr(_context(), key): diff --git a/mindspore/train/serialization.py b/mindspore/train/serialization.py index ce776d6821..c4fba9ba8a 100644 --- a/mindspore/train/serialization.py +++ b/mindspore/train/serialization.py @@ -29,8 +29,7 @@ from mindspore.common.api import _executor from mindspore.common import dtype as mstype from mindspore._checkparam import check_input_data - -__all__ = ["save_checkpoint", "load_checkpoint", "load_param_into_net", "export"] +__all__ = ["save_checkpoint", "load_checkpoint", "load_param_into_net", "export", "parse_print"] tensor_to_ms_type = {"Int8": mstype.int8, "Uint8": mstype.uint8, "Int16": mstype.int16, "Uint16": mstype.uint16, "Int32": mstype.int32, "Uint32": mstype.uint32, "Int64": mstype.int64, "Uint64": mstype.uint64, @@ -513,6 +512,13 @@ def parse_print(print_file_name): tensor_list.append(Tensor(param_value, ms_type)) # Scale type else: + data_type_ = data_type.lower() + if 'float' in data_type_: + param_data = float(param_data[0]) + elif 'int' in data_type_: + param_data = int(param_data[0]) + elif 'bool' in data_type_: + param_data = bool(param_data[0]) tensor_list.append(Tensor(param_data, ms_type)) except BaseException as e: From 22eeabdc5f37fe239993fbc590f66749dda2cedb Mon Sep 17 00:00:00 2001 From: liuwenhao4 Date: Tue, 23 Jun 2020 22:53:45 +0800 Subject: [PATCH 053/254] Fix some mistakes of ArgMaxWithValue vm ops --- mindspore/ops/operations/array_ops.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mindspore/ops/operations/array_ops.py b/mindspore/ops/operations/array_ops.py index 1bb39d1547..f59a3a37e5 100644 --- a/mindspore/ops/operations/array_ops.py +++ b/mindspore/ops/operations/array_ops.py @@ -1123,7 +1123,7 @@ class ArgMaxWithValue(PrimitiveWithInfer): - output_x (Tensor) - The maximum value of input tensor, the shape same as index. Examples: - >>> input_x = Tensor(np.random.rand(5)) + >>> input_x = Tensor(np.random.rand(5), mindspore.float32) >>> index, output = P.ArgMaxWithValue()(input_x) """ From 078738adc5c91c6c80e9a19392283bab00ad6b38 Mon Sep 17 00:00:00 2001 From: kingfo Date: Tue, 23 Jun 2020 19:32:58 +0800 Subject: [PATCH 054/254] add tensor mod & floordiv operation --- mindspore/common/api.py | 4 +++- mindspore/common/tensor.py | 21 +++++++++++++------ mindspore/ops/functional.py | 2 ++ tests/ut/python/ir/test_tensor.py | 10 ++++++++- .../ge/model/test_lenet_model.py | 14 +------------ 5 files changed, 30 insertions(+), 21 deletions(-) diff --git a/mindspore/common/api.py b/mindspore/common/api.py index 4fad3e455b..050baf9f79 100644 --- a/mindspore/common/api.py +++ b/mindspore/common/api.py @@ -158,7 +158,9 @@ class _MindSporeFunction: # replace key with obj info and object ext info when fn is a method if self.obj is not None: self.obj.__parse_method__ = method_name - generate_name = self.obj.__module__ + "." + str(self.obj.create_time) + generate_name = self.obj.__module__ + "." + if self.obj.__class__.__name__ != "ClipByNorm": + generate_name = generate_name + str(self.obj.create_time) if self.identify_obj is not None: generate_name = generate_name + str(id(self.identify_obj)) diff --git a/mindspore/common/tensor.py b/mindspore/common/tensor.py index 0a631b954f..7ad3962ed1 100644 --- a/mindspore/common/tensor.py +++ b/mindspore/common/tensor.py @@ -102,16 +102,14 @@ class Tensor(Tensor_): return out def __iadd__(self, other): - out = self.__add__(other) - return out + return self.__add__(other) def __radd__(self, other): out = tensor_operator_registry.get('__add__')(self, other) return out def __imul__(self, other): - out = self.__mul__(other) - return out + return self.__mul__(other) def __rmul__(self, other): out = tensor_operator_registry.get('__mul__')(self, other) @@ -130,8 +128,7 @@ class Tensor(Tensor_): return out def __isub__(self, other): - out = self.__sub__(other) - return out + return self.__sub__(other) def __rsub__(self, other): out = tensor_operator_registry.get('__sub__')(other, self) @@ -168,6 +165,18 @@ class Tensor(Tensor_): return 1 return out[0] + def __mod__(self, other): + return tensor_operator_registry.get('__mod__')(self, other) + + def __imod__(self, other): + return self.__mod__(other) + + def __floordiv__(self, other): + return tensor_operator_registry.get('__floordiv__')(self, other) + + def __ifloordiv__(self, other): + return self.__floordiv__(other) + def __str__(self): if self.dtype == mstype.type_none: return "Unknown Tensor type!" diff --git a/mindspore/ops/functional.py b/mindspore/ops/functional.py index 5637274bfb..840c4e745e 100644 --- a/mindspore/ops/functional.py +++ b/mindspore/ops/functional.py @@ -157,6 +157,8 @@ tensor_operator_registry.register('__add__', tensor_add) tensor_operator_registry.register('__sub__', tensor_sub) tensor_operator_registry.register('__mul__', tensor_mul) tensor_operator_registry.register('__truediv__', tensor_div) +tensor_operator_registry.register('__mod__', tensor_mod) +tensor_operator_registry.register('__floordiv__', tensor_floordiv) #ms cannot support Tensor(True) compare tensor_operator_registry.register('__eq__', equal) tensor_operator_registry.register('__ne__', not_equal) diff --git a/tests/ut/python/ir/test_tensor.py b/tests/ut/python/ir/test_tensor.py index ff0a5c971f..357187cddd 100644 --- a/tests/ut/python/ir/test_tensor.py +++ b/tests/ut/python/ir/test_tensor.py @@ -24,13 +24,15 @@ import pytest import mindspore as ms import mindspore.common.api as me import mindspore.nn as nn -from mindspore import Tensor +from mindspore import Tensor, context from mindspore.common.initializer import initializer from mindspore.common.parameter import Parameter from ..ut_filter import non_graph_engine ndarr = np.ones((2, 3)) +context.set_context(mode=context.GRAPH_MODE) + def test_tensor_flatten(): with pytest.raises(AttributeError): @@ -452,5 +454,11 @@ def test_tensor_operation(): assert np.all(res.asnumpy() == np.ones((3, 3)) * 2) res = 8 / x assert np.all(res.asnumpy() == np.ones((3, 3)) * 2) + res = x % 3 + assert np.all(res.asnumpy() == np.ones((3, 3))) + res = x // 3 + assert np.all(res.asnumpy() == np.ones((3, 3))) + x %= 3 + assert np.all(x.asnumpy() == np.ones((3, 3))) with pytest.raises(ValueError): res = x * (2, 3) diff --git a/tests/ut/python/pynative_mode/ge/model/test_lenet_model.py b/tests/ut/python/pynative_mode/ge/model/test_lenet_model.py index 23999c398e..b882273aab 100644 --- a/tests/ut/python/pynative_mode/ge/model/test_lenet_model.py +++ b/tests/ut/python/pynative_mode/ge/model/test_lenet_model.py @@ -18,8 +18,7 @@ import pytest import mindspore.nn as nn from mindspore.common.tensor import Tensor -from mindspore.nn import WithGradCell, WithLossCell -from mindspore.nn.optim import Momentum +from mindspore.nn import WithGradCell from mindspore.ops import operations as P @@ -63,17 +62,6 @@ def test_lenet_pynative_train_net(): loss_fn = nn.SoftmaxCrossEntropyWithLogits(is_grad=False) grad_fn = nn.SoftmaxCrossEntropyWithLogits() grad_net = WithGradCell(net, grad_fn, sens=dout) - gradients = grad_net(data, label) - - # update parameters - opt = Momentum(net.trainable_params(), learning_rate=0.1, momentum=0.9) - opt(gradients) - - # verification - if i == verification_step: - loss_net = WithLossCell(net, loss_fn) - loss_output = loss_net(data, label) - print("The loss of %s-th iteration is %s" % (i, loss_output.asnumpy())) def test_lenet_pynative_train_model(): From 6d491b90739e7dce1c8daf8aba217bcd05d95d87 Mon Sep 17 00:00:00 2001 From: Xian Weizhao Date: Wed, 24 Jun 2020 16:51:26 +0800 Subject: [PATCH 055/254] relax the exception of control depend on value node --- mindspore/ccsrc/transform/convert.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mindspore/ccsrc/transform/convert.cc b/mindspore/ccsrc/transform/convert.cc index 32333a06ae..3f6b31303c 100644 --- a/mindspore/ccsrc/transform/convert.cc +++ b/mindspore/ccsrc/transform/convert.cc @@ -1646,7 +1646,7 @@ bool DfGraphConvertor::GetControlDependList(const CNodePtr &node, dst_ops_list->insert(dst_ops_list->end(), converted_list.begin(), converted_list.end()); } if (src_ops_list->empty() || dst_ops_list->empty()) { - MS_LOG(WARNING) << "Control depend node's src or dest node is not a apply node, ignore it"; + MS_LOG(DEBUG) << "Control depend node's src or dest node is not a CNode, ignore it"; error_ = SUCCESS; } return true; @@ -1690,6 +1690,8 @@ void DfGraphConvertor::ConvertControlDependNode(const CNodePtr node) { }); } else if (src_ops_list->size() == 1 && dst_ops_list->size() == 1) { control_edges.push_back({(*src_ops_list)[0], (*dst_ops_list)[0]}); + } else if (src_ops_list->empty() || dst_ops_list->empty()) { + MS_LOG(DEBUG) << "Depend list of src or dst is empty, ignore it"; } else { MS_LOG(ERROR) << "Convert control depend node to operator failed, depend src:" << src_ops_list->size() << " -> dst:" << dst_ops_list->size(); From 1767c6386d14427fa12a50434d1ad432c237c436 Mon Sep 17 00:00:00 2001 From: hongxing Date: Wed, 24 Jun 2020 10:52:23 +0200 Subject: [PATCH 056/254] fix cppcheck error --- .../parallel/auto_parallel/rec_core/rec_generate_strategy.cc | 2 -- .../ccsrc/parallel/auto_parallel/rec_core/rec_parse_graph.cc | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_generate_strategy.cc b/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_generate_strategy.cc index 630833f4a6..b8a57ae997 100644 --- a/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_generate_strategy.cc +++ b/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_generate_strategy.cc @@ -370,8 +370,6 @@ std::vector CopyIncomingOperatorOutputStrategy(const std::shared_ptrnodes[iter_graph].tensor_parm.tensor_str.str_c); s.push_back(1 / graph->nodes[iter_graph].tensor_parm.tensor_str.str_h); s.push_back(1 / graph->nodes[iter_graph].tensor_parm.tensor_str.str_w); - } else if (input_stra_dim == 0) { - s = {}; } else { MS_LOG(EXCEPTION) << ops[iter_ops]->name() << ": Tensor's shape is unknown."; } diff --git a/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_parse_graph.cc b/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_parse_graph.cc index 58884be9db..190a716063 100644 --- a/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_parse_graph.cc +++ b/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_parse_graph.cc @@ -215,7 +215,7 @@ std::shared_ptr EliminateGraph(const std::shared_ptr &graph, const std::shared_ptr>> &eli_list, const std::shared_ptr> &index_list) { MS_EXCEPTION_IF_NULL(graph); - const std::set elementwise_type = { + static const std::set elementwise_type = { OperatorType::kRecReLU, OperatorType::kRecLog, OperatorType::kRecExp, OperatorType::kRecAdd, OperatorType::kRecElmWiseOp, OperatorType::kRecBiasAdd, OperatorType::kRecSub, OperatorType::kRecMul, OperatorType::kRecDiv, OperatorType::kRecSqueeze, OperatorType::kRecReduce, OperatorType::kRecCast, From 0332e4a85438beeb8762e99a10eea868a120da06 Mon Sep 17 00:00:00 2001 From: jinyaohui Date: Wed, 24 Jun 2020 17:11:37 +0800 Subject: [PATCH 057/254] add ENABLE_GE --- mindspore/ccsrc/pipeline/pipeline.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mindspore/ccsrc/pipeline/pipeline.cc b/mindspore/ccsrc/pipeline/pipeline.cc index 5ddd150593..d346c980ea 100644 --- a/mindspore/ccsrc/pipeline/pipeline.cc +++ b/mindspore/ccsrc/pipeline/pipeline.cc @@ -245,7 +245,9 @@ py::dict ExecutorPy::GetAllreduceFusion(const std::string &phase) { } void ExecutorPy::DelNetRes(const std::string &id) { +#ifdef ENABLE_GE FinalizeBackend(); +#endif if (executor_ != nullptr) { bool flag = false; auto tmp_info = info_; From 90639a2a44568269d33f9e1397b01eaa78e07fa9 Mon Sep 17 00:00:00 2001 From: guohongzilong <2713219276@qq.com> Date: Fri, 19 Jun 2020 16:23:37 +0800 Subject: [PATCH 058/254] fix params KeyError in group params --- mindspore/nn/optim/optimizer.py | 23 ++++++++++++++++++++--- mindspore/ops/operations/debug_ops.py | 6 ------ 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/mindspore/nn/optim/optimizer.py b/mindspore/nn/optim/optimizer.py index 9bfc3a284b..40642226ad 100755 --- a/mindspore/nn/optim/optimizer.py +++ b/mindspore/nn/optim/optimizer.py @@ -219,8 +219,28 @@ class Optimizer(Cell): raise TypeError("Learning rate should be float, Tensor or Iterable.") return lr + def _check_group_params(self, parameters): + """Check group params.""" + parse_keys = ['params', 'lr', 'weight_decay', 'order_params'] + for group_param in parameters: + invalid_key = list(filter(lambda x: x not in parse_keys, group_param.keys())) + if invalid_key: + raise KeyError(f'The key "{invalid_key}" cannot be recognized in group params.') + + if 'order_params' in group_param.keys(): + if len(group_param.keys()) > 1: + raise ValueError("The order params dict in group parameters should " + "only include the 'order_params' key.") + if not isinstance(group_param['order_params'], Iterable): + raise TypeError("The value of 'order_params' should be an Iterable type.") + continue + + if not group_param['params']: + raise ValueError("Optimizer got an empty group parameter list.") + def _parse_group_params(self, parameters, learning_rate): """Parse group params.""" + self._check_group_params(parameters) if self.dynamic_lr: dynamic_lr_length = learning_rate.size() else: @@ -250,9 +270,6 @@ class Optimizer(Cell): if dynamic_lr_length not in (lr_length, 0): raise ValueError("The dynamic learning rate in group should be the same size.") - if not group_param['params']: - raise ValueError("Optimizer got an empty group parameter list.") - dynamic_lr_length = lr_length self.dynamic_lr_length = dynamic_lr_length diff --git a/mindspore/ops/operations/debug_ops.py b/mindspore/ops/operations/debug_ops.py index c4fbddd38e..47b70688a8 100644 --- a/mindspore/ops/operations/debug_ops.py +++ b/mindspore/ops/operations/debug_ops.py @@ -302,12 +302,6 @@ class Print(PrimitiveWithInfer): Output tensor or string to stdout. Note: - The print operation cannot support the following cases currently. - - 1. The type of tensor is float64 or bool. - - 2. The data of tensor is a scalar type. - In pynative mode, please use python print function. Inputs: From 4b3c98ccd5343e90c26bf317f5ab1c0386aad8a3 Mon Sep 17 00:00:00 2001 From: huangdongrun Date: Wed, 24 Jun 2020 17:34:11 +0800 Subject: [PATCH 059/254] add back assignvalue --- mindspore/ccsrc/ir/tensor.cc | 12 +++++++++++- mindspore/ccsrc/ir/tensor.h | 3 +++ mindspore/ccsrc/ir/tensor_py.cc | 13 +++++++++++++ 3 files changed, 27 insertions(+), 1 deletion(-) diff --git a/mindspore/ccsrc/ir/tensor.cc b/mindspore/ccsrc/ir/tensor.cc index c43f5423f1..673a8da842 100644 --- a/mindspore/ccsrc/ir/tensor.cc +++ b/mindspore/ccsrc/ir/tensor.cc @@ -272,7 +272,17 @@ bool Tensor::operator==(const Tensor &tensor) const { bool Tensor::ValueEqual(const Tensor &tensor) const { return (&tensor == this || (MetaTensor::operator==(tensor) && data_->equals(*tensor.data_))); } - +// assgin value to this tensor +Tensor &Tensor::AssignValue(const Tensor &tensor) { + if (this != &tensor) { + MetaTensor::operator=(tensor); + dirty_ = tensor.is_dirty(); + device_address_ = tensor.device_address(); + data_ = tensor.data_; + id_ = tensor.id(); + } + return *this; +} abstract::AbstractBasePtr Tensor::ToAbstract() { auto tens = shared_from_base(); auto dtype = tens->Dtype(); diff --git a/mindspore/ccsrc/ir/tensor.h b/mindspore/ccsrc/ir/tensor.h index cd30827129..5be8a063c1 100644 --- a/mindspore/ccsrc/ir/tensor.h +++ b/mindspore/ccsrc/ir/tensor.h @@ -147,6 +147,9 @@ class Tensor : public MetaTensor { // it do real value comparison. bool ValueEqual(const Tensor &tensor) const; + // assgin value to this tensor + Tensor &AssignValue(const Tensor &tensor); + bool operator==(const Value &other) const override { if (other.isa()) { auto &other_ = static_cast(other); diff --git a/mindspore/ccsrc/ir/tensor_py.cc b/mindspore/ccsrc/ir/tensor_py.cc index 1c763d48f4..11a000cef7 100644 --- a/mindspore/ccsrc/ir/tensor_py.cc +++ b/mindspore/ccsrc/ir/tensor_py.cc @@ -327,6 +327,19 @@ REGISTER_PYBIND_DEFINE(Tensor, ([](const py::module *m) { >>> data.dim() 2 )mydelimiter") + .def("assign_value", &Tensor::AssignValue, R"mydelimiter( + Assign another tensor value to this. + + Arg: + value (:class:`mindspore.tensor`): The value tensor. + + Examples: + >>> data = mindspore.Tensor(np.ones((1, 2), np.float32)) + >>> data2 = mindspore.Tensor(np.ones((2, 2), np.float32)) + >>> data.assign_value(data2) + >>> data.shape + (2, 2) + )mydelimiter") .def("set_dtype", &Tensor::SetDtype, R"mydelimiter( Set the tensor's data type. From 0a0ca3cfdc25cdcef44cfdf7df47897519372df5 Mon Sep 17 00:00:00 2001 From: rick_sanchez Date: Wed, 24 Jun 2020 17:46:44 +0800 Subject: [PATCH 060/254] Fix the bug of pynative mode catching the exception. --- mindspore/ccsrc/pynative/pynative_execute.cc | 106 +++++++++++++++++-- mindspore/ccsrc/pynative/pynative_execute.h | 6 +- mindspore/nn/cell.py | 2 +- 3 files changed, 103 insertions(+), 11 deletions(-) diff --git a/mindspore/ccsrc/pynative/pynative_execute.cc b/mindspore/ccsrc/pynative/pynative_execute.cc index 2b8d504f1a..efe95f2566 100644 --- a/mindspore/ccsrc/pynative/pynative_execute.cc +++ b/mindspore/ccsrc/pynative/pynative_execute.cc @@ -22,6 +22,7 @@ #include #include +#include "debug/trace.h" #include "ir/tensor_py.h" #include "ir/param_value_py.h" #include "utils/any.h" @@ -66,6 +67,42 @@ PynativeExecutorPtr PynativeExecutor::executor_ = nullptr; std::mutex PynativeExecutor::instance_lock_; ResourcePtr PynativeExecutor::resource_; +template +void PynativeExecutorTry(PynativeExecutor *const executor, void (PynativeExecutor::*method)(Args...), Args &&... args) { + try { + (executor->*method)(args...); + } catch (const py::error_already_set &ex) { + // print function call stack info before release + std::ostringstream oss; + trace::TraceGraphEval(); + trace::GetEvalStackInfo(oss); + // call py::print to output function call stack to STDOUT, in case of output the log to file, the user can see + // these info from screen, no need to open log file to find these info + py::print(oss.str()); + MS_LOG(ERROR) << oss.str(); + PynativeExecutor::GetInstance()->Clean(); + // re-throw this exception to Python interpreter to handle it + throw(py::error_already_set(ex)); + } catch (const py::type_error &ex) { + PynativeExecutor::GetInstance()->Clean(); + throw py::type_error(ex); + } catch (const py::value_error &ex) { + PynativeExecutor::GetInstance()->Clean(); + throw py::value_error(ex); + } catch (const py::index_error &ex) { + PynativeExecutor::GetInstance()->Clean(); + throw py::index_error(ex); + } catch (const std::exception &ex) { + PynativeExecutor::GetInstance()->Clean(); + // re-throw this exception to Python interpreter to handle it + throw(std::runtime_error(ex.what())); + } catch (...) { + PynativeExecutor::GetInstance()->Clean(); + std::string exName(abi::__cxa_current_exception_type()->name()); + MS_LOG(EXCEPTION) << "Error occurred when compile graph. Exception name: " << exName; + } +} + inline ValuePtr PyAttrValue(const py::object &obj) { ValuePtr converted_ret = parse::data_converter::PyDataToValue(obj); if (!converted_ret) { @@ -144,7 +181,7 @@ std::map GetDstType(const py::tuple &py_args, } py::tuple ConvertInputs(const PrimitivePyPtr &prim, const py::list &args, py::tuple *const out_args, - py::list *out_args_list) { + py::list *const out_args_list) { auto &py_args = *out_args; py::tuple input_mask(args.size()); for (size_t i = 0; i < args.size(); ++i) { @@ -564,7 +601,7 @@ AnfNodePtr PynativeExecutor::GetObjNode(const py::object &obj) { return node; } -py::tuple RunOp(const OpExecInfoPtr &op_exec_info, const py::args &args) { +py::tuple RunOpInner(const OpExecInfoPtr &op_exec_info, const py::args &args) { MS_LOG(INFO) << "RunOp start, op name is: " << op_exec_info->op_name; mindspore::parse::python_adapter::set_python_env_flag(true); MsBackendPolicy backend_policy; @@ -603,7 +640,7 @@ py::tuple RunOp(const OpExecInfoPtr &op_exec_info, const py::args &args) { return result; } -py::tuple RunOp(const py::args &args) { +py::tuple RunOpInner(const py::args &args) { MS_LOG(DEBUG) << "RunOp start" << args.size(); py::list args_input = args[PY_INPUTS]; @@ -623,7 +660,42 @@ py::tuple RunOp(const py::args &args) { return value_ret; } } - return RunOp(op_exec_info, args_input); + return RunOpInner(op_exec_info, args_input); +} + +py::tuple RunOp(const py::args &args) { + try { + return RunOpInner(args); + } catch (const py::error_already_set &ex) { + // print function call stack info before release + std::ostringstream oss; + trace::TraceGraphEval(); + trace::GetEvalStackInfo(oss); + // call py::print to output function call stack to STDOUT, in case of output the log to file, the user can see + // these info from screen, no need to open log file to find these info + py::print(oss.str()); + MS_LOG(ERROR) << oss.str(); + PynativeExecutor::GetInstance()->Clean(); + // re-throw this exception to Python interpreter to handle it + throw(py::error_already_set(ex)); + } catch (const py::type_error &ex) { + PynativeExecutor::GetInstance()->Clean(); + throw py::type_error(ex); + } catch (const py::value_error &ex) { + PynativeExecutor::GetInstance()->Clean(); + throw py::value_error(ex); + } catch (const py::index_error &ex) { + PynativeExecutor::GetInstance()->Clean(); + throw py::index_error(ex); + } catch (const std::exception &ex) { + PynativeExecutor::GetInstance()->Clean(); + // re-throw this exception to Python interpreter to handle it + throw(std::runtime_error(ex.what())); + } catch (...) { + PynativeExecutor::GetInstance()->Clean(); + std::string exName(abi::__cxa_current_exception_type()->name()); + MS_LOG(EXCEPTION) << "Error occurred when compile graph. Exception name: " << exName; + } } void ClearPyNativeSession() { session = nullptr; } @@ -632,7 +704,7 @@ PynativeExecutor::~PynativeExecutor() { ClearRes(); } PynativeExecutor::PynativeExecutor() { grad_flag_ = false; } -void PynativeExecutor::NewGraph(const py::object &cell, const py::args &args) { +void PynativeExecutor::NewGraphInner(const py::object &cell, const py::args &args) { auto cell_id = GetId(cell); if (cell_graph_map_.count(cell_id) != 0) { MS_LOG(DEBUG) << "Newgraph already compiled"; @@ -753,7 +825,7 @@ void PynativeExecutor::Popp() { graph_p_.pop(); } -void PynativeExecutor::EndGraph(const py::object &cell, const py::object &out, const py::args &args) { +void PynativeExecutor::EndGraphInner(const py::object &cell, const py::object &out, const py::args &args) { auto cell_id = GetId(cell); if (cell_graph_map_.count(cell_id) != 0) { MS_LOG(DEBUG) << "Endgraph already compiled"; @@ -892,8 +964,8 @@ abstract::AbstractBasePtrList PynativeExecutor::GetArgsSpec(const py::args &args return args_spec; } -void PynativeExecutor::GradNet(const GradOperationPtr &grad, const py::object &cell, const py::object &weights, - const py::args &args) { +void PynativeExecutor::GradNetInner(const GradOperationPtr &grad, const py::object &cell, const py::object &weights, + const py::args &args) { MS_LOG(INFO) << "GradNet start" << args.size(); std::size_t size = args.size(); @@ -939,8 +1011,10 @@ void PynativeExecutor::GradNet(const GradOperationPtr &grad, const py::object &c } void PynativeExecutor::Clear(const std::string &flag) { - if (flag == "resource") { + if (!flag.empty()) { MS_LOG(INFO) << "Clear res"; + (void)graph_map_.erase(flag); + (void)cell_graph_map_.erase(flag); Clean(); // Maybe exit in the pynative runing op, so need reset pynative flag. auto ms_context = MsContext::GetInstance(); @@ -949,6 +1023,7 @@ void PynativeExecutor::Clear(const std::string &flag) { } return; } + MS_LOG(INFO) << "Clear"; top_g_ = nullptr; curr_g_ = nullptr; @@ -1010,6 +1085,19 @@ FuncGraphPtr PynativeExecutor::GradGraph(FuncGraphPtr g, const GradOperationPtr return df_builder_; } +void PynativeExecutor::NewGraph(const py::object &cell, const py::args &args) { + PynativeExecutorTry(this, &PynativeExecutor::NewGraphInner, cell, args); +} + +void PynativeExecutor::EndGraph(const py::object &cell, const py::object &out, const py::args &args) { + PynativeExecutorTry(this, &PynativeExecutor::EndGraphInner, cell, out, args); +} + +void PynativeExecutor::GradNet(const GradOperationPtr &grad, const py::object &cell, const py::object &weights, + const py::args &args) { + PynativeExecutorTry(this, &PynativeExecutor::GradNetInner, grad, cell, weights, args); +} + REGISTER_PYBIND_DEFINE(PynativeExecutor_, ([](const py::module *m) { (void)py::class_>(*m, "PynativeExecutor_") .def_static("get_instance", &PynativeExecutor::GetInstance, "PynativeExecutor get_instance.") diff --git a/mindspore/ccsrc/pynative/pynative_execute.h b/mindspore/ccsrc/pynative/pynative_execute.h index 3b030f7035..83cbea88d4 100644 --- a/mindspore/ccsrc/pynative/pynative_execute.h +++ b/mindspore/ccsrc/pynative/pynative_execute.h @@ -46,7 +46,7 @@ py::object RunOpInVM(const OpExecInfoPtr &op_exec_info, PynativeStatusCode *stat py::tuple RunOp(const py::args &args); py::tuple ConvertInputs(const PrimitivePyPtr &prim, const py::list &py_args, py::tuple *const out_args, - py::list *out_args_list); + py::list *const out_args_list); void ClearPyNativeSession(); @@ -68,11 +68,15 @@ class PynativeExecutor : public std::enable_shared_from_this { return executor_; } void NewGraph(const py::object &cell, const py::args &args); + void NewGraphInner(const py::object &cell, const py::args &args); void EndGraph(const py::object &cell, const py::object &out, const py::args &args); + void EndGraphInner(const py::object &cell, const py::object &out, const py::args &args); void EndGraphByOutId(const std::string &out_id, const py::object &cell, const py::object &out, const py::args &args); std::vector GetWeightsArgs(const py::object &weights); abstract::AbstractBasePtrList GetArgsSpec(const py::args &args); void GradNet(const GradOperationPtr &grad, const py::object &cell, const py::object &weights, const py::args &args); + void GradNetInner(const GradOperationPtr &grad, const py::object &cell, const py::object &weights, + const py::args &args); void Clear(const std::string &flag = ""); void Clean(); void ClearRes(); diff --git a/mindspore/nn/cell.py b/mindspore/nn/cell.py index c046c2e1bf..0f8c75955b 100755 --- a/mindspore/nn/cell.py +++ b/mindspore/nn/cell.py @@ -186,7 +186,7 @@ class Cell: raise AttributeError("'{}' object has no attribute '{}'.".format(type(self).__name__, name)) def __del__(self): - _pynative_exec.clear("resource") + _pynative_exec.clear(str(id(self))) if hasattr(self, "_create_time"): _executor.del_net_res(str(self._create_time)) From 4fa6d5621c6c81202a00ce03535908950fe7faf7 Mon Sep 17 00:00:00 2001 From: Lixia Chen Date: Fri, 19 Jun 2020 14:34:54 -0400 Subject: [PATCH 061/254] Compute column map before launch the tree. --- .../ccsrc/dataset/engine/dataset_iterator.cc | 7 +- .../ccsrc/dataset/engine/dataset_iterator.h | 1 - .../dataset/engine/datasetops/barrier_op.cc | 3 - .../dataset/engine/datasetops/batch_op.cc | 1 - .../datasetops/bucket_batch_by_length_op.cc | 1 - .../engine/datasetops/build_vocab_op.cc | 1 - .../dataset/engine/datasetops/concat_op.cc | 34 +++-- .../dataset/engine/datasetops/concat_op.h | 4 + .../dataset/engine/datasetops/dataset_op.cc | 36 +++--- .../dataset/engine/datasetops/dataset_op.h | 10 +- .../dataset/engine/datasetops/filter_op.cc | 3 - .../ccsrc/dataset/engine/datasetops/map_op.cc | 116 ++++++++---------- .../ccsrc/dataset/engine/datasetops/map_op.h | 13 +- .../dataset/engine/datasetops/project_op.cc | 40 +++--- .../dataset/engine/datasetops/project_op.h | 4 + .../dataset/engine/datasetops/rename_op.cc | 87 ++++++------- .../dataset/engine/datasetops/rename_op.h | 4 +- .../dataset/engine/datasetops/repeat_op.cc | 2 - .../dataset/engine/datasetops/shuffle_op.cc | 3 - .../dataset/engine/datasetops/skip_op.cc | 3 - .../engine/datasetops/source/celeba_op.cc | 17 ++- .../engine/datasetops/source/celeba_op.h | 4 + .../engine/datasetops/source/cifar_op.cc | 16 ++- .../engine/datasetops/source/cifar_op.h | 4 + .../engine/datasetops/source/clue_op.cc | 21 ++-- .../engine/datasetops/source/clue_op.h | 4 + .../engine/datasetops/source/coco_op.cc | 16 ++- .../engine/datasetops/source/coco_op.h | 4 + .../engine/datasetops/source/generator_op.cc | 18 ++- .../engine/datasetops/source/generator_op.h | 4 + .../datasetops/source/image_folder_op.cc | 16 ++- .../datasetops/source/image_folder_op.h | 4 + .../engine/datasetops/source/manifest_op.cc | 16 ++- .../engine/datasetops/source/manifest_op.h | 4 + .../engine/datasetops/source/mindrecord_op.cc | 15 ++- .../engine/datasetops/source/mindrecord_op.h | 4 + .../engine/datasetops/source/mnist_op.cc | 16 ++- .../engine/datasetops/source/mnist_op.h | 4 + .../datasetops/source/random_data_op.cc | 13 +- .../engine/datasetops/source/random_data_op.h | 4 + .../engine/datasetops/source/text_file_op.cc | 17 ++- .../engine/datasetops/source/text_file_op.h | 4 + .../engine/datasetops/source/tf_reader_op.cc | 17 ++- .../engine/datasetops/source/tf_reader_op.h | 4 + .../engine/datasetops/source/voc_op.cc | 16 ++- .../dataset/engine/datasetops/source/voc_op.h | 4 + .../dataset/engine/datasetops/take_op.cc | 1 - .../ccsrc/dataset/engine/datasetops/zip_op.cc | 43 ++++--- .../ccsrc/dataset/engine/datasetops/zip_op.h | 4 + 49 files changed, 419 insertions(+), 268 deletions(-) diff --git a/mindspore/ccsrc/dataset/engine/dataset_iterator.cc b/mindspore/ccsrc/dataset/engine/dataset_iterator.cc index 7eb38785aa..be333741b1 100644 --- a/mindspore/ccsrc/dataset/engine/dataset_iterator.cc +++ b/mindspore/ccsrc/dataset/engine/dataset_iterator.cc @@ -27,7 +27,7 @@ namespace mindspore { namespace dataset { // Constructor of the IteratorBase -IteratorBase::IteratorBase() : curr_buffer_(nullptr), eof_handled_(false), first_row_(true) {} +IteratorBase::IteratorBase() : curr_buffer_(nullptr), eof_handled_(false) {} IteratorBase::~IteratorBase() = default; @@ -51,13 +51,10 @@ Status IteratorBase::GetNextAsMap(TensorMap *out_map) { // The column name mapping comes from the source operator that is producing the data into the iterator. // To avoid having to fetch this for every time, we'll take a local copy of the column name id mapping // and save in the iterator. We only have to do this once. All subsequent iterations use the same mapping. - // Note: This can only be done after the first row has been produced, as this guarantees the the child has - // it's column mapping set up. - if (first_row_) { + if (col_name_id_map_.empty()) { // Determine the column name map by calling the derived class method to retrieve the column // name map col_name_id_map_ = this->GetColumnNameMap(); - first_row_ = false; } // Populate the out map from the row and return it diff --git a/mindspore/ccsrc/dataset/engine/dataset_iterator.h b/mindspore/ccsrc/dataset/engine/dataset_iterator.h index ada2b0ffb6..4e40e77c74 100644 --- a/mindspore/ccsrc/dataset/engine/dataset_iterator.h +++ b/mindspore/ccsrc/dataset/engine/dataset_iterator.h @@ -72,7 +72,6 @@ class IteratorBase { protected: std::unique_ptr curr_buffer_; // holds the current buffer bool eof_handled_; // T/F if this op got an eof - bool first_row_; // internal tracking for first row case std::unordered_map col_name_id_map_; }; diff --git a/mindspore/ccsrc/dataset/engine/datasetops/barrier_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/barrier_op.cc index 5ab2df4ac4..6fc276a75e 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/barrier_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/barrier_op.cc @@ -144,9 +144,6 @@ Status BarrierOp::prepare(TensorQTable *const table) { table->push_back(std::move(new_row)); - // Assign the column name id map - RETURN_IF_NOT_OK(DatasetOp::AssignColMapFromChild()); - // the update code below shouldn't do anything bad if the column name already exists. return Status::OK(); } diff --git a/mindspore/ccsrc/dataset/engine/datasetops/batch_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/batch_op.cc index 60643c90ba..f311c90c33 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/batch_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/batch_op.cc @@ -76,7 +76,6 @@ Status BatchOp::operator()() { std::unique_ptr table = std::make_unique(); child_iterator_ = std::make_unique(this, 0, 0); RETURN_IF_NOT_OK(child_iterator_->FetchNextTensorRow(&new_row)); - RETURN_IF_NOT_OK(DatasetOp::AssignColMapFromChild()); // must come after the first fetch above int32_t cur_batch_size = 0; RETURN_IF_NOT_OK(GetBatchSize(&cur_batch_size, CBatchInfo(0, 0, 0))); while (child_iterator_->eof_handled() == false) { diff --git a/mindspore/ccsrc/dataset/engine/datasetops/bucket_batch_by_length_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/bucket_batch_by_length_op.cc index def2ea0fee..5e143b700f 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/bucket_batch_by_length_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/bucket_batch_by_length_op.cc @@ -115,7 +115,6 @@ Status BucketBatchByLengthOp::operator()() { TensorRow current_row; child_iterator_ = std::make_unique(this, 0, 0); RETURN_IF_NOT_OK(child_iterator_->FetchNextTensorRow(¤t_row)); - RETURN_IF_NOT_OK(AssignColMapFromChild()); while (!child_iterator_->eof_handled()) { while (!current_row.empty()) { int32_t element_length; diff --git a/mindspore/ccsrc/dataset/engine/datasetops/build_vocab_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/build_vocab_op.cc index f99804ec9b..ceb5058593 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/build_vocab_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/build_vocab_op.cc @@ -86,7 +86,6 @@ Status BuildVocabOp::operator()() { child_iterator_ = std::make_unique(this, 0, 0); TensorRow new_row; RETURN_IF_NOT_OK(child_iterator_->FetchNextTensorRow(&new_row)); - RETURN_IF_NOT_OK(AssignColMapFromChild()); if (!col_names_.empty()) { col_ids_.reserve(col_names_.size()); for (std::string col : col_names_) { diff --git a/mindspore/ccsrc/dataset/engine/datasetops/concat_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/concat_op.cc index 7162dc0b47..c5aac523d2 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/concat_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/concat_op.cc @@ -66,12 +66,6 @@ Status ConcatOp::operator()() { std::unique_ptr buf; RETURN_IF_NOT_OK(child_[0]->GetNextBuffer(&buf)); - // Obtain columns_name_id_map from child_[0] - column_name_id_map_ = child_[0]->column_name_id_map(); - if (column_name_id_map_.empty()) { - RETURN_STATUS_UNEXPECTED("Child column name map cannot be empty!"); - } - int eof_count = 0; while (eof_count != children_num_) { for (int i = 0; i < children_num_; i++) { @@ -115,17 +109,13 @@ Status ConcatOp::Verify(int32_t id, const std::unique_ptr &buf) { buf->GetRow(0, &new_row); if (id == 0) { - // Obtain the column name, data type and data rank in child[0] - column_name_id_ = child_[id]->column_name_id_map(); + // Obtain the data type and data rank in child[0] for (auto item : new_row) { data_type_.push_back(item->type()); data_rank_.push_back(item->Rank()); } } else { - // Compare the column name, data type and data rank with these in child[0] - if (child_[id]->column_name_id_map() != column_name_id_) { - RETURN_STATUS_UNEXPECTED("The column name or column order is not the same with previous dataset."); - } + // Compare the data type and data rank with these in child[0] int32_t index = 0; for (auto item : new_row) { if ((item->type() != data_type_[index]) || item->Rank() != data_rank_[index++]) { @@ -141,5 +131,25 @@ Status ConcatOp::PrepareNodePostAction() { tree_->AddToRepeatStack(shared_from_this()); return Status::OK(); } + +// We need to overwrite the super class ComputeColMap here because the number of children is more than 1. +Status ConcatOp::ComputeColMap() { + if (column_name_id_map_.empty()) { + // Obtain columns_name_id_map from child_[0] + column_name_id_map_ = child_[0]->column_name_id_map(); + if (column_name_id_map_.empty()) { + RETURN_STATUS_UNEXPECTED("Child column name map cannot be empty!"); + } + // Verify all children have the same column name map + for (int32_t i = 0; i < child_.size(); ++i) { + if (child_[i]->column_name_id_map() != column_name_id_map_) { + RETURN_STATUS_UNEXPECTED("The column name or column order is not the same with previous dataset."); + } + } + } else { + MS_LOG(WARNING) << "Column name map is already set!"; + } + return Status::OK(); +} } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/concat_op.h b/mindspore/ccsrc/dataset/engine/datasetops/concat_op.h index 0fb8ec8362..4bcfdbf6c6 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/concat_op.h +++ b/mindspore/ccsrc/dataset/engine/datasetops/concat_op.h @@ -85,6 +85,10 @@ class ConcatOp : public PipelineOp { // @return Name of the current Op std::string Name() const override { return "ConcatOp"; } + // Private function for computing the assignment of the column name map. + // @return - Status + Status ComputeColMap() override; + private: Status Verify(int32_t id, const std::unique_ptr &buf); diff --git a/mindspore/ccsrc/dataset/engine/datasetops/dataset_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/dataset_op.cc index bf991ea7d9..d5625c8d06 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/dataset_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/dataset_op.cc @@ -39,8 +39,7 @@ DatasetOp::DatasetOp(int32_t op_connector_size) tree_(nullptr), state_(OpState::kDeOpIdle), op_ctrl_flags_(kDeOpNone), - out_connector_(nullptr), - first_fetch_(true) { + out_connector_(nullptr) { // The operator starts out with an invalid operator id. The only way to // get it out of invalid state is to assign the operator to an execution tree. } @@ -240,6 +239,10 @@ Status DatasetOp::PrepareNodePostAction() { RETURN_IF_NOT_OK(out_connector_->Register(tree_->AllTasks())); } RETURN_IF_NOT_OK(this->RegisterWorkerConnectors()); + + // Generate the column name map for the current op. + RETURN_IF_NOT_OK(this->ComputeColMap()); + return Status::OK(); } @@ -262,30 +265,21 @@ std::string DatasetOp::ColumnNameMapAsString() const { return outStr; } -// A helper function for providing assignment of the column name map. -// This grabs the map from child 0 and assigns it into this op. -// Can only be used if number of children is 1. -Status DatasetOp::AssignColMapFromChild() { +// Computing the assignment of the column name map. +// This just inherits the column map from its first child, can only be used if the number of children is 1. +// Operations changing the column map must overwrite this function. +Status DatasetOp::ComputeColMap() { if (child_.size() > 1) { RETURN_STATUS_UNEXPECTED("Assigning column name map from child only works for single-child operators."); } - // Assign the correct column name map to this op by taking it from the input child. - // This must be done AFTER the first fetch, but only needs to be done once by the first worker to - // do the first fetch. - if (first_fetch_) { - // If there was a single worker, or this is being called from a master thread in a parallel op, - // then the mutex is not really needed here, although it's harmless. - std::unique_lock lock(column_name_map_mutex_); - // If the map has not been set up yet, then we are the first one in to set it up. The first_fetch_ (dirty read) - // bool allows us to avoid acquiring the lock if the map has already been set. + if (column_name_id_map_.empty()) { + column_name_id_map_ = child_[0]->column_name_id_map(); if (column_name_id_map_.empty()) { - column_name_id_map_ = child_[0]->column_name_id_map(); - first_fetch_ = false; - if (column_name_id_map_.empty()) { - RETURN_STATUS_UNEXPECTED("Child column name map cannot be empty!"); - } + RETURN_STATUS_UNEXPECTED("Child column name map cannot be empty!"); } - MS_LOG(DEBUG) << "Setting column map after first fetch:\n" << DatasetOp::ColumnNameMapAsString(); + MS_LOG(DEBUG) << "Setting column map:\n" << DatasetOp::ColumnNameMapAsString(); + } else { + MS_LOG(WARNING) << "Column name map is already set!"; } return Status::OK(); } diff --git a/mindspore/ccsrc/dataset/engine/datasetops/dataset_op.h b/mindspore/ccsrc/dataset/engine/datasetops/dataset_op.h index 973b5be962..c444004b79 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/dataset_op.h +++ b/mindspore/ccsrc/dataset/engine/datasetops/dataset_op.h @@ -277,11 +277,12 @@ class DatasetOp : public std::enable_shared_from_this { // @param parent - The parent node to remove void RemoveParent(DatasetOp *parent); - // A helper function for providing an assignment of the column name map. - // This grabs the map from child 0 and assigns it into this op. - // Can only be used if number of children is 1. + // Compute the current op's column map using its child's column map. + // Get called during the tree post-prepare phase in PrepareNodePostAction. + // This base implementation just inherits the map from child 0, and can only be used if the number of children is 1. + // Operations changing the column map it inherits from the child must overwrite this function. // @return - Status - Status AssignColMapFromChild(); + virtual Status ComputeColMap(); std::vector> child_; // Child nodes std::vector parent_; // Parent nodes. No ownership @@ -292,7 +293,6 @@ class DatasetOp : public std::enable_shared_from_this { uint32_t op_ctrl_flags_; // Flags for the operator std::unique_ptr out_connector_; // Output Connector std::unordered_map column_name_id_map_; // Mapping between col index and col name - bool first_fetch_; // For use when setting column map std::mutex column_name_map_mutex_; // For protecting shared access to the column map private: diff --git a/mindspore/ccsrc/dataset/engine/datasetops/filter_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/filter_op.cc index 26b99080c8..a1c5ed0070 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/filter_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/filter_op.cc @@ -126,9 +126,6 @@ Status FilterOp::WorkerEntry(int32_t worker_id) { continue; } - // Now that the first fetch is in, use the helper function to assign the column name map to this op. - RETURN_IF_NOT_OK(DatasetOp::AssignColMapFromChild()); - RETURN_IF_NOT_OK(CheckColumns(in_buffer.get(), &in_columns_)); // if the databuffer was all filtered, it is marked as kFilterEmpty. diff --git a/mindspore/ccsrc/dataset/engine/datasetops/map_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/map_op.cc index 9918260201..053559f88b 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/map_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/map_op.cc @@ -156,14 +156,15 @@ Status MapOp::WorkerEntry(int32_t worker_id) { // initializations that happen after the first fetch. RETURN_IF_NOT_OK(FetchNextBuffer(&in_buffer, worker_id)); - // Initialize details related to column selections and column map by calling WorkerEntryInit. - // WorkerEntryInit contains thread-safe lock to ensure that this init work is only performed once - // by the first worker to enter the codepath. All other threads will share the const info that - // gets set up here going forward. + // Sanity check the databuffer. // Special case: if there's more threads than buffers, some threads simply get the final control - // messages (eoe/eof), and so they will not perform the init work. + // messages (eoe/eof), and so they will not perform the check. if (!in_buffer->eoe() && !in_buffer->eof()) { - RETURN_IF_NOT_OK(WorkerEntryInit(in_buffer.get())); + int32_t num_rows = in_buffer->NumRows(); + int32_t num_cols = in_buffer->NumCols(); + if (num_rows == 0 || num_cols == 0) { + RETURN_STATUS_UNEXPECTED("MapOp is getting an empty DataBuffer."); + } } // Now that init work is done, drop into the main fetching loop. @@ -258,63 +259,18 @@ Status MapOp::WorkerCompute(DataBuffer *in_buffer, TensorQTable *new_tensor_tabl return Status::OK(); } -// initialize some internal data structure used by WorkerEntry() -Status MapOp::WorkerEntryInit(const DataBuffer *in_buf) { - int32_t num_rows = in_buf->NumRows(); - int32_t num_cols = in_buf->NumCols(); - if (num_rows == 0 || num_cols == 0) { - RETURN_STATUS_UNEXPECTED("MapOp is getting an empty DataBuffer."); +Status MapOp::ComputeColMap() { + // If the map has not been set up yet in the base class, then set it up + if (column_name_id_map_.empty()) { + std::unordered_map current_name_id_map = child_[0]->column_name_id_map(); + // Initialize private variables + RETURN_IF_NOT_OK(InitPrivateVariable(¤t_name_id_map)); + // Create the final column name to index mapping in the base class field + CreateFinalColMap(¤t_name_id_map); + MS_LOG(DEBUG) << "Column name map for map op set: " << this->ColumnNameMapAsString(); + } else { + MS_LOG(WARNING) << "Column name map is already set!"; } - - // We can't use AssignColMapFromChild() here since we need to modify the column map. We need to be threadsafe - // though for saving the final map in the op, so use the lock here. - if (first_fetch_) { - std::unique_lock lock(column_name_map_mutex_); - // If the map has not been set up yet in the base class, then we are the first one in to set it up - // (and we are under protection of the mutex lock) - if (column_name_id_map_.empty()) { - std::unordered_map current_name_id_map = child_[0]->column_name_id_map(); - - // If input_columns is empty(), The col at index-0 will be picked. - if (in_columns_.empty()) { - for (const auto &pair : current_name_id_map) { - if (pair.second == 0) { - MS_LOG(INFO) << "Input columns empty for map op, will apply to the first column in the current table."; - in_columns_.push_back(pair.first); - break; - } - } - - // If caller didn't specify the out_col_names, assume they are same as the input_columns. - // This was done in the constructor, but if input columns was empty to start we have to redo it here. - if (out_columns_.empty() || out_columns_[0].empty()) { - out_columns_ = in_columns_; - } - } - - // Before we continue, issue a sanity check to make sure the input columns from user and the incoming - // columns from child are correct - RETURN_IF_NOT_OK(this->ValidateInColumns(current_name_id_map)); - - // initialize keep_input_columns, true means to keep the column. - keep_input_columns_.resize(num_cols, true); - for (const auto &col_name : in_columns_) { - int32_t missed = current_name_id_map[col_name]; - keep_input_columns_[missed] = false; - } - - // initialize to_process_indices. - for (const auto &col_name : in_columns_) { - to_process_indices_.push_back(current_name_id_map[col_name]); - } - - // Create the final column name to index mapping in the base class field - CreateFinalColMap(¤t_name_id_map); - first_fetch_ = false; - } - } // mutex lock will release here - - MS_LOG(DEBUG) << "Column name map for map op set: " << this->ColumnNameMapAsString(); return Status::OK(); } @@ -330,6 +286,42 @@ Status MapOp::ValidateInColumns(const std::unordered_map & return Status::OK(); } +Status MapOp::InitPrivateVariable(std::unordered_map *col_name_id_map) { + // If input_columns is empty(), The col at index-0 will be picked. + if (in_columns_.empty()) { + for (const auto &pair : *col_name_id_map) { + if (pair.second == 0) { + MS_LOG(INFO) << "Input columns empty for map op, will apply to the first column in the current table."; + in_columns_.push_back(pair.first); + break; + } + } + + // If caller didn't specify the out_col_names, assume they are same as the input_columns. + // This was done in the constructor, but if input columns was empty to start we have to redo it here. + if (out_columns_.empty() || out_columns_[0].empty()) { + out_columns_ = in_columns_; + } + } + + // Before we continue, issue a sanity check to make sure the input columns from user and the incoming + // columns from child are correct + RETURN_IF_NOT_OK(this->ValidateInColumns(*col_name_id_map)); + + // initialize keep_input_columns, true means to keep the column. + keep_input_columns_.resize(col_name_id_map->size(), true); + for (const auto &col_name : in_columns_) { + int32_t missed = (*col_name_id_map)[col_name]; + keep_input_columns_[missed] = false; + } + + // initialize to_process_indices. + for (const auto &col_name : in_columns_) { + to_process_indices_.push_back((*col_name_id_map)[col_name]); + } + return Status::OK(); +} + // Create the final column name to index mapping and get indices of the columns this mapop does not use. void MapOp::CreateFinalColMap(std::unordered_map *col_name_id_map) { std::unordered_map final_col_name_id_map; diff --git a/mindspore/ccsrc/dataset/engine/datasetops/map_op.h b/mindspore/ccsrc/dataset/engine/datasetops/map_op.h index 4d7ffd1204..94569bd41f 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/map_op.h +++ b/mindspore/ccsrc/dataset/engine/datasetops/map_op.h @@ -258,15 +258,18 @@ class MapOp : public ParallelOp { // @param col_name_id_map The column name to index mapping obtained from child operator void CreateFinalColMap(std::unordered_map *col_name_id_map); - // Private function that initialize some internal data structure used by WorkerEntry() - // @param in_buf A raw pointer to the DataBuffer. A raw pointer is fine because this function does not manage memory - // and is not shared with other threads. - Status WorkerEntryInit(const DataBuffer *in_buf); - // Validating if each of the input_columns exists in the DataBuffer. // @param - the column map to check // @return - status return code Status ValidateInColumns(const std::unordered_map &col_name_id_map); + + // Private function for computing the assignment of the column name map. + // @return - Status + Status ComputeColMap() override; + + // Private function for initializing private variables such as in_columns_, out_columns_. + // @return - Status + Status InitPrivateVariable(std::unordered_map *col_name_id_map); }; } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/project_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/project_op.cc index 063c618aeb..14b064bab9 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/project_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/project_op.cc @@ -74,24 +74,6 @@ void ProjectOp::Print(std::ostream &out, bool show_all) const { Status ProjectOp::GetNextBuffer(std::unique_ptr *p_buffer, int32_t worker_id, bool retry_if_eoe) { RETURN_IF_NOT_OK(child_[0]->GetNextBuffer(p_buffer, worker_id, retry_if_eoe)); if (!((*p_buffer)->eoe()) && !((*p_buffer)->eof())) { - // Only for the first buffer fetched, get the column map of the incoming data and save it - // into our own column name map after making the appropriate mods - // We cannot use the super class AssignColMapFromChild here because we're making a modification of the - // map from the child map. - if (first_fetch_) { - std::unordered_map child_column_name_mapping = child_[0]->column_name_id_map(); - for (size_t i = 0; i < columns_to_project_.size(); i++) { - std::string ¤t_column = columns_to_project_[i]; - if (child_column_name_mapping.find(current_column) == child_column_name_mapping.end()) { - std::string err_msg = "ProjectOp: column " + current_column + " does not exist in child operator."; - RETURN_STATUS_UNEXPECTED(err_msg); - } - // Setup the new column name mapping for ourself (base class field) - column_name_id_map_[current_column] = i; - projected_column_indices_.push_back(child_column_name_mapping[current_column]); - } - first_fetch_ = false; // we only need to do this path once - } RETURN_IF_NOT_OK(Project(p_buffer)); } return Status::OK(); @@ -151,5 +133,27 @@ Status ProjectOp::Accept(NodePass *p, bool *modified) { // Downcast shared pointer then call visitor return p->RunOnNode(std::static_pointer_cast(shared_from_this()), modified); } + +// Compute the column map and save it into our own column name map +// We cannot use the super class ComputeColMap here because we're making a modification of the +// map from the child map. +Status ProjectOp::ComputeColMap() { + if (column_name_id_map_.empty()) { + std::unordered_map child_column_name_mapping = child_[0]->column_name_id_map(); + for (size_t i = 0; i < columns_to_project_.size(); i++) { + std::string ¤t_column = columns_to_project_[i]; + if (child_column_name_mapping.find(current_column) == child_column_name_mapping.end()) { + std::string err_msg = "ProjectOp: column " + current_column + " does not exist in child operator."; + RETURN_STATUS_UNEXPECTED(err_msg); + } + // Setup the new column name mapping for ourself (base class field) + column_name_id_map_[current_column] = i; + projected_column_indices_.push_back(child_column_name_mapping[current_column]); + } + } else { + MS_LOG(WARNING) << "Column name map is already set!"; + } + return Status::OK(); +} } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/project_op.h b/mindspore/ccsrc/dataset/engine/datasetops/project_op.h index ced0f9e5a9..628c1342ba 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/project_op.h +++ b/mindspore/ccsrc/dataset/engine/datasetops/project_op.h @@ -116,6 +116,10 @@ class ProjectOp : public PipelineOp { std::vector projected_column_indices_; Status Project(std::unique_ptr *data_buffer); + + // Computing the assignment of the column name map. + // @return - Status + Status ComputeColMap() override; }; } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/rename_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/rename_op.cc index c026aac4fa..bebca780ff 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/rename_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/rename_op.cc @@ -69,12 +69,6 @@ Status RenameOp::operator()() { RETURN_STATUS_UNEXPECTED(err_msg); } - // First, populate the column map from the input child. - // This will not be the final map for output from this op. - RETURN_IF_NOT_OK(DatasetOp::AssignColMapFromChild()); - // core rename functionality only needs to happen once, to identify the new column names/indexes - RETURN_IF_NOT_OK(RenameColumns()); - while (curr_buffer->eof() == false) { while (curr_buffer->eoe() == false) { // push the renamed input buffer @@ -95,45 +89,52 @@ Status RenameOp::operator()() { return Status::OK(); } -// renames the columns -Status RenameOp::RenameColumns() { - // iterate over my index in input vector, find the corresponding position - std::unordered_map new_col_name_id_map = {}; - // parameter for input check - size_t found = 0; - - // iterate over all the pairs and if there is a name match with rename, rename the column and add it to new map - // by doing it this way we recreate a new ColNameIdMap and allow for switching - for (const auto &pair : column_name_id_map_) { - std::string name = pair.first; - int32_t id = pair.second; - // find name - std::vector::iterator it; - it = std::find(in_columns_.begin(), in_columns_.end(), name); - // for c input checks here we have to count the number of times we find the stuff in in_columns_ - // because we iterate over the mInputList n times - if (it != in_columns_.end()) { - // found - found += 1; - int index = std::distance(in_columns_.begin(), it); - MS_LOG(DEBUG) << "Rename operator index found " << index << " value " << id << "."; - - new_col_name_id_map[out_columns_[index]] = id; - } else { - // not found - MS_LOG(DEBUG) << "Rename operator index not found: " << id << " is the column id."; - new_col_name_id_map[name] = id; +// Rename core functionality to compute the new column name id map. +// We need to overwrite the super class ComputeColMap here because we're making a modification of the +// map from the child map. +Status RenameOp::ComputeColMap() { + if (column_name_id_map_.empty()) { + column_name_id_map_ = child_[0]->column_name_id_map(); + // iterate over my index in input vector, find the corresponding position + std::unordered_map new_col_name_id_map = {}; + // parameter for input check + size_t found = 0; + + // iterate over all the pairs and if there is a name match with rename, rename the column and add it to new map + // by doing it this way we recreate a new ColNameIdMap and allow for switching + for (const auto &pair : column_name_id_map_) { + std::string name = pair.first; + int32_t id = pair.second; + // find name + std::vector::iterator it; + it = std::find(in_columns_.begin(), in_columns_.end(), name); + // for c input checks here we have to count the number of times we find the stuff in in_columns_ + // because we iterate over the mInputList n times + if (it != in_columns_.end()) { + // found + found += 1; + int index = std::distance(in_columns_.begin(), it); + MS_LOG(DEBUG) << "Rename operator index found " << index << " value " << id << "."; + + new_col_name_id_map[out_columns_[index]] = id; + } else { + // not found + MS_LOG(DEBUG) << "Rename operator index not found: " << id << " is the column id."; + new_col_name_id_map[name] = id; + } + } + // only checks number of renamed columns have been found, this input check doesn't check everything + if (found != in_columns_.size()) { + MS_LOG(DEBUG) << "Rename operator column names found: " << found << " out of " << in_columns_.size() << "."; + std::string err_msg = "Renamed column doesn't exist in dataset"; + RETURN_STATUS_UNEXPECTED(err_msg); } - } - // only checks number of renamed columns have been found, this input check doesn't check everything - if (found != in_columns_.size()) { - MS_LOG(DEBUG) << "Rename operator column names found: " << found << " out of " << in_columns_.size() << "."; - std::string err_msg = "Renamed column doesn't exist in dataset"; - RETURN_STATUS_UNEXPECTED(err_msg); - } - // Now, overwrite our column map with the new renamed columns/id's - column_name_id_map_ = new_col_name_id_map; + // Now, overwrite our column map with the new renamed columns/id's + column_name_id_map_ = new_col_name_id_map; + } else { + MS_LOG(WARNING) << "Column name map is already set!"; + } return Status::OK(); } diff --git a/mindspore/ccsrc/dataset/engine/datasetops/rename_op.h b/mindspore/ccsrc/dataset/engine/datasetops/rename_op.h index eaca20ccc8..e209c075d6 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/rename_op.h +++ b/mindspore/ccsrc/dataset/engine/datasetops/rename_op.h @@ -122,7 +122,9 @@ class RenameOp : public PipelineOp { protected: // Rename core functionality - Status RenameColumns(); + // Computing the assignment of the new column name map. + // @return - Status + Status ComputeColMap() override; // Variable to store the input column names std::vector in_columns_; diff --git a/mindspore/ccsrc/dataset/engine/datasetops/repeat_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/repeat_op.cc index 524603fd94..66e2177636 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/repeat_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/repeat_op.cc @@ -123,8 +123,6 @@ Status RepeatOp::GetNextBuffer(std::unique_ptr *p_buffer, int32_t wo if (buf->eof()) { RETURN_IF_NOT_OK(EofReceived(worker_id)); } - // Update the column name map if needed - RETURN_IF_NOT_OK(DatasetOp::AssignColMapFromChild()); *p_buffer = std::move(buf); return Status::OK(); } diff --git a/mindspore/ccsrc/dataset/engine/datasetops/shuffle_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/shuffle_op.cc index 69f7e09986..c16f3f9625 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/shuffle_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/shuffle_op.cc @@ -266,9 +266,6 @@ Status ShuffleOp::InitShuffleBuffer() { RETURN_STATUS_UNEXPECTED("Unable to fetch a single row for shuffle buffer."); } - // Now that a first fetch is done, assign the column map for this operator - RETURN_IF_NOT_OK(DatasetOp::AssignColMapFromChild()); - // Now fill the rest of the shuffle buffer until we are unable to get the next row or we reached // the desired shuffle buffer size. while (!new_row.empty() && shuffle_buffer_->size() < static_cast(shuffle_size_ - 1)) { diff --git a/mindspore/ccsrc/dataset/engine/datasetops/skip_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/skip_op.cc index 35fae8d091..c00fd486b7 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/skip_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/skip_op.cc @@ -86,9 +86,6 @@ Status SkipOp::operator()() { std::unique_ptr curr_buffer; RETURN_IF_NOT_OK(GetNextInput(&curr_buffer)); - // After the first buffer fetch above we can do the one-time assign of the column name map - RETURN_IF_NOT_OK(DatasetOp::AssignColMapFromChild()); - while (curr_buffer->eof() == false) { // Reset count skip_count_ = 0; diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/celeba_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/celeba_op.cc index 4b32201d6d..7889362555 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/celeba_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/celeba_op.cc @@ -79,11 +79,6 @@ CelebAOp::CelebAOp(int32_t num_workers, int32_t rows_per_buffer, const std::stri sampler_(std::move(sampler)), num_rows_in_attr_file_(0), dataset_type_(dataset_type) { - // Set the column name map (base class field) - for (int32_t index = 0; index < data_schema_->NumColumns(); index++) { - column_name_id_map_[data_schema_->column(index).name()] = index; - } - attr_info_queue_ = std::make_unique>>(queue_size); io_block_queues_.Init(num_workers_, queue_size); } @@ -413,5 +408,17 @@ Status CelebAOp::Reset() { wp_.Set(); // wake up master thread after reset is done return Status::OK(); } + +Status CelebAOp::ComputeColMap() { + // Set the column name map (base class field) + if (column_name_id_map_.empty()) { + for (int32_t index = 0; index < data_schema_->NumColumns(); index++) { + column_name_id_map_[data_schema_->column(index).name()] = index; + } + } else { + MS_LOG(WARNING) << "Column name map is already set!"; + } + return Status::OK(); +} } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/celeba_op.h b/mindspore/ccsrc/dataset/engine/datasetops/source/celeba_op.h index f4b5d040ca..f8a49dabb2 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/celeba_op.h +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/celeba_op.h @@ -212,6 +212,10 @@ class CelebAOp : public ParallelOp, RandomAccessOp { // @return Status - The error code return Status Reset() override; + // Private function for computing the assignment of the column name map. + // @return - Status + Status ComputeColMap() override; + int32_t rows_per_buffer_; std::string folder_path_; // directory of celeba folder bool decode_; diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/cifar_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/cifar_op.cc index ad87e394eb..e7c418b146 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/cifar_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/cifar_op.cc @@ -87,10 +87,6 @@ CifarOp::CifarOp(CifarType type, int32_t num_works, int32_t rows_per_buf, const sampler_(std::move(sampler)), row_cnt_(0), buf_cnt_(0) { - // set the column name map (base class field) - for (uint32_t i = 0; i < data_schema_->NumColumns(); ++i) { - column_name_id_map_[data_schema_->column(i).name()] = i; - } constexpr uint64_t kUtilQueueSize = 512; cifar_raw_data_block_ = std::make_unique>>(kUtilQueueSize); io_block_queues_.Init(num_workers_, queue_size); @@ -454,5 +450,17 @@ Status CifarOp::CountTotalRows(const std::string &dir, bool isCIFAR10, int64_t * return Status::OK(); } } + +Status CifarOp::ComputeColMap() { + // set the column name map (base class field) + if (column_name_id_map_.empty()) { + for (uint32_t i = 0; i < data_schema_->NumColumns(); ++i) { + column_name_id_map_[data_schema_->column(i).name()] = i; + } + } else { + MS_LOG(WARNING) << "Column name map is already set!"; + } + return Status::OK(); +} } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/cifar_op.h b/mindspore/ccsrc/dataset/engine/datasetops/source/cifar_op.h index 62c20ac401..21ed80ceab 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/cifar_op.h +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/cifar_op.h @@ -208,6 +208,10 @@ class CifarOp : public ParallelOp, public RandomAccessOp { // @return Status - The error code return Status GetClassIds(std::map> *cls_ids) const override; + // Private function for computing the assignment of the column name map. + // @return - Status + Status ComputeColMap() override; + CifarType cifar_type_; int32_t rows_per_buffer_; std::string folder_path_; diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/clue_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/clue_op.cc index e92ca0d26c..d863de15ad 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/clue_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/clue_op.cc @@ -112,13 +112,6 @@ Status ClueOp::Init() { int32_t safe_queue_size = static_cast(std::ceil(clue_files_list_.size() / num_workers_) + 1); io_block_queues_.Init(num_workers_, safe_queue_size); - // Set the column name mapping (base class field) - int count = 0; - for (auto &p : cols_to_keyword_) { - column_name_id_map_[p.first] = count; - count++; - } - RETURN_IF_NOT_OK(ParallelOp::CreateWorkerConnector(worker_connector_size_)); jagged_buffer_connector_ = std::make_unique(num_workers_, 1, worker_connector_size_); @@ -549,5 +542,19 @@ Status ClueOp::CountAllFileRows(const std::vector &files, int64_t * } return Status::OK(); } + +Status ClueOp::ComputeColMap() { + // Set the column name mapping (base class field) + if (column_name_id_map_.empty()) { + int count = 0; + for (auto &p : cols_to_keyword_) { + column_name_id_map_[p.first] = count; + count++; + } + } else { + MS_LOG(WARNING) << "Column name map is already set!"; + } + return Status::OK(); +} } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/clue_op.h b/mindspore/ccsrc/dataset/engine/datasetops/source/clue_op.h index b6a797d3f4..f41abd020c 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/clue_op.h +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/clue_op.h @@ -263,6 +263,10 @@ class ClueOp : public ParallelOp { // @return Status - the error code returned. Status GetValue(const nlohmann::json &js, std::vector key_chain, std::shared_ptr *t); + // Private function for computing the assignment of the column name map. + // @return - Status + Status ComputeColMap() override; + int32_t device_id_; bool shuffle_files_; bool shuffle_global_; diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/coco_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/coco_op.cc index 8d352bbd6c..92f6794769 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/coco_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/coco_op.cc @@ -129,10 +129,6 @@ CocoOp::CocoOp(const TaskType &task_type, const std::string &image_folder_path, rows_per_buffer_(rows_per_buffer), sampler_(std::move(sampler)), data_schema_(std::move(data_schema)) { - // Set the column name map (base class field) - for (int32_t i = 0; i < data_schema_->NumColumns(); ++i) { - column_name_id_map_[data_schema_->column(i).name()] = i; - } io_block_queues_.Init(num_workers_, queue_size); } @@ -627,5 +623,17 @@ Status CocoOp::GetClassIndexing(const std::string &dir, const std::string &file, *output_class_indexing = op->label_index_; return Status::OK(); } + +Status CocoOp::ComputeColMap() { + // Set the column name map (base class field) + if (column_name_id_map_.empty()) { + for (int32_t i = 0; i < data_schema_->NumColumns(); ++i) { + column_name_id_map_[data_schema_->column(i).name()] = i; + } + } else { + MS_LOG(WARNING) << "Column name map is already set!"; + } + return Status::OK(); +} } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/coco_op.h b/mindspore/ccsrc/dataset/engine/datasetops/source/coco_op.h index f5abeed72e..3791853798 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/coco_op.h +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/coco_op.h @@ -306,6 +306,10 @@ class CocoOp : public ParallelOp, public RandomAccessOp { template Status SearchNodeInJson(nlohmann::json input_tree, std::string node_name, T *output_node); + // Private function for computing the assignment of the column name map. + // @return - Status + Status ComputeColMap() override; + bool decode_; int64_t row_cnt_; int64_t buf_cnt_; diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/generator_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/generator_op.cc index d316524c04..eb5ba32642 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/generator_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/generator_op.cc @@ -94,12 +94,6 @@ void GeneratorOp::Dealloc() noexcept { Status GeneratorOp::Init() { // Reset BufferID buffer_id_ = 0; - // Setup column names map (base class field) - if (column_name_id_map_.empty()) { - for (int i = 0; i < column_names_.size(); ++i) { - column_name_id_map_[column_names_[i]] = i; - } - } Status ret; { // Acquire Python GIL @@ -257,5 +251,17 @@ Status GeneratorOp::Accept(NodePass *p, bool *modified) { // Downcast shared pointer then call visitor return p->RunOnNode(std::static_pointer_cast(shared_from_this()), modified); } + +Status GeneratorOp::ComputeColMap() { + // Setup column names map (base class field) + if (column_name_id_map_.empty()) { + for (int i = 0; i < column_names_.size(); ++i) { + column_name_id_map_[column_names_[i]] = i; + } + } else { + MS_LOG(WARNING) << "Column name map is already set!"; + } + return Status::OK(); +} } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/generator_op.h b/mindspore/ccsrc/dataset/engine/datasetops/source/generator_op.h index 82b395d6de..98dd2d70a1 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/generator_op.h +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/generator_op.h @@ -150,6 +150,10 @@ class GeneratorOp : public PipelineOp { Status PyRowToTensorRow(py::object py_data, TensorRow *tensor_row); Status FillBuffer(TensorQTable *tt); + + // Private function for computing the assignment of the column name map. + // @return - Status + Status ComputeColMap() override; }; #pragma GCC visibility pop diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/image_folder_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/image_folder_op.cc index 5cdfa8bb76..b2611a67fc 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/image_folder_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/image_folder_op.cc @@ -78,10 +78,6 @@ ImageFolderOp::ImageFolderOp(int32_t num_wkrs, int32_t rows_per_buffer, std::str buf_cnt_(0), sampler_ind_(0), dirname_offset_(0) { - // Set the column name map (base class field) - for (int32_t i = 0; i < data_schema_->NumColumns(); ++i) { - column_name_id_map_[data_schema_->column(i).name()] = i; - } folder_name_queue_ = std::make_unique>(num_wkrs * queue_size); image_name_queue_ = std::make_unique>(num_wkrs * queue_size); io_block_queues_.Init(num_workers_, queue_size); @@ -418,5 +414,17 @@ Status ImageFolderOp::Accept(NodePass *p, bool *modified) { // Downcast shared pointer then call visitor return p->RunOnNode(std::static_pointer_cast(shared_from_this()), modified); } + +Status ImageFolderOp::ComputeColMap() { + // Set the column name map (base class field) + if (column_name_id_map_.empty()) { + for (int32_t i = 0; i < data_schema_->NumColumns(); ++i) { + column_name_id_map_[data_schema_->column(i).name()] = i; + } + } else { + MS_LOG(WARNING) << "Column name map is already set!"; + } + return Status::OK(); +} } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/image_folder_op.h b/mindspore/ccsrc/dataset/engine/datasetops/source/image_folder_op.h index e1d578e034..06f39deee0 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/image_folder_op.h +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/image_folder_op.h @@ -248,6 +248,10 @@ class ImageFolderOp : public ParallelOp, public RandomAccessOp { // @return Status - The error code return Status Reset() override; + // Private function for computing the assignment of the column name map. + // @return - Status + Status ComputeColMap() override; + int32_t rows_per_buffer_; std::string folder_path_; // directory of image folder bool recursive_; diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/manifest_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/manifest_op.cc index 0762f36d5a..e26bb7de65 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/manifest_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/manifest_op.cc @@ -76,10 +76,6 @@ ManifestOp::ManifestOp(int32_t num_works, int32_t rows_per_buffer, std::string f decode_(decode), usage_(usage), buf_cnt_(0) { - // Set the column name map (base class field) - for (int32_t i = 0; i < data_schema_->NumColumns(); ++i) { - column_name_id_map_[data_schema_->column(i).name()] = i; - } io_block_queues_.Init(num_workers_, queue_size); (void)std::transform(usage_.begin(), usage_.end(), usage_.begin(), ::tolower); } @@ -420,5 +416,17 @@ Status ManifestOp::GetClassIndexing(const std::string &file, const py::dict &dic return Status::OK(); } + +Status ManifestOp::ComputeColMap() { + // Set the column name map (base class field) + if (column_name_id_map_.empty()) { + for (int32_t i = 0; i < data_schema_->NumColumns(); ++i) { + column_name_id_map_[data_schema_->column(i).name()] = i; + } + } else { + MS_LOG(WARNING) << "Column name map is already set!"; + } + return Status::OK(); +} } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/manifest_op.h b/mindspore/ccsrc/dataset/engine/datasetops/source/manifest_op.h index edfdbb51ae..1bdf683084 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/manifest_op.h +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/manifest_op.h @@ -219,6 +219,10 @@ class ManifestOp : public ParallelOp, public RandomAccessOp { // @return Status - The error code return Status CountDatasetInfo(); + // Private function for computing the assignment of the column name map. + // @return - Status + Status ComputeColMap() override; + int32_t rows_per_buffer_; int64_t io_block_pushed_; int64_t row_cnt_; diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/mindrecord_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/mindrecord_op.cc index 0f762386af..3c95b9b054 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/mindrecord_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/mindrecord_op.cc @@ -196,10 +196,6 @@ Status MindRecordOp::Init() { data_schema_ = std::move(tmp_schema); } - for (int i = 0; i < static_cast(columns_to_load_.size()); i++) { - column_name_id_map_[columns_to_load_[i]] = i; - } - return Status::OK(); } @@ -502,5 +498,16 @@ Status MindRecordOp::Accept(NodePass *p, bool *modified) { // Downcast shared pointer then call visitor return p->RunOnNode(std::static_pointer_cast(shared_from_this()), modified); } + +Status MindRecordOp::ComputeColMap() { + if (column_name_id_map_.empty()) { + for (int i = 0; i < static_cast(columns_to_load_.size()); i++) { + column_name_id_map_[columns_to_load_[i]] = i; + } + } else { + MS_LOG(WARNING) << "Column name map is already set!"; + } + return Status::OK(); +} } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/mindrecord_op.h b/mindspore/ccsrc/dataset/engine/datasetops/source/mindrecord_op.h index b704240aaa..af405a8f5b 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/mindrecord_op.h +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/mindrecord_op.h @@ -234,6 +234,10 @@ class MindRecordOp : public ParallelOp { Status FetchBlockBuffer(const int32_t &buffer_id); + // Private function for computing the assignment of the column name map. + // @return - Status + Status ComputeColMap() override; + int32_t rows_per_buffer_; // The number of requested rows per buffer. std::vector dataset_file_; // dataset files bool load_dataset_; // load dataset from single file or not diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/mnist_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/mnist_op.cc index eacd9daf75..67e7757da5 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/mnist_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/mnist_op.cc @@ -73,10 +73,6 @@ MnistOp::MnistOp(int32_t num_workers, int32_t rows_per_buffer, std::string folde rows_per_buffer_(rows_per_buffer), sampler_(std::move(sampler)), data_schema_(std::move(data_schema)) { - // set the column name map (base class field) - for (int32_t i = 0; i < data_schema_->NumColumns(); ++i) { - column_name_id_map_[data_schema_->column(i).name()] = i; - } io_block_queues_.Init(num_workers, queue_size); } @@ -432,5 +428,17 @@ Status MnistOp::CountTotalRows(const std::string &dir, int64_t *count) { return Status::OK(); } + +Status MnistOp::ComputeColMap() { + // set the column name map (base class field) + if (column_name_id_map_.empty()) { + for (int32_t i = 0; i < data_schema_->NumColumns(); ++i) { + column_name_id_map_[data_schema_->column(i).name()] = i; + } + } else { + MS_LOG(WARNING) << "Column name map is already set!"; + } + return Status::OK(); +} } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/mnist_op.h b/mindspore/ccsrc/dataset/engine/datasetops/source/mnist_op.h index 909ac22124..c22ee24acd 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/mnist_op.h +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/mnist_op.h @@ -226,6 +226,10 @@ class MnistOp : public ParallelOp, public RandomAccessOp { // @return Status - The error code return Status Reset() override; + // Private function for computing the assignment of the column name map. + // @return - Status + Status ComputeColMap() override; + int64_t buf_cnt_; int64_t row_cnt_; WaitPost wp_; diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/random_data_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/random_data_op.cc index 9e3d1140a7..afd7ebcc55 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/random_data_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/random_data_op.cc @@ -53,9 +53,6 @@ Status RandomDataOp::Builder::Build(std::shared_ptr *out_op) { RETURN_IF_NOT_OK((*out_op)->GenerateSchema()); } - // Extract the column name mapping from the schema and save it in the class. - RETURN_IF_NOT_OK((*out_op)->data_schema_->GetColumnNameMap(&((*out_op)->column_name_id_map_))); - return Status::OK(); } @@ -405,5 +402,15 @@ Status RandomDataOp::Reset() { return Status::OK(); } + +Status RandomDataOp::ComputeColMap() { + // Extract the column name mapping from the schema and save it in the class. + if (column_name_id_map_.empty()) { + RETURN_IF_NOT_OK(data_schema_->GetColumnNameMap(&(column_name_id_map_))); + } else { + MS_LOG(WARNING) << "Column name map is already set!"; + } + return Status::OK(); +} } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/random_data_op.h b/mindspore/ccsrc/dataset/engine/datasetops/source/random_data_op.h index 48cfb0be51..020c9a6e09 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/random_data_op.h +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/random_data_op.h @@ -250,6 +250,10 @@ class RandomDataOp : public ParallelOp { return ++buffer_id_; } + // Private function for computing the assignment of the column name map. + // @return - Status + Status ComputeColMap() override; + int32_t buffer_id_; int64_t rows_per_buffer_; int64_t total_rows_; diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/text_file_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/text_file_op.cc index 26058cc8b8..5ae950b803 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/text_file_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/text_file_op.cc @@ -127,11 +127,6 @@ Status TextFileOp::Init() { int32_t safe_queue_size = static_cast(std::ceil(text_files_list_.size() / num_workers_) + 1); io_block_queues_.Init(num_workers_, safe_queue_size); - // Set the column name mapping (base class field) - for (int32_t i = 0; i < data_schema_->NumColumns(); ++i) { - column_name_id_map_[data_schema_->column(i).name()] = i; - } - RETURN_IF_NOT_OK(ParallelOp::CreateWorkerConnector(worker_connector_size_)); jagged_buffer_connector_ = std::make_unique(num_workers_, 1, worker_connector_size_); @@ -488,5 +483,17 @@ Status TextFileOp::CountAllFileRows(const std::vector &files, int64 } return Status::OK(); } + +Status TextFileOp::ComputeColMap() { + // Set the column name mapping (base class field) + if (column_name_id_map_.empty()) { + for (int32_t i = 0; i < data_schema_->NumColumns(); ++i) { + column_name_id_map_[data_schema_->column(i).name()] = i; + } + } else { + MS_LOG(WARNING) << "Column name map is already set!"; + } + return Status::OK(); +} } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/text_file_op.h b/mindspore/ccsrc/dataset/engine/datasetops/source/text_file_op.h index dd258d914e..31224cb299 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/text_file_op.h +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/text_file_op.h @@ -264,6 +264,10 @@ class TextFileOp : public ParallelOp { // @return Status - the error code returned. Status PostEndOfEpoch(int32_t queue_index); + // Private function for computing the assignment of the column name map. + // @return - Status + Status ComputeColMap() override; + int32_t device_id_; int32_t num_devices_; int64_t rows_per_buffer_; diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/tf_reader_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/tf_reader_op.cc index 23dce8dc10..8b92d19249 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/tf_reader_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/tf_reader_op.cc @@ -195,11 +195,6 @@ Status TFReaderOp::Init() { RETURN_IF_NOT_OK(CreateSchema(dataset_files_list_[0], columns_to_load_)); } - // Construct the column name map for this operator (base class field) - for (int32_t i = 0; i < data_schema_->NumColumns(); ++i) { - column_name_id_map_[data_schema_->column(i).name()] = i; - } - if (total_rows_ == 0) { total_rows_ = data_schema_->num_rows(); } @@ -1015,5 +1010,17 @@ Status TFReaderOp::Accept(NodePass *p, bool *modified) { // Downcast shared pointer then call visitor return p->RunOnNode(std::static_pointer_cast(shared_from_this()), modified); } + +Status TFReaderOp::ComputeColMap() { + // Construct the column name map for this operator (base class field) + if (column_name_id_map_.empty()) { + for (int32_t i = 0; i < data_schema_->NumColumns(); ++i) { + column_name_id_map_[data_schema_->column(i).name()] = i; + } + } else { + MS_LOG(WARNING) << "Column name map is already set!"; + } + return Status::OK(); +} } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/tf_reader_op.h b/mindspore/ccsrc/dataset/engine/datasetops/source/tf_reader_op.h index 9c92d6d4be..417cd8bef0 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/tf_reader_op.h +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/tf_reader_op.h @@ -381,6 +381,10 @@ class TFReaderOp : public ParallelOp { // @return Status - the error code returned. Status CalculateNumRowsPerShard(); + // Private function for computing the assignment of the column name map. + // @return - Status + Status ComputeColMap() override; + int32_t device_id_; int32_t num_devices_; int64_t rows_per_buffer_; diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/voc_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/voc_op.cc index d3c7ff397f..5d9f0ee92c 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/voc_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/voc_op.cc @@ -99,10 +99,6 @@ VOCOp::VOCOp(const TaskType &task_type, const std::string &task_mode, const std: rows_per_buffer_(rows_per_buffer), sampler_(std::move(sampler)), data_schema_(std::move(data_schema)) { - // Set the column name map (base class field) - for (int32_t i = 0; i < data_schema_->NumColumns(); ++i) { - column_name_id_map_[data_schema_->column(i).name()] = i; - } io_block_queues_.Init(num_workers_, queue_size); } @@ -454,5 +450,17 @@ Status VOCOp::GetClassIndexing(const std::string &dir, const std::string &task_t return Status::OK(); } + +Status VOCOp::ComputeColMap() { + // Set the column name map (base class field) + if (column_name_id_map_.empty()) { + for (int32_t i = 0; i < data_schema_->NumColumns(); ++i) { + column_name_id_map_[data_schema_->column(i).name()] = i; + } + } else { + MS_LOG(WARNING) << "Column name map is already set!"; + } + return Status::OK(); +} } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/voc_op.h b/mindspore/ccsrc/dataset/engine/datasetops/source/voc_op.h index bce82a43c9..a0f5eba4d6 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/voc_op.h +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/voc_op.h @@ -263,6 +263,10 @@ class VOCOp : public ParallelOp, public RandomAccessOp { // @return Status - The error code return Status Reset() override; + // Private function for computing the assignment of the column name map. + // @return - Status + Status ComputeColMap() override; + bool decode_; int64_t row_cnt_; int64_t buf_cnt_; diff --git a/mindspore/ccsrc/dataset/engine/datasetops/take_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/take_op.cc index f9ff49a5b7..05c224ee2e 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/take_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/take_op.cc @@ -73,7 +73,6 @@ Status TakeOp::operator()() { TaskManager::FindMe()->Post(); std::unique_ptr buf; RETURN_IF_NOT_OK(child_[0]->GetNextBuffer(&buf)); - RETURN_IF_NOT_OK(DatasetOp::AssignColMapFromChild()); while (buf->eof() == false) { if (take_count_ == max_takes_) { diff --git a/mindspore/ccsrc/dataset/engine/datasetops/zip_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/zip_op.cc index bb438552f3..55734324fc 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/zip_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/zip_op.cc @@ -139,24 +139,6 @@ Status ZipOp::prepare(TensorQTable *const table) { // Pack this first row into our tensor table table->push_back(std::move(new_row)); - // At this point we have at least 1 row produced, so all child iterators have their column names such that we - // can produce our column name map now. - column_name_id_map_ = {}; - for (int32_t i = 0; i < children_num_; ++i) { - // Initializing col_name_id_map_ from the first data buffer. - const std::unordered_map col_name_id_map = child_iterators_[i]->GetColumnNameMap(); - int32_t colsCurrent = column_name_id_map_.size(); - // the update code below shouldn't do anything bad if the column name already exists. - for (const auto &pair : col_name_id_map) { - std::string name = pair.first; - int32_t old_id = pair.second; - // check if name already exists in column name descriptor - if (column_name_id_map_.count(name) == 1) { - RETURN_STATUS_UNEXPECTED("key already exists when zipping datasets"); - } - column_name_id_map_[name] = old_id + colsCurrent; - } - } return Status::OK(); } @@ -257,5 +239,30 @@ Status ZipOp::Accept(NodePass *p, bool *modified) { // Downcast shared pointer then call visitor return p->RunOnNode(std::static_pointer_cast(shared_from_this()), modified); } + +Status ZipOp::ComputeColMap() { + if (column_name_id_map_.empty()) { + column_name_id_map_ = {}; + for (int32_t i = 0; i < child_.size(); ++i) { + // Initializing col_name_id_map from the child. + const std::unordered_map col_name_id_map = child_[i]->column_name_id_map(); + int32_t colsCurrent = column_name_id_map_.size(); + // the update code below shouldn't do anything bad if the column name already exists. + for (const auto &pair : col_name_id_map) { + std::string name = pair.first; + int32_t old_id = pair.second; + // check if name already exists in column name descriptor + if (column_name_id_map_.count(name) == 1) { + RETURN_STATUS_UNEXPECTED("key already exists when zipping datasets"); + } + column_name_id_map_[name] = old_id + colsCurrent; + } + } + MS_LOG(DEBUG) << "Setting column map:\n" << this->ColumnNameMapAsString(); + } else { + MS_LOG(WARNING) << "Column name map is already set!"; + } + return Status::OK(); +} } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/zip_op.h b/mindspore/ccsrc/dataset/engine/datasetops/zip_op.h index 08b93c18b5..fad3c22eaa 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/zip_op.h +++ b/mindspore/ccsrc/dataset/engine/datasetops/zip_op.h @@ -141,6 +141,10 @@ class ZipOp : public PipelineOp { // 1, a, T Status getNextTensorRow(TensorRow *const new_zip_row); + // Computing the assignment of the column name map. + // @return - Status + Status ComputeColMap() override; + int32_t children_num_; int32_t rows_per_buffer_; int32_t buffer_id_; From c7d32e1e5536df03a4afdbad6b9dbace50eb678a Mon Sep 17 00:00:00 2001 From: Jesse Lee Date: Mon, 22 Jun 2020 13:29:15 -0400 Subject: [PATCH 062/254] CacheOp branch infrastructure --- mindspore/ccsrc/dataset/util/CMakeLists.txt | 6 + mindspore/ccsrc/dataset/util/allocator.h | 87 ++++ mindspore/ccsrc/dataset/util/buddy.cc | 388 ++++++++++++++++++ mindspore/ccsrc/dataset/util/buddy.h | 133 ++++++ mindspore/ccsrc/dataset/util/cache_pool.cc | 202 +++++++++ mindspore/ccsrc/dataset/util/cache_pool.h | 139 +++++++ mindspore/ccsrc/dataset/util/list.h | 18 + mindspore/ccsrc/dataset/util/memory_pool.h | 14 - mindspore/ccsrc/dataset/util/path.cc | 118 +++++- mindspore/ccsrc/dataset/util/path.h | 14 + mindspore/ccsrc/dataset/util/semaphore.cc | 41 ++ mindspore/ccsrc/dataset/util/semaphore.h | 54 +++ mindspore/ccsrc/dataset/util/slice.cc | 38 ++ mindspore/ccsrc/dataset/util/slice.h | 122 ++++++ .../ccsrc/dataset/util/storage_container.cc | 164 ++++++++ .../ccsrc/dataset/util/storage_container.h | 79 ++++ .../ccsrc/dataset/util/storage_manager.cc | 167 ++++++++ .../ccsrc/dataset/util/storage_manager.h | 76 ++++ mindspore/ccsrc/dataset/util/system_pool.h | 7 + 19 files changed, 1850 insertions(+), 17 deletions(-) create mode 100644 mindspore/ccsrc/dataset/util/buddy.cc create mode 100644 mindspore/ccsrc/dataset/util/buddy.h create mode 100644 mindspore/ccsrc/dataset/util/cache_pool.cc create mode 100644 mindspore/ccsrc/dataset/util/cache_pool.h create mode 100644 mindspore/ccsrc/dataset/util/semaphore.cc create mode 100644 mindspore/ccsrc/dataset/util/semaphore.h create mode 100644 mindspore/ccsrc/dataset/util/slice.cc create mode 100644 mindspore/ccsrc/dataset/util/slice.h create mode 100644 mindspore/ccsrc/dataset/util/storage_container.cc create mode 100644 mindspore/ccsrc/dataset/util/storage_container.h create mode 100644 mindspore/ccsrc/dataset/util/storage_manager.cc create mode 100644 mindspore/ccsrc/dataset/util/storage_manager.h diff --git a/mindspore/ccsrc/dataset/util/CMakeLists.txt b/mindspore/ccsrc/dataset/util/CMakeLists.txt index b36d612435..96489add07 100644 --- a/mindspore/ccsrc/dataset/util/CMakeLists.txt +++ b/mindspore/ccsrc/dataset/util/CMakeLists.txt @@ -2,6 +2,8 @@ file(GLOB_RECURSE _CURRENT_SRC_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cc" set_property(SOURCE ${_CURRENT_SRC_FILES} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_MD) add_library(utils OBJECT arena.cc + buddy.cc + cache_pool.cc circular_pool.cc memory_pool.cc cond_var.cc @@ -11,7 +13,11 @@ add_library(utils OBJECT service.cc services.cc lock.cc + semaphore.cc status.cc + storage_container.cc + storage_manager.cc + slice.cc path.cc wait_post.cc sig_handler.cc) diff --git a/mindspore/ccsrc/dataset/util/allocator.h b/mindspore/ccsrc/dataset/util/allocator.h index ba6c7786df..50a9cadbe3 100644 --- a/mindspore/ccsrc/dataset/util/allocator.h +++ b/mindspore/ccsrc/dataset/util/allocator.h @@ -17,8 +17,10 @@ #define DATASET_UTIL_ALLOCATOR_H_ #include +#include #include #include +#include #include "dataset/util/memory_pool.h" namespace mindspore { @@ -84,6 +86,91 @@ class Allocator { private: std::shared_ptr pool_; }; +/// \brief It is a wrapper of unique_ptr with a custom allocator and acts like std::lock_guard such that the memory will +/// be released when the object goes out of scope \tparam T The type of object to be allocated \tparam C Allocator. +/// Default to std::allocator +template > +class MemGuard { + public: + using allocator = C; + MemGuard() : n_(0) {} + explicit MemGuard(allocator a) : n_(0), alloc_(a) {} + // There is no copy constructor nor assignment operator because the memory is solely owned by this object. + MemGuard(const MemGuard &) = delete; + MemGuard &operator=(const MemGuard &) = delete; + // On the other hand, We can support move constructor + MemGuard(MemGuard &&lhs) noexcept : alloc_(std::move(lhs.alloc_)), ptr_(std::move(lhs.ptr_)), n_(lhs.n_) {} + MemGuard &operator=(MemGuard &&lhs) noexcept { + if (this != &lhs) { + this->deallocate(); + n_ = lhs.n_; + alloc_ = std::move(lhs.alloc_); + ptr_ = std::move(lhs.ptr_); + } + return *this; + } + /// \brief Explicitly deallocate the memory if allocated + void deallocate() { + if (ptr_) { + auto *p = ptr_.release(); + if (!std::is_arithmetic::value && std::is_destructible::value) { + for (auto i = 0; i < n_; ++i) { + p[i].~T(); + } + } + alloc_.deallocate(p, n_); + n_ = 0; + } + } + /// \brief Allocate memory (with emplace feature). Previous one will be released. If size is 0, no new memory is + /// allocated. + /// \param n Number of objects of type T to be allocated + /// \tparam Args Extra arguments pass to the constructor of T + template + Status allocate(size_t n, Args &&... args) noexcept { + try { + deallocate(); + if (n > 0) { + T *data = alloc_.allocate(n); + if (!std::is_arithmetic::value) { + for (auto i = 0; i < n; i++) { + std::allocator_traits::construct(alloc_, &(data[i]), std::forward(args)...); + } + } + ptr_ = std::unique_ptr(data); + n_ = n; + } + } catch (const std::bad_alloc &e) { + return Status(StatusCode::kOutOfMemory); + } catch (std::exception &e) { + RETURN_STATUS_UNEXPECTED(e.what()); + } + return Status::OK(); + } + ~MemGuard() noexcept { deallocate(); } + /// \brief Getter function + /// \return The pointer to the memory allocated + T *GetPointer() const { return ptr_.get(); } + /// \brief Getter function + /// \return The pointer to the memory allocated + T *GetMutablePointer() { return ptr_.get(); } + /// \brief Overload [] operator to access a particular element + /// \param x index to the element. Must be less than number of element allocated. + /// \return pointer to the x-th element + T *operator[](size_t x) { return GetMutablePointer() + x; } + /// \brief Overload [] operator to access a particular element + /// \param x index to the element. Must be less than number of element allocated. + /// \return pointer to the x-th element + T *operator[](size_t x) const { return GetPointer() + x; } + /// \brief Return how many bytes are allocated in total + /// \return Number of bytes allocated in total + size_t GetSizeInBytes() const { return n_ * sizeof(T); } + + private: + allocator alloc_; + std::unique_ptr> ptr_; + size_t n_; +}; } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/dataset/util/buddy.cc b/mindspore/ccsrc/dataset/util/buddy.cc new file mode 100644 index 0000000000..3a14258419 --- /dev/null +++ b/mindspore/ccsrc/dataset/util/buddy.cc @@ -0,0 +1,388 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "dataset/util/buddy.h" +#include +#include +#include "dataset/util/de_error.h" +#include "dataset/util/memory_pool.h" +#include "dataset/util/system_pool.h" +#include "./securec.h" + +inline uint64_t BitLeftShift(uint64_t v, uint64_t n) { return (v << n); } + +inline uint64_t BitRightShift(uint64_t v, uint64_t n) { return (v >> n); } + +inline uint64_t BitOr(uint64_t rhs, uint64_t lhs) { return rhs | lhs; } + +inline uint64_t BitEx(uint64_t rhs, uint64_t lhs) { return rhs ^ lhs; } + +inline uint64_t BitAnd(uint64_t rhs, uint64_t lhs) { return rhs & lhs; } + +namespace mindspore { +namespace dataset { +Status BuddySpace::Init() { + if (log_min_ < 0) { + return Status(StatusCode::kUnexpectedError, __LINE__, __FILE__, + "log_min must be positive : " + std::to_string(log_min_)); + } + if (num_lvl_ < 3 || num_lvl_ > 18) { + return Status(StatusCode::kUnexpectedError, __LINE__, __FILE__, + "num_lvl must be between 3 and 18 : " + std::to_string(num_lvl_)); + } + min_ = BitLeftShift(1, log_min_); + max_ = BitLeftShift(1, log_min_ + num_lvl_ - 1); + size_t offset_1 = sizeof(rel_addr_t) * num_lvl_; + size_t offset_2 = sizeof(int) * num_lvl_ + offset_1; + size_t offset_3 = sizeof(char) * BitLeftShift(1, num_lvl_ - 3) + offset_2; + RETURN_IF_NOT_OK(DeMalloc(offset_3, &ptr_, true)); + hint_ = reinterpret_cast(ptr_); + count_ = reinterpret_cast((reinterpret_cast(ptr_) + offset_1)); + map_ = reinterpret_cast(ptr_) + offset_2; + count_[num_lvl_ - 1] = 1; + map_[0] = BitOr(MORE_BIT, num_lvl_ - 3); + return Status::OK(); +} + +Status BuddySpace::Alloc(const uint64_t sz, BSpaceDescriptor *desc, addr_t *p) noexcept { + std::lock_guard lock(mutex_); + addr_t addr = AllocNoLock(sz, desc); + if (addr != NOSPACE) { + *p = addr; + return Status::OK(); + } else { + return Status(StatusCode::kNoSpace, "BuddySpace full. Not an error. Please ignore."); + } +} + +addr_t BuddySpace::AllocNoLock(const uint64_t sz, BSpaceDescriptor *desc) noexcept { + DS_ASSERT(sz <= max_); + uint32_t reqSize = SizeToBlock(sz); + rel_addr_t rel_addr = AllocBuddySeg(reqSize); + if (rel_addr != static_cast(NOSPACE)) { + (void)memset_s(desc, sizeof(BSpaceDescriptor), 0, sizeof(BSpaceDescriptor)); + desc->sig = static_cast(0xDEADBEEF); + desc->addr = rel_addr; + desc->req_size = reqSize; + desc->blk_size = NextPowerOf2(reqSize); + return static_cast(rel_addr * min_); + } else { + return NOSPACE; + } +} + +void BuddySpace::FreeNoLock(const BSpaceDescriptor *desc) { + DS_ASSERT(desc->sig == 0XDEADBEEF); + rel_addr_t rel_addr = desc->addr; + size_t blk_size = desc->blk_size; + size_t req_size = desc->req_size; + FreeBuddySeg(rel_addr, blk_size, req_size); +} + +void BuddySpace::Free(const BSpaceDescriptor *desc) { + std::lock_guard lock(mutex_); + return FreeNoLock(desc); +} + +std::ostream &operator<<(std::ostream &os, const BuddySpace &s) { + os << "1 unit = " << s.GetMinSize() << "\n" + << "Size of buddy space = " << s.GetMaxSize() << "\n" + << "Number of levels = " << s.num_lvl_ << "\n\n" + << "Percent free = " << s.PercentFree() << "\n" + << "Dumping count array : " + << "\n"; + for (int i = 0; i < s.num_lvl_; i++) { + os << "[" << i << "] = " << s.count_[i] << " "; + if (((i + 1) % 4) == 0) { + os << "\n"; + } + } + os << "\n"; + os << "Dumping allocation info:" + << "\n"; + auto max_addr = static_cast(BitLeftShift(1, s.num_lvl_ - 1)); + rel_addr_t addr = 0; + while (addr < max_addr) { + size_t sz = 0; + BuddySpace::STATE st; + s.GetBuddySegState(addr, &sz, &st); + os << "Address : " << std::left << std::setw(8) << addr << " Size : " << std::setw(8) << sz << " State : " + << ((st == BuddySpace::STATE::kAlloc) ? "ALLOC" : ((st == BuddySpace::STATE::kFree) ? "FREE" : "Unkonwn")) + << "\n"; + addr += sz; + } + return os; +} + +void BuddySpace::GetBuddySegState(const rel_addr_t rel_addr, size_t *rel_sz, STATE *st) const { + char byte; + int pos; + int offset; + uint64_t val = 0; + int shift; + pos = BitRightShift(rel_addr, 2); + offset = rel_addr % 4; + shift = offset * 2; + byte = map_[pos]; + switch (offset) { + case 0: + val = byte; + break; + case 1: + case 3: + if (offset == 1) { + val = BitLeftShift(BitAnd(byte, 0x30), shift); + } else { + val = BitLeftShift(BitAnd(byte, 0x03), shift); + } + break; + case 2: + val = BitLeftShift(BitAnd(byte, 0x0F), shift); + break; + } + if (BitAnd(val, ONE_BIT)) { + *rel_sz = 1; + } else if (BitAnd(val, TWO_BIT)) { + *rel_sz = 2; + } else if (BitAnd(val, MORE_BIT)) { + log_t lg = BitAnd(val, 0x0F); + *rel_sz = BitLeftShift(1, lg + 2); + } else { + *st = STATE::kEmpty; + return; + } + *st = BitAnd(val, ALLOC_BIT) ? STATE::kAlloc : STATE::kFree; +} + +void BuddySpace::SetBuddySegState(rel_addr_t rel_addr, size_t rel_sz, STATE st) { + int clr; + int mask; + int pos; + int offset; + int val = 0; + int shift; + auto log_sz = static_cast(Log2(rel_sz)); + pos = BitRightShift(rel_addr, 2); + offset = rel_addr % 4; + shift = offset * 2; + if (rel_sz == 1) { + val = ONE_BIT; + mask = 0xC0; + } else if (rel_sz == 2) { + val = TWO_BIT; + mask = 0xF0; + } else { + val = BitOr(log_sz - 2, MORE_BIT); + mask = 0xFF; + } + if (st == STATE::kAlloc) { + val = BitOr(val, ALLOC_BIT); + } else if (st == STATE::kFree) { + val = BitAnd(val, ~(static_cast(ALLOC_BIT))); + } else if (st == STATE::kEmpty) { + val = 0; + } + clr = static_cast(~(BitRightShift(mask, shift))); + map_[pos] = static_cast(BitAnd(map_[pos], clr)); + map_[pos] = static_cast(BitOr(map_[pos], BitRightShift(val, shift))); + if (st == STATE::kAlloc) { + count_[log_sz]--; + } else if (st == STATE::kFree) { + count_[log_sz]++; + if (rel_addr < hint_[log_sz]) { + hint_[log_sz] = rel_addr; + } + } +} + +void BuddySpace::JoinBuddySeg(rel_addr_t addr, size_t blk_sz) { + while (blk_sz < BitLeftShift(1, num_lvl_)) { + rel_addr_t buddy = BitEx(addr, blk_sz); + size_t sz = 0; + STATE st; + GetBuddySegState(buddy, &sz, &st); + if (st == STATE::kFree && sz == blk_sz) { + auto log_sz = static_cast(Log2(blk_sz)); + rel_addr_t left = (buddy < addr) ? buddy : addr; + rel_addr_t right = left + blk_sz; + DS_ASSERT(count_[log_sz] >= 2); + count_[log_sz] -= 2; + SetBuddySegState(right, blk_sz, STATE::kEmpty); + SetBuddySegState(left, BitLeftShift(blk_sz, 1), STATE::kFree); + for (int i = 0; i < log_sz; i++) { + if (hint_[i] == right) { + hint_[i] = left; + } + } + addr = left; + blk_sz <<= 1u; + } else { + break; + } + } +} + +void BuddySpace::TrimBuddySeg(rel_addr_t addr, size_t blk_sz, size_t ask_sz) { + DS_ASSERT(ask_sz < blk_sz); + uint32_t inx = Log2(blk_sz); + size_t remaining_sz = ask_sz; + for (int i = inx; i > 0; i--) { + size_t b_size = BitLeftShift(1, i); + size_t half_sz = BitRightShift(b_size, 1); + count_[i]--; + SetBuddySegState(addr, half_sz, STATE::kFree); + SetBuddySegState(addr + half_sz, half_sz, STATE::kFree); + if (remaining_sz >= half_sz) { + SetBuddySegState(addr, half_sz, STATE::kAlloc); + remaining_sz -= half_sz; + if (remaining_sz == 0) { + break; + } + addr += half_sz; + } + } +} + +void BuddySpace::UnTrimBuddySeg(rel_addr_t addr, size_t blk_sz, size_t ask_sz) { + DS_ASSERT(ask_sz < blk_sz); + uint32_t inx = Log2(blk_sz); + size_t remaining_sz = ask_sz; + for (int i = inx; i > 0; i--) { + size_t b_size = BitLeftShift(1, i); + size_t half_sz = BitRightShift(b_size, 1); + if (remaining_sz >= half_sz) { +#ifdef DEBUG + { + size_t sz = 0; + STATE st; + GetBuddySegState(addr, &sz, &st); + DS_ASSERT(sz == half_sz && st == STATE::kAlloc); + } +#endif + SetBuddySegState(addr, half_sz, STATE::kFree); + remaining_sz -= half_sz; + if (remaining_sz == 0) { + JoinBuddySeg(addr, half_sz); + break; + } + addr += half_sz; + } + } +} + +rel_addr_t BuddySpace::AllocBuddySeg(uint32_t req_size) noexcept { + uint32_t blk_size = NextPowerOf2(req_size); + int start_inx = static_cast(Log2(blk_size)); + bool found = false; + rel_addr_t ask_addr = 0; + auto max_addr = static_cast(BitLeftShift(1, num_lvl_ - 1)); + STATE st; + size_t sz = 0; + for (int i = start_inx; !found && i < num_lvl_; i++) { + DS_ASSERT(count_[i] >= 0); + if (count_[i] == 0) { + continue; + } + auto blk_sz = static_cast(BitLeftShift(1, i)); + ask_addr = hint_[i]; + while (ask_addr < max_addr && !found) { + GetBuddySegState(ask_addr, &sz, &st); + if (st == STATE::kFree && sz == blk_sz) { + found = true; + } else { + DS_ASSERT(st != STATE::kEmpty); + ask_addr += ((sz > blk_sz) ? sz : blk_sz); + } + } + } + if (found) { + if (sz > req_size) { + TrimBuddySeg(ask_addr, sz, req_size); + } else { + SetBuddySegState(ask_addr, sz, STATE::kAlloc); + hint_[start_inx] = ask_addr; + } + return ask_addr; + } else { + return static_cast(NOSPACE); + } +} + +void BuddySpace::FreeBuddySeg(rel_addr_t addr, size_t blk_size, size_t req_size) { + if (req_size == blk_size) { +#ifdef DEBUG + { + size_t sz = 0; + STATE st; + GetBuddySegState(addr, &sz, &st); + } +#endif + SetBuddySegState(addr, blk_size, STATE::kFree); + JoinBuddySeg(addr, blk_size); + } else { + UnTrimBuddySeg(addr, blk_size, req_size); + } +} + +int BuddySpace::PercentFree() const { + uint64_t total_free_sz = 0; + uint64_t max_sz_in_unit = BitLeftShift(1, num_lvl_ - 1); + // Go through the count array without lock + for (int i = 0; i < num_lvl_; i++) { + int cnt = count_[i]; + if (cnt == 0) { + continue; + } + uint64_t blk_sz = BitLeftShift(1, i); + total_free_sz += (blk_sz * cnt); + } + return static_cast(static_cast(total_free_sz) / static_cast(max_sz_in_unit) * 100); +} + +BuddySpace::BuddySpace(int log_min, int num_lvl) + : hint_(nullptr), + count_(nullptr), + map_(nullptr), + log_min_(log_min), + num_lvl_(num_lvl), + min_(0), + max_(0), + ptr_(nullptr) {} + +BuddySpace::~BuddySpace() { + if (ptr_ != nullptr) { + free(ptr_); + } + hint_ = nullptr; + count_ = nullptr; + map_ = nullptr; +} + +Status BuddySpace::CreateBuddySpace(std::unique_ptr *out_bs, int log_min, int num_lvl) { + Status rc; + auto bs = new (std::nothrow) BuddySpace(log_min, num_lvl); + if (bs == nullptr) { + return Status(StatusCode::kOutOfMemory); + } + rc = bs->Init(); + if (rc.IsOk()) { + (*out_bs).reset(bs); + } else { + delete bs; + } + return rc; +} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/util/buddy.h b/mindspore/ccsrc/dataset/util/buddy.h new file mode 100644 index 0000000000..08c05cbbdb --- /dev/null +++ b/mindspore/ccsrc/dataset/util/buddy.h @@ -0,0 +1,133 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_UTIL_BUDDY_H_ +#define DATASET_UTIL_BUDDY_H_ + +#include +#include +#include +#include +#include +#include +#include "dataset/util/status.h" + +using addr_t = int64_t; +using rel_addr_t = int32_t; +using log_t = int; +#define ALLOC_BIT 0x80 +#define ONE_BIT 0x40 +#define TWO_BIT 0x20 +#define MORE_BIT 0x10 +#define NOSPACE ((addr_t)(-1)) +namespace mindspore { +namespace dataset { +struct BSpaceDescriptor { + int32_t sig; + rel_addr_t addr; + size_t req_size; + size_t blk_size; +}; + +class BuddySpace { + public: + // C++11 feature. Change STATE into a type safe class with + // the keyword. Don't take out the keyword 'class' + enum class STATE { kFree, kAlloc, kEmpty }; + + BuddySpace(const BuddySpace &) = delete; + + BuddySpace &operator=(const BuddySpace &) = delete; + + virtual ~BuddySpace(); + + Status Alloc(uint64_t sz, BSpaceDescriptor *desc, addr_t *) noexcept; + + void Free(const BSpaceDescriptor *desc); + + uint64_t GetMinSize() const { return min_; } + + uint64_t GetMaxSize() const { return max_; } + + int PercentFree() const; + + friend std::ostream &operator<<(std::ostream &os, const BuddySpace &s); + + static uint64_t NextPowerOf2(uint64_t n) { + if (n <= 1) { + return 1; + } + n = n - 1; + while (n & (n - 1)) { + n = n & (n - 1); + } + return n << 1; + } + + static uint32_t Log2(uint64_t n) { + uint32_t cnt = 0; + while (n >>= 1) { + cnt++; + } + return cnt; + } + + static Status CreateBuddySpace(std::unique_ptr *out_bs, int log_min = 15, int num_lvl = 18); + + private: + rel_addr_t *hint_; + int *count_; + char *map_; + int log_min_; + int num_lvl_; + uint64_t min_; + uint64_t max_; + void *ptr_; + std::mutex mutex_; + + explicit BuddySpace(int log_min = 15, int num_lvl = 18); + + Status Init(); + + addr_t AllocNoLock(const uint64_t sz, BSpaceDescriptor *desc) noexcept; + + void FreeNoLock(const BSpaceDescriptor *desc); + + uint32_t SizeToBlock(const uint64_t sz) const { + uint32_t reqSize = (sz / min_); + if (sz % min_) { + reqSize++; + } + return reqSize; + } + + void GetBuddySegState(const rel_addr_t rel_addr, size_t *rel_sz, STATE *st) const; + + void SetBuddySegState(rel_addr_t rel_addr, size_t rel_sz, STATE st); + + void JoinBuddySeg(rel_addr_t addr, size_t blk_sz); + + void TrimBuddySeg(rel_addr_t addr, size_t blk_sz, size_t ask_sz); + + void UnTrimBuddySeg(rel_addr_t addr, size_t blk_sz, size_t ask_sz); + + rel_addr_t AllocBuddySeg(uint32_t req_size) noexcept; + + void FreeBuddySeg(rel_addr_t addr, size_t blk_size, size_t req_size); +}; +} // namespace dataset +} // namespace mindspore + +#endif // DATASET_UTIL_BUDDY_H_ diff --git a/mindspore/ccsrc/dataset/util/cache_pool.cc b/mindspore/ccsrc/dataset/util/cache_pool.cc new file mode 100644 index 0000000000..92504cd063 --- /dev/null +++ b/mindspore/ccsrc/dataset/util/cache_pool.cc @@ -0,0 +1,202 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include "common/utils.h" +#include "dataset/util/cache_pool.h" +#include "dataset/util/services.h" + +namespace mindspore { +namespace dataset { +CachePool::CachePool(const value_allocator &alloc, const std::string &root) + : alloc_(alloc), root_(root), subfolder_(Services::GetUniqueID()), sm_(nullptr), tree_(nullptr) {} + +Status CachePool::DoServiceStart() { + tree_ = std::make_shared(); + // If we are given a disk path, set up the StorageManager + if (!root_.toString().empty()) { + Path spill = GetSpillPath(); + RETURN_IF_NOT_OK(spill.CreateDirectories()); + sm_ = std::make_shared(spill); + RETURN_IF_NOT_OK(sm_->ServiceStart()); + MS_LOG(INFO) << "CachePool will use disk folder: " << common::SafeCStr(spill.toString()); + } + return Status::OK(); +} +Status CachePool::DoServiceStop() { + Status rc; + Status rc2; + if (sm_ != nullptr) { + rc = sm_->ServiceStop(); + if (rc.IsError()) { + rc2 = rc; + } + } + sm_.reset(); + for (auto &bl : *tree_) { + if (bl.ptr != nullptr) { + alloc_.deallocate(bl.ptr, bl.sz); + } + } + tree_.reset(); + if (!root_.toString().empty()) { + Path spill = GetSpillPath(); + auto it = Path::DirIterator::OpenDirectory(&spill); + while (it->hasNext()) { + rc = it->next().Remove(); + if (rc.IsError() && rc2.IsOk()) { + rc2 = rc; + } + } + rc = spill.Remove(); + if (rc.IsError() && rc2.IsOk()) { + rc2 = rc; + } + } + return rc2; +} +CachePool::~CachePool() noexcept { (void)ServiceStop(); } +Status CachePool::Insert(const std::vector &buf, CachePool::key_type *key) { + DataLocator bl; + Status rc; + size_t sz = 0; + // We will consolidate all the slices into one piece. + for (auto &v : buf) { + sz += v.GetSize(); + } + bl.sz = sz; + try { + bl.ptr = alloc_.allocate(sz); + // We will do a piecewise copy. + WritableSlice dest(bl.ptr, bl.sz); + size_t pos = 0; + for (auto &v : buf) { + WritableSlice out(dest, pos); + rc = WritableSlice::Copy(&out, v); + if (rc.IsError()) { + break; + } + pos += v.GetSize(); + } + if (rc.IsError()) { + alloc_.deallocate(bl.ptr, sz); + bl.ptr = nullptr; + return rc; + } + } catch (std::bad_alloc &e) { + if (sm_ != nullptr) { + RETURN_IF_NOT_OK(sm_->Write(&bl.storage_key, buf)); + // We have an assumption 0 is not a valid key from the design of AutoIndexObj. + // Make sure it is not 0. + if (bl.storage_key == 0) { + RETURN_STATUS_UNEXPECTED("Key 0 is returned which is unexpected"); + } + } else { + return Status(StatusCode::kOutOfMemory, __LINE__, __FILE__); + } + } + rc = tree_->insert(bl, key); + if (rc.IsError() && bl.ptr != nullptr) { + alloc_.deallocate(bl.ptr, sz); + } + return rc; +} +Status CachePool::Read(CachePool::key_type key, WritableSlice *dest, size_t *bytesRead) const { + RETURN_UNEXPECTED_IF_NULL(dest); + auto r = tree_->Search(key); + if (r.second) { + auto &it = r.first; + if (it->ptr != nullptr) { + ReadableSlice src(it->ptr, it->sz); + RETURN_IF_NOT_OK(WritableSlice::Copy(dest, src)); + } else if (sm_ != nullptr) { + size_t expectedLength = 0; + RETURN_IF_NOT_OK(sm_->Read(it->storage_key, dest, &expectedLength)); + if (expectedLength != it->sz) { + MS_LOG(ERROR) << "Unexpected length. Read " << expectedLength << ". Expected " << it->sz << "." + << " Internal key: " << key << "\n"; + RETURN_STATUS_UNEXPECTED("Length mismatch. See log file for details."); + } + } + if (bytesRead != nullptr) { + *bytesRead = it->sz; + } + } else { + RETURN_STATUS_UNEXPECTED("Key not found"); + } + return Status::OK(); +} +const CachePool::value_allocator &CachePool::get_allocator() const { return alloc_; } +Path CachePool::GetSpillPath() const { + auto spill = Path(root_) / subfolder_; + return spill; +} +CachePool::CacheStat CachePool::GetStat() const { + CacheStat cs{0}; + for (auto &it : *tree_) { + if (it.ptr != nullptr) { + ++cs.num_mem_cached; + } else { + ++cs.num_disk_cached; + } + } + return cs; +} +Status CachePool::Spill(CachePool::DataLocator *dl) { + if (sm_ == nullptr) { + RETURN_STATUS_UNEXPECTED("No disk storage to spill"); + } + RETURN_UNEXPECTED_IF_NULL(dl); + RETURN_UNEXPECTED_IF_NULL(dl->ptr); + if (dl->storage_key == 0) { + ReadableSlice data(dl->ptr, dl->sz); + RETURN_IF_NOT_OK(sm_->Write(&dl->storage_key, {data})); + } + alloc_.deallocate(dl->ptr, dl->sz); + dl->ptr = nullptr; + return Status::OK(); +} +Status CachePool::Locate(CachePool::DataLocator *dl) { + RETURN_UNEXPECTED_IF_NULL(dl); + if (dl->ptr == nullptr) { + if (sm_ == nullptr) { + RETURN_STATUS_UNEXPECTED("No disk storage to locate the data"); + } + try { + dl->ptr = alloc_.allocate(dl->sz); + WritableSlice dest(dl->ptr, dl->sz); + Status rc = Read(dl->storage_key, &dest); + if (rc.IsError()) { + alloc_.deallocate(dl->ptr, dl->sz); + dl->ptr = nullptr; + return rc; + } + } catch (const std::bad_alloc &e) { + return Status(StatusCode::kOutOfMemory, __LINE__, __FILE__); + } + } + return Status::OK(); +} +size_t CachePool::GetSize(CachePool::key_type key) const { + auto r = tree_->Search(key); + if (r.second) { + auto &it = r.first; + return it->sz; + } else { + return 0; + } +} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/util/cache_pool.h b/mindspore/ccsrc/dataset/util/cache_pool.h new file mode 100644 index 0000000000..d35617d0e4 --- /dev/null +++ b/mindspore/ccsrc/dataset/util/cache_pool.h @@ -0,0 +1,139 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_UTIL_CACHE_POOL_H_ +#define DATASET_UTIL_CACHE_POOL_H_ + +#include +#include +#include +#include +#include "dataset/util/allocator.h" +#include "dataset/util/service.h" +#include "dataset/util/slice.h" +#include "dataset/util/storage_manager.h" +#include "dataset/util/auto_index.h" + +namespace mindspore { +namespace dataset { +/// \brief A CachePool provides service for backup/restore a buffer. A buffer can be represented in a form of vector of +/// ReadableSlice where all memory blocks will be copied to one contiguous block which can be in memory or spilled to +/// disk (if a disk directory is provided). Every buffer insert will return a generated key which can be used to +/// restore the buffer. +/// \see ReadableSlice +class CachePool : public Service { + public: + using base_type = uint8_t; + using pointer = base_type *; + using const_pointer = const base_type *; + using reference = base_type &; + using const_reference = const base_type &; + using value_allocator = Allocator; + + // An internal class to locate the whereabouts of a backed up buffer which can be either in + class DataLocator { + public: + DataLocator() : ptr(nullptr), sz(0), storage_key(0) {} + ~DataLocator() = default; + DataLocator(const DataLocator &other) = default; + DataLocator &operator=(const DataLocator &other) = default; + DataLocator(DataLocator &&other) noexcept { + ptr = other.ptr; + sz = other.sz; + storage_key = other.storage_key; + other.ptr = nullptr; + other.sz = 0; + other.storage_key = 0; + } + DataLocator &operator=(DataLocator &&other) noexcept { + if (&other != this) { + ptr = other.ptr; + sz = other.sz; + storage_key = other.storage_key; + other.ptr = nullptr; + other.sz = 0; + other.storage_key = 0; + } + return *this; + } + pointer ptr; + size_t sz; + StorageManager::key_type storage_key; + }; + + using data_index = AutoIndexObj; + using key_type = data_index::key_type; + using bl_alloc_type = typename value_allocator::template rebind::other; + + /// \brief Simple statistics returned from CachePool like how many elements are cached in memory and + /// how many elements are spilled to disk. + struct CacheStat { + int64_t num_mem_cached; + int64_t num_disk_cached; + }; + + /// \brief Constructor + /// \param alloc Allocator to allocate memory from + /// \param root Optional disk folder to spill + explicit CachePool(const value_allocator &alloc, const std::string &root = ""); + + CachePool(const CachePool &) = delete; + CachePool(CachePool &&) = delete; + CachePool &operator=(const CachePool &) = delete; + CachePool &operator=(CachePool &&) = delete; + ~CachePool() noexcept; + + Status DoServiceStart() override; + Status DoServiceStop() override; + + Path GetSpillPath() const; + + /// \brief Insert a sequence of ReadableSlice objects into the pool. + /// All memory blocks will be consolidated into one contiguous block and be cached in either memory or on disk. + /// \param[in] buf A sequence of ReadableSlice objects. + /// \param[out] key Generated key + /// \return Error code + Status Insert(const std::vector &buf, key_type *key); + /// \brief Restore a cached buffer (from memory or disk) + /// \param[in] key A previous key returned from Insert + /// \param[out] dest The cached buffer will be copied to this destination represented by a WritableSlice + /// \param[out] bytesRead Optional. Number of bytes read. + /// \return Error code + Status Read(key_type key, WritableSlice *dest, size_t *bytesRead = nullptr) const; + + Status Spill(DataLocator *dl); + + Status Locate(DataLocator *dl); + + size_t GetSize(key_type key) const; + + /// \brief Get statistics. + /// \return CacheStat object + CacheStat GetStat() const; + + const value_allocator &get_allocator() const; + + std::string MyName() const { return subfolder_; } + + private: + value_allocator alloc_; + Path root_; + const std::string subfolder_; + std::shared_ptr sm_; + std::shared_ptr tree_; +}; +} // namespace dataset +} // namespace mindspore +#endif diff --git a/mindspore/ccsrc/dataset/util/list.h b/mindspore/ccsrc/dataset/util/list.h index 5a08f4514e..a4c15daa0e 100644 --- a/mindspore/ccsrc/dataset/util/list.h +++ b/mindspore/ccsrc/dataset/util/list.h @@ -106,6 +106,24 @@ struct List { ++count; } + // Insert elem2 before elem1 in the list. + virtual void InsertBefore(pointer elem1, pointer elem2) { + DS_ASSERT(elem1 != elem2); + Node &elem1_node = elem1->*node; + Node &elem2_node = elem2->*node; + elem2_node.next = elem1; + elem2_node.prev = elem1_node.prev; + if (elem1_node.prev != nullptr) { + Node &prev_node = elem1_node.prev->*node; + prev_node.next = elem2; + } + elem1_node.prev = elem2; + if (head == elem1) { + head = elem2; + } + ++count; + } + // Remove an element in the list virtual void Remove(pointer elem) noexcept { Node &elem_node = elem->*node; diff --git a/mindspore/ccsrc/dataset/util/memory_pool.h b/mindspore/ccsrc/dataset/util/memory_pool.h index 70876a8141..ee1da3bda1 100644 --- a/mindspore/ccsrc/dataset/util/memory_pool.h +++ b/mindspore/ccsrc/dataset/util/memory_pool.h @@ -44,20 +44,6 @@ class MemoryPool { virtual ~MemoryPool() {} }; -// Used by unique_ptr -template -class Deleter { - public: - explicit Deleter(std::shared_ptr &mp) : mp_(mp) {} - - ~Deleter() = default; - - void operator()(T *ptr) const { mp_->Deallocate(ptr); } - - private: - std::shared_ptr mp_; -}; - Status DeMalloc(std::size_t s, void **p, bool); } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/dataset/util/path.cc b/mindspore/ccsrc/dataset/util/path.cc index c37fdc17f1..59e5e5232c 100644 --- a/mindspore/ccsrc/dataset/util/path.cc +++ b/mindspore/ccsrc/dataset/util/path.cc @@ -16,6 +16,8 @@ #include "dataset/util/path.h" #include +#include +#include #include #include #include @@ -26,7 +28,7 @@ namespace mindspore { namespace dataset { -#ifdef _WIN32 +#if defined(_WIN32) || defined(_WIN64) char Path::separator_ = '\\'; #else char Path::separator_ = '/'; @@ -132,7 +134,7 @@ Status Path::CreateDirectory() { #if defined(_WIN32) || defined(_WIN64) int rc = mkdir(common::SafeCStr(path_)); #else - int rc = mkdir(common::SafeCStr(path_), 0700); + int rc = mkdir(common::SafeCStr(path_), S_IRUSR | S_IWUSR | S_IXUSR); #endif if (rc) { std::ostringstream oss; @@ -182,6 +184,111 @@ Status Path::CreateDirectories() { return Status::OK(); } +Status Path::Remove() { + if (Exists()) { + if (IsDirectory()) { + errno_t err = rmdir(common::SafeCStr(path_)); + if (err == -1) { + std::ostringstream oss; + oss << "Unable to delete directory " << path_ << ". Errno = " << errno; + RETURN_STATUS_UNEXPECTED(oss.str()); + } + } else { + errno_t err = unlink(common::SafeCStr(path_)); + if (err == -1) { + std::ostringstream oss; + oss << "Unable to delete file " << path_ << ". Errno = " << errno; + RETURN_STATUS_UNEXPECTED(oss.str()); + } + } + } + return Status::OK(); +} + +Status Path::CreateFile(int *file_descriptor) { return OpenFile(file_descriptor, true); } + +Status Path::OpenFile(int *file_descriptor, bool create) { + int fd; + if (file_descriptor == nullptr) { + RETURN_STATUS_UNEXPECTED("null pointer"); + } + if (IsDirectory()) { + std::ostringstream oss; + oss << "Unable to create file " << path_ << " which is a directory."; + RETURN_STATUS_UNEXPECTED(oss.str()); + } + // Convert to canonical form. + if (strlen(common::SafeCStr(path_)) > PATH_MAX) { + RETURN_STATUS_UNEXPECTED(strerror(errno)); + } + char canonical_path[PATH_MAX + 1] = {0x00}; +#if defined(_WIN32) || defined(_WIN64) + if (_fullpath(canonical_path, common::SafeCStr(path_), PATH_MAX) == nullptr) { +#else + if (realpath(common::SafeCStr(path_), canonical_path) == nullptr) { +#endif + if (errno == ENOENT && create) { + // File doesn't exist and we are to create it. Let's break it down. + auto file_part = Basename(); + auto parent_part = ParentPath(); +#if defined(_WIN32) || defined(_WIN64) + if (_fullpath(canonical_path, common::SafeCStr(parent_part), PATH_MAX) == nullptr) { +#else + if (realpath(common::SafeCStr(parent_part), canonical_path) == nullptr) { +#endif + RETURN_STATUS_UNEXPECTED(strerror(errno)); + } + auto cur_inx = strlen(canonical_path); + if ((cur_inx + file_part.length() + 1) > PATH_MAX) { + RETURN_STATUS_UNEXPECTED(strerror(errno)); + } + canonical_path[cur_inx++] = separator_; + if (strncpy_s(canonical_path + cur_inx, PATH_MAX - cur_inx, common::SafeCStr(file_part), file_part.length()) != + EOK) { + RETURN_STATUS_UNEXPECTED(strerror(errno)); + } + } else { + RETURN_STATUS_UNEXPECTED(strerror(errno)); + } + } + if (create) { + fd = open(canonical_path, O_CREAT | O_TRUNC | O_RDWR, S_IRUSR | S_IWUSR | S_IRGRP); + } else { + fd = open(canonical_path, O_RDWR); + } + if (fd == -1) { + RETURN_STATUS_UNEXPECTED(strerror(errno)); + } + *file_descriptor = fd; + return Status::OK(); +} + +Status Path::CloseFile(int fd) const { + if (close(fd) < 0) { + RETURN_STATUS_UNEXPECTED(strerror(errno)); + } + return Status::OK(); +} + +Status Path::TruncateFile(int fd) const { + int rc; + rc = ftruncate(fd, 0); + if (rc == 0) { + return Status::OK(); + } else { + RETURN_STATUS_UNEXPECTED(strerror(errno)); + } +} + +std::string Path::Basename() { + std::size_t found = path_.find_last_of(separator_); + if (found != std::string::npos) { + return path_.substr(found + 1); + } else { + return path_; + } +} + std::shared_ptr Path::DirIterator::OpenDirectory(Path *f) { auto it = new (std::nothrow) DirIterator(f); @@ -208,7 +315,7 @@ Path::DirIterator::~DirIterator() { Path::DirIterator::DirIterator(Path *f) : dir_(f), dp_(nullptr), entry_(nullptr) { MS_LOG(DEBUG) << "Open directory " << f->toString() << "."; - dp_ = opendir(common::SafeCStr(f->toString())); + dp_ = opendir(f->toString().c_str()); } bool Path::DirIterator::hasNext() { @@ -225,5 +332,10 @@ bool Path::DirIterator::hasNext() { } Path Path::DirIterator::next() { return (*(this->dir_) / Path(entry_->d_name)); } + +std::ostream &operator<<(std::ostream &os, const Path &s) { + os << s.path_; + return os; +} } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/dataset/util/path.h b/mindspore/ccsrc/dataset/util/path.h index efe01a7d16..fbf65b8c23 100644 --- a/mindspore/ccsrc/dataset/util/path.h +++ b/mindspore/ccsrc/dataset/util/path.h @@ -90,6 +90,20 @@ class Path { std::string ParentPath(); + Status Remove(); + + Status CreateFile(int *fd); + + Status OpenFile(int *fd, bool create = false); + + Status CloseFile(int fd) const; + + Status TruncateFile(int fd) const; + + std::string Basename(); + + friend std::ostream &operator<<(std::ostream &os, const Path &s); + private: static char separator_; std::string path_; diff --git a/mindspore/ccsrc/dataset/util/semaphore.cc b/mindspore/ccsrc/dataset/util/semaphore.cc new file mode 100644 index 0000000000..36ddf5511d --- /dev/null +++ b/mindspore/ccsrc/dataset/util/semaphore.cc @@ -0,0 +1,41 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "dataset/util/semaphore.h" +#include "dataset/util/task_manager.h" + +namespace mindspore { +namespace dataset { +Status Semaphore::P() { + std::unique_lock lck(mutex_); + RETURN_IF_NOT_OK(wait_cond_.Wait(&lck, [this]() { return value_ > 0; })); + --value_; + return Status::OK(); +} +void Semaphore::V() { + std::unique_lock lck(mutex_); + ++value_; + wait_cond_.NotifyOne(); +} +int Semaphore::Peek() { + std::unique_lock lck(mutex_); + return value_; +} +Status Semaphore::Register(TaskGroup *vg) { return wait_cond_.Register(vg->GetIntrpService()); } +Status Semaphore::Deregister() { return (wait_cond_.Deregister()); } +void Semaphore::ResetIntrpState() { wait_cond_.ResetIntrpState(); } + +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/util/semaphore.h b/mindspore/ccsrc/dataset/util/semaphore.h new file mode 100644 index 0000000000..07b9e83e7f --- /dev/null +++ b/mindspore/ccsrc/dataset/util/semaphore.h @@ -0,0 +1,54 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_UTIL_SEMAPHORE_H_ +#define DATASET_UTIL_SEMAPHORE_H_ + +#include "dataset/util/cond_var.h" + +namespace mindspore { +namespace dataset { +class TaskGroup; + +/// \brief A counting semaphore. There are two external functions P and V. P decrements the internal count and will be +/// blocked if the count is 0 (zero). V increments the internal count and wake up one of the waiters. +class Semaphore { + public: + /// \brief Constructor + /// \param init Initial value of the internal counter. + explicit Semaphore(int init) : value_(init) {} + + virtual ~Semaphore() {} + /// \brief Decrement the internal counter. Will be blocked if the value is 0. + /// \return Error code. Can get interrupt. + Status P(); + /// \brief Increment the internal counter. Wakeup on of the watiers if any. + void V(); + /// \brief Peek the internal value + /// \return The internal value + int Peek(); + Status Register(TaskGroup *vg); + Status Deregister(); + void ResetIntrpState(); + + private: + int value_; + + std::mutex mutex_; + CondVar wait_cond_; +}; +} // namespace dataset +} // namespace mindspore +#endif // DATASET_UTIL_SEMAPHORE_H_ diff --git a/mindspore/ccsrc/dataset/util/slice.cc b/mindspore/ccsrc/dataset/util/slice.cc new file mode 100644 index 0000000000..f1798b4f44 --- /dev/null +++ b/mindspore/ccsrc/dataset/util/slice.cc @@ -0,0 +1,38 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + + * http://www.apache.org/licenses/LICENSE-2.0 + + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ +#include "dataset/util/slice.h" + +namespace mindspore { +namespace dataset { +WritableSlice::WritableSlice(const WritableSlice &src, off64_t offset, size_t len) : ReadableSlice(src, offset, len) { + mutable_data_ = static_cast(src.mutable_data_) + offset; +} +WritableSlice::WritableSlice(const WritableSlice &src, off64_t offset) + : WritableSlice(src, offset, src.GetSize() - offset) {} +Status WritableSlice::Copy(WritableSlice *dest, const ReadableSlice &src) { + RETURN_UNEXPECTED_IF_NULL(dest); + RETURN_UNEXPECTED_IF_NULL(dest->GetMutablePointer()); + if (dest->GetSize() <= 0) { + RETURN_STATUS_UNEXPECTED("Destination length is non-positive"); + } + auto err = memcpy_s(dest->GetMutablePointer(), dest->GetSize(), src.GetPointer(), src.GetSize()); + if (err) { + RETURN_STATUS_UNEXPECTED(std::to_string(err)); + } + return Status::OK(); +} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/util/slice.h b/mindspore/ccsrc/dataset/util/slice.h new file mode 100644 index 0000000000..127df23cfa --- /dev/null +++ b/mindspore/ccsrc/dataset/util/slice.h @@ -0,0 +1,122 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_UTIL_SLICE_H_ +#define DATASET_UTIL_SLICE_H_ + +#include +#include +#include +#include "./securec.h" +#include "dataset/util/allocator.h" +#include "dataset/util/status.h" +namespace mindspore { +namespace dataset { +/// \brief A ReadableSlice wraps a const pointer in memory and its size. +/// \see WritableSlice for a non-const version +/// +class ReadableSlice { + public: + ReadableSlice() : ptr_(nullptr), sz_(0) {} + ReadableSlice(const void *ptr, size_t sz) : ptr_(ptr), sz_(sz) {} + ReadableSlice(const ReadableSlice &src, off64_t offset, size_t len) { + ptr_ = static_cast(src.GetPointer()) + offset; + sz_ = len; + } + ReadableSlice(const ReadableSlice &src, off64_t offset) : ReadableSlice(src, offset, src.sz_ - offset) {} + ReadableSlice(const ReadableSlice &lhs) { + ptr_ = lhs.ptr_; + sz_ = lhs.sz_; + } + ReadableSlice &operator=(const ReadableSlice &lhs) { + if (this != &lhs) { + ptr_ = lhs.ptr_; + sz_ = lhs.sz_; + } + return *this; + } + ReadableSlice(ReadableSlice &&lhs) noexcept { + if (this != &lhs) { + ptr_ = lhs.ptr_; + sz_ = lhs.sz_; + lhs.ptr_ = nullptr; + lhs.sz_ = 0; + } + } + ReadableSlice &operator=(ReadableSlice &&lhs) noexcept { + if (this != &lhs) { + ptr_ = lhs.ptr_; + sz_ = lhs.sz_; + lhs.ptr_ = nullptr; + lhs.sz_ = 0; + } + return *this; + } + /// \brief Getter function + /// \return Const version of the pointer + const void *GetPointer() const { return ptr_; } + /// \brief Getter function + /// \return Size of the slice + size_t GetSize() const { return sz_; } + bool empty() const { return ptr_ == nullptr; } + + private: + const void *ptr_; + size_t sz_; +}; +/// \brief A WritableSlice inherits from ReadableSlice to allow +/// one to write to the address pointed to by the pointer. +/// +class WritableSlice : public ReadableSlice { + public: + friend class StorageContainer; + /// \brief Default constructor + WritableSlice() : ReadableSlice(), mutable_data_(nullptr) {} + /// \brief This form of a constructor takes a pointer and its size. + WritableSlice(void *ptr, size_t sz) : ReadableSlice(ptr, sz), mutable_data_(ptr) {} + WritableSlice(const WritableSlice &src, off64_t offset, size_t len); + WritableSlice(const WritableSlice &src, off64_t offset); + WritableSlice(const WritableSlice &lhs) : ReadableSlice(lhs) { mutable_data_ = lhs.mutable_data_; } + WritableSlice &operator=(const WritableSlice &lhs) { + if (this != &lhs) { + mutable_data_ = lhs.mutable_data_; + ReadableSlice::operator=(lhs); + } + return *this; + } + WritableSlice(WritableSlice &&lhs) noexcept : ReadableSlice(std::move(lhs)) { + if (this != &lhs) { + mutable_data_ = lhs.mutable_data_; + lhs.mutable_data_ = nullptr; + } + } + WritableSlice &operator=(WritableSlice &&lhs) noexcept { + if (this != &lhs) { + mutable_data_ = lhs.mutable_data_; + lhs.mutable_data_ = nullptr; + ReadableSlice::operator=(std::move(lhs)); + } + return *this; + } + /// \brief Copy the content from one slice onto another. + static Status Copy(WritableSlice *dest, const ReadableSlice &src); + + private: + void *mutable_data_; + void *GetMutablePointer() { return mutable_data_; } +}; +} // namespace dataset +} // namespace mindspore +#endif // DATASET_UTIL_SLICE_H_ diff --git a/mindspore/ccsrc/dataset/util/storage_container.cc b/mindspore/ccsrc/dataset/util/storage_container.cc new file mode 100644 index 0000000000..96f5b45d0c --- /dev/null +++ b/mindspore/ccsrc/dataset/util/storage_container.cc @@ -0,0 +1,164 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "dataset/util/storage_container.h" + +#include +#include +#include +#include +#include "common/utils.h" +#include "dataset/util/de_error.h" +#include "dataset/util/path.h" +#include "dataset/util/status.h" +#include "utils/log_adapter.h" + +namespace mindspore { +namespace dataset { +Status StorageContainer::Create() { + RETURN_IF_NOT_OK(BuddySpace::CreateBuddySpace(&bs_)); + RETURN_IF_NOT_OK(cont_.CreateFile(&fd_)); + is_open_ = true; + MS_LOG(INFO) << "Container " << cont_ << " created"; + return Status::OK(); +} + +Status StorageContainer::Open() noexcept { + std::lock_guard lck(mutex_); + // Check again + if (!is_open_) { + RETURN_IF_NOT_OK(cont_.OpenFile(&fd_)); + is_open_ = true; + } + return Status::OK(); +} + +Status StorageContainer::Close() noexcept { + if (is_open_) { + std::lock_guard lck(mutex_); + // Check again + if (is_open_) { + RETURN_IF_NOT_OK(cont_.CloseFile(fd_)); + is_open_ = false; + fd_ = -1; + } + } + return Status::OK(); +} + +Status StorageContainer::Read(WritableSlice *dest, off64_t offset) const noexcept { + DS_ASSERT(is_open_); + RETURN_UNEXPECTED_IF_NULL(dest); + auto sz = dest->GetSize(); +#if defined(_WIN32) || defined(_WIN64) + // Doesn't seem there is any pread64 on mingw. + // So we will do a seek and then a read under + // a protection of mutex. + std::lock_guard lck(mutex_); + auto seek_err = lseek(fd_, offset, SEEK_SET); + if (seek_err < 0) { + RETURN_STATUS_UNEXPECTED(strerror(errno)); + } + auto r_sz = read(fd_, dest->GetMutablePointer(), sz); +#else + auto r_sz = pread64(fd_, dest->GetMutablePointer(), sz, offset); +#endif + if (r_sz != sz) { + errno_t err = (r_sz == 0) ? EOF : errno; + RETURN_STATUS_UNEXPECTED(strerror(err)); + } + return Status::OK(); +} + +Status StorageContainer::Write(const ReadableSlice &dest, off64_t offset) const noexcept { + DS_ASSERT(is_open_); + auto sz = dest.GetSize(); +#if defined(_WIN32) || defined(_WIN64) + // Doesn't seem there is any pwrite64 on mingw. + // So we will do a seek and then a read under + // a protection of mutex. + std::lock_guard lck(mutex_); + auto seek_err = lseek(fd_, offset, SEEK_SET); + if (seek_err < 0) { + RETURN_STATUS_UNEXPECTED(strerror(errno)); + } + auto r_sz = write(fd_, dest.GetPointer(), sz); +#else + auto r_sz = pwrite64(fd_, dest.GetPointer(), sz, offset); +#endif + if (r_sz != sz) { + errno_t err = (r_sz == 0) ? EOF : errno; + RETURN_STATUS_UNEXPECTED(strerror(err)); + } + return Status::OK(); +} + +Status StorageContainer::Insert(const std::vector &buf, off64_t *offset) noexcept { + size_t sz = 0; + for (auto &v : buf) { + sz += v.GetSize(); + } + if (sz == 0) { + RETURN_STATUS_UNEXPECTED("Unexpected 0 length"); + } + if (sz > bs_->GetMaxSize()) { + RETURN_STATUS_UNEXPECTED("Request size too big"); + } + BSpaceDescriptor bspd{0}; + addr_t addr = 0; + RETURN_IF_NOT_OK(bs_->Alloc(sz, &bspd, &addr)); + *offset = static_cast(addr); + // We will do piecewise copy of the data to disk. + for (auto &v : buf) { + RETURN_IF_NOT_OK(Write(v, addr)); + addr += v.GetSize(); + } + return Status::OK(); +} + +Status StorageContainer::Truncate() const noexcept { + if (is_open_) { + RETURN_IF_NOT_OK(cont_.TruncateFile(fd_)); + MS_LOG(INFO) << "Container " << cont_ << " truncated"; + } + return Status::OK(); +} + +StorageContainer::~StorageContainer() noexcept { + (void)Truncate(); + (void)Close(); +} + +std::ostream &operator<<(std::ostream &os, const StorageContainer &s) { + os << "File path : " << s.cont_ << "\n" << *(s.bs_.get()); + return os; +} + +Status StorageContainer::CreateStorageContainer(std::shared_ptr *out_sc, const std::string &path) { + Status rc; + auto sc = new (std::nothrow) StorageContainer(path); + if (sc == nullptr) { + return Status(StatusCode::kOutOfMemory); + } + rc = sc->Create(); + if (rc.IsOk()) { + (*out_sc).reset(sc); + } else { + delete sc; + } + return rc; +} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/util/storage_container.h b/mindspore/ccsrc/dataset/util/storage_container.h new file mode 100644 index 0000000000..07e41bd66a --- /dev/null +++ b/mindspore/ccsrc/dataset/util/storage_container.h @@ -0,0 +1,79 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_UTIL_STORAGE_CONTAINER_H_ +#define DATASET_UTIL_STORAGE_CONTAINER_H_ + +#include +#include +#include +#include +#include +#include +#include "dataset/util/system_pool.h" +#include "dataset/util/buddy.h" +#include "dataset/util/path.h" +#include "dataset/util/slice.h" +#include "dataset/util/status.h" + +namespace mindspore { +namespace dataset { +class StorageManager; + +class StorageContainer { + public: + friend class StorageManager; + + ~StorageContainer() noexcept; + + StorageContainer(const StorageContainer &) = delete; + + StorageContainer &operator=(const StorageContainer &) = delete; + + friend std::ostream &operator<<(std::ostream &os, const StorageContainer &s); + + Status Open() noexcept; + + Status Close() noexcept; + + Status Insert(const std::vector &buf, off64_t *offset) noexcept; + + Status Write(const ReadableSlice &dest, off64_t offset) const noexcept; + + Status Read(WritableSlice *dest, off64_t offset) const noexcept; + + Status Truncate() const noexcept; + + bool IsOpen() const { return is_open_; } + + static Status CreateStorageContainer(std::shared_ptr *out_sc, const std::string &path); + + private: + mutable std::mutex mutex_; + Path cont_; + int fd_; + bool is_open_; + std::unique_ptr bs_; + + // Use the default value of BuddySpace + // which can map upto 4G of space. + explicit StorageContainer(const std::string &path) : cont_(path), fd_(-1), is_open_(false), bs_(nullptr) {} + + Status Create(); +}; +} // namespace dataset +} // namespace mindspore + +#endif // DATASET_UTIL_STORAGE_CONTAINER_H_ diff --git a/mindspore/ccsrc/dataset/util/storage_manager.cc b/mindspore/ccsrc/dataset/util/storage_manager.cc new file mode 100644 index 0000000000..8b7a6044e9 --- /dev/null +++ b/mindspore/ccsrc/dataset/util/storage_manager.cc @@ -0,0 +1,167 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "dataset/util/storage_manager.h" + +#include +#include +#include +#include +#include "common/utils.h" +#include "dataset/util/path.h" +#include "dataset/util/services.h" +#include "dataset/util//de_error.h" +#include "utils/log_adapter.h" + +namespace mindspore { +namespace dataset { +std::string StorageManager::GetBaseName(const std::string &prefix, int32_t file_id) { + std::ostringstream oss; + oss << prefix << std::setfill('0') << std::setw(5) << file_id; + return oss.str(); +} + +std::string StorageManager::ConstructFileName(const std::string &prefix, int32_t file_id, const std::string &suffix) { + std::string base_name = GetBaseName(prefix, file_id); + return (base_name + "." + suffix); +} + +Status StorageManager::AddOneContainer() { + const std::string kPrefix = "IMG"; + const std::string kSuffix = "LB"; + Path container_name = root_ / ConstructFileName(kPrefix, file_id_, kSuffix); + std::shared_ptr sc; + RETURN_IF_NOT_OK(StorageContainer::CreateStorageContainer(&sc, container_name.toString())); + containers_.push_back(sc); + file_id_++; + return Status::OK(); +} + +Status StorageManager::DoServiceStart() { + containers_.reserve(1000); + if (root_.IsDirectory()) { + RETURN_IF_NOT_OK(AddOneContainer()); + } else { + RETURN_STATUS_UNEXPECTED("Not a directory"); + } + return Status::OK(); +} + +Status StorageManager::Write(key_type *key, const std::vector &buf) { + RETURN_UNEXPECTED_IF_NULL(key); + size_t sz = 0; + for (auto &v : buf) { + sz += v.GetSize(); + } + if (sz == 0) { + RETURN_STATUS_UNEXPECTED("Unexpected 0 length"); + } + std::shared_ptr cont; + key_type out_key; + value_type out_value; + bool create_new_container = false; + do { + SharedLock lock_s(&rw_lock_); + size_t num_containers = containers_.size(); + if (create_new_container) { + // Upgrade to exclusvie lock. + lock_s.Upgrade(); + create_new_container = false; + // Check again if someone has already added a + // new container after we got the x lock + if (containers_.size() == num_containers) { + RETURN_IF_NOT_OK(AddOneContainer()); + } + // Refresh how many containers there are. + num_containers = containers_.size(); + // Downgrade back to shared lock + lock_s.Downgrade(); + } + if (num_containers == 0) { + RETURN_STATUS_UNEXPECTED("num_containers is zero"); + } + // Go to the last container to insert. + cont = containers_.at(num_containers - 1); + off64_t offset; + Status rc = cont->Insert(buf, &offset); + if (rc.IsNoSpace()) { + create_new_container = true; + } else if (rc.IsOk()) { + out_value = std::make_pair(num_containers - 1, std::make_pair(offset, sz)); + RETURN_IF_NOT_OK(index_.insert(out_value, &out_key)); + *key = out_key; + break; + } else { + return rc; + } + } while (true); + return Status::OK(); +} + +Status StorageManager::Read(StorageManager::key_type key, WritableSlice *dest, size_t *bytesRead) const { + RETURN_UNEXPECTED_IF_NULL(dest); + auto r = index_.Search(key); + if (r.second) { + auto &it = r.first; + value_type v = *it; + int container_inx = v.first; + off_t offset = v.second.first; + size_t sz = v.second.second; + if (dest->GetSize() < sz) { + std::string errMsg = "Destination buffer too small. Expect at least " + std::to_string(sz) + + " but length = " + std::to_string(dest->GetSize()); + RETURN_STATUS_UNEXPECTED(errMsg); + } + if (bytesRead != nullptr) { + *bytesRead = sz; + } + auto cont = containers_.at(container_inx); + RETURN_IF_NOT_OK(cont->Read(dest, offset)); + } else { + RETURN_STATUS_UNEXPECTED("Key not found"); + } + return Status::OK(); +} + +Status StorageManager::DoServiceStop() noexcept { + Status rc; + Status rc1; + for (auto const &p : containers_) { + // The destructor of StorageContainer is not called automatically until the use + // count drops to 0. But it is not always the case. We will do it ourselves. + rc = p.get()->Truncate(); + if (rc.IsError()) { + rc1 = rc; + } + } + containers_.clear(); + file_id_ = 0; + return rc1; +} + +StorageManager::StorageManager(const Path &root) : root_(root), file_id_(0), index_() {} + +StorageManager::~StorageManager() { (void)StorageManager::DoServiceStop(); } + +std::ostream &operator<<(std::ostream &os, const StorageManager &s) { + os << "Dumping all containers ..." + << "\n"; + for (auto const &p : s.containers_) { + os << *(p.get()); + } + return os; +} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/util/storage_manager.h b/mindspore/ccsrc/dataset/util/storage_manager.h new file mode 100644 index 0000000000..075ac713d2 --- /dev/null +++ b/mindspore/ccsrc/dataset/util/storage_manager.h @@ -0,0 +1,76 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_UTIL_STORAGE_MANAGER_H_ +#define DATASET_UTIL_STORAGE_MANAGER_H_ + +#include +#include +#include +#include +#include +#include "dataset/util/allocator.h" +#include "dataset/util/auto_index.h" +#include "dataset/util/lock.h" +#include "dataset/util/memory_pool.h" +#include "dataset/util/path.h" +#include "dataset/util/service.h" +#include "dataset/util/slice.h" +#include "dataset/util/storage_container.h" + +using ListOfContainers = std::vector>; +namespace mindspore { +namespace dataset { +class StorageManager : public Service { + public: + using storage_index = AutoIndexObj>>; + using key_type = storage_index::key_type; + using value_type = storage_index::value_type; + + explicit StorageManager(const Path &); + + ~StorageManager() override; + + StorageManager(const StorageManager &) = delete; + + StorageManager &operator=(const StorageManager &) = delete; + + Status Write(key_type *out_key, const std::vector &buf); + + Status Read(key_type key, WritableSlice *dest, size_t *bytesRead) const; + + Status DoServiceStart() override; + + Status DoServiceStop() noexcept override; + + friend std::ostream &operator<<(std::ostream &os, const StorageManager &s); + + private: + Path root_; + ListOfContainers containers_; + int file_id_; + RWLock rw_lock_; + storage_index index_; + + std::string GetBaseName(const std::string &prefix, int32_t file_id); + + std::string ConstructFileName(const std::string &prefix, int32_t file_id, const std::string &suffix); + + Status AddOneContainer(); +}; +} // namespace dataset +} // namespace mindspore + +#endif // DATASET_UTIL_STORAGE_MANAGER_H_ diff --git a/mindspore/ccsrc/dataset/util/system_pool.h b/mindspore/ccsrc/dataset/util/system_pool.h index bd15ad11dd..286e30a615 100644 --- a/mindspore/ccsrc/dataset/util/system_pool.h +++ b/mindspore/ccsrc/dataset/util/system_pool.h @@ -19,8 +19,10 @@ #include #include #include +#include #include #include "./securec.h" +#include "dataset/util/allocator.h" #include "dataset/util/memory_pool.h" namespace mindspore { @@ -61,6 +63,11 @@ class SystemPool : public MemoryPool { uint64_t get_max_size() const override { return std::numeric_limits::max(); } int PercentFree() const override { return 100; } + + template + static Allocator GetAllocator() { + return Allocator(std::make_shared()); + } }; } // namespace dataset } // namespace mindspore From a8f392ca4280af55c8e7026e54ed856267acd54b Mon Sep 17 00:00:00 2001 From: Danish Farid Date: Wed, 24 Jun 2020 12:41:41 -0400 Subject: [PATCH 063/254] updated test util file + new BoundingBoxCheck + fixed VOCDataset annotations Style Error fix fixed PyLint problem reverting testVOC2012 due to CI break for existing test reverting testVOC2012 due to CI break for existing test-2 updated old error messages to confirm with global standard addressing PR 2355 Comments - 1 addressing PR 2355 Comments - 2 addressing PR 2355 Comments - 3 --- .../random_vertical_flip_with_bbox_op.cc | 2 +- mindspore/ccsrc/dataset/kernels/tensor_op.h | 8 +- .../dataset/testVOC2012/Annotations/129.xml | 27 +++ .../ut/data/dataset/testVOC2012_2/README.txt | 2 + .../test_random_crop_and_resize_with_bbox.py | 4 +- .../dataset/test_random_crop_with_bbox.py | 24 +-- .../test_random_vertical_flip_with_bbox.py | 204 +++--------------- tests/ut/python/dataset/util.py | 123 +++++++++++ 8 files changed, 197 insertions(+), 197 deletions(-) create mode 100644 tests/ut/data/dataset/testVOC2012/Annotations/129.xml create mode 100644 tests/ut/data/dataset/testVOC2012_2/README.txt diff --git a/mindspore/ccsrc/dataset/kernels/image/random_vertical_flip_with_bbox_op.cc b/mindspore/ccsrc/dataset/kernels/image/random_vertical_flip_with_bbox_op.cc index d88c009559..c6aa8450a8 100644 --- a/mindspore/ccsrc/dataset/kernels/image/random_vertical_flip_with_bbox_op.cc +++ b/mindspore/ccsrc/dataset/kernels/image/random_vertical_flip_with_bbox_op.cc @@ -41,7 +41,7 @@ Status RandomVerticalFlipWithBBoxOp::Compute(const TensorRow &input, TensorRow * RETURN_IF_NOT_OK(input[1]->GetUnsignedIntAt(&boxHeight, {i, 3})); // get height of bbox // subtract (curCorner + height) from (max) for new Corner position - newBoxCorner_y = (imHeight - 1) - (boxCorner_y + boxHeight); + newBoxCorner_y = (imHeight - 1) - ((boxCorner_y + boxHeight) - 1); RETURN_IF_NOT_OK(input[1]->SetItemAt({i, 1}, newBoxCorner_y)); } diff --git a/mindspore/ccsrc/dataset/kernels/tensor_op.h b/mindspore/ccsrc/dataset/kernels/tensor_op.h index 293d4a4f99..9aae50d6b0 100644 --- a/mindspore/ccsrc/dataset/kernels/tensor_op.h +++ b/mindspore/ccsrc/dataset/kernels/tensor_op.h @@ -45,14 +45,18 @@ #define BOUNDING_BOX_CHECK(input) \ do { \ + if (input.size() != 2) { \ + return Status(StatusCode::kBoundingBoxInvalidShape, __LINE__, __FILE__, \ + "Requires Image and Bounding Boxes, likely missed bounding boxes."); \ + } \ if (input[1]->shape().Size() < 2) { \ return Status(StatusCode::kBoundingBoxInvalidShape, __LINE__, __FILE__, \ - "Bounding boxes shape should have at least two dims"); \ + "Bounding boxes shape should have at least two dimensions."); \ } \ uint32_t num_of_features = input[1]->shape()[1]; \ if (num_of_features < 4) { \ return Status(StatusCode::kBoundingBoxInvalidShape, __LINE__, __FILE__, \ - "Bounding boxes should be have at least 4 features"); \ + "Bounding boxes should be have at least 4 features."); \ } \ uint32_t num_of_boxes = input[1]->shape()[0]; \ uint32_t img_h = input[0]->shape()[0]; \ diff --git a/tests/ut/data/dataset/testVOC2012/Annotations/129.xml b/tests/ut/data/dataset/testVOC2012/Annotations/129.xml new file mode 100644 index 0000000000..3bb822f545 --- /dev/null +++ b/tests/ut/data/dataset/testVOC2012/Annotations/129.xml @@ -0,0 +1,27 @@ + + VOC2012 + 129.jpg + + simulate VOC2007 Database + simulate VOC2007 + flickr + + + 500 + 375 + 3 + + 1 + + dog + Frontal + 0 + 0 + + 1124 + 437 + 1684 + 2669 + + + diff --git a/tests/ut/data/dataset/testVOC2012_2/README.txt b/tests/ut/data/dataset/testVOC2012_2/README.txt new file mode 100644 index 0000000000..6410067c69 --- /dev/null +++ b/tests/ut/data/dataset/testVOC2012_2/README.txt @@ -0,0 +1,2 @@ +Custom VOC2012-like dataset with valid annotations for images. +Created to test BoundingBox Augmentation Ops - June 2020. \ No newline at end of file diff --git a/tests/ut/python/dataset/test_random_crop_and_resize_with_bbox.py b/tests/ut/python/dataset/test_random_crop_and_resize_with_bbox.py index 3dd97d2512..90269a7027 100644 --- a/tests/ut/python/dataset/test_random_crop_and_resize_with_bbox.py +++ b/tests/ut/python/dataset/test_random_crop_and_resize_with_bbox.py @@ -305,8 +305,8 @@ def test_c_random_resized_crop_with_bbox_op_bad(): if __name__ == "__main__": - test_c_random_resized_crop_with_bbox_op(False) - test_c_random_resized_crop_with_bbox_op_edge(False) + test_c_random_resized_crop_with_bbox_op(plot_vis=True) + test_c_random_resized_crop_with_bbox_op_edge(plot_vis=True) test_c_random_resized_crop_with_bbox_op_invalid() test_c_random_resized_crop_with_bbox_op_invalid2() test_c_random_resized_crop_with_bbox_op_bad() diff --git a/tests/ut/python/dataset/test_random_crop_with_bbox.py b/tests/ut/python/dataset/test_random_crop_with_bbox.py index d1e2e08419..7f5fa46512 100644 --- a/tests/ut/python/dataset/test_random_crop_with_bbox.py +++ b/tests/ut/python/dataset/test_random_crop_with_bbox.py @@ -142,7 +142,7 @@ def gen_bbox_edge(im, bbox): return im, bbox -def c_random_crop_with_bbox_op(plot_vis=False): +def test_random_crop_with_bbox_op_c(plot_vis=False): """ Prints images side by side with and without Aug applied + bboxes """ @@ -176,7 +176,7 @@ def c_random_crop_with_bbox_op(plot_vis=False): visualize(unaugSamp, augSamp) -def c_random_crop_with_bbox_op2(plot_vis=False): +def test_random_crop_with_bbox_op2_c(plot_vis=False): """ Prints images side by side with and without Aug applied + bboxes With Fill Value @@ -212,7 +212,7 @@ def c_random_crop_with_bbox_op2(plot_vis=False): visualize(unaugSamp, augSamp) -def c_random_crop_with_bbox_op3(plot_vis=False): +def test_random_crop_with_bbox_op3_c(plot_vis=False): """ Prints images side by side with and without Aug applied + bboxes With Padding Mode passed @@ -247,7 +247,7 @@ def c_random_crop_with_bbox_op3(plot_vis=False): visualize(unaugSamp, augSamp) -def c_random_crop_with_bbox_op_edge(plot_vis=False): +def test_random_crop_with_bbox_op_edge_c(plot_vis=False): """ Prints images side by side with and without Aug applied + bboxes Testing for an Edge case @@ -289,7 +289,7 @@ def c_random_crop_with_bbox_op_edge(plot_vis=False): visualize(unaugSamp, augSamp) -def c_random_crop_with_bbox_op_invalid(): +def test_random_crop_with_bbox_op_invalid_c(): """ Checking for invalid params passed to Aug Constructor """ @@ -319,7 +319,7 @@ def c_random_crop_with_bbox_op_invalid(): assert "Size should be a single integer" in str(err) -def c_random_crop_with_bbox_op_bad(): +def test_random_crop_with_bbox_op_bad_c(): # Should Fail - Errors logged to logger for ix, badFunc in enumerate(badGenFuncs): try: @@ -352,9 +352,9 @@ def c_random_crop_with_bbox_op_bad(): if __name__ == "__main__": - c_random_crop_with_bbox_op(False) - c_random_crop_with_bbox_op2(False) - c_random_crop_with_bbox_op3(False) - c_random_crop_with_bbox_op_edge(False) - c_random_crop_with_bbox_op_invalid() - c_random_crop_with_bbox_op_bad() + test_random_crop_with_bbox_op_c(plot_vis=True) + test_random_crop_with_bbox_op2_c(plot_vis=True) + test_random_crop_with_bbox_op3_c(plot_vis=True) + test_random_crop_with_bbox_op_edge_c(plot_vis=True) + test_random_crop_with_bbox_op_invalid_c() + test_random_crop_with_bbox_op_bad_c() diff --git a/tests/ut/python/dataset/test_random_vertical_flip_with_bbox.py b/tests/ut/python/dataset/test_random_vertical_flip_with_bbox.py index e0c8b455f4..b1bb4bc459 100644 --- a/tests/ut/python/dataset/test_random_vertical_flip_with_bbox.py +++ b/tests/ut/python/dataset/test_random_vertical_flip_with_bbox.py @@ -15,15 +15,12 @@ """ Testing RandomVerticalFlipWithBBox op """ -import numpy as np - -import matplotlib.pyplot as plt -import matplotlib.patches as patches import mindspore.dataset as ds import mindspore.dataset.transforms.vision.c_transforms as c_vision from mindspore import log as logger +from util import visualize_with_bounding_boxes, InvalidBBoxType, check_bad_bbox # updated VOC dataset with correct annotations DATA_DIR = "../data/dataset/testVOC2012_2" @@ -46,106 +43,11 @@ def fix_annotate(bboxes): return bboxes -def add_bounding_boxes(ax, bboxes): - for bbox in bboxes: - rect = patches.Rectangle((bbox[0], bbox[1]), - bbox[2], bbox[3], - linewidth=1, edgecolor='r', facecolor='none') - # Add the patch to the Axes - ax.add_patch(rect) - - -def vis_check(orig, aug): - if not isinstance(orig, list) or not isinstance(aug, list): - return False - if len(orig) != len(aug): - return False - return True - - -def visualize(orig, aug): - - if not vis_check(orig, aug): - return - - plotrows = 3 - compset = int(len(orig)/plotrows) - - orig, aug = np.array(orig), np.array(aug) - - orig = np.split(orig[:compset*plotrows], compset) + [orig[compset*plotrows:]] - aug = np.split(aug[:compset*plotrows], compset) + [aug[compset*plotrows:]] - - for ix, allData in enumerate(zip(orig, aug)): - base_ix = ix * plotrows # will signal what base level we're on - fig, axs = plt.subplots(len(allData[0]), 2) - fig.tight_layout(pad=1.5) - - for x, (dataA, dataB) in enumerate(zip(allData[0], allData[1])): - cur_ix = base_ix + x - - axs[x, 0].imshow(dataA["image"]) - add_bounding_boxes(axs[x, 0], dataA["annotation"]) - axs[x, 0].title.set_text("Original" + str(cur_ix+1)) - print("Original **\n ", str(cur_ix+1), " :", dataA["annotation"]) - - axs[x, 1].imshow(dataB["image"]) - add_bounding_boxes(axs[x, 1], dataB["annotation"]) - axs[x, 1].title.set_text("Augmented" + str(cur_ix+1)) - print("Augmented **\n", str(cur_ix+1), " ", dataB["annotation"], "\n") - - plt.show() - -# Functions to pass to Gen for creating invalid bounding boxes - - -def gen_bad_bbox_neg_xy(im, bbox): - im_h, im_w = im.shape[0], im.shape[1] - bbox[0][:4] = [-50, -50, im_w - 10, im_h - 10] - return im, bbox - - -def gen_bad_bbox_overflow_width(im, bbox): - im_h, im_w = im.shape[0], im.shape[1] - bbox[0][:4] = [0, 0, im_w + 10, im_h - 10] - return im, bbox - - -def gen_bad_bbox_overflow_height(im, bbox): - im_h, im_w = im.shape[0], im.shape[1] - bbox[0][:4] = [0, 0, im_w - 10, im_h + 10] - return im, bbox - - -def gen_bad_bbox_wrong_shape(im, bbox): - bbox = np.array([[0, 0, 0]]).astype(bbox.dtype) - return im, bbox - - -badGenFuncs = [gen_bad_bbox_neg_xy, - gen_bad_bbox_overflow_width, - gen_bad_bbox_overflow_height, - gen_bad_bbox_wrong_shape] - -assertVal = ["min_x", - "is out of bounds of the image", - "is out of bounds of the image", - "4 features"] - - -# Gen Edge case BBox -def gen_bbox_edge(im, bbox): - im_h, im_w = im.shape[0], im.shape[1] - bbox[0][:4] = [0, 0, im_w, im_h] - return im, bbox - - -def c_random_vertical_flip_with_bbox_op(plot_vis=False): +def test_random_vertical_flip_with_bbox_op_c(plot_vis=False): """ Prints images side by side with and without Aug applied + bboxes to compare and test """ - # Load dataset dataVoc1 = ds.VOCDataset(DATA_DIR, task="Detection", mode="train", decode=True, shuffle=False) @@ -175,10 +77,10 @@ def c_random_vertical_flip_with_bbox_op(plot_vis=False): augSamp.append(Aug) if plot_vis: - visualize(unaugSamp, augSamp) + visualize_with_bounding_boxes(unaugSamp, augSamp) -def c_random_vertical_flip_with_bbox_op_rand(plot_vis=False): +def test_random_vertical_flip_with_bbox_op_rand_c(plot_vis=False): """ Prints images side by side with and without Aug applied + bboxes to compare and test @@ -213,54 +115,12 @@ def c_random_vertical_flip_with_bbox_op_rand(plot_vis=False): augSamp.append(Aug) if plot_vis: - visualize(unaugSamp, augSamp) - + visualize_with_bounding_boxes(unaugSamp, augSamp) -def c_random_vertical_flip_with_bbox_op_edge(plot_vis=False): - # Should Pass - # Load dataset - dataVoc1 = ds.VOCDataset(DATA_DIR, task="Detection", mode="train", - decode=True, shuffle=False) - - dataVoc2 = ds.VOCDataset(DATA_DIR, task="Detection", mode="train", - decode=True, shuffle=False) - test_op = c_vision.RandomVerticalFlipWithBBox(0.6) - - # maps to fix annotations to HQ standard - dataVoc1 = dataVoc1.map(input_columns=["annotation"], - output_columns=["annotation"], - operations=fix_annotate) - dataVoc2 = dataVoc2.map(input_columns=["annotation"], - output_columns=["annotation"], - operations=fix_annotate) - - # Modify BBoxes to serve as valid edge cases - dataVoc2 = dataVoc2.map(input_columns=["image", "annotation"], - output_columns=["image", "annotation"], - columns_order=["image", "annotation"], - operations=[gen_bbox_edge]) - - # map to apply ops - dataVoc2 = dataVoc2.map(input_columns=["image", "annotation"], - output_columns=["image", "annotation"], - columns_order=["image", "annotation"], - operations=[test_op]) - - unaugSamp, augSamp = [], [] - - for unAug, Aug in zip(dataVoc1.create_dict_iterator(), dataVoc2.create_dict_iterator()): - unaugSamp.append(unAug) - augSamp.append(Aug) - - if plot_vis: - visualize(unaugSamp, augSamp) - - -def c_random_vertical_flip_with_bbox_op_invalid(): +def test_random_vertical_flip_with_bbox_op_invalid_c(): # Should Fail # Load dataset - dataVoc2 = ds.VOCDataset(DATA_DIR, task="Detection", mode="train", decode=True, shuffle=False) @@ -286,41 +146,25 @@ def c_random_vertical_flip_with_bbox_op_invalid(): assert "Input is not" in str(err) -def c_random_vertical_flip_with_bbox_op_bad(): - # Should Fail - Errors logged to logger - for ix, badFunc in enumerate(badGenFuncs): - try: - dataVoc2 = ds.VOCDataset(DATA_DIR, task="Detection", mode="train", - decode=True, shuffle=False) - - test_op = c_vision.RandomVerticalFlipWithBBox(1) - - dataVoc2 = dataVoc2.map(input_columns=["annotation"], - output_columns=["annotation"], - operations=fix_annotate) - - dataVoc2 = dataVoc2.map(input_columns=["image", "annotation"], - output_columns=["image", "annotation"], - columns_order=["image", "annotation"], - operations=[badFunc]) - - # map to apply ops - dataVoc2 = dataVoc2.map(input_columns=["image", "annotation"], - output_columns=["image", "annotation"], - columns_order=["image", "annotation"], - operations=[test_op]) - - for _ in dataVoc2.create_dict_iterator(): - break # first sample will cause exception +def test_random_vertical_flip_with_bbox_op_bad_c(): + """ + Test RandomHorizontalFlipWithBBox op with invalid bounding boxes + """ + logger.info("test_random_horizontal_bbox_invalid_bounds_c") + test_op = c_vision.RandomVerticalFlipWithBBox(1) - except RuntimeError as err: - logger.info("Got an exception in DE: {}".format(str(err))) - assert assertVal[ix] in str(err) + data_voc2 = ds.VOCDataset(DATA_DIR, task="Detection", mode="train", decode=True, shuffle=False) + check_bad_bbox(data_voc2, test_op, InvalidBBoxType.WidthOverflow, "bounding boxes is out of bounds of the image") + data_voc2 = ds.VOCDataset(DATA_DIR, task="Detection", mode="train", decode=True, shuffle=False) + check_bad_bbox(data_voc2, test_op, InvalidBBoxType.HeightOverflow, "bounding boxes is out of bounds of the image") + data_voc2 = ds.VOCDataset(DATA_DIR, task="Detection", mode="train", decode=True, shuffle=False) + check_bad_bbox(data_voc2, test_op, InvalidBBoxType.NegativeXY, "min_x") + data_voc2 = ds.VOCDataset(DATA_DIR, task="Detection", mode="train", decode=True, shuffle=False) + check_bad_bbox(data_voc2, test_op, InvalidBBoxType.WrongShape, "4 features") if __name__ == "__main__": - c_random_vertical_flip_with_bbox_op(False) - c_random_vertical_flip_with_bbox_op_rand(False) - c_random_vertical_flip_with_bbox_op_edge(False) - c_random_vertical_flip_with_bbox_op_invalid() - c_random_vertical_flip_with_bbox_op_bad() + test_random_vertical_flip_with_bbox_op_c(plot_vis=True) + test_random_vertical_flip_with_bbox_op_rand_c(plot_vis=True) + test_random_vertical_flip_with_bbox_op_invalid_c() + test_random_vertical_flip_with_bbox_op_bad_c() diff --git a/tests/ut/python/dataset/util.py b/tests/ut/python/dataset/util.py index 11335e120b..00a2c7ef57 100644 --- a/tests/ut/python/dataset/util.py +++ b/tests/ut/python/dataset/util.py @@ -16,7 +16,9 @@ import hashlib import json import os +from enum import Enum import matplotlib.pyplot as plt +import matplotlib.patches as patches import numpy as np # import jsbeautifier import mindspore.dataset as ds @@ -284,3 +286,124 @@ def config_get_set_num_parallel_workers(num_parallel_workers_new): logger.info("num_parallel_workers: original = {} new = {} ".format(num_parallel_workers_original, num_parallel_workers_new)) return num_parallel_workers_original + + +def visualize_with_bounding_boxes(orig, aug, plot_rows=3): + """ + Take a list of un-augmented and augmented images with "annotation" bounding boxes + Plot images to compare test correct BBox augment functionality + :param orig: list of original images and bboxes (without aug) + :param aug: list of augmented images and bboxes + :param plot_rows: number of rows on plot (rows = samples on one plot) + :return: None + """ + + def add_bounding_boxes(ax, bboxes): + for bbox in bboxes: + rect = patches.Rectangle((bbox[0], bbox[1]), + bbox[2], bbox[3], + linewidth=1, edgecolor='r', facecolor='none') + # Add the patch to the Axes + ax.add_patch(rect) + + # Quick check to confirm correct input parameters + if not isinstance(orig, list) or not isinstance(aug, list): + return + if len(orig) != len(aug) or not orig: + return + + comp_set = int(len(orig)/plot_rows) + orig, aug = np.array(orig), np.array(aug) + + if len(orig) > plot_rows: + orig = np.split(orig[:comp_set*plot_rows], comp_set) + [orig[comp_set*plot_rows:]] + aug = np.split(aug[:comp_set*plot_rows], comp_set) + [aug[comp_set*plot_rows:]] + else: + orig = [orig] + aug = [aug] + + for ix, allData in enumerate(zip(orig, aug)): + base_ix = ix * plot_rows # will signal what base level we're on + + sub_plot_count = 2 if (len(allData[0]) < 2) else len(allData[0]) # if 1 image remains, create subplot for 2 to simplify axis selection + fig, axs = plt.subplots(sub_plot_count, 2) + fig.tight_layout(pad=1.5) + + for x, (dataA, dataB) in enumerate(zip(allData[0], allData[1])): + cur_ix = base_ix + x + + axs[x, 0].imshow(dataA["image"]) + add_bounding_boxes(axs[x, 0], dataA["annotation"]) + axs[x, 0].title.set_text("Original" + str(cur_ix+1)) + logger.info("Original **\n{} : {}".format(str(cur_ix+1), dataA["annotation"])) + + axs[x, 1].imshow(dataB["image"]) + add_bounding_boxes(axs[x, 1], dataB["annotation"]) + axs[x, 1].title.set_text("Augmented" + str(cur_ix+1)) + logger.info("Augmented **\n{} : {}\n".format(str(cur_ix+1), dataB["annotation"])) + + plt.show() + + +class InvalidBBoxType(Enum): + """ + Defines Invalid Bounding Bbox types for test cases + """ + WidthOverflow = 1 + HeightOverflow = 2 + NegativeXY = 3 + WrongShape = 4 + + +def check_bad_bbox(data, test_op, invalid_bbox_type, expected_error): + """ + :param data: de object detection pipeline + :param test_op: Augmentation Op to test on image + :param invalid_bbox_type: type of bad box + :param expected_error: error expected to get due to bad box + :return: None + """ + + def add_bad_annotation(img, bboxes, invalid_bbox_type_): + """ + Used to generate erroneous bounding box examples on given img. + :param img: image where the bounding boxes are. + :param bboxes: in [x_min, y_min, w, h, label, truncate, difficult] format + :param box_type_: type of bad box + :return: bboxes with bad examples added + """ + height = img.shape[0] + width = img.shape[1] + if invalid_bbox_type_ == InvalidBBoxType.WidthOverflow: + # use box that overflows on width + return img, np.array([[0, 0, width + 1, height, 0, 0, 0]]).astype(np.uint32) + + if invalid_bbox_type_ == InvalidBBoxType.HeightOverflow: + # use box that overflows on height + return img, np.array([[0, 0, width, height + 1, 0, 0, 0]]).astype(np.uint32) + + if invalid_bbox_type_ == InvalidBBoxType.NegativeXY: + # use box with negative xy + return img, np.array([[-10, -10, width, height, 0, 0, 0]]).astype(np.uint32) + + if invalid_bbox_type_ == InvalidBBoxType.WrongShape: + # use box that has incorrect shape + return img, np.array([[0, 0, width - 1]]).astype(np.uint32) + return img, bboxes + + try: + # map to use selected invalid bounding box type + data = data.map(input_columns=["image", "annotation"], + output_columns=["image", "annotation"], + columns_order=["image", "annotation"], + operations=lambda img, bboxes: add_bad_annotation(img, bboxes, invalid_bbox_type)) + # map to apply ops + data = data.map(input_columns=["image", "annotation"], + output_columns=["image", "annotation"], + columns_order=["image", "annotation"], + operations=[test_op]) # Add column for "annotation" + for _, _ in enumerate(data.create_dict_iterator()): + break + except RuntimeError as error: + logger.info("Got an exception in DE: {}".format(str(error))) + assert expected_error in str(error) From ef13a4b6fb7fe8705e09e0851687b67c20ca3100 Mon Sep 17 00:00:00 2001 From: Wei Luning Date: Thu, 25 Jun 2020 00:45:10 +0800 Subject: [PATCH 064/254] adjust cse code when op has side effect. --- mindspore/ccsrc/optimizer/cse.cc | 80 ++++++++++++------- mindspore/ccsrc/optimizer/cse.h | 2 +- .../pass/common_subexpression_elimination.cc | 2 +- .../pass/common_subexpression_elimination.h | 2 +- mindspore/ccsrc/pybind_api/export_flags.cc | 1 + mindspore/ccsrc/pybind_api/export_flags.h | 2 +- mindspore/nn/wrap/cell_wrapper.py | 4 +- mindspore/ops/operations/debug_ops.py | 2 +- tests/ut/python/utils/test_serialize.py | 1 + 9 files changed, 63 insertions(+), 33 deletions(-) diff --git a/mindspore/ccsrc/optimizer/cse.cc b/mindspore/ccsrc/optimizer/cse.cc index 1af08ea3e1..0b675cca72 100644 --- a/mindspore/ccsrc/optimizer/cse.cc +++ b/mindspore/ccsrc/optimizer/cse.cc @@ -89,15 +89,28 @@ bool CSE::BuildOrderGroupAndDoReplace(const FuncGraphManagerPtr manager) const { return changed; } - +// The op like print, summary, or the op do not has true output, and always as a depend node input. +static bool HasSideEffect(const AnfNodePtr &node) { + auto prim = GetCNodePrimitive(node); + if (prim == nullptr) { + return false; + } + auto side_effect_v = prim->GetAttr(GRAPH_FLAG_SIDE_EFFECT); + if (side_effect_v != nullptr && side_effect_v->isa()) { + return GetValue(side_effect_v); + } + return false; +} +// If true do not merge the node. bool CSE::CheckRandomEffect(const AnfNodePtr &main, const AnfNodePtr &node) const { bool has_random_effect = false; auto prim_main = GetCNodePrimitive(main); auto prim_node = GetCNodePrimitive(node); - if (prim_main == prim_node) { - return false; - } + // if has random effect, when generate by different op (not same object), do not merge. if (prim_main != nullptr) { + if (prim_main == prim_node) { + return false; + } auto effect_val = prim_main->GetAttr(GRAPH_FLAG_RANDOM_EFFECT); if (effect_val != nullptr && effect_val->isa()) { has_random_effect = GetValue(effect_val); @@ -106,45 +119,58 @@ bool CSE::CheckRandomEffect(const AnfNodePtr &main, const AnfNodePtr &node) cons return has_random_effect; } -bool CSE::CheckReplace(const AnfNodePtr &main, const AnfNodePtr &node) const { +bool CSE::CheckReplace(const AnfNodePtr &main, const AnfNodePtr &node, bool check_side_effect) const { MS_EXCEPTION_IF_NULL(main); MS_EXCEPTION_IF_NULL(node); - bool replace = false; if (main->isa() && node->isa()) { auto main_value = GetValueNode(main); auto node_value = GetValueNode(node); - replace = (AbsOf(main) == AbsOf(node)) && (*main_value == *node_value); + return (AbsOf(main) == AbsOf(node)) && (*main_value == *node_value); } else if (main->isa() && node->isa()) { auto c_main = main->cast(); auto c_node = node->cast(); + // When appsame is true, check if has side effect, do not merge. + if (check_side_effect && HasSideEffect(main)) { + return false; + } const auto &inp1 = c_main->inputs(); const auto &inp2 = c_node->inputs(); - if (inp1.size() == inp2.size()) { - bool appsame = true; - for (size_t j = 0; j < inp1.size(); j++) { - MS_EXCEPTION_IF_NULL(inp1[j]); - MS_EXCEPTION_IF_NULL(inp2[j]); - if (!(*inp1[j] == *inp2[j])) { - // Handle the case of two different Tensor, but with the same value - if (IsValueNode(inp1[j]) && IsValueNode(inp2[j])) { - auto tensor1 = GetValueNode(inp1[j]); - auto tensor2 = GetValueNode(inp2[j]); - if (tensor1->ValueEqual(*tensor2)) { - continue; - } + if (inp1.size() != inp2.size()) { + return false; + } + for (size_t j = 0; j < inp1.size(); j++) { + auto inp1_j = inp1[j]; + auto inp2_j = inp2[j]; + MS_EXCEPTION_IF_NULL(inp1_j); + MS_EXCEPTION_IF_NULL(inp2_j); + if (!(*inp1_j == *inp2_j)) { + // Handle the case of two different Tensor, but with the same value + if (IsValueNode(inp1_j) && IsValueNode(inp2_j)) { + auto tensor1 = GetValueNode(inp1_j); + auto tensor2 = GetValueNode(inp2_j); + if (tensor1->ValueEqual(*tensor2)) { + continue; + } + } else if (HasSideEffect(inp1_j) && HasSideEffect(inp2_j)) { + // When the same side effect node as another two nodes' inputs, we still merge the node. + // Because the node only can be the inputs of `depend`, when the `depend` is duplicated merge the depend the + // node. + if (CheckReplace(inp1_j, inp2_j, false)) { + continue; } - appsame = false; - break; } + return false; } - if (CheckRandomEffect(c_main, c_node)) { - appsame = false; - } - replace = appsame; } + // When appsame is true, check if has random effect do not merge + if (CheckRandomEffect(c_main, c_node)) { + return false; + } + return true; } - return replace; + // a parameter node. + return false; } bool CSE::DoReplace(const FuncGraphManagerPtr manager, const std::vector &order_group, diff --git a/mindspore/ccsrc/optimizer/cse.h b/mindspore/ccsrc/optimizer/cse.h index fd90f61eeb..57163cc5c9 100644 --- a/mindspore/ccsrc/optimizer/cse.h +++ b/mindspore/ccsrc/optimizer/cse.h @@ -41,7 +41,7 @@ class CSE { return chg && report_changes_; } - virtual bool CheckReplace(const AnfNodePtr &main, const AnfNodePtr &node) const; + virtual bool CheckReplace(const AnfNodePtr &main, const AnfNodePtr &node, bool check_side_effect = true) const; virtual bool CheckRandomEffect(const AnfNodePtr &main, const AnfNodePtr &node) const; diff --git a/mindspore/ccsrc/pre_activate/pass/common_subexpression_elimination.cc b/mindspore/ccsrc/pre_activate/pass/common_subexpression_elimination.cc index 9af50eac33..297a167aa8 100644 --- a/mindspore/ccsrc/pre_activate/pass/common_subexpression_elimination.cc +++ b/mindspore/ccsrc/pre_activate/pass/common_subexpression_elimination.cc @@ -35,7 +35,7 @@ bool CheckEqualKernelBuildInfo(const AnfNodePtr &main, const AnfNodePtr &node) { } } // namespace -bool BackendCSE::CheckReplace(const AnfNodePtr &main, const AnfNodePtr &node) const { +bool BackendCSE::CheckReplace(const AnfNodePtr &main, const AnfNodePtr &node, bool) const { MS_EXCEPTION_IF_NULL(main); MS_EXCEPTION_IF_NULL(node); diff --git a/mindspore/ccsrc/pre_activate/pass/common_subexpression_elimination.h b/mindspore/ccsrc/pre_activate/pass/common_subexpression_elimination.h index 8e1768ea99..18f433ab95 100644 --- a/mindspore/ccsrc/pre_activate/pass/common_subexpression_elimination.h +++ b/mindspore/ccsrc/pre_activate/pass/common_subexpression_elimination.h @@ -31,7 +31,7 @@ class BackendCSE : public CSE { public: BackendCSE() = default; ~BackendCSE() override = default; - bool CheckReplace(const AnfNodePtr &main, const AnfNodePtr &node) const override; + bool CheckReplace(const AnfNodePtr &main, const AnfNodePtr &node, bool check_side_effect = true) const override; }; } // namespace opt } // namespace mindspore diff --git a/mindspore/ccsrc/pybind_api/export_flags.cc b/mindspore/ccsrc/pybind_api/export_flags.cc index 83392784f3..253e271e52 100644 --- a/mindspore/ccsrc/pybind_api/export_flags.cc +++ b/mindspore/ccsrc/pybind_api/export_flags.cc @@ -33,5 +33,6 @@ const char GRAPH_FLAG_LOOP_CAN_UNROLL[] = "loop_can_unroll"; const char GRAPH_FLAG_HAS_EFFECT[] = "has_effect"; const char GRAPH_FLAG_EFFECT_PATIAL_ORDER[] = "_effect_patial_order"; const char GRAPH_FLAG_RANDOM_EFFECT[] = "_random_effect"; +const char GRAPH_FLAG_SIDE_EFFECT[] = "_side_effect"; } // namespace mindspore diff --git a/mindspore/ccsrc/pybind_api/export_flags.h b/mindspore/ccsrc/pybind_api/export_flags.h index 74c27ff35d..6ea584e66d 100644 --- a/mindspore/ccsrc/pybind_api/export_flags.h +++ b/mindspore/ccsrc/pybind_api/export_flags.h @@ -34,7 +34,7 @@ extern const char GRAPH_FLAG_LOOP_CAN_UNROLL[]; extern const char GRAPH_FLAG_HAS_EFFECT[]; extern const char GRAPH_FLAG_EFFECT_PATIAL_ORDER[]; extern const char GRAPH_FLAG_RANDOM_EFFECT[]; - +extern const char GRAPH_FLAG_SIDE_EFFECT[]; } // namespace mindspore #endif // PYBIND_API_EXPORT_FLAGS_H_ diff --git a/mindspore/nn/wrap/cell_wrapper.py b/mindspore/nn/wrap/cell_wrapper.py index f0d920f51f..9e3d00cc95 100644 --- a/mindspore/nn/wrap/cell_wrapper.py +++ b/mindspore/nn/wrap/cell_wrapper.py @@ -220,7 +220,9 @@ class DataWrapper(Cell): def __init__(self, network, dataset_types, dataset_shapes, queue_name): super(DataWrapper, self).__init__(auto_prefix=False, flags=network.get_flags()) - + # Also copy the flag in `network` construct + flags = getattr(network.__class__.construct, "_mindspore_flags", {}) + self.add_flags(**flags) self.get_next = P.GetNext(dataset_types, dataset_shapes, len(dataset_types), queue_name) self.network = network diff --git a/mindspore/ops/operations/debug_ops.py b/mindspore/ops/operations/debug_ops.py index bafc72897e..c62b3f1ab8 100644 --- a/mindspore/ops/operations/debug_ops.py +++ b/mindspore/ops/operations/debug_ops.py @@ -334,7 +334,7 @@ class Print(PrimitiveWithInfer): @prim_attr_register def __init__(self): - pass + self.add_prim_attr("_side_effect", True) def __call__(self, *args): for arg in args: diff --git a/tests/ut/python/utils/test_serialize.py b/tests/ut/python/utils/test_serialize.py index c5b4586566..035ea87845 100644 --- a/tests/ut/python/utils/test_serialize.py +++ b/tests/ut/python/utils/test_serialize.py @@ -336,6 +336,7 @@ class PrintNet(nn.Cell): def construct(self, int8, uint8, int16, uint16, int32, uint32, int64, uint64, flt16, flt32, flt64, bool_, scale1, scale2): self.print('============tensor int8:==============', int8) + self.print('============tensor int8:==============', int8) self.print('============tensor uint8:==============', uint8) self.print('============tensor int16:==============', int16) self.print('============tensor uint16:==============', uint16) From aabec55c79139580f4be7b7f274dd03e268fd6c2 Mon Sep 17 00:00:00 2001 From: Giancarlo Colmenares Date: Tue, 23 Jun 2020 09:25:47 -0400 Subject: [PATCH 065/254] Removing TransformFuncType --- mindspore/ccsrc/ir/optimizer_caller.h | 12 +- mindspore/ccsrc/optimizer/irpass.cc | 164 ++++++++++-------- .../optimizer/irpass/arithmetic_simplify.h | 59 +++---- .../ccsrc/optimizer/irpass/cast_eliminate.h | 6 +- .../optimizer/irpass/env_item_eliminate.h | 30 ++-- .../optimizer/irpass/incorporate_getitem.h | 27 +-- .../optimizer/irpass/item_tuple_eliminate.h | 33 ++-- .../ccsrc/optimizer/irpass/ref_eliminate.h | 4 +- .../optimizer/irpass/reshape_eliminate.h | 11 +- .../optimizer/irpass/special_op_eliminate.h | 36 ++-- mindspore/ccsrc/optimizer/opt.cc | 19 +- mindspore/ccsrc/optimizer/opt.h | 24 +-- tests/ut/cpp/optimizer/opt_test.cc | 8 +- 13 files changed, 226 insertions(+), 207 deletions(-) diff --git a/mindspore/ccsrc/ir/optimizer_caller.h b/mindspore/ccsrc/ir/optimizer_caller.h index bd30454147..036f4ab510 100644 --- a/mindspore/ccsrc/ir/optimizer_caller.h +++ b/mindspore/ccsrc/ir/optimizer_caller.h @@ -17,13 +17,23 @@ #ifndef MINDSPORE_CCSRC_IR_OPTIMIZER_CALLER_H_ #define MINDSPORE_CCSRC_IR_OPTIMIZER_CALLER_H_ +#include + #include "ir/anf.h" -#include "optimizer/opt.h" namespace mindspore { +namespace opt { +class Optimizer; +using OptimizerPtr = std::shared_ptr; +using OptimizerWeakPtr = std::weak_ptr; + +using PredicateFuncType = std::function; +} // namespace opt + class OptimizerCaller { public: virtual AnfNodePtr operator()(const opt::OptimizerPtr &, const AnfNodePtr &) { return nullptr; } }; +using OptimizerCallerPtr = std::shared_ptr; } // namespace mindspore #endif // MINDSPORE_CCSRC_IR_OPTIMIZER_CALLER_H_ diff --git a/mindspore/ccsrc/optimizer/irpass.cc b/mindspore/ccsrc/optimizer/irpass.cc index 0033e386d8..0996abee2c 100644 --- a/mindspore/ccsrc/optimizer/irpass.cc +++ b/mindspore/ccsrc/optimizer/irpass.cc @@ -14,140 +14,154 @@ * limitations under the License. */ -#include "optimizer/irpass.h" - #include -#include "optimizer/irpass/symbol_resolver.h" +#include "optimizer/irpass.h" #include "optimizer/irpass/arithmetic_simplify.h" -#include "optimizer/irpass/special_op_eliminate.h" -#include "optimizer/irpass/item_tuple_eliminate.h" -#include "optimizer/irpass/env_item_eliminate.h" -#include "optimizer/irpass/tile_eliminate.h" -#include "optimizer/irpass/cast_eliminate.h" -#include "optimizer/irpass/reshape_eliminate.h" -#include "optimizer/irpass/transpose_eliminate.h" -#include "optimizer/irpass/reduce_eliminate.h" -#include "optimizer/irpass/partial_eliminate.h" -#include "optimizer/irpass/ref_eliminate.h" -#include "optimizer/irpass/merge_addn.h" #include "optimizer/irpass/branch_culling.h" +#include "optimizer/irpass/cast_eliminate.h" +#include "optimizer/irpass/convert.h" +#include "optimizer/irpass/env_item_eliminate.h" +#include "optimizer/irpass/grad_var_prepare.h" #include "optimizer/irpass/gradient_eliminate.h" -#include "optimizer/irpass/minmax_grad.h" #include "optimizer/irpass/inline.h" -#include "optimizer/irpass/convert.h" -#include "optimizer/irpass/specialize_transform.h" -#include "optimizer/irpass/incorporate_getitem.h" #include "optimizer/irpass/incorporate_call.h" -#include "optimizer/irpass/grad_var_prepare.h" -#include "optimizer/irpass/param_replace.h" +#include "optimizer/irpass/incorporate_getitem.h" +#include "optimizer/irpass/item_tuple_eliminate.h" #include "optimizer/irpass/mark_interface_fusion.h" +#include "optimizer/irpass/merge_addn.h" +#include "optimizer/irpass/minmax_grad.h" +#include "optimizer/irpass/param_replace.h" +#include "optimizer/irpass/partial_eliminate.h" +#include "optimizer/irpass/reduce_eliminate.h" +#include "optimizer/irpass/ref_eliminate.h" +#include "optimizer/irpass/reshape_eliminate.h" +#include "optimizer/irpass/special_op_eliminate.h" +#include "optimizer/irpass/specialize_transform.h" +#include "optimizer/irpass/symbol_resolver.h" +#include "optimizer/irpass/tile_eliminate.h" +#include "optimizer/irpass/transpose_eliminate.h" #include "optimizer/opt.h" namespace mindspore { namespace opt { namespace irpass { OptimizeIRPassLib::OptimizeIRPassLib() { - arithmetic_simplify_ = MakeSubstitution(ArithmeticSimplify(), "arithmetic_simplify", + arithmetic_simplify_ = MakeSubstitution(std::make_shared(), "arithmetic_simplify", {prim::kPrimScalarAdd, prim::kPrimScalarMul, prim::kPrimTensorAdd, prim::kPrimIdentity, prim::kPrimMomentum, prim::kPrimMul, prim::kPrimPow}); - arithmetic_simplify2_ = MakeSubstitution(ArithmeticSimplify2(), "arithmetic_simplify2", {prim::kPrimMul}); + arithmetic_simplify2_ = + MakeSubstitution(std::make_shared(), "arithmetic_simplify2", {prim::kPrimMul}); special_op_eliminate_ = - MakeSubstitution(SpecialOpEliminater(), "special_op_eliminate", + MakeSubstitution(std::make_shared(), "special_op_eliminate", {prim::kPrimInsertGradientOf, prim::kPrimStopGradient, prim::kPrimHookBackward, prim::kPrimPrintShapeType, prim::kPrimGetRefKey, prim::kPrimMirror, prim::kPrimVirtualDiv}); - zero_like_fill_zero_ = MakeSubstitution(ZeroLikeFillZero(), "zero_like_fill_zero", prim::kPrimZerosLike); - adjust_all_reduce_mul_add_ = MakeSubstitution(AdjustAllReduceMulAdd(), "adjust_all_reduce_mul_add", prim::kPrimAddN); + zero_like_fill_zero_ = + MakeSubstitution(std::make_shared(), "zero_like_fill_zero", prim::kPrimZerosLike); + adjust_all_reduce_mul_add_ = + MakeSubstitution(std::make_shared(), "adjust_all_reduce_mul_add", prim::kPrimAddN); // ops eliminate - item_tuple_eliminate_ = - MakeSubstitution(ItemTupleEliminater(), "item_tuple_eliminate", {prim::kPrimTupleGetItem, prim::kPrimTupleSetItem}); - tile_eliminate_ = MakeSubstitution(TileMultiplyByOne(), "tile_eliminate", prim::kPrimTile); - cast_eliminate_ = MakeSubstitution(CastEliminater(), "cast_eliminate", prim::kPrimCast); - reshape_eliminate_ = MakeSubstitution(ReshapeEliminater(), "reshape_eliminate", prim::kPrimReshape); - transpose_eliminate_ = MakeSubstitution(TransposeSameIOEliminater(), "transpose_eliminate", prim::kPrimTranspose); + item_tuple_eliminate_ = MakeSubstitution(std::make_shared(), "item_tuple_eliminate", + {prim::kPrimTupleGetItem, prim::kPrimTupleSetItem}); + tile_eliminate_ = MakeSubstitution(std::make_shared(), "tile_eliminate", prim::kPrimTile); + cast_eliminate_ = MakeSubstitution(std::make_shared(), "cast_eliminate", prim::kPrimCast); + reshape_eliminate_ = MakeSubstitution(std::make_shared(), "reshape_eliminate", prim::kPrimReshape); + transpose_eliminate_ = + MakeSubstitution(std::make_shared(), "transpose_eliminate", prim::kPrimTranspose); reduce_eliminate_ = MakeSubstitution( - ReduceOneEliminater(), "reduce_eliminate", + std::make_shared(), "reduce_eliminate", {prim::kPrimReduceMean, prim::kPrimReduceAll, prim::kPrimReduceSum, prim::kPrimReduceMax, prim::kPrimReduceMin}); - partial_eliminate_ = MakeSubstitution(PartialEliminater(), "partial_eliminate", IsCNodeDup); - same_eliminate_ = MakeSubstitution(SameEliminater(), "same_eliminate", prim::kPrimSameTypeShape); - check_bprop_eliminate_ = MakeSubstitution(CheckBpropEliminater(), "check_bprop_eliminate", prim::kPrimCheckBprop); - reset_defer_inline_ = MakeSubstitution(ResetDeferInline(), "reset_defer_inline", IsValueNode); - depend_value_elim_ = MakeSubstitution(DependValueElim(), "depend_value_elim", prim::kPrimDepend); + partial_eliminate_ = MakeSubstitution(std::make_shared(), "partial_eliminate", IsCNodeDup); + same_eliminate_ = MakeSubstitution(std::make_shared(), "same_eliminate", prim::kPrimSameTypeShape); + check_bprop_eliminate_ = + MakeSubstitution(std::make_shared(), "check_bprop_eliminate", prim::kPrimCheckBprop); + reset_defer_inline_ = + MakeSubstitution(std::make_shared(), "reset_defer_inline", IsValueNode); + depend_value_elim_ = MakeSubstitution(std::make_shared(), "depend_value_elim", prim::kPrimDepend); // Env Item Eliminate - env_get_item_eliminate_ = MakeSubstitution(EnvGetItemEliminater(), "env_get_item_eliminate", prim::kPrimEnvGetItem); - new_env_get_item_ = MakeSubstitution(NewEnvGetItem(), "new_env_get_item", prim::kPrimEnvGetItem); + env_get_item_eliminate_ = + MakeSubstitution(std::make_shared(), "env_get_item_eliminate", prim::kPrimEnvGetItem); + new_env_get_item_ = MakeSubstitution(std::make_shared(), "new_env_get_item", prim::kPrimEnvGetItem); incorporate_env_getitem_ = - MakeSubstitution(IncorporateEnvGetitem(), "incorporate_env_get_item", prim::kPrimEnvGetItem); - incorporate_env_getitem_switch_ = - MakeSubstitution(IncorporateEnvGetitemSwitch(), "incorporate_env_getitem_switch", prim::kPrimEnvGetItem); + MakeSubstitution(std::make_shared(), "incorporate_env_get_item", prim::kPrimEnvGetItem); + incorporate_env_getitem_switch_ = MakeSubstitution(std::make_shared(), + "incorporate_env_getitem_switch", prim::kPrimEnvGetItem); // Ref eliminate - make_ref_eliminate_ = MakeSubstitution(MakeRefEliminater(), "make_ref_eliminate", prim::kPrimMakeRef); - get_ref_param_eliminate_ = MakeSubstitution(GetRefParamEliminater(), "get_ref_param_eliminate", + make_ref_eliminate_ = + MakeSubstitution(std::make_shared(), "make_ref_eliminate", prim::kPrimMakeRef); + get_ref_param_eliminate_ = MakeSubstitution(std::make_shared(), "get_ref_param_eliminate", {prim::kPrimGetRefValue, prim::kPrimGetRefOrigin}); - get_make_ref_eliminate_ = MakeSubstitution(GetMakeRefEliminater(), "get_make_ref_eliminate", + get_make_ref_eliminate_ = MakeSubstitution(std::make_shared(), "get_make_ref_eliminate", {prim::kPrimGetRefKey, prim::kPrimGetRefValue, prim::kPrimGetRefOrigin}); - replace_refkey_by_param_ = - MakeSubstitution(ReplaceRefkeyByParam(), "replace_refkey_by_param", IsValueNode, opt::FORCE_RENORM); - replace_old_param_ = MakeSubstitution(ReplaceOldParam(), "replace_old_param", IsParam); + replace_refkey_by_param_ = MakeSubstitution(std::make_shared(), "replace_refkey_by_param", + IsValueNode, opt::FORCE_RENORM); + replace_old_param_ = MakeSubstitution(std::make_shared(), "replace_old_param", IsParam); // Gradient transforms - expand_jprim_ = MakeSubstitution(ExpandJPrim(), "expand_jprim", prim::kPrimJ); - minmaximum_grad_ = MakeSubstitution(MinMaximumGrad(), "minmaximum_grad", prim::kPrimTupleGetItem); + expand_jprim_ = MakeSubstitution(std::make_shared(), "expand_jprim", prim::kPrimJ); + minmaximum_grad_ = MakeSubstitution(std::make_shared(), "minmaximum_grad", prim::kPrimTupleGetItem); // branch culling - switch_simplify_ = MakeSubstitution(SwitchSimplify(), "switch_simplify", prim::kPrimSwitch); - float_tuple_getitem_switch_ = - MakeSubstitution(FloatTupleGetItemSwitch(), "float_tuple_getitem_switch", prim::kPrimTupleGetItem); + switch_simplify_ = MakeSubstitution(std::make_shared(), "switch_simplify", prim::kPrimSwitch); + float_tuple_getitem_switch_ = MakeSubstitution(std::make_shared(), + "float_tuple_getitem_switch", prim::kPrimTupleGetItem); float_env_getitem_switch_ = - MakeSubstitution(FloatEnvGetItemSwitch(), "float_env_getitem_switch", prim::kPrimEnvGetItem); - convert_switch_replacement_ = MakeSubstitution(ConvertSwitchReplacement(), "convert_switch_replacement", IsCNodeDup); + MakeSubstitution(std::make_shared(), "float_env_getitem_switch", prim::kPrimEnvGetItem); + convert_switch_replacement_ = + MakeSubstitution(std::make_shared(), "convert_switch_replacement", IsCNodeDup); // Addn - merge_addn_ = MakeSubstitution(MergeAddN(), "merge_addn", prim::kPrimAddN); - addn_zero_filter_ = MakeSubstitution(AddNZeroFilter(), "addn_zero_filter", prim::kPrimAddN); + merge_addn_ = MakeSubstitution(std::make_shared(), "merge_addn", prim::kPrimAddN); + addn_zero_filter_ = MakeSubstitution(std::make_shared(), "addn_zero_filter", prim::kPrimAddN); // inline - inline_ = MakeSubstitution(Inliner(), "inline", IsCNodeGraph); - replace_applicator_ = MakeSubstitution(ReplaceApplicator(), "replace_applicator", IsValueNode); - specialize_transform_ = MakeSubstitution(SpecializeOnGraphArguments(), "specialize_transform", IsCNodeGraph); + inline_ = MakeSubstitution(std::make_shared(), "inline", IsCNodeGraph); + replace_applicator_ = + MakeSubstitution(std::make_shared(), "replace_applicator", IsValueNode); + specialize_transform_ = + MakeSubstitution(std::make_shared(), "specialize_transform", IsCNodeGraph); // Incorporation incorporate_getitem_set_ = - MakeSubstitution(IncorporateGetitemSet(), "incorporate_getitem_set", prim::kPrimTupleGetItem); - incorporate_getitem_from_param_ = - MakeSubstitution(IncorporateGetitemFromParam(), "incorporate_getitem_from_param", IsCNodeGraphKernel); - incorporate_call_ = MakeSubstitution(IncorporateCall(), "incorporate_call", IsCNodeDup); - incorporate_call_switch_ = MakeSubstitution(IncorporateCallSwitch(), "incorporate_call_switch", IsCNodeDup); + MakeSubstitution(std::make_shared(), "incorporate_getitem_set", prim::kPrimTupleGetItem); + incorporate_getitem_from_param_ = MakeSubstitution(std::make_shared(), + "incorporate_getitem_from_param", IsCNodeGraphKernel); + incorporate_call_ = MakeSubstitution(std::make_shared(), "incorporate_call", IsCNodeDup); + incorporate_call_switch_ = + MakeSubstitution(std::make_shared(), "incorporate_call_switch", IsCNodeDup); // Virtual Dataset - virtual_dataset_eliminate_ = - MakeSubstitution(VirtualDatasetEliminater(), "virtual_dataset_eliminate", prim::kPrimVirtualDataset); + virtual_dataset_eliminate_ = MakeSubstitution(std::make_shared(), + "virtual_dataset_eliminate", prim::kPrimVirtualDataset); // Convert - print_tuple_wrapper_ = MakeSubstitution(PrintTupleWrapper(), "print_tuple_wrapper", prim::kPrimPrint); + print_tuple_wrapper_ = + MakeSubstitution(std::make_shared(), "print_tuple_wrapper", prim::kPrimPrint); // Unused parameter eliminate unused_parameter_eliminate_ = - MakeSubstitution(UnusedParasEliminater(), "unused_parameter_eliminate", IsCNodeGraphKernel); - unused_output_eliminate_ = MakeSubstitution(UnusedOutputEliminater(), "unused_output_eliminate", IsCNodeGraphKernel); + MakeSubstitution(std::make_shared(), "unused_parameter_eliminate", IsCNodeGraphKernel); + unused_output_eliminate_ = + MakeSubstitution(std::make_shared(), "unused_output_eliminate", IsCNodeGraphKernel); // AddN eliminate - addn_eliminate_ = MakeSubstitution(AddNEliminater(), "addn_eliminate", IsCNodeGraphKernel); + addn_eliminate_ = MakeSubstitution(std::make_shared(), "addn_eliminate", IsCNodeGraphKernel); // Mark interface fusion - mark_interface_fusion_ = MakeSubstitution(MarkInterfaceFusion(), "mark_interface_fusion", prim::kPrimSelect); + mark_interface_fusion_ = + MakeSubstitution(std::make_shared(), "mark_interface_fusion", prim::kPrimSelect); } ResolveIRPassLib::ResolveIRPassLib() { - resolver_resolve_ = MakeSubstitution(ResolverResolve(), "resolver_resolve", prim::kPrimResolve); - resolver_getattr_ = MakeSubstitution(ResolverGetattr(), "resolver_getattr", prim::kPrimGetAttr); + resolver_resolve_ = MakeSubstitution(std::make_shared(), "resolver_resolve", prim::kPrimResolve); + resolver_getattr_ = MakeSubstitution(std::make_shared(), "resolver_getattr", prim::kPrimGetAttr); } InferenceOptPrepareLib::InferenceOptPrepareLib() { - grad_var_prepare_ = MakeSubstitution(GradVarPrepare(), "grad_var_prepare", IsCNode); + grad_var_prepare_ = MakeSubstitution(std::make_shared(), "grad_var_prepare", IsCNode); } } // namespace irpass } // namespace opt diff --git a/mindspore/ccsrc/optimizer/irpass/arithmetic_simplify.h b/mindspore/ccsrc/optimizer/irpass/arithmetic_simplify.h index 270db8305f..a26b81e952 100644 --- a/mindspore/ccsrc/optimizer/irpass/arithmetic_simplify.h +++ b/mindspore/ccsrc/optimizer/irpass/arithmetic_simplify.h @@ -17,15 +17,16 @@ #ifndef MINDSPORE_CCSRC_OPTIMIZER_IRPASS_ARITHMETIC_SIMPLIFY_H_ #define MINDSPORE_CCSRC_OPTIMIZER_IRPASS_ARITHMETIC_SIMPLIFY_H_ -#include -#include #include +#include +#include -#include "optimizer/optimizer.h" -#include "optimizer/irpass.h" -#include "optimizer/irpass/prim_eliminate.h" +#include "ir/optimizer_caller.h" #include "ir/visitor.h" #include "operator/ops.h" +#include "optimizer/irpass.h" +#include "optimizer/irpass/prim_eliminate.h" +#include "optimizer/optimizer.h" namespace mindspore { namespace opt { @@ -739,17 +740,17 @@ class AdjustAllReduceMulAdd : public AnfVisitor { FuncGraphPtr all_reduce_fg_{nullptr}; }; -class ArithmeticSimplify { +class ArithmeticSimplify : public OptimizerCaller { public: ArithmeticSimplify() - : multiply_by_zero_or_one_(), - tensor_multiply_by_one_(), - add_by_zero_(), - tensor_add_by_zero_(), - identity_(prim::kPrimIdentity), - opt_update_zero_tensor_(), - constant_duplicate_mul_(), - power_one_() { + : multiply_by_zero_or_one_(std::make_shared()), + tensor_multiply_by_one_(std::make_shared()), + add_by_zero_(std::make_shared()), + tensor_add_by_zero_(std::make_shared()), + identity_(std::make_shared(prim::kPrimIdentity)), + opt_update_zero_tensor_(std::make_shared()), + constant_duplicate_mul_(std::make_shared()), + power_one_(std::make_shared()) { eliminaters_.emplace_back(multiply_by_zero_or_one_); eliminaters_.emplace_back(tensor_multiply_by_one_); eliminaters_.emplace_back(add_by_zero_); @@ -761,10 +762,10 @@ class ArithmeticSimplify { } ~ArithmeticSimplify() = default; - AnfNodePtr operator()(const OptimizerPtr &optimizer, const AnfNodePtr &node) { + AnfNodePtr operator()(const OptimizerPtr &optimizer, const AnfNodePtr &node) override { AnfNodePtr new_node; for (auto &eliminater : eliminaters_) { - new_node = eliminater(optimizer, node); + new_node = (*eliminater)(optimizer, node); if (new_node != nullptr) { return new_node; } @@ -773,15 +774,9 @@ class ArithmeticSimplify { } private: - MultiplyByZeroOrOne multiply_by_zero_or_one_; - TensorMultiplyByOne tensor_multiply_by_one_; - AddByZero add_by_zero_; - TensorAddByZero tensor_add_by_zero_; - PrimEliminater identity_; - OptUpdateZeroTensor opt_update_zero_tensor_; - ConstantDuplicateMul constant_duplicate_mul_; - PowerOneEliminate power_one_; - std::vector eliminaters_{}; + OptimizerCallerPtr multiply_by_zero_or_one_, tensor_multiply_by_one_, add_by_zero_, tensor_add_by_zero_, identity_, + opt_update_zero_tensor_, constant_duplicate_mul_, power_one_; + std::vector eliminaters_{}; }; // Arithmetic Simplifications should be done after step_parallel. @@ -789,15 +784,17 @@ class ArithmeticSimplify { // with shape(weight), but after step_parallel, shape of weight may be changed, so the // shape of the constant tensor should also be changed. So this pass is seperated from // ArithmeticSimplify and deferred until step_parallel. -class ArithmeticSimplify2 { +class ArithmeticSimplify2 : public OptimizerCaller { public: - ArithmeticSimplify2() : tensor_multiply_by_zero_() { eliminaters_.emplace_back(tensor_multiply_by_zero_); } + ArithmeticSimplify2() : tensor_multiply_by_zero_(std::make_shared()) { + eliminaters_.emplace_back(tensor_multiply_by_zero_); + } ~ArithmeticSimplify2() = default; - AnfNodePtr operator()(const OptimizerPtr &optimizer, const AnfNodePtr &node) { + AnfNodePtr operator()(const OptimizerPtr &optimizer, const AnfNodePtr &node) override { AnfNodePtr new_node; for (auto &eliminater : eliminaters_) { - new_node = eliminater(optimizer, node); + new_node = (*eliminater)(optimizer, node); if (new_node != nullptr) { return new_node; } @@ -806,8 +803,8 @@ class ArithmeticSimplify2 { } private: - TensorMultiplyByZero tensor_multiply_by_zero_; - std::vector eliminaters_{}; + OptimizerCallerPtr tensor_multiply_by_zero_; + std::vector eliminaters_{}; }; } // namespace irpass } // namespace opt diff --git a/mindspore/ccsrc/optimizer/irpass/cast_eliminate.h b/mindspore/ccsrc/optimizer/irpass/cast_eliminate.h index 734d88cb10..d98d0b677b 100644 --- a/mindspore/ccsrc/optimizer/irpass/cast_eliminate.h +++ b/mindspore/ccsrc/optimizer/irpass/cast_eliminate.h @@ -17,9 +17,9 @@ #ifndef MINDSPORE_CCSRC_OPTIMIZER_IRPASS_CAST_ELIMINATE_H_ #define MINDSPORE_CCSRC_OPTIMIZER_IRPASS_CAST_ELIMINATE_H_ +#include "ir/visitor.h" #include "optimizer/irpass.h" #include "optimizer/optimizer.h" -#include "ir/visitor.h" namespace mindspore { namespace opt { @@ -52,12 +52,12 @@ class TwoCastEliminater : public AnfVisitor { AnfNodePtr x_{nullptr}, t_{nullptr}; }; -class CastEliminater { +class CastEliminater : public OptimizerCaller { public: CastEliminater() : cast_same_type_eliminater_(), two_cast_eliminater_() {} ~CastEliminater() = default; - AnfNodePtr operator()(const OptimizerPtr &optimizer, const AnfNodePtr &node) { + AnfNodePtr operator()(const OptimizerPtr &optimizer, const AnfNodePtr &node) override { auto new_node = cast_same_type_eliminater_(optimizer, node); if (new_node != nullptr) { return new_node; diff --git a/mindspore/ccsrc/optimizer/irpass/env_item_eliminate.h b/mindspore/ccsrc/optimizer/irpass/env_item_eliminate.h index 0f59c69fef..3f100dcaec 100644 --- a/mindspore/ccsrc/optimizer/irpass/env_item_eliminate.h +++ b/mindspore/ccsrc/optimizer/irpass/env_item_eliminate.h @@ -17,18 +17,19 @@ #ifndef MINDSPORE_CCSRC_OPTIMIZER_IRPASS_ENV_ITEM_ELIMINATE_H_ #define MINDSPORE_CCSRC_OPTIMIZER_IRPASS_ENV_ITEM_ELIMINATE_H_ -#include -#include #include -#include #include +#include +#include +#include -#include "optimizer/irpass.h" -#include "optimizer/optimizer.h" -#include "ir/visitor.h" #include "ir/func_graph.h" #include "ir/func_graph_cloner.h" +#include "ir/optimizer_caller.h" +#include "ir/visitor.h" #include "operator/ops.h" +#include "optimizer/irpass.h" +#include "optimizer/optimizer.h" #include "utils/symbolic.h" namespace mindspore { @@ -225,19 +226,22 @@ class EnvGetSetItem : public AnfVisitor { bool is_match_{false}; }; -class EnvGetItemEliminater { +class EnvGetItemEliminater : public OptimizerCaller { public: - EnvGetItemEliminater() : new_env_get_item_(), add_env_get_item_(), env_get_set_item_() { + EnvGetItemEliminater() + : new_env_get_item_(std::make_shared()), + add_env_get_item_(std::make_shared()), + env_get_set_item_(std::make_shared()) { eliminaters_.emplace_back(new_env_get_item_); eliminaters_.emplace_back(add_env_get_item_); eliminaters_.emplace_back(env_get_set_item_); } ~EnvGetItemEliminater() = default; - AnfNodePtr operator()(const OptimizerPtr &optimizer, const AnfNodePtr &node) { + AnfNodePtr operator()(const OptimizerPtr &optimizer, const AnfNodePtr &node) override { AnfNodePtr new_node; for (auto &eliminater : eliminaters_) { - new_node = eliminater(optimizer, node); + new_node = (*eliminater)(optimizer, node); if (new_node != nullptr) { return new_node; } @@ -246,10 +250,8 @@ class EnvGetItemEliminater { } private: - NewEnvGetItem new_env_get_item_; - AddEnvGetItem add_env_get_item_; - EnvGetSetItem env_get_set_item_; - std::vector eliminaters_{}; + OptimizerCallerPtr new_env_get_item_, add_env_get_item_, env_get_set_item_; + std::vector eliminaters_{}; }; // {prim::kPrimEnvGetItem, {G, Xs}, C, Y} diff --git a/mindspore/ccsrc/optimizer/irpass/incorporate_getitem.h b/mindspore/ccsrc/optimizer/irpass/incorporate_getitem.h index 5afee45e95..b6c8fb0e18 100644 --- a/mindspore/ccsrc/optimizer/irpass/incorporate_getitem.h +++ b/mindspore/ccsrc/optimizer/irpass/incorporate_getitem.h @@ -17,18 +17,20 @@ #ifndef MINDSPORE_CCSRC_OPTIMIZER_IRPASS_INCORPORATE_GETITEM_H_ #define MINDSPORE_CCSRC_OPTIMIZER_IRPASS_INCORPORATE_GETITEM_H_ -#include #include -#include #include +#include #include +#include -#include "optimizer/irpass.h" -#include "optimizer/optimizer.h" -#include "ir/visitor.h" #include "ir/func_graph.h" #include "ir/func_graph_cloner.h" +#include "ir/optimizer_caller.h" +#include "ir/visitor.h" #include "operator/ops.h" +#include "optimizer/irpass.h" +#include "optimizer/optimizer.h" + namespace mindspore { namespace opt { namespace irpass { @@ -383,18 +385,20 @@ class IncorporateGetitemSwitch : public AnfVisitor { internal::GetitemTransform getitem_transform_; }; -class IncorporateGetitemSet { +class IncorporateGetitemSet : public OptimizerCaller { public: - IncorporateGetitemSet() : incorporate_getitem_(), incorporate_getitem_switch_() { + IncorporateGetitemSet() + : incorporate_getitem_(std::make_shared()), + incorporate_getitem_switch_(std::make_shared()) { eliminaters_.emplace_back(incorporate_getitem_); eliminaters_.emplace_back(incorporate_getitem_switch_); } ~IncorporateGetitemSet() = default; - AnfNodePtr operator()(const OptimizerPtr &optimizer, const AnfNodePtr &node) { + AnfNodePtr operator()(const OptimizerPtr &optimizer, const AnfNodePtr &node) override { AnfNodePtr new_node; for (auto &eliminater : eliminaters_) { - new_node = eliminater(optimizer, node); + new_node = (*eliminater)(optimizer, node); if (new_node != nullptr) { return new_node; } @@ -403,9 +407,8 @@ class IncorporateGetitemSet { } private: - IncorporateGetitem incorporate_getitem_; - IncorporateGetitemSwitch incorporate_getitem_switch_; - std::vector eliminaters_{}; + OptimizerCallerPtr incorporate_getitem_, incorporate_getitem_switch_; + std::vector eliminaters_{}; }; } // namespace irpass } // namespace opt diff --git a/mindspore/ccsrc/optimizer/irpass/item_tuple_eliminate.h b/mindspore/ccsrc/optimizer/irpass/item_tuple_eliminate.h index 21cdff51ad..202951a254 100644 --- a/mindspore/ccsrc/optimizer/irpass/item_tuple_eliminate.h +++ b/mindspore/ccsrc/optimizer/irpass/item_tuple_eliminate.h @@ -17,13 +17,15 @@ #ifndef MINDSPORE_CCSRC_OPTIMIZER_IRPASS_ITEM_TUPLE_ELIMINATE_H_ #define MINDSPORE_CCSRC_OPTIMIZER_IRPASS_ITEM_TUPLE_ELIMINATE_H_ -#include #include +#include +#include -#include "optimizer/irpass.h" -#include "optimizer/optimizer.h" +#include "ir/optimizer_caller.h" #include "ir/visitor.h" #include "operator/ops.h" +#include "optimizer/irpass.h" +#include "optimizer/optimizer.h" namespace mindspore { namespace opt { @@ -261,14 +263,14 @@ class GetitemDependReorder : public AnfVisitor { AnfNodePtr x_{nullptr}, y_{nullptr}, c_{nullptr}; }; -class ItemTupleEliminater { +class ItemTupleEliminater : public OptimizerCaller { public: ItemTupleEliminater() - : get_item_eliminater_(), - get_item_const_eliminater_(), - set_item_eliminater_(), - get_set_item_eliminater_(), - get_item_depend_reorder_() { + : get_item_eliminater_(std::make_shared()), + get_item_const_eliminater_(std::make_shared()), + set_item_eliminater_(std::make_shared()), + get_set_item_eliminater_(std::make_shared()), + get_item_depend_reorder_(std::make_shared()) { eliminaters_.emplace_back(get_item_eliminater_); eliminaters_.emplace_back(get_item_const_eliminater_); eliminaters_.emplace_back(set_item_eliminater_); @@ -277,10 +279,10 @@ class ItemTupleEliminater { } ~ItemTupleEliminater() = default; - AnfNodePtr operator()(const OptimizerPtr &optimizer, const AnfNodePtr &node) { + AnfNodePtr operator()(const OptimizerPtr &optimizer, const AnfNodePtr &node) override { AnfNodePtr new_node; for (auto &eliminater : eliminaters_) { - new_node = eliminater(optimizer, node); + new_node = (*eliminater)(optimizer, node); if (new_node != nullptr) { return new_node; } @@ -289,12 +291,9 @@ class ItemTupleEliminater { } private: - GetitemEliminater get_item_eliminater_; - GetitemConstEliminater get_item_const_eliminater_; - SetitemEliminater set_item_eliminater_; - GetSetitemEliminater get_set_item_eliminater_; - GetitemDependReorder get_item_depend_reorder_; - std::vector eliminaters_{}; + OptimizerCallerPtr get_item_eliminater_, get_item_const_eliminater_, set_item_eliminater_, get_set_item_eliminater_, + get_item_depend_reorder_; + std::vector eliminaters_{}; }; } // namespace irpass } // namespace opt diff --git a/mindspore/ccsrc/optimizer/irpass/ref_eliminate.h b/mindspore/ccsrc/optimizer/irpass/ref_eliminate.h index 41f379221c..6d81b401c3 100644 --- a/mindspore/ccsrc/optimizer/irpass/ref_eliminate.h +++ b/mindspore/ccsrc/optimizer/irpass/ref_eliminate.h @@ -19,9 +19,9 @@ #include -#include "optimizer/optimizer.h" -#include "optimizer/irpass.h" #include "ir/pattern_matcher.h" +#include "optimizer/irpass.h" +#include "optimizer/optimizer.h" namespace mindspore { namespace opt { diff --git a/mindspore/ccsrc/optimizer/irpass/reshape_eliminate.h b/mindspore/ccsrc/optimizer/irpass/reshape_eliminate.h index fb43f6ffd8..cafc8b796c 100644 --- a/mindspore/ccsrc/optimizer/irpass/reshape_eliminate.h +++ b/mindspore/ccsrc/optimizer/irpass/reshape_eliminate.h @@ -19,11 +19,12 @@ #include -#include "optimizer/irpass.h" -#include "optimizer/optimizer.h" -#include "ir/visitor.h" #include "ir/func_graph.h" +#include "ir/optimizer_caller.h" +#include "ir/visitor.h" #include "operator/ops.h" +#include "optimizer/irpass.h" +#include "optimizer/optimizer.h" #include "pipeline/static_analysis/dshape.h" namespace mindspore { @@ -124,12 +125,12 @@ class TwoReshapeEliminater : public AnfVisitor { AnfNodePtr x_{nullptr}, shape_{nullptr}; }; -class ReshapeEliminater { +class ReshapeEliminater : public OptimizerCaller { public: ReshapeEliminater() : reshape_same_shape_eliminater_(), two_reshape_eliminater_() {} ~ReshapeEliminater() = default; - AnfNodePtr operator()(const OptimizerPtr &optimizer, const AnfNodePtr &node) { + AnfNodePtr operator()(const OptimizerPtr &optimizer, const AnfNodePtr &node) override { auto new_node = reshape_same_shape_eliminater_(optimizer, node); if (new_node != nullptr) { return new_node; diff --git a/mindspore/ccsrc/optimizer/irpass/special_op_eliminate.h b/mindspore/ccsrc/optimizer/irpass/special_op_eliminate.h index dcba80431a..b6a4e1c852 100644 --- a/mindspore/ccsrc/optimizer/irpass/special_op_eliminate.h +++ b/mindspore/ccsrc/optimizer/irpass/special_op_eliminate.h @@ -18,31 +18,31 @@ #define MINDSPORE_CCSRC_OPTIMIZER_IRPASS_SPECIAL_OP_ELIMINATE_H_ #include -#include -#include #include +#include +#include -#include "optimizer/optimizer.h" -#include "optimizer/irpass.h" #include "ir/optimizer_caller.h" -#include "optimizer/irpass/prim_eliminate.h" +#include "ir/pattern_matcher.h" #include "ir/visitor.h" #include "operator/ops.h" -#include "ir/pattern_matcher.h" +#include "optimizer/irpass.h" +#include "optimizer/irpass/prim_eliminate.h" +#include "optimizer/optimizer.h" namespace mindspore { namespace opt { namespace irpass { -class SpecialOpEliminater { +class SpecialOpEliminater : public OptimizerCaller { public: SpecialOpEliminater() - : insert_gradient_of_(prim::kPrimInsertGradientOf), - stop_gradient_(prim::kPrimStopGradient), - hook_backward_(prim::kPrimHookBackward), - print_shape_type_(prim::kPrimPrintShapeType), - get_ref_value_(prim::kPrimGetRefValue), - mirror_(prim::kPrimMirror), - virtual_div_(prim::kPrimVirtualDiv) { + : insert_gradient_of_(std::make_shared(prim::kPrimInsertGradientOf)), + stop_gradient_(std::make_shared(prim::kPrimStopGradient)), + hook_backward_(std::make_shared(prim::kPrimHookBackward)), + print_shape_type_(std::make_shared(prim::kPrimPrintShapeType)), + get_ref_value_(std::make_shared(prim::kPrimGetRefValue)), + mirror_(std::make_shared(prim::kPrimMirror)), + virtual_div_(std::make_shared(prim::kPrimVirtualDiv)) { eliminaters_.emplace_back(insert_gradient_of_); eliminaters_.emplace_back(stop_gradient_); eliminaters_.emplace_back(hook_backward_); @@ -53,10 +53,10 @@ class SpecialOpEliminater { } ~SpecialOpEliminater() = default; - AnfNodePtr operator()(const OptimizerPtr &optimizer, const AnfNodePtr &node) { + AnfNodePtr operator()(const OptimizerPtr &optimizer, const AnfNodePtr &node) override { AnfNodePtr new_node; for (auto &eliminater : eliminaters_) { - new_node = eliminater(optimizer, node); + new_node = (*eliminater)(optimizer, node); if (new_node != nullptr) { return new_node; } @@ -65,9 +65,9 @@ class SpecialOpEliminater { } private: - PrimEliminater insert_gradient_of_, stop_gradient_, hook_backward_, print_shape_type_, get_ref_value_, mirror_, + OptimizerCallerPtr insert_gradient_of_, stop_gradient_, hook_backward_, print_shape_type_, get_ref_value_, mirror_, virtual_div_; - std::vector eliminaters_{}; + std::vector eliminaters_{}; }; // {PrimVirtualDataset, X} -> X diff --git a/mindspore/ccsrc/optimizer/opt.cc b/mindspore/ccsrc/optimizer/opt.cc index 82fbcc2036..4c2e85157f 100644 --- a/mindspore/ccsrc/optimizer/opt.cc +++ b/mindspore/ccsrc/optimizer/opt.cc @@ -16,28 +16,27 @@ #include "optimizer/opt.h" +#include +#include #include #include -#include -#include #include "ir/anf.h" #include "ir/manager.h" -#include "utils/ordered_set.h" - -#include "utils/log_adapter.h" #include "optimizer/optimizer.h" +#include "utils/log_adapter.h" +#include "utils/ordered_set.h" namespace mindspore { /* namespace to support opt */ namespace opt { -SubstitutionPtr MakeSubstitution(const TransformFuncType &transform, const std::string &name, const PrimitivePtr &prim, +SubstitutionPtr MakeSubstitution(const OptimizerCallerPtr &transform, const std::string &name, const PrimitivePtr &prim, const RenormAction &renorm_action) { auto fn = [prim](const AnfNodePtr &node) -> bool { return IsPrimitiveCNode(node, prim); }; return std::make_shared(transform, name, fn, renorm_action); } -SubstitutionPtr MakeSubstitution(const TransformFuncType &transform, const std::string &name, +SubstitutionPtr MakeSubstitution(const OptimizerCallerPtr &transform, const std::string &name, const std::vector &prims, const RenormAction &renorm_action) { auto fn = [prims](const AnfNodePtr &node) -> bool { if (!node->isa()) { @@ -64,16 +63,16 @@ SubstitutionPtr MakeSubstitution(const TransformFuncType &transform, const std:: return std::make_shared(transform, name, fn, renorm_action); } -SubstitutionPtr MakeSubstitution(const TransformFuncType &transform, const std::string &name, +SubstitutionPtr MakeSubstitution(const OptimizerCallerPtr &transform, const std::string &name, const PredicateFuncType &predicate, const RenormAction &renorm_action) { return std::make_shared(transform, name, predicate, renorm_action); } -AnfNodePtr Substitution::operator()(const OptimizerPtr &optimizer, const AnfNodePtr &node) const { +AnfNodePtr Substitution::operator()(const OptimizerPtr &optimizer, const AnfNodePtr &node) { #ifdef ENABLE_PROFILE double t = GetTime(); #endif - AnfNodePtr result = transform_(optimizer, node); + AnfNodePtr result = (*transform_)(optimizer, node); #ifdef ENABLE_PROFILE if (optimizer != nullptr) { auto time = GetTime(); diff --git a/mindspore/ccsrc/optimizer/opt.h b/mindspore/ccsrc/optimizer/opt.h index fb0bdc58be..6601d969d2 100644 --- a/mindspore/ccsrc/optimizer/opt.h +++ b/mindspore/ccsrc/optimizer/opt.h @@ -17,24 +17,18 @@ #ifndef MINDSPORE_CCSRC_OPTIMIZER_OPT_H_ #define MINDSPORE_CCSRC_OPTIMIZER_OPT_H_ -#include -#include #include +#include +#include #include "ir/anf.h" #include "ir/func_graph.h" +#include "ir/optimizer_caller.h" #include "operator/ops.h" namespace mindspore { /* namespace to support opt */ namespace opt { -class Optimizer; - -using OptimizerPtr = std::shared_ptr; -using OptimizerWeakPtr = std::weak_ptr; - -using PredicateFuncType = std::function; -using TransformFuncType = std::function; // Define the interaction mode between an Optimize pass and Renormalize pass // FORCE_RENORM: if the pass modified the graph then the next Renormalize will be executed @@ -43,26 +37,26 @@ enum RenormAction : int { FORCE_RENORM = 0, CHECK_RENORM }; class Substitution { public: - TransformFuncType transform_{nullptr}; + OptimizerCallerPtr transform_; std::string name_; PredicateFuncType predicate_{nullptr}; // an enum to mark this Substitution relation to renormalize pass RenormAction renorm_action_; - Substitution(const TransformFuncType &transform, const std::string &name, const PredicateFuncType &predicate, + Substitution(const OptimizerCallerPtr &transform, const std::string &name, const PredicateFuncType &predicate, const RenormAction &renorm_action) : transform_(transform), name_(name), predicate_(predicate), renorm_action_(renorm_action) {} ~Substitution() = default; - AnfNodePtr operator()(const OptimizerPtr &optimizer, const AnfNodePtr &node) const; + AnfNodePtr operator()(const OptimizerPtr &optimizer, const AnfNodePtr &node); }; using SubstitutionPtr = std::shared_ptr; -SubstitutionPtr MakeSubstitution(const TransformFuncType &transform, const std::string &name, const PrimitivePtr &prim, +SubstitutionPtr MakeSubstitution(const OptimizerCallerPtr &transform, const std::string &name, const PrimitivePtr &prim, const RenormAction &action_renorm = CHECK_RENORM); -SubstitutionPtr MakeSubstitution(const TransformFuncType &transform, const std::string &name, +SubstitutionPtr MakeSubstitution(const OptimizerCallerPtr &transform, const std::string &name, const std::vector &prims, const RenormAction &action_renorm = CHECK_RENORM); -SubstitutionPtr MakeSubstitution(const TransformFuncType &transform, const std::string &name, +SubstitutionPtr MakeSubstitution(const OptimizerCallerPtr &transform, const std::string &name, const PredicateFuncType &predicate, const RenormAction &action_renorm = CHECK_RENORM); class SubstitutionList { diff --git a/tests/ut/cpp/optimizer/opt_test.cc b/tests/ut/cpp/optimizer/opt_test.cc index 05e7e6b978..2428d0dddb 100644 --- a/tests/ut/cpp/optimizer/opt_test.cc +++ b/tests/ut/cpp/optimizer/opt_test.cc @@ -77,10 +77,10 @@ class TestOptOpt : public UT::Common { }; void SetUp() { - elim_Z = MakeSubstitution(irpass::AddByZero(), "elim_Z", prim::kPrimScalarAdd); - elim_R = MakeSubstitution(irpass::PrimEliminater(R), "elim_R", R); - idempotent_P = MakeSubstitution(IdempotentEliminater(), "idempotent_P", P); - Qct_to_P = MakeSubstitution(QctToP(), "Qct_to_P", Q); + elim_Z = MakeSubstitution(std::make_shared(), "elim_Z", prim::kPrimScalarAdd); + elim_R = MakeSubstitution(std::make_shared(R), "elim_R", R); + idempotent_P = MakeSubstitution(std::make_shared(), "idempotent_P", P); + Qct_to_P = MakeSubstitution(std::make_shared(), "Qct_to_P", Q); } bool CheckTransform(FuncGraphPtr gbefore, FuncGraphPtr gafter, const SubstitutionList &transform) { From c22eac743d92b5ba27e9b213f70591a556f8a8f5 Mon Sep 17 00:00:00 2001 From: Jamie Nisbet Date: Tue, 23 Jun 2020 14:58:10 -0400 Subject: [PATCH 066/254] subtree creation in python apis updates fix fix cpplint fix --- mindspore/ccsrc/dataset/api/de_pipeline.cc | 370 +++++++++++++----- mindspore/ccsrc/dataset/api/de_pipeline.h | 80 ++-- .../ccsrc/dataset/api/python_bindings.cc | 6 +- .../ccsrc/dataset/engine/datasetops/map_op.cc | 9 +- .../ccsrc/dataset/engine/datasetops/map_op.h | 20 +- .../engine/datasetops/source/clue_op.cc | 13 +- .../engine/datasetops/source/clue_op.h | 16 +- .../engine/datasetops/source/text_file_op.cc | 12 +- .../engine/datasetops/source/text_file_op.h | 16 +- .../engine/datasetops/source/tf_reader_op.cc | 10 +- .../engine/datasetops/source/tf_reader_op.h | 16 +- .../ccsrc/dataset/engine/execution_tree.cc | 10 +- .../ccsrc/dataset/engine/opt/CMakeLists.txt | 4 +- .../dataset/engine/opt/pre/global_shuffle.cc | 98 ----- .../dataset/engine/opt/pre/global_shuffle.h | 35 -- .../engine/opt/pre/map_column_reorder.cc | 51 --- .../engine/opt/pre/map_column_reorder.h | 35 -- mindspore/dataset/engine/iterators.py | 6 +- tests/ut/cpp/dataset/map_op_test.cc | 69 ---- tests/ut/python/dataset/test_opt_pass.py | 24 +- 20 files changed, 378 insertions(+), 522 deletions(-) delete mode 100644 mindspore/ccsrc/dataset/engine/opt/pre/global_shuffle.cc delete mode 100644 mindspore/ccsrc/dataset/engine/opt/pre/global_shuffle.h delete mode 100644 mindspore/ccsrc/dataset/engine/opt/pre/map_column_reorder.cc delete mode 100644 mindspore/ccsrc/dataset/engine/opt/pre/map_column_reorder.h diff --git a/mindspore/ccsrc/dataset/api/de_pipeline.cc b/mindspore/ccsrc/dataset/api/de_pipeline.cc index ce70476423..78fcdb7dd4 100644 --- a/mindspore/ccsrc/dataset/api/de_pipeline.cc +++ b/mindspore/ccsrc/dataset/api/de_pipeline.cc @@ -15,6 +15,7 @@ */ #include "dataset/api/de_pipeline.h" +#include #include #include @@ -45,7 +46,7 @@ namespace mindspore { namespace dataset { -using pFunction = Status (DEPipeline::*)(const py::dict &, std::shared_ptr *); +using pFunction = Status (DEPipeline::*)(const py::dict &, std::shared_ptr *, std::shared_ptr *); static std::unordered_map g_parse_op_func_ = { {kShuffle, &DEPipeline::ParseShuffleOp}, @@ -107,18 +108,44 @@ DEPipeline::~DEPipeline() { } // Function to add a Node to the Execution Tree. -Status DEPipeline::AddNodeToTree(const OpName &op_name, const py::dict &args, DsOpPtr *out) { - // For each operator, Parse through the list of arguments, - // then call the respective builder/constructor. +Status DEPipeline::AddNodeToTree(const OpName &op_name, const py::dict &args, py::dict *output) { + // For each operator, Parse through the list of arguments, then call the respective builder/constructor. + // Note that each call to the parse function may result in building more than one dataset operator. + // For example, one call to ParseNNNOp may result in multiple internal C nodes: + // nodeA + // | + // nodeB + // | + // nodeC + // However, the python side dataset is more abstract, and it does not know about the potential subtree that + // is being built here. Since the python api is hooking tree nodes together (parent/child hookups), the + // python side needs to know about nodeA and NodeC to be able to appropriately hook up parents and child + // to this subtee. + // Thus, it is required that both the top-most parent and bottom-most child are returned from the parse + // function. + DsOpPtr top = nullptr; + DsOpPtr bottom = nullptr; auto iter = g_parse_op_func_.find(op_name); if (iter != g_parse_op_func_.end()) { pFunction func = iter->second; - RETURN_IF_NOT_OK((this->*func)(args, out)); + RETURN_IF_NOT_OK((this->*func)(args, &top, &bottom)); + + if (top == nullptr) { + RETURN_STATUS_UNEXPECTED("An operator was parsed but it did not produce a C node."); + } + + // It is not required that the parse function always produces the bottom pointer. If it's still null, + // then set top and bottom to be the same operator + if (bottom == nullptr) bottom = top; + + // Pack these pointers into a py dict so that we can return both back to python. + (*output)["top"] = top; + (*output)["bottom"] = bottom; } else { RETURN_STATUS_UNEXPECTED("No such Op"); } // Associate current dataset op node with the tree. - RETURN_IF_NOT_OK(tree_->AssociateNode(*out)); + RETURN_IF_NOT_OK(tree_->AssociateNode(top)); return Status::OK(); } // Function to add a child and parent relationship. @@ -300,7 +327,8 @@ Status DEPipeline::SetBatchParameters(const py::dict &args) { return Status::OK(); } -Status DEPipeline::ParseShuffleOp(const py::dict &args, std::shared_ptr *ptr) { +Status DEPipeline::ParseShuffleOp(const py::dict &args, std::shared_ptr *top, + std::shared_ptr *bottom) { std::shared_ptr builder = std::make_shared(); if (!args["buffer_size"].is_none()) { (void)builder->SetShuffleSize(ToInt(args["buffer_size"])); @@ -322,7 +350,7 @@ Status DEPipeline::ParseShuffleOp(const py::dict &args, std::shared_ptr op; RETURN_IF_NOT_OK(builder->Build(&op)); - *ptr = op; + *top = op; return Status::OK(); } @@ -350,7 +378,8 @@ Status DEPipeline::BuildMindrecordSamplerChain(const py::handle &handle, return Status::OK(); } -Status DEPipeline::ParseMindRecordOp(const py::dict &args, std::shared_ptr *ptr) { +Status DEPipeline::ParseMindRecordOp(const py::dict &args, std::shared_ptr *top, + std::shared_ptr *bottom) { if (args["dataset_file"].is_none()) { std::string err_msg = "Error: at least one of dataset_files is missing"; RETURN_STATUS_UNEXPECTED(err_msg); @@ -403,13 +432,15 @@ Status DEPipeline::ParseMindRecordOp(const py::dict &args, std::shared_ptr op; RETURN_IF_NOT_OK(builder->Build(&op)); num_rows_ = op->num_rows(); - *ptr = op; + *top = op; return Status::OK(); } -Status DEPipeline::ParseMapOp(const py::dict &args, std::shared_ptr *ptr) { - std::shared_ptr builder = std::make_shared(); +Status DEPipeline::ParseMapOp(const py::dict &args, std::shared_ptr *top, + std::shared_ptr *bottom) { + MapOp::Builder map_builder; std::vector> tensor_op_list; + std::vector project_columns; if (args["operations"].is_none()) RETURN_STATUS_UNEXPECTED("Error: 'operations' is not set. \n"); @@ -419,15 +450,15 @@ Status DEPipeline::ParseMapOp(const py::dict &args, std::shared_ptr * if (!value.is_none()) { if (key == "input_columns") { std::vector in_col_names = ToStringVector(args["input_columns"]); - (void)builder->SetInColNames(in_col_names); + (void)map_builder.SetInColNames(in_col_names); } else if (key == "output_columns") { - (void)builder->SetOutColNames(ToStringVector(value)); + (void)map_builder.SetOutColNames(ToStringVector(value)); } else if (key == "columns_order") { - (void)builder->SetColOrder(ToStringVector(value)); + project_columns = ToStringVector(value); } else if (key == "num_parallel_workers") { - (void)builder->SetNumWorkers(ToInt(value)); + (void)map_builder.SetNumWorkers(ToInt(value)); } else if (key == "prefetch_size") { - (void)builder->SetOpConnectorSize(ToInt(value)); + (void)map_builder.SetOpConnectorSize(ToInt(value)); } else if (key == "operations") { py::handle tensor_ops = args["operations"]; // operation can be a list of TensorOps or a single TensorOp. @@ -445,20 +476,34 @@ Status DEPipeline::ParseMapOp(const py::dict &args, std::shared_ptr * } } if (tensor_op_list.empty()) RETURN_STATUS_UNEXPECTED("Error: tensor_op is invalid or not set."); - (void)builder->SetTensorFuncs(std::move(tensor_op_list)); + (void)map_builder.SetTensorFuncs(std::move(tensor_op_list)); } else { RETURN_STATUS_UNEXPECTED("Error: Unhandled key: " + key); } } } - std::shared_ptr op; - RETURN_IF_NOT_OK(builder->Build(&op)); - *ptr = op; + std::shared_ptr map_op; + RETURN_IF_NOT_OK(map_builder.Build(&map_op)); + RETURN_IF_NOT_OK(tree_->AssociateNode(map_op)); + *top = map_op; + + // Add a project op over top of the map if the user wanted to reposition the columns + if (!project_columns.empty()) { + ProjectOp::Builder proj_builder(project_columns); + std::shared_ptr proj_op; + RETURN_IF_NOT_OK(proj_builder.Build(&proj_op)); + RETURN_IF_NOT_OK(tree_->AssociateNode(proj_op)); + RETURN_IF_NOT_OK(proj_op->AddChild(map_op)); + *top = proj_op; + *bottom = map_op; + } + return Status::OK(); } -Status DEPipeline::ParseFilterOp(const py::dict &args, std::shared_ptr *ptr) { +Status DEPipeline::ParseFilterOp(const py::dict &args, std::shared_ptr *top, + std::shared_ptr *bottom) { std::shared_ptr builder = std::make_shared(); if (args["predicate"].is_none()) { @@ -489,11 +534,12 @@ Status DEPipeline::ParseFilterOp(const py::dict &args, std::shared_ptr op; RETURN_IF_NOT_OK(builder->Build(&op)); - *ptr = op; + *top = op; return Status::OK(); } -Status DEPipeline::ParseRepeatOp(const py::dict &args, std::shared_ptr *ptr) { +Status DEPipeline::ParseRepeatOp(const py::dict &args, std::shared_ptr *top, + std::shared_ptr *bottom) { if (args["count"].is_none()) { std::string err_msg = "Error: count is invalid or not set."; RETURN_STATUS_UNEXPECTED(err_msg); @@ -501,22 +547,24 @@ Status DEPipeline::ParseRepeatOp(const py::dict &args, std::shared_ptr op; RETURN_IF_NOT_OK(RepeatOp::Builder(ToInt(args["count"])).Build(&op)); - *ptr = op; + *top = op; return Status::OK(); } -Status DEPipeline::ParseSkipOp(const py::dict &args, std::shared_ptr *ptr) { +Status DEPipeline::ParseSkipOp(const py::dict &args, std::shared_ptr *top, + std::shared_ptr *bottom) { if (args["count"].is_none()) { std::string err_msg = "Error: count is invalid or not set."; RETURN_STATUS_UNEXPECTED(err_msg); } std::shared_ptr op; RETURN_IF_NOT_OK(SkipOp::Builder(ToInt(args["count"])).Build(&op)); - *ptr = op; + *top = op; return Status::OK(); } -Status DEPipeline::ParseGeneratorOp(const py::dict &args, std::shared_ptr *ptr) { +Status DEPipeline::ParseGeneratorOp(const py::dict &args, std::shared_ptr *top, + std::shared_ptr *bottom) { std::shared_ptr builder = std::make_shared(); for (auto arg : args) { std::string key = py::str(arg.first); @@ -538,11 +586,12 @@ Status DEPipeline::ParseGeneratorOp(const py::dict &args, std::shared_ptr op; RETURN_IF_NOT_OK(builder->Build(&op)); - *ptr = op; + *top = op; return Status::OK(); } -Status DEPipeline::ParseBatchOp(const py::dict &args, std::shared_ptr *ptr) { +Status DEPipeline::ParseBatchOp(const py::dict &args, std::shared_ptr *top, + std::shared_ptr *bottom) { std::shared_ptr builder; if (py::isinstance(args["batch_size"])) { batch_size_ = ToInt(args["batch_size"]); @@ -582,11 +631,12 @@ Status DEPipeline::ParseBatchOp(const py::dict &args, std::shared_ptr std::shared_ptr op; RETURN_IF_NOT_OK(builder->Build(&op)); - *ptr = op; + *top = op; return Status::OK(); } -Status DEPipeline::ParseBucketBatchByLengthOp(const py::dict &args, std::shared_ptr *ptr) { +Status DEPipeline::ParseBucketBatchByLengthOp(const py::dict &args, std::shared_ptr *top, + std::shared_ptr *bottom) { std::vector mandatory_arguments = {"length_dependent_columns", "bucket_boundaries", "bucket_batch_sizes"}; for (auto name : mandatory_arguments) { @@ -632,11 +682,12 @@ Status DEPipeline::ParseBucketBatchByLengthOp(const py::dict &args, std::shared_ std::shared_ptr op; RETURN_IF_NOT_OK(builder->Build(&op)); - *ptr = op; + *top = op; return Status::OK(); } -Status DEPipeline::ParseBarrierOp(const py::dict &args, std::shared_ptr *ptr) { +Status DEPipeline::ParseBarrierOp(const py::dict &args, std::shared_ptr *top, + std::shared_ptr *bottom) { std::shared_ptr builder = std::make_shared(); // Right now barrier should only take num_rows_per_buffer = 1 // The reason for this is because having it otherwise can lead to blocking issues @@ -656,11 +707,12 @@ Status DEPipeline::ParseBarrierOp(const py::dict &args, std::shared_ptr op; RETURN_IF_NOT_OK(builder->Build(&op)); - *ptr = op; + *top = op; return Status::OK(); } -Status DEPipeline::ParseDeviceQueueOp(const py::dict &args, std::shared_ptr *ptr) { +Status DEPipeline::ParseDeviceQueueOp(const py::dict &args, std::shared_ptr *top, + std::shared_ptr *bottom) { int32_t prefetch_size = 0; if (args.contains("prefetch_size")) { if (args["prefetch_size"].is_none()) { @@ -687,11 +739,12 @@ Status DEPipeline::ParseDeviceQueueOp(const py::dict &args, std::shared_ptr op; RETURN_IF_NOT_OK(builder->Build(&op)); - *ptr = op; + *top = op; return Status::OK(); } -Status DEPipeline::ParseRenameOp(const py::dict &args, std::shared_ptr *ptr) { +Status DEPipeline::ParseRenameOp(const py::dict &args, std::shared_ptr *top, + std::shared_ptr *bottom) { std::vector in_col_names; std::vector out_col_names; std::shared_ptr builder = std::make_shared(); @@ -718,48 +771,57 @@ Status DEPipeline::ParseRenameOp(const py::dict &args, std::shared_ptrSetOutColNames(out_col_names); std::shared_ptr op; RETURN_IF_NOT_OK(builder->Build(&op)); - *ptr = op; + *top = op; return Status::OK(); } -Status DEPipeline::ParseTakeOp(const py::dict &args, std::shared_ptr *ptr) { +Status DEPipeline::ParseTakeOp(const py::dict &args, std::shared_ptr *top, + std::shared_ptr *bottom) { if (args["count"].is_none()) { std::string err_msg = "Error: count is invalid or not set."; RETURN_STATUS_UNEXPECTED(err_msg); } std::shared_ptr op; RETURN_IF_NOT_OK(TakeOp::Builder(ToInt(args["count"])).Build(&op)); - *ptr = op; + *top = op; return Status::OK(); } -Status DEPipeline::ParseZipOp(const py::dict &args, std::shared_ptr *ptr) { +Status DEPipeline::ParseZipOp(const py::dict &args, std::shared_ptr *top, + std::shared_ptr *bottom) { std::shared_ptr builder = std::make_shared(); std::shared_ptr op; RETURN_IF_NOT_OK(builder->Build(&op)); - *ptr = op; + *top = op; return Status::OK(); } -Status DEPipeline::ParseConcatOp(const py::dict &args, std::shared_ptr *ptr) { +Status DEPipeline::ParseConcatOp(const py::dict &args, std::shared_ptr *top, + std::shared_ptr *bottom) { std::shared_ptr builder = std::make_shared(); std::shared_ptr op; RETURN_IF_NOT_OK(builder->Build(&op)); - *ptr = op; + *top = op; return Status::OK(); } -Status DEPipeline::ParseTFReaderOp(const py::dict &args, std::shared_ptr *ptr) { +Status DEPipeline::ParseTFReaderOp(const py::dict &args, std::shared_ptr *top, + std::shared_ptr *bottom) { // Required arguments + std::vector files_list; std::shared_ptr builder = std::make_shared(); if (!args["dataset_files"].is_none()) { - (void)builder->SetDatasetFilesList(ToStringVector(args["dataset_files"])); + files_list = ToStringVector(args["dataset_files"]); + (void)builder->SetDatasetFilesList(files_list); } else { std::string err_msg = "Error: at least one of dataset_files or schema_file is missing"; RETURN_STATUS_UNEXPECTED(err_msg); } std::vector columns_to_load; bool schema_exists = false; + bool shuffle_required = false; + int64_t num_devices = 0; + int64_t total_rows = 0; // Optional arguments for (auto arg : args) { std::string key = py::str(arg.first); @@ -773,13 +835,15 @@ Status DEPipeline::ParseTFReaderOp(const py::dict &args, std::shared_ptrSetShuffleFiles(ToBool(value)); } else if (key == "shuffle_global") { - (void)builder->SetShuffleGlobal(ToBool(value)); + shuffle_required = ToBool(value); } else if (key == "schema_file_path" || key == "schema_json_string") { schema_exists = true; } else if (key == "num_samples") { - (void)builder->setTotalRows(ToInt(value)); + total_rows = ToInt(value); + (void)builder->setTotalRows(total_rows); } else if (key == "num_shards") { - (void)builder->SetNumDevices(ToInt(value)); + num_devices = ToInt(value); + (void)builder->SetNumDevices(num_devices); } else if (key == "shard_id") { (void)builder->SetDeviceId(ToInt(value)); } else if (key == "shard_equal_rows") { @@ -796,13 +860,33 @@ Status DEPipeline::ParseTFReaderOp(const py::dict &args, std::shared_ptrSetDataSchema(std::move(schema)); } - std::shared_ptr op; - RETURN_IF_NOT_OK(builder->Build(&op)); - *ptr = op; + std::shared_ptr tf_op; + RETURN_IF_NOT_OK(builder->Build(&tf_op)); + RETURN_IF_NOT_OK(tree_->AssociateNode(tf_op)); + *top = tf_op; + + if (shuffle_required) { + const boolean estimate = true; + const int64_t workers = 8; + std::shared_ptr shuffle_op = nullptr; + int64_t shuffle_size = 0; + int64_t num_rows = 0; + + // First, get the number of rows in the dataset via estimate and then compute the shuffle size + RETURN_IF_NOT_OK(TFReaderOp::CountTotalRows(&num_rows, files_list, workers, estimate)); + RETURN_IF_NOT_OK(ComputeShuffleSize(files_list.size(), num_devices, num_rows, total_rows, &shuffle_size)); + + // Add the shuffle op over top of this op and return the subtree (top/bottom) to caller + RETURN_IF_NOT_OK(AddShuffleOp(shuffle_size, tf_op, &shuffle_op)); + *top = shuffle_op; + *bottom = tf_op; + } + return Status::OK(); } -Status DEPipeline::ParseProjectOp(const py::dict &args, std::shared_ptr *ptr) { +Status DEPipeline::ParseProjectOp(const py::dict &args, std::shared_ptr *top, + std::shared_ptr *bottom) { if (args["columns"].is_none()) { std::string err_msg = "Error: columns is missing"; RETURN_STATUS_UNEXPECTED(err_msg); @@ -811,11 +895,12 @@ Status DEPipeline::ParseProjectOp(const py::dict &args, std::shared_ptr builder = std::make_shared(columns_to_project); std::shared_ptr op; RETURN_IF_NOT_OK(builder->Build(&op)); - *ptr = op; + *top = op; return Status::OK(); } -Status DEPipeline::ParseImageFolderOp(const py::dict &args, std::shared_ptr *ptr) { +Status DEPipeline::ParseImageFolderOp(const py::dict &args, std::shared_ptr *top, + std::shared_ptr *bottom) { // Required arguments if (args["dataset_dir"].is_none()) { std::string err_msg = "Error: No dataset path specified"; @@ -846,11 +931,12 @@ Status DEPipeline::ParseImageFolderOp(const py::dict &args, std::shared_ptr op; RETURN_IF_NOT_OK(builder->Build(&op)); - *ptr = op; + *top = op; return Status::OK(); } -Status DEPipeline::ParseManifestOp(const py::dict &args, std::shared_ptr *ptr) { +Status DEPipeline::ParseManifestOp(const py::dict &args, std::shared_ptr *top, + std::shared_ptr *bottom) { // Required arguments if (args["dataset_file"].is_none()) { std::string err_msg = "Error: No dataset files specified for manifest"; @@ -881,11 +967,12 @@ Status DEPipeline::ParseManifestOp(const py::dict &args, std::shared_ptr op; RETURN_IF_NOT_OK(builder->Build(&op)); - *ptr = op; + *top = op; return Status::OK(); } -Status DEPipeline::ParseVOCOp(const py::dict &args, std::shared_ptr *ptr) { +Status DEPipeline::ParseVOCOp(const py::dict &args, std::shared_ptr *top, + std::shared_ptr *bottom) { if (args["dataset_dir"].is_none()) { std::string err_msg = "Error: No dataset path specified"; RETURN_STATUS_UNEXPECTED(err_msg); @@ -924,11 +1011,13 @@ Status DEPipeline::ParseVOCOp(const py::dict &args, std::shared_ptr * } std::shared_ptr op; RETURN_IF_NOT_OK(builder->Build(&op)); - *ptr = op; + *top = op; + return Status::OK(); } -Status DEPipeline::ParseCocoOp(const py::dict &args, std::shared_ptr *ptr) { +Status DEPipeline::ParseCocoOp(const py::dict &args, std::shared_ptr *top, + std::shared_ptr *bottom) { if (args["dataset_dir"].is_none()) { std::string err_msg = "Error: No dataset path specified"; RETURN_STATUS_UNEXPECTED(err_msg); @@ -965,11 +1054,12 @@ Status DEPipeline::ParseCocoOp(const py::dict &args, std::shared_ptr } std::shared_ptr op; RETURN_IF_NOT_OK(builder->Build(&op)); - *ptr = op; + *top = op; return Status::OK(); } -Status DEPipeline::ParseCifar10Op(const py::dict &args, std::shared_ptr *ptr) { +Status DEPipeline::ParseCifar10Op(const py::dict &args, std::shared_ptr *top, + std::shared_ptr *bottom) { // Required arguments if (args["dataset_dir"].is_none()) { std::string err_msg = "Error: No dataset path specified"; @@ -998,11 +1088,12 @@ Status DEPipeline::ParseCifar10Op(const py::dict &args, std::shared_ptr op; RETURN_IF_NOT_OK(builder->Build(&op)); - *ptr = op; + *top = op; return Status::OK(); } -Status DEPipeline::ParseCifar100Op(const py::dict &args, std::shared_ptr *ptr) { +Status DEPipeline::ParseCifar100Op(const py::dict &args, std::shared_ptr *top, + std::shared_ptr *bottom) { // Required arguments if (args["dataset_dir"].is_none()) { std::string err_msg = "Error: No dataset path specified"; @@ -1031,11 +1122,12 @@ Status DEPipeline::ParseCifar100Op(const py::dict &args, std::shared_ptr op; RETURN_IF_NOT_OK(builder->Build(&op)); - *ptr = op; + *top = op; return Status::OK(); } -Status DEPipeline::ParseRandomDataOp(const py::dict &args, std::shared_ptr *ptr) { +Status DEPipeline::ParseRandomDataOp(const py::dict &args, std::shared_ptr *top, + std::shared_ptr *bottom) { // Required arguments RandomDataOp::Builder builder; @@ -1072,13 +1164,14 @@ Status DEPipeline::ParseRandomDataOp(const py::dict &args, std::shared_ptr op; RETURN_IF_NOT_OK(builder.Build(&op)); - *ptr = op; + *top = op; return Status::OK(); } int32_t DEPipeline::GetNumClasses() const { return num_classes_; } -Status DEPipeline::ParseMnistOp(const py::dict &args, std::shared_ptr *ptr) { +Status DEPipeline::ParseMnistOp(const py::dict &args, std::shared_ptr *top, + std::shared_ptr *bottom) { // Required arguments if (args["dataset_dir"].is_none()) { std::string err_msg = "Error: No dataset path specified"; @@ -1104,11 +1197,12 @@ Status DEPipeline::ParseMnistOp(const py::dict &args, std::shared_ptr } std::shared_ptr op; RETURN_IF_NOT_OK(builder->Build(&op)); - *ptr = op; + *top = op; return Status::OK(); } -Status DEPipeline::ParseCelebAOp(const py::dict &args, std::shared_ptr *ptr) { +Status DEPipeline::ParseCelebAOp(const py::dict &args, std::shared_ptr *top, + std::shared_ptr *bottom) { // Required arguments if (args["dataset_dir"].is_none()) { std::string err_msg = "Error: No dataset path specified"; @@ -1143,19 +1237,24 @@ Status DEPipeline::ParseCelebAOp(const py::dict &args, std::shared_ptr op; RETURN_IF_NOT_OK(builder->Build(&op)); - *ptr = op; + *top = op; return Status::OK(); } -Status DEPipeline::ParseTextFileOp(const py::dict &args, std::shared_ptr *ptr) { +Status DEPipeline::ParseTextFileOp(const py::dict &args, std::shared_ptr *top, + std::shared_ptr *bottom) { // Required arguments + std::vector files_list; std::shared_ptr builder = std::make_shared(); if (!args["dataset_files"].is_none()) { - (void)builder->SetTextFilesList(ToStringVector(args["dataset_files"])); + files_list = ToStringVector(args["dataset_files"]); + (void)builder->SetTextFilesList(files_list); } else { RETURN_STATUS_UNEXPECTED("Error: dataset_files is missing"); } // Optional arguments + bool shuffle_required = false; + int64_t num_devices = 0; for (auto arg : args) { std::string key = py::str(arg.first); py::handle value = arg.second; @@ -1165,19 +1264,38 @@ Status DEPipeline::ParseTextFileOp(const py::dict &args, std::shared_ptrSetShuffleFiles(ToBool(value)); } else if (key == "shuffle_global") { - (void)builder->SetShuffleGlobal(ToBool(value)); + shuffle_required = ToBool(value); } else if (key == "num_samples") { (void)builder->SetTotalRows(ToInt(value)); } else if (key == "num_shards") { - (void)builder->SetNumDevices(ToInt(value)); + num_devices = ToInt(value); + (void)builder->SetNumDevices(num_devices); } else if (key == "shard_id") { (void)builder->SetDeviceId(ToInt(value)); } } } - std::shared_ptr op; - RETURN_IF_NOT_OK(builder->Build(&op)); - *ptr = op; + + std::shared_ptr txt_op; + RETURN_IF_NOT_OK(builder->Build(&txt_op)); + RETURN_IF_NOT_OK(tree_->AssociateNode(txt_op)); + *top = txt_op; + + if (shuffle_required) { + std::shared_ptr shuffle_op = nullptr; + int64_t shuffle_size = 0; + int64_t num_rows = 0; + + // First, get the number of rows in the dataset and then compute the shuffle size + RETURN_IF_NOT_OK(TextFileOp::CountAllFileRows(files_list, &num_rows)); + RETURN_IF_NOT_OK(ComputeShuffleSize(files_list.size(), num_devices, num_rows, 0, &shuffle_size)); + + // Add the shuffle op over top of this op and return the subtree (top/bottom) to caller + RETURN_IF_NOT_OK(AddShuffleOp(shuffle_size, txt_op, &shuffle_op)); + *top = shuffle_op; + *bottom = txt_op; + } + return Status::OK(); } @@ -1208,7 +1326,8 @@ Status DEPipeline::ParsePadInfo(py::handle value, PadInfo *pad_info) { return Status::OK(); } -Status DEPipeline::ParseBuildVocabOp(const py::dict &args, std::shared_ptr *ptr) { +Status DEPipeline::ParseBuildVocabOp(const py::dict &args, std::shared_ptr *top, + std::shared_ptr *bottom) { std::shared_ptr builder = std::make_shared(); for (auto arg : args) { std::string key = py::str(arg.first); @@ -1235,18 +1354,23 @@ Status DEPipeline::ParseBuildVocabOp(const py::dict &args, std::shared_ptr op; RETURN_IF_NOT_OK(builder->Build(&op)); - *ptr = op; + *top = op; return Status::OK(); } -Status DEPipeline::ParseClueOp(const py::dict &args, std::shared_ptr *ptr) { +Status DEPipeline::ParseClueOp(const py::dict &args, std::shared_ptr *top, + std::shared_ptr *bottom) { + std::vector files_list; std::shared_ptr builder = std::make_shared(); if (!args["dataset_files"].is_none()) { - (void)builder->SetClueFilesList(ToStringVector(args["dataset_files"])); + files_list = ToStringVector(args["dataset_files"]); + (void)builder->SetClueFilesList(files_list); } else { RETURN_STATUS_UNEXPECTED("Error: dataset_files is missing"); } // Optional arguments + bool shuffle_required = false; + int64_t num_devices = 0; for (auto arg : args) { std::string key = py::str(arg.first); py::handle value = arg.second; @@ -1256,11 +1380,12 @@ Status DEPipeline::ParseClueOp(const py::dict &args, std::shared_ptr } else if (key == "shuffle_files") { (void)builder->SetShuffleFiles(ToBool(value)); } else if (key == "shuffle_global") { - (void)builder->SetShuffleGlobal(ToBool(value)); + shuffle_required = ToBool(value); } else if (key == "num_samples") { (void)builder->SetNumSamples(ToInt(value)); } else if (key == "num_shards") { - (void)builder->SetNumDevices(ToInt(value)); + num_devices = ToInt(value); + (void)builder->SetNumDevices(num_devices); } else if (key == "shard_id") { (void)builder->SetDeviceId(ToInt(value)); } else if (key == "cols_to_keyword") { @@ -1276,9 +1401,76 @@ Status DEPipeline::ParseClueOp(const py::dict &args, std::shared_ptr } } } - std::shared_ptr op; - RETURN_IF_NOT_OK(builder->Build(&op)); - *ptr = op; + + std::shared_ptr clue_op; + RETURN_IF_NOT_OK(builder->Build(&clue_op)); + RETURN_IF_NOT_OK(tree_->AssociateNode(clue_op)); + *top = clue_op; + + if (shuffle_required) { + std::shared_ptr shuffle_op = nullptr; + int64_t shuffle_size = 0; + int64_t num_rows = 0; + + // First, get the number of rows in the dataset and then compute the shuffle size + RETURN_IF_NOT_OK(ClueOp::CountAllFileRows(files_list, &num_rows)); + RETURN_IF_NOT_OK(ComputeShuffleSize(files_list.size(), num_devices, num_rows, 0, &shuffle_size)); + + // Add the shuffle op over top of this op and return the subtree (top/bottom) to caller + RETURN_IF_NOT_OK(AddShuffleOp(shuffle_size, clue_op, &shuffle_op)); + *top = shuffle_op; + *bottom = clue_op; + } + + return Status::OK(); +} + +// Helper function to inject a shuffle operator over top of the current operation being built. +Status DEPipeline::AddShuffleOp(int64_t shuffle_size, std::shared_ptr input_op, + std::shared_ptr *shuffle_op) { + std::shared_ptr new_shuffle_op = nullptr; + ShuffleOp::Builder shuffle_builder; + + (void)shuffle_builder.SetShuffleSize(shuffle_size); + RETURN_IF_NOT_OK(shuffle_builder.Build(&new_shuffle_op)); + RETURN_IF_NOT_OK(tree_->AssociateNode(new_shuffle_op)); + RETURN_IF_NOT_OK(new_shuffle_op->AddChild(input_op)); + // We have now created: + // + // ShuffleOp + // | + // input_op + // + *shuffle_op = new_shuffle_op; + + return Status::OK(); +} + +// Common code for computing a default shuffle size +Status DEPipeline::ComputeShuffleSize(int64_t num_files, int64_t num_devices, int64_t num_rows, int64_t total_rows, + int64_t *shuffle_size) { + const int64_t average_files_multiplier = 4; + const int64_t shuffle_max = 10000; + int64_t avg_rows_per_file = 0; + + // Adjust the num rows per shard if sharding was given + if (num_devices > 0) { + if (num_rows % num_devices == 0) { + num_rows = num_rows / num_devices; + } else { + num_rows = (num_rows / num_devices) + 1; + } + } + + // Cap based on total rows directive. Some ops do not have this and give value of 0. + if (total_rows > 0) { + num_rows = std::min(num_rows, total_rows); + } + + // get the average per file + avg_rows_per_file = num_rows / num_files; + + *shuffle_size = std::max(avg_rows_per_file * average_files_multiplier, shuffle_max); return Status::OK(); } } // namespace dataset diff --git a/mindspore/ccsrc/dataset/api/de_pipeline.h b/mindspore/ccsrc/dataset/api/de_pipeline.h index d6127d5d44..7cfc73307c 100644 --- a/mindspore/ccsrc/dataset/api/de_pipeline.h +++ b/mindspore/ccsrc/dataset/api/de_pipeline.h @@ -77,7 +77,7 @@ class DEPipeline { ~DEPipeline(); // Function to add a Node to the Execution Tree. - Status AddNodeToTree(const OpName &op_name, const py::dict &args, DsOpPtr *out); + Status AddNodeToTree(const OpName &op_name, const py::dict &args, py::dict *output); // Function to add a child and parent relationship. static Status AddChildToParentNode(const DsOpPtr &child_op, const DsOpPtr &parent_op); @@ -104,73 +104,74 @@ class DEPipeline { int GetRepeatCount() const; - Status ParseShuffleOp(const py::dict &args, std::shared_ptr *ptr); + Status ParseShuffleOp(const py::dict &args, std::shared_ptr *top, std::shared_ptr *bottom); - Status ParseMindRecordOp(const py::dict &args, std::shared_ptr *ptr); + Status ParseMindRecordOp(const py::dict &args, std::shared_ptr *top, std::shared_ptr *bottom); Status BuildMindrecordSamplerChain(const py::handle &handle, std::vector> *operators, int num_padded); - Status ParseMapOp(const py::dict &args, std::shared_ptr *ptr); + Status ParseMapOp(const py::dict &args, std::shared_ptr *top, std::shared_ptr *bottom); - Status ParseFilterOp(const py::dict &args, std::shared_ptr *ptr); + Status ParseFilterOp(const py::dict &args, std::shared_ptr *top, std::shared_ptr *bottom); - Status ParseRepeatOp(const py::dict &args, std::shared_ptr *ptr); + Status ParseRepeatOp(const py::dict &args, std::shared_ptr *top, std::shared_ptr *bottom); - Status ParseSkipOp(const py::dict &args, std::shared_ptr *ptr); + Status ParseSkipOp(const py::dict &args, std::shared_ptr *top, std::shared_ptr *bottom); - Status ParseBatchOp(const py::dict &args, std::shared_ptr *ptr); + Status ParseBatchOp(const py::dict &args, std::shared_ptr *top, std::shared_ptr *bottom); - Status ParseBucketBatchByLengthOp(const py::dict &args, std::shared_ptr *ptr); + Status ParseBucketBatchByLengthOp(const py::dict &args, std::shared_ptr *top, + std::shared_ptr *bottom); - Status ParseBarrierOp(const py::dict &args, std::shared_ptr *ptr); + Status ParseBarrierOp(const py::dict &args, std::shared_ptr *top, std::shared_ptr *bottom); - Status ParseGeneratorOp(const py::dict &args, std::shared_ptr *ptr); + Status ParseGeneratorOp(const py::dict &args, std::shared_ptr *top, std::shared_ptr *bottom); - Status ParseRenameOp(const py::dict &args, std::shared_ptr *ptr); + Status ParseRenameOp(const py::dict &args, std::shared_ptr *top, std::shared_ptr *bottom); - Status ParseTakeOp(const py::dict &args, std::shared_ptr *ptr); + Status ParseTakeOp(const py::dict &args, std::shared_ptr *top, std::shared_ptr *bottom); - Status ParseZipOp(const py::dict &args, std::shared_ptr *ptr); + Status ParseZipOp(const py::dict &args, std::shared_ptr *top, std::shared_ptr *bottom); - Status ParseConcatOp(const py::dict &args, std::shared_ptr *ptr); + Status ParseConcatOp(const py::dict &args, std::shared_ptr *top, std::shared_ptr *bottom); - Status ParseDeviceQueueOp(const py::dict &args, std::shared_ptr *ptr); + Status ParseDeviceQueueOp(const py::dict &args, std::shared_ptr *top, std::shared_ptr *bottom); - Status ParseTFReaderOp(const py::dict &args, std::shared_ptr *ptr); + Status ParseTFReaderOp(const py::dict &args, std::shared_ptr *top, std::shared_ptr *bottom); - Status ParseProjectOp(const py::dict &args, std::shared_ptr *ptr); + Status ParseProjectOp(const py::dict &args, std::shared_ptr *top, std::shared_ptr *bottom); - Status ParseImageFolderOp(const py::dict &args, std::shared_ptr *ptr); + Status ParseImageFolderOp(const py::dict &args, std::shared_ptr *top, std::shared_ptr *bottom); - Status ParseManifestOp(const py::dict &args, std::shared_ptr *ptr); + Status ParseManifestOp(const py::dict &args, std::shared_ptr *top, std::shared_ptr *bottom); - Status ParseVOCOp(const py::dict &args, std::shared_ptr *ptr); + Status ParseVOCOp(const py::dict &args, std::shared_ptr *top, std::shared_ptr *bottom); - Status ParseCocoOp(const py::dict &args, std::shared_ptr *ptr); + Status ParseCocoOp(const py::dict &args, std::shared_ptr *top, std::shared_ptr *bottom); - Status ParseCifar10Op(const py::dict &args, std::shared_ptr *ptr); + Status ParseCifar10Op(const py::dict &args, std::shared_ptr *top, std::shared_ptr *bottom); - Status ParseCifar100Op(const py::dict &args, std::shared_ptr *ptr); + Status ParseCifar100Op(const py::dict &args, std::shared_ptr *top, std::shared_ptr *bottom); - Status ParseRandomDataOp(const py::dict &args, std::shared_ptr *ptr); + Status ParseRandomDataOp(const py::dict &args, std::shared_ptr *top, std::shared_ptr *bottom); void PrintTree(); int32_t GetNumClasses() const; - Status ParseMnistOp(const py::dict &args, std::shared_ptr *ptr); + Status ParseMnistOp(const py::dict &args, std::shared_ptr *top, std::shared_ptr *bottom); Status SetBatchParameters(const py::dict &args); - Status ParseCelebAOp(const py::dict &args, std::shared_ptr *ptr); + Status ParseCelebAOp(const py::dict &args, std::shared_ptr *top, std::shared_ptr *bottom); - Status ParseTextFileOp(const py::dict &args, std::shared_ptr *ptr); + Status ParseTextFileOp(const py::dict &args, std::shared_ptr *top, std::shared_ptr *bottom); - Status ParseBuildVocabOp(const py::dict &args, std::shared_ptr *ptr); + Status ParseBuildVocabOp(const py::dict &args, std::shared_ptr *top, std::shared_ptr *bottom); - Status ParseClueOp(const py::dict &args, std::shared_ptr *ptr); + Status ParseClueOp(const py::dict &args, std::shared_ptr *top, std::shared_ptr *bottom); private: // Execution tree that links the dataset operators. @@ -180,6 +181,25 @@ class DEPipeline { static Status ParsePadInfo(py::handle value, PadInfo *pad_info); + /// \brief Helper function to inject a shuffle operator over top of the current operation being built. + /// \param[in] shuffle_size The size to use in the shuffle buffer + /// \param[in] input_op The operator to build shuffle on top of + /// \param[out] shuffle_op The top node of the created subtree (subtree contains two nodes). In this case it will be + /// the shuffle operator + /// \return Status return code + Status AddShuffleOp(int64_t shuffle_size, std::shared_ptr input_op, + std::shared_ptr *shuffle_op); + + /// \brief Helper function to compute the shuffle size + /// \param[in] num_files The number of files in the dataset + /// \param[in] num_devices The number of devices in the dataset + /// \param[in] num_rows The number of rows in the dataset + /// \param[in] total_rows An upper bound on the total rows in the dataset + /// \param[out] shuffle_size The resultant computed shuffle size + /// \return Status return code + Status ComputeShuffleSize(int64_t num_files, int64_t num_devices, int64_t num_rows, int64_t total_rows, + int64_t *shuffle_size); + int batch_size_; int repeat_num_; int num_rows_; diff --git a/mindspore/ccsrc/dataset/api/python_bindings.cc b/mindspore/ccsrc/dataset/api/python_bindings.cc index 51f2be49d5..7bed870f1a 100644 --- a/mindspore/ccsrc/dataset/api/python_bindings.cc +++ b/mindspore/ccsrc/dataset/api/python_bindings.cc @@ -116,9 +116,9 @@ void bindDEPipeline(py::module *m) { .def( "AddNodeToTree", [](DEPipeline &de, const OpName &op_name, const py::dict &args) { - DsOpPtr op; - THROW_IF_ERROR(de.AddNodeToTree(op_name, args, &op)); - return op; + py::dict out; + THROW_IF_ERROR(de.AddNodeToTree(op_name, args, &out)); + return out; }, py::return_value_policy::reference) .def_static("AddChildToParentNode", diff --git a/mindspore/ccsrc/dataset/engine/datasetops/map_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/map_op.cc index 053559f88b..fcb2e357e8 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/map_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/map_op.cc @@ -54,20 +54,19 @@ Status MapOp::Builder::sanityCheck() const { Status MapOp::Builder::Build(std::shared_ptr *ptr) { RETURN_IF_NOT_OK(sanityCheck()); *ptr = std::make_shared(std::move(build_in_col_names_), std::move(build_out_col_names_), - std::move(build_tensor_funcs_), std::move(build_col_order_), build_num_workers_, - build_op_connector_size_, build_perf_mode_); + std::move(build_tensor_funcs_), build_num_workers_, build_op_connector_size_, + build_perf_mode_); return Status::OK(); } // Constructor of MapOp MapOp::MapOp(const std::vector &in_col_names, const std::vector &out_col_names, - std::vector> tensor_funcs, const std::vector &columns_order, - int32_t num_workers, int32_t op_connector_size, bool perf_mode) + std::vector> tensor_funcs, int32_t num_workers, int32_t op_connector_size, + bool perf_mode) : ParallelOp(num_workers, op_connector_size), tfuncs_(std::move(tensor_funcs)), in_columns_(in_col_names), out_columns_(out_col_names), - columns_order_(columns_order), perf_mode_(perf_mode) { // If caller didn't specify the out_col_names, assume they are same as the in_columns. if (out_columns_.empty() || out_columns_[0].empty()) { diff --git a/mindspore/ccsrc/dataset/engine/datasetops/map_op.h b/mindspore/ccsrc/dataset/engine/datasetops/map_op.h index 94569bd41f..371d865196 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/map_op.h +++ b/mindspore/ccsrc/dataset/engine/datasetops/map_op.h @@ -93,13 +93,6 @@ class MapOp : public ParallelOp { return *this; } - // Setter method. - // @return Builder setter method returns reference to the builder. - Builder &SetColOrder(const std::vector &col_order_) { - build_col_order_ = col_order_; - return *this; - } - // Setter method. // @return Builder setter method returns reference to the builder. Builder &SetNumWorkers(int32_t num_workers) { @@ -130,7 +123,6 @@ class MapOp : public ParallelOp { std::vector build_in_col_names_; std::vector build_out_col_names_; std::vector> build_tensor_funcs_; - std::vector build_col_order_; int32_t build_num_workers_; int32_t build_op_connector_size_; bool build_perf_mode_; // Default true. @@ -145,12 +137,11 @@ class MapOp : public ParallelOp { // @param in_col_names A list of input column names (should match the input/output \p tensorFuncs). // @param out_col_names A list of output column names (should match the input/output \p tensorFuncs). // @param tensor_funcs A list of TensorOp pointers for MapOp to apply to each data. - // @param columns_order names A full list of column names (should match the whole dataset view post \p tensorFuncs). // @param num_workers The number of worker threads. // @param op_connector_size The size of each queue in the connector. MapOp(const std::vector &in_col_names, const std::vector &out_col_names, - std::vector> tensor_funcs, const std::vector &columns_order, - int32_t num_workers, int32_t op_connector_size, bool perf_mode); + std::vector> tensor_funcs, int32_t num_workers, int32_t op_connector_size, + bool perf_mode); // Destructor ~MapOp() = default; @@ -190,10 +181,6 @@ class MapOp : public ParallelOp { // @return Name of the current Op std::string Name() const override { return "MapOp"; } - // Columns order getter - // @return The post map columns order - std::vector const &ColumnsOrder() const { return columns_order_; } - private: // Local queues where worker threads can pop from. // Popping directly from the Connector can block if the previous designated threads haven't pop. @@ -215,9 +202,6 @@ class MapOp : public ParallelOp { // Indices of the columns to process. std::vector to_process_indices_; - // Variable to store the column_order of all columns post tensorOps - std::vector columns_order_; - // Performance mode is when the main thread creates local queues, pulls databuffers from the previous // op's Connector and distributes them to the local queues. Workers pull from the local queues. // If this flag is false, each worker pulls directly from the Connector. This use less resources diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/clue_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/clue_op.cc index d863de15ad..9fceb6f333 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/clue_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/clue_op.cc @@ -31,11 +31,7 @@ namespace mindspore { namespace dataset { ClueOp::Builder::Builder() - : builder_device_id_(0), - builder_num_devices_(1), - builder_num_samples_(0), - builder_shuffle_files_(false), - builder_shuffle_global_(false) { + : builder_device_id_(0), builder_num_devices_(1), builder_num_samples_(0), builder_shuffle_files_(false) { std::shared_ptr config_manager = GlobalContext::config_manager(); builder_num_workers_ = config_manager->num_parallel_workers(); builder_op_connector_size_ = config_manager->op_connector_size(); @@ -66,8 +62,8 @@ Status ClueOp::Builder::Build(std::shared_ptr *op) { std::shared_ptr clue_op = std::make_shared( builder_num_workers_, builder_rows_per_buffer_, builder_num_samples_, builder_worker_connector_size_, ck_map, - builder_clue_files_list_, builder_op_connector_size_, builder_shuffle_files_, builder_shuffle_global_, - builder_num_devices_, builder_device_id_); + builder_clue_files_list_, builder_op_connector_size_, builder_shuffle_files_, builder_num_devices_, + builder_device_id_); RETURN_IF_NOT_OK(clue_op->Init()); *op = std::move(clue_op); @@ -87,7 +83,7 @@ std::vector ClueOp::Builder::split(const std::string &s, char delim ClueOp::ClueOp(int32_t num_workers, int64_t rows_per_buffer, int64_t num_samples, int32_t worker_connector_size, ColKeyMap cols_to_keyword, std::vector clue_files_list, int32_t op_connector_size, - bool shuffle_files, bool shuffle_global, int32_t num_device, int32_t device_id) + bool shuffle_files, int32_t num_device, int32_t device_id) : ParallelOp(num_workers, op_connector_size), rows_per_buffer_(rows_per_buffer), num_rows_per_shard_(0), @@ -98,7 +94,6 @@ ClueOp::ClueOp(int32_t num_workers, int64_t rows_per_buffer, int64_t num_samples load_jagged_connector_(true), cols_to_keyword_(cols_to_keyword), shuffle_files_(shuffle_files), - shuffle_global_(shuffle_global), finished_reading_dataset_(false), num_devices_(num_device), device_id_(device_id), diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/clue_op.h b/mindspore/ccsrc/dataset/engine/datasetops/source/clue_op.h index f41abd020c..487ed0d47f 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/clue_op.h +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/clue_op.h @@ -104,13 +104,6 @@ class ClueOp : public ParallelOp { return *this; } - // Setter method. - // @return Builder - setter method returns reference to the builder. - Builder &SetShuffleGlobal(bool shuffle_global) { - builder_shuffle_global_ = shuffle_global; - return *this; - } - // Setter method. // @return Builder - setter method returns reference to the builder. Builder &SetNumSamples(int64_t num_samples) { @@ -139,15 +132,13 @@ class ClueOp : public ParallelOp { int32_t builder_worker_connector_size_; std::vector builder_clue_files_list_; bool builder_shuffle_files_; - bool builder_shuffle_global_; std::map builder_cols_to_keyword_; }; // Constructor of ClueOp - // @param shuffle_global - whether or not to shuffle the entire dataset. ClueOp(int32_t num_workers, int64_t rows_per_buffer, int64_t num_samples, int32_t worker_connector_size, ColKeyMap cols_to_keyword, std::vector clue_files_list, int32_t op_connector_size, - bool shuffle_files, bool shuffle_global, int32_t num_devices, int32_t device_id); + bool shuffle_files, int32_t num_devices, int32_t device_id); // Default destructor ~ClueOp() = default; @@ -182,10 +173,6 @@ class ClueOp : public ParallelOp { // @return Vector of the input file names std::vector FileNames() { return clue_files_list_; } - // Global shuffle flag getter - // @return Bool - whether this Op requires global shuffle - bool RequireGlobalShuffle() { return shuffle_global_; } - private: // The entry point for when workers are launched. // @param worker_id - the id of the worker that is executing this function. @@ -269,7 +256,6 @@ class ClueOp : public ParallelOp { int32_t device_id_; bool shuffle_files_; - bool shuffle_global_; bool finished_reading_dataset_; int32_t num_devices_; int64_t rows_per_buffer_; diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/text_file_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/text_file_op.cc index 5ae950b803..fbba73de21 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/text_file_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/text_file_op.cc @@ -33,11 +33,7 @@ namespace mindspore { namespace dataset { TextFileOp::Builder::Builder() - : builder_device_id_(0), - builder_num_devices_(1), - builder_total_rows_(0), - builder_shuffle_files_(false), - builder_shuffle_global_(false) { + : builder_device_id_(0), builder_num_devices_(1), builder_total_rows_(0), builder_shuffle_files_(false) { std::shared_ptr config_manager = GlobalContext::config_manager(); builder_num_workers_ = config_manager->num_parallel_workers(); builder_op_connector_size_ = config_manager->op_connector_size(); @@ -68,7 +64,7 @@ Status TextFileOp::Builder::Build(std::shared_ptr *op) { std::shared_ptr text_file_op = std::make_shared( builder_num_workers_, builder_rows_per_buffer_, builder_total_rows_, builder_worker_connector_size_, std::move(builder_schema_), builder_text_files_list_, builder_op_connector_size_, builder_shuffle_files_, - builder_shuffle_global_, builder_num_devices_, builder_device_id_); + builder_num_devices_, builder_device_id_); RETURN_IF_NOT_OK(text_file_op->Init()); *op = std::move(text_file_op); @@ -77,8 +73,7 @@ Status TextFileOp::Builder::Build(std::shared_ptr *op) { TextFileOp::TextFileOp(int32_t num_workers, int64_t rows_per_buffer, int64_t total_rows, int32_t worker_connector_size, std::unique_ptr schema, std::vector text_files_list, - int32_t op_connector_size, bool shuffle_files, bool shuffle_global, int32_t num_device, - int32_t device_id) + int32_t op_connector_size, bool shuffle_files, int32_t num_device, int32_t device_id) : ParallelOp(num_workers, op_connector_size), device_id_(device_id), num_devices_(num_device), @@ -86,7 +81,6 @@ TextFileOp::TextFileOp(int32_t num_workers, int64_t rows_per_buffer, int64_t tot total_rows_(total_rows), text_files_list_(std::move(text_files_list)), shuffle_files_(shuffle_files), - shuffle_global_(shuffle_global), data_schema_(std::move(schema)), all_num_rows_(0), num_rows_per_shard_(0), diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/text_file_op.h b/mindspore/ccsrc/dataset/engine/datasetops/source/text_file_op.h index 31224cb299..5379263979 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/text_file_op.h +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/text_file_op.h @@ -105,13 +105,6 @@ class TextFileOp : public ParallelOp { return *this; } - // Setter method. - // @return Builder - setter method returns reference to the builder. - Builder &SetShuffleGlobal(bool shuffle_global) { - builder_shuffle_global_ = shuffle_global; - return *this; - } - // Setter method. // @return Builder - setter method returns reference to the builder. Builder &SetTotalRows(int64_t total_rows) { @@ -129,7 +122,6 @@ class TextFileOp : public ParallelOp { int32_t builder_worker_connector_size_; std::vector builder_text_files_list_; bool builder_shuffle_files_; - bool builder_shuffle_global_; std::unique_ptr builder_schema_; }; @@ -143,11 +135,10 @@ class TextFileOp : public ParallelOp { // @param op_connector_size - size of each queue in the connector that the child operator pulls from. // @param columns_to_load - the names of the columns to load data from. // @param shuffle_files - whether or not to shuffle the files before reading data. - // @param shuffle_global - whether or not to shuffle the entire dataset. // @param equal_rows_per_shard - whether or not to get equal rows for each process. TextFileOp(int32_t num_workers, int64_t rows_per_buffer, int64_t total_rows, int32_t worker_connector_size, std::unique_ptr, std::vector text_files_list, int32_t op_connector_size, - bool shuffle_files, bool shuffle_global, int32_t num_devices, int32_t device_id); + bool shuffle_files, int32_t num_devices, int32_t device_id); // Default destructor ~TextFileOp() = default; @@ -186,10 +177,6 @@ class TextFileOp : public ParallelOp { // @return Vector of the input file names std::vector FileNames() { return text_files_list_; } - // Global shuffle flag getter - // @return Bool - whether this Op requires global shuffle - bool RequireGlobalShuffle() { return shuffle_global_; } - private: // The entry point for when workers are launched. // @param worker_id - the id of the worker that is executing this function. @@ -274,7 +261,6 @@ class TextFileOp : public ParallelOp { int64_t total_rows_; std::vector text_files_list_; bool shuffle_files_; - bool shuffle_global_; std::unique_ptr data_schema_; int64_t all_num_rows_; int64_t num_rows_per_shard_; diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/tf_reader_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/tf_reader_op.cc index 8b92d19249..b05fa54978 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/tf_reader_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/tf_reader_op.cc @@ -55,7 +55,6 @@ TFReaderOp::Builder::Builder() builder_op_connector_size_ = config_manager->op_connector_size(); builder_rows_per_buffer_ = config_manager->rows_per_buffer(); builder_shuffle_files_ = false; - builder_shuffle_global_ = false; builder_data_schema_ = std::make_unique(); } @@ -126,8 +125,7 @@ Status TFReaderOp::Builder::Build(std::shared_ptr *out_tf_reader_op) std::shared_ptr new_tf_reader_op = std::make_shared( builder_num_workers_, builder_worker_connector_size_, builder_rows_per_buffer_, builder_total_rows_, builder_dataset_files_list_, std::move(builder_data_schema_), builder_op_connector_size_, builder_columns_to_load_, - builder_shuffle_files_, builder_shuffle_global_, builder_num_devices_, builder_device_id_, - builder_equal_rows_per_shard_); + builder_shuffle_files_, builder_num_devices_, builder_device_id_, builder_equal_rows_per_shard_); RETURN_IF_NOT_OK(new_tf_reader_op->Init()); *out_tf_reader_op = std::move(new_tf_reader_op); @@ -137,8 +135,8 @@ Status TFReaderOp::Builder::Build(std::shared_ptr *out_tf_reader_op) TFReaderOp::TFReaderOp(int32_t num_workers, int32_t worker_connector_size, int64_t rows_per_buffer, int64_t total_num_rows, std::vector dataset_files_list, std::unique_ptr data_schema, int32_t op_connector_size, - std::vector columns_to_load, bool shuffle_files, bool shuffle_global, - int32_t num_device, int32_t device_id, bool equal_rows_per_shard) + std::vector columns_to_load, bool shuffle_files, int32_t num_device, + int32_t device_id, bool equal_rows_per_shard) : ParallelOp(num_workers, op_connector_size), device_id_(device_id), num_devices_(num_device), @@ -148,7 +146,6 @@ TFReaderOp::TFReaderOp(int32_t num_workers, int32_t worker_connector_size, int64 columns_to_load_(std::move(columns_to_load)), finished_reading_dataset_(false), shuffle_files_(shuffle_files), - shuffle_global_(shuffle_global), data_schema_(std::move(data_schema)), filename_index_(std::make_unique()), load_io_block_queue_(true), @@ -174,7 +171,6 @@ void TFReaderOp::Print(std::ostream &out, bool show_all) const { // Then show any custom derived-internal stuff out << "\nRows per buffer: " << rows_per_buffer_ << "\nTotal rows: " << total_rows_ << "\nDevice id: " << device_id_ << "\nNumber of devices: " << num_devices_ << "\nShuffle files: " << ((shuffle_files_) ? "yes" : "no") - << "\nShuffle global: " << ((shuffle_global_) ? "yes" : "no") << "\nDataset files list: Size: " << dataset_files_list_.size() << "\n"; for (int i = 0; i < dataset_files_list_.size(); ++i) { out << " " << dataset_files_list_[i]; diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/tf_reader_op.h b/mindspore/ccsrc/dataset/engine/datasetops/source/tf_reader_op.h index 417cd8bef0..9d2e38ec6b 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/tf_reader_op.h +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/tf_reader_op.h @@ -146,13 +146,6 @@ class TFReaderOp : public ParallelOp { return *this; } - // Setter method. - // @return Builder - setter method returns reference to the builder. - Builder &SetShuffleGlobal(bool shuffle_global) { - builder_shuffle_global_ = shuffle_global; - return *this; - } - // Setter method. // @return Builder - setter method returns reference to the builder. Builder &SetShardEqualRows(bool shard_equal_rows) { @@ -172,7 +165,6 @@ class TFReaderOp : public ParallelOp { std::vector builder_dataset_files_list_; std::vector builder_columns_to_load_; bool builder_shuffle_files_; - bool builder_shuffle_global_; bool builder_equal_rows_per_shard_; }; @@ -187,12 +179,11 @@ class TFReaderOp : public ParallelOp { // @param op_connector_size - size of each queue in the connector that the child operator pulls from. // @param columns_to_load - the names of the columns to load data from. // @param shuffle_files - whether or not to shuffle the files before reading data. - // @param shuffle_global - whether or not to shuffle the entire dataset. // @param equal_rows_per_shard - whether or not to get equal rows for each process. TFReaderOp(int32_t num_workers, int32_t worker_connector_size, int64_t rows_per_buffer, int64_t total_num_rows, std::vector dataset_files_list, std::unique_ptr data_schema, int32_t op_connector_size, std::vector columns_to_load, bool shuffle_files, - bool shuffle_global, int32_t num_devices, int32_t device_id, bool equal_rows_per_shard); + int32_t num_devices, int32_t device_id, bool equal_rows_per_shard); // Default destructor ~TFReaderOp() = default; @@ -245,10 +236,6 @@ class TFReaderOp : public ParallelOp { // @return Vector of the input file names std::vector FileNames() { return dataset_files_list_; } - // Global shuffle flag getter - // @return Bool - whether this Op requires global shuffle - bool RequireGlobalShuffle() { return shuffle_global_; } - private: // The entry point for when workers are launched. // @param worker_id - the id of the worker that is executing this function. @@ -393,7 +380,6 @@ class TFReaderOp : public ParallelOp { std::vector columns_to_load_; bool finished_reading_dataset_; bool shuffle_files_; - bool shuffle_global_; std::unique_ptr data_schema_; std::unique_ptr filename_index_; bool load_io_block_queue_; diff --git a/mindspore/ccsrc/dataset/engine/execution_tree.cc b/mindspore/ccsrc/dataset/engine/execution_tree.cc index 80a11aca02..8dd622912b 100644 --- a/mindspore/ccsrc/dataset/engine/execution_tree.cc +++ b/mindspore/ccsrc/dataset/engine/execution_tree.cc @@ -19,8 +19,7 @@ #include "dataset/engine/datasetops/dataset_op.h" #include "dataset/engine/datasetops/shuffle_op.h" #include "dataset/util/task_manager.h" -#include "dataset/engine/opt/pre/map_column_reorder.h" -#include "dataset/engine/opt/pre/global_shuffle.h" +#include "dataset/engine/opt/pass.h" #include "dataset/engine/perf/profiling.h" #include "dataset/engine/perf/monitor.h" @@ -42,6 +41,10 @@ ExecutionTree::~ExecutionTree() { (void)tg_->ServiceStop(); } // provides it with a link to the tree. A node cannot form any relationships (parent/child) with // other nodes unless they are associated with the same tree. Status ExecutionTree::AssociateNode(const std::shared_ptr &op) { + // If we are already a part of the tree, no-op + if (op->tree_ == this) { + return Status::OK(); + } if (tree_state_ != kDeTStateInit && tree_state_ != kDeTStateBuilding) { std::string err_msg = "Invalid tree state for adding a node. Current state: " + std::to_string(static_cast(tree_state_)) + @@ -211,8 +214,7 @@ Status ExecutionTree::PrepareTreePreAction() { bool modified = false; std::vector> pre_actions; // Construct pre actions - pre_actions.push_back(std::make_unique()); - pre_actions.push_back(std::make_unique()); + // example: pre_actions.push_back(new SomePass()); // Apply pre action passes for (auto &pass : pre_actions) { RETURN_IF_NOT_OK(pass->Run(this, &modified)); diff --git a/mindspore/ccsrc/dataset/engine/opt/CMakeLists.txt b/mindspore/ccsrc/dataset/engine/opt/CMakeLists.txt index 170cbb55e5..af0a8918db 100644 --- a/mindspore/ccsrc/dataset/engine/opt/CMakeLists.txt +++ b/mindspore/ccsrc/dataset/engine/opt/CMakeLists.txt @@ -2,7 +2,5 @@ file(GLOB_RECURSE _CURRENT_SRC_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cc" set_property(SOURCE ${_CURRENT_SRC_FILES} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_MD) add_library(engine-opt OBJECT pass.cc - pre/map_column_reorder.cc - pre/global_shuffle.cc util/printer_pass.cc - ) \ No newline at end of file + ) diff --git a/mindspore/ccsrc/dataset/engine/opt/pre/global_shuffle.cc b/mindspore/ccsrc/dataset/engine/opt/pre/global_shuffle.cc deleted file mode 100644 index 2adf734a6c..0000000000 --- a/mindspore/ccsrc/dataset/engine/opt/pre/global_shuffle.cc +++ /dev/null @@ -1,98 +0,0 @@ -/** - * Copyright 2020 Huawei Technologies Co., Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include "dataset/engine/opt/pre/global_shuffle.h" -#include "dataset/engine/execution_tree.h" -#include "dataset/engine/datasetops/shuffle_op.h" -#include "dataset/engine/datasetops/source/tf_reader_op.h" -#include "dataset/engine/datasetops/source/text_file_op.h" -#include "dataset/engine/datasetops/source/clue_op.h" - -namespace mindspore { -namespace dataset { - -Status GlobalShufflePass::RunOnTree(ExecutionTree *tree, bool *modified) { - std::vector> tf_readers; - std::vector> text_files; - std::vector> clues; - - // Pass 1, search for all sources which requires global shuffle - for (auto &op : *tree) { - if (auto ptr = std::dynamic_pointer_cast(op.shared_from_this())) { - if (ptr->RequireGlobalShuffle()) { - tf_readers.push_back(ptr); - continue; - } - } - if (auto ptr = std::dynamic_pointer_cast(op.shared_from_this())) { - if (ptr->RequireGlobalShuffle()) { - text_files.push_back(ptr); - continue; - } - } - if (auto ptr = std::dynamic_pointer_cast(op.shared_from_this())) { - if (ptr->RequireGlobalShuffle()) { - clues.push_back(ptr); - continue; - } - } - } - - // Pass 2, insert shuffle nodes - // The following blocks can be implemented with template if we unify the CountTotalRows across all source nodes . - for (auto node : tf_readers) { - std::shared_ptr builder = std::make_shared(); - int64_t total_rows = 0; - TFReaderOp::CountTotalRows(&total_rows, node->FileNames(), 8, true); - int32_t avg_file_size = total_rows / (node->FileNames().size()); - builder->SetShuffleSize(std::max(avg_file_size * 4, 10000)); - std::shared_ptr op; - RETURN_IF_NOT_OK(builder->Build(&op)); - RETURN_IF_NOT_OK(tree->AssociateNode(op)); - RETURN_IF_NOT_OK(node->InsertAsParent(op)); - } - - for (auto node : text_files) { - std::shared_ptr builder = std::make_shared(); - int64_t total_rows = 0; - TextFileOp::CountAllFileRows(node->FileNames(), &total_rows); - int32_t avg_file_size = total_rows / (node->FileNames().size()); - builder->SetShuffleSize(std::max(avg_file_size * 4, 10000)); - std::shared_ptr op; - RETURN_IF_NOT_OK(builder->Build(&op)); - RETURN_IF_NOT_OK(tree->AssociateNode(op)); - RETURN_IF_NOT_OK(node->InsertAsParent(op)); - } - - for (auto node : clues) { - std::shared_ptr builder = std::make_shared(); - int64_t total_rows = 0; - ClueOp::CountAllFileRows(node->FileNames(), &total_rows); - int32_t avg_file_size = total_rows / (node->FileNames().size()); - builder->SetShuffleSize(std::max(avg_file_size * 4, 10000)); - std::shared_ptr op; - RETURN_IF_NOT_OK(builder->Build(&op)); - RETURN_IF_NOT_OK(tree->AssociateNode(op)); - RETURN_IF_NOT_OK(node->InsertAsParent(op)); - } - - return Status::OK(); -} - -} // namespace dataset -} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/opt/pre/global_shuffle.h b/mindspore/ccsrc/dataset/engine/opt/pre/global_shuffle.h deleted file mode 100644 index 6865ac9391..0000000000 --- a/mindspore/ccsrc/dataset/engine/opt/pre/global_shuffle.h +++ /dev/null @@ -1,35 +0,0 @@ -/** - * Copyright 2020 Huawei Technologies Co., Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef DATASET_ENGINE_OPT_PASS_PRE_GLOBALSHUFFLE_H -#define DATASET_ENGINE_OPT_PASS_PRE_GLOBALSHUFFLE_H - -#include -#include "dataset/engine/opt/pass.h" - -namespace mindspore { -namespace dataset { -// Global Shuffle Pass will insert ShuffleOp when the leaf nodes requires global shuffle. -// Example: -// Input Tree: TFReader(GLOBAL_SHUFFLE) -> Batch -// Output Tree: TFReader -> Shuffle -> Batch -class GlobalShufflePass : public TreePass { - Status RunOnTree(ExecutionTree *tree, bool *modified) override; -}; -} // namespace dataset -} // namespace mindspore - -#endif // DATASET_ENGINE_OPT_PASS_PRE_GLOBALSHUFFLE_H diff --git a/mindspore/ccsrc/dataset/engine/opt/pre/map_column_reorder.cc b/mindspore/ccsrc/dataset/engine/opt/pre/map_column_reorder.cc deleted file mode 100644 index a3dbbfcc54..0000000000 --- a/mindspore/ccsrc/dataset/engine/opt/pre/map_column_reorder.cc +++ /dev/null @@ -1,51 +0,0 @@ -/** - * Copyright 2020 Huawei Technologies Co., Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include "dataset/engine/opt/pre/map_column_reorder.h" -#include "dataset/engine/execution_tree.h" -#include "dataset/engine/datasetops/map_op.h" -#include "dataset/engine/datasetops/project_op.h" - -namespace mindspore { -namespace dataset { - -Status MapColumnReorder::RunOnTree(ExecutionTree *tree, bool *modified) { - std::vector> to_process; - - // Pass 1, search for all MapOp with column orders - for (auto &op : *tree) { - if (auto mapOp = std::dynamic_pointer_cast(op.shared_from_this())) { - if (mapOp->ColumnsOrder().size() != 0) { - to_process.push_back(mapOp); - } - } - } - - // Pass 2, insert nodes for all MapOp - for (auto node : to_process) { - std::shared_ptr builder = std::make_shared(node->ColumnsOrder()); - std::shared_ptr op; - RETURN_IF_NOT_OK(builder->Build(&op)); - RETURN_IF_NOT_OK(tree->AssociateNode(op)); - RETURN_IF_NOT_OK(node->InsertAsParent(op)); - } - return Status::OK(); -} - -} // namespace dataset -} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/opt/pre/map_column_reorder.h b/mindspore/ccsrc/dataset/engine/opt/pre/map_column_reorder.h deleted file mode 100644 index 84274db3d5..0000000000 --- a/mindspore/ccsrc/dataset/engine/opt/pre/map_column_reorder.h +++ /dev/null @@ -1,35 +0,0 @@ -/** - * Copyright 2020 Huawei Technologies Co., Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef DATASET_ENGINE_OPT_PASS_PRE_MAPCOLREORDER_H -#define DATASET_ENGINE_OPT_PASS_PRE_MAPCOLREORDER_H - -#include -#include "dataset/engine/opt/pass.h" - -namespace mindspore { -namespace dataset { -// Map Column Recorder Pass will insert ProjectOp when MapOp requires a full output columns reorder. -// Example: -// Input Tree: TFReader -> MapOp(with col_order) -> Batch -// Output Tree: TFReader -> MapOp -> ProjectOp(col_order) -> Batch -class MapColumnReorder : public TreePass { - Status RunOnTree(ExecutionTree *tree, bool *modified) override; -}; -} // namespace dataset -} // namespace mindspore - -#endif // DATASET_ENGINE_OPT_PASS_PRE_MAPCOLREORDER_H diff --git a/mindspore/dataset/engine/iterators.py b/mindspore/dataset/engine/iterators.py index 4946fb3252..1d2d28c1c0 100644 --- a/mindspore/dataset/engine/iterators.py +++ b/mindspore/dataset/engine/iterators.py @@ -172,13 +172,13 @@ class Iterator: # Convert python node into C node and add to C layer execution tree in postorder traversal. def __convert_node_postorder(self, node): op_type = self.__get_dataset_type(node) - c_node = self.depipeline.AddNodeToTree(op_type, node.get_args()) + c_nodes = self.depipeline.AddNodeToTree(op_type, node.get_args()) for py_child in node.children: c_child = self.__convert_node_postorder(py_child) - self.depipeline.AddChildToParentNode(c_child, c_node) + self.depipeline.AddChildToParentNode(c_child, c_nodes["bottom"]) - return c_node + return c_nodes["top"] def __batch_node(self, dataset, level): """Recursively get batch node in the dataset tree.""" diff --git a/tests/ut/cpp/dataset/map_op_test.cc b/tests/ut/cpp/dataset/map_op_test.cc index b01b4a6df6..8b6a152488 100644 --- a/tests/ut/cpp/dataset/map_op_test.cc +++ b/tests/ut/cpp/dataset/map_op_test.cc @@ -130,75 +130,6 @@ std::shared_ptr ImageFolder(int64_t num_works, int64_t rows, int6 std::shared_ptr Build(std::vector> ops); -// TestByPosition scenario: -// TFReaderOp reads a dataset that have column ordering |image|label|A|B|. -// A TensorOp that does nothing picks the label column and output a column also named label. -// Thus, based on the new MapOp behaviour, the column ordering will be |image|label|A|B|. -// Verify the column ordering based on the Tensor properties matching to that of in the schema file. -TEST_F(MindDataTestMapOp, TestByPosition) { - Status rc; - MS_LOG(INFO) << "Doing TestByPosition."; - - // Note: The above TFReader config yields 5 buffers, each with 2 rows, for a total - // of 10 rows. - auto my_tfreader_op = this->CreateTFReaderOp(); - rc = my_tree_->AssociateNode(my_tfreader_op); - EXPECT_TRUE(rc.IsOk()); - auto my_no_op = std::make_shared(); - std::vector> my_func_list; - my_func_list.push_back(my_no_op); - std::shared_ptr my_map_op; - MapOp::Builder builder; - builder.SetInColNames({"label"}) - .SetOutColNames({}) - .SetColOrder({"image", "label", "A", "B"}) - .SetTensorFuncs(std::move(my_func_list)) - .SetNumWorkers(100); - rc = builder.Build(&my_map_op); - EXPECT_TRUE(rc.IsOk()); - rc = my_tree_->AssociateNode(my_map_op); - EXPECT_TRUE(rc.IsOk()); - rc = my_map_op->AddChild(my_tfreader_op); - EXPECT_TRUE(rc.IsOk()); - rc = my_tree_->AssignRoot(my_map_op); - EXPECT_TRUE(rc.IsOk()); - rc = my_tree_->Prepare(); - EXPECT_TRUE(rc.IsOk()); - rc = my_tree_->Launch(); - EXPECT_TRUE(rc.IsOk()); - - - // Based on the schema file, create the golden result to compare with. - std::vector golden_types({ - DataType::Type::DE_UINT8, - DataType::Type::DE_INT64, - DataType::Type::DE_FLOAT32, - DataType::Type::DE_INT64} - ); - - std::vector golden_ranks({3, 1, 4, 1}); - - std::vector golden_shapes({ - TensorShape({3, 4, 2}), - TensorShape({7}), - TensorShape({1, 13, 14, 12}), - TensorShape({9})} - ); - - // Start the loop of reading tensors from our pipeline - DatasetIterator di(my_tree_); - TensorRow tensor_list; - rc = di.FetchNextTensorRow(&tensor_list); - EXPECT_TRUE(rc.IsOk()); - EXPECT_EQ(tensor_list.size(), 4); - for (uint32_t i = 0; i < tensor_list.size(); i++) { - EXPECT_EQ(tensor_list[i]->type(), golden_types[i]); - EXPECT_EQ(tensor_list[i]->Rank(), golden_ranks[i]); - EXPECT_EQ(tensor_list[i]->shape(), golden_shapes[i]); - EXPECT_NE(tensor_list[i]->GetBuffer(), nullptr); - } -} - // TestAsMap scenario: // TFReaderOp reads a dataset that have column ordering |image|label|A|B|. // A TensorOp that does nothing picks the "image" column and produces a column named "X". diff --git a/tests/ut/python/dataset/test_opt_pass.py b/tests/ut/python/dataset/test_opt_pass.py index bab881e283..480bfcbeab 100644 --- a/tests/ut/python/dataset/test_opt_pass.py +++ b/tests/ut/python/dataset/test_opt_pass.py @@ -16,8 +16,10 @@ import numpy as np import mindspore.dataset as ds - -def test_map_reorder_pass_0(): +# tests the construction of multiple ops from a single dataset. +# map dataset with columns order arguments should produce a ProjectOp over MapOp +# This test does not utilize the compiling passes at this time. +def test_map_reorder0(): def generator_mc(maxid=1): for _ in range(maxid): yield (np.array([0]), np.array([1])) @@ -31,8 +33,10 @@ def test_map_reorder_pass_0(): for item in data0.create_tuple_iterator(): # each data is a dictionary assert item == [np.array(1), np.array(0)] - -def test_map_reorder_pass_1(): +# tests the construction of multiple ops from a single dataset. +# map dataset with columns order arguments should produce a ProjectOp over MapOp +# This test does not utilize the compiling passes at this time. +def test_map_reorder1(): def generator_mc(maxid=1): for _ in range(maxid): yield (np.array([0]), np.array([1]), np.array([2])) @@ -48,8 +52,10 @@ def test_map_reorder_pass_1(): for item in data2.create_tuple_iterator(): assert item == [np.array(2), np.array(2), np.array(1), np.array(1), np.array(0), np.array(0)] - -def test_global_shuffle_pass(): +# tests the construction of multiple ops from a single dataset. +# TFRecordDataset with global shuffle should produce a ShuffleOp over TfReaderOp. +# This test does not utilize the compiling passes at this time. +def test_shuffle(): FILES = ["../data/dataset/testTFTestAllTypes/test.data"] SCHEMA_FILE = "../data/dataset/testTFTestAllTypes/datasetSchema.json" @@ -85,6 +91,6 @@ def test_global_shuffle_pass(): if __name__ == "__main__": - test_map_reorder_pass_0() - test_map_reorder_pass_1() - test_global_shuffle_pass() + test_map_reorder0() + test_map_reorder1() + test_global_shuffle() From 587e260236210428f68d0776b518948ccc4d5a63 Mon Sep 17 00:00:00 2001 From: avakh Date: Tue, 23 Jun 2020 15:42:02 -0400 Subject: [PATCH 067/254] addressing comments --- .../ccsrc/dataset/api/python_bindings.cc | 14 + .../dataset/kernels/image/CMakeLists.txt | 2 + .../image/random_resize_with_bbox_op.cc | 33 ++ .../image/random_resize_with_bbox_op.h | 56 ++++ .../kernels/image/resize_with_bbox_op.cc | 53 ++++ .../kernels/image/resize_with_bbox_op.h | 43 +++ .../dataset/transforms/vision/c_transforms.py | 53 ++++ .../dataset/test_random_resize_with_bbox.py | 265 ++++++++++++++++ .../python/dataset/test_resize_with_bbox.py | 295 ++++++++++++++++++ 9 files changed, 814 insertions(+) create mode 100644 mindspore/ccsrc/dataset/kernels/image/random_resize_with_bbox_op.cc create mode 100644 mindspore/ccsrc/dataset/kernels/image/random_resize_with_bbox_op.h create mode 100644 mindspore/ccsrc/dataset/kernels/image/resize_with_bbox_op.cc create mode 100644 mindspore/ccsrc/dataset/kernels/image/resize_with_bbox_op.h create mode 100644 tests/ut/python/dataset/test_random_resize_with_bbox.py create mode 100644 tests/ut/python/dataset/test_resize_with_bbox.py diff --git a/mindspore/ccsrc/dataset/api/python_bindings.cc b/mindspore/ccsrc/dataset/api/python_bindings.cc index 51f2be49d5..852087872f 100644 --- a/mindspore/ccsrc/dataset/api/python_bindings.cc +++ b/mindspore/ccsrc/dataset/api/python_bindings.cc @@ -63,12 +63,14 @@ #include "dataset/kernels/image/random_horizontal_flip_bbox_op.h" #include "dataset/kernels/image/random_horizontal_flip_op.h" #include "dataset/kernels/image/random_resize_op.h" +#include "dataset/kernels/image/random_resize_with_bbox_op.h" #include "dataset/kernels/image/random_rotation_op.h" #include "dataset/kernels/image/random_vertical_flip_op.h" #include "dataset/kernels/image/random_vertical_flip_with_bbox_op.h" #include "dataset/kernels/image/rescale_op.h" #include "dataset/kernels/image/resize_bilinear_op.h" #include "dataset/kernels/image/resize_op.h" +#include "dataset/kernels/image/resize_with_bbox_op.h" #include "dataset/kernels/image/uniform_aug_op.h" #include "dataset/kernels/no_op.h" #include "dataset/text/kernels/jieba_tokenizer_op.h" @@ -348,6 +350,18 @@ void bindTensorOps1(py::module *m) { .def(py::init(), py::arg("targetHeight"), py::arg("targetWidth") = ResizeOp::kDefWidth, py::arg("interpolation") = ResizeOp::kDefInterpolation); + (void)py::class_>( + *m, "ResizeWithBBoxOp", "Tensor operation to resize an image. Takes height, width and mode.") + .def(py::init(), py::arg("targetHeight"), + py::arg("targetWidth") = ResizeWithBBoxOp::kDefWidth, + py::arg("interpolation") = ResizeWithBBoxOp::kDefInterpolation); + + (void)py::class_>( + *m, "RandomResizeWithBBoxOp", + "Tensor operation to resize an image using a randomly selected interpolation. Takes height and width.") + .def(py::init(), py::arg("targetHeight"), + py::arg("targetWidth") = RandomResizeWithBBoxOp::kDefTargetWidth); + (void)py::class_>( *m, "UniformAugOp", "Tensor operation to apply random augmentation(s).") .def(py::init>, int32_t>(), py::arg("operations"), diff --git a/mindspore/ccsrc/dataset/kernels/image/CMakeLists.txt b/mindspore/ccsrc/dataset/kernels/image/CMakeLists.txt index 3d88d9989c..fef698912c 100644 --- a/mindspore/ccsrc/dataset/kernels/image/CMakeLists.txt +++ b/mindspore/ccsrc/dataset/kernels/image/CMakeLists.txt @@ -25,4 +25,6 @@ add_library(kernels-image OBJECT resize_bilinear_op.cc resize_op.cc uniform_aug_op.cc + resize_with_bbox_op.cc + random_resize_with_bbox_op.cc ) diff --git a/mindspore/ccsrc/dataset/kernels/image/random_resize_with_bbox_op.cc b/mindspore/ccsrc/dataset/kernels/image/random_resize_with_bbox_op.cc new file mode 100644 index 0000000000..de69c02e39 --- /dev/null +++ b/mindspore/ccsrc/dataset/kernels/image/random_resize_with_bbox_op.cc @@ -0,0 +1,33 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "dataset/kernels/image/random_resize_with_bbox_op.h" +#include "dataset/kernels/image/resize_with_bbox_op.h" +#include "dataset/util/status.h" + +namespace mindspore { +namespace dataset { +const int32_t RandomResizeWithBBoxOp::kDefTargetWidth = 0; + +Status RandomResizeWithBBoxOp::Compute(const TensorRow &input, TensorRow *output) { + // Randomly selects from the following four interpolation methods + // 0-bilinear, 1-nearest_neighbor, 2-bicubic, 3-area + interpolation_ = static_cast(distribution_(random_generator_)); + RETURN_IF_NOT_OK(ResizeWithBBoxOp::Compute(input, output)); + return Status::OK(); +} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/kernels/image/random_resize_with_bbox_op.h b/mindspore/ccsrc/dataset/kernels/image/random_resize_with_bbox_op.h new file mode 100644 index 0000000000..4a7614525f --- /dev/null +++ b/mindspore/ccsrc/dataset/kernels/image/random_resize_with_bbox_op.h @@ -0,0 +1,56 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef DATASET_KERNELS_IMAGE_RANDOM_RESIZE_WITH_BBOX_OP_H +#define DATASET_KERNELS_IMAGE_RANDOM_RESIZE_WITH_BBOX_OP_H + +#include +#include + +#include "dataset/core/tensor.h" +#include "dataset/kernels/image/resize_op.h" +#include "dataset/kernels/image/resize_with_bbox_op.h" +#include "dataset/kernels/tensor_op.h" +#include "dataset/util/random.h" +#include "dataset/util/status.h" + +namespace mindspore { +namespace dataset { +class RandomResizeWithBBoxOp : public ResizeWithBBoxOp { + public: + // Default values, also used by python_bindings.cc + static const int32_t kDefTargetWidth; + explicit RandomResizeWithBBoxOp(int32_t size_1, int32_t size_2 = kDefTargetWidth) : ResizeWithBBoxOp(size_1, size_2) { + random_generator_.seed(GetSeed()); + } + + ~RandomResizeWithBBoxOp() = default; + + // Description: A function that prints info about the node + void Print(std::ostream &out) const override { + out << "RandomResizeWithBBoxOp: " << ResizeWithBBoxOp::size1_ << " " << ResizeWithBBoxOp::size2_; + } + + Status Compute(const TensorRow &input, TensorRow *output) override; + + private: + std::mt19937 random_generator_; + std::uniform_int_distribution distribution_{0, 3}; +}; +} // namespace dataset +} // namespace mindspore + +#endif // DATASET_KERNELS_IMAGE_RANDOM_RESIZE_WITH_BBOX_OP_H diff --git a/mindspore/ccsrc/dataset/kernels/image/resize_with_bbox_op.cc b/mindspore/ccsrc/dataset/kernels/image/resize_with_bbox_op.cc new file mode 100644 index 0000000000..8a633d5678 --- /dev/null +++ b/mindspore/ccsrc/dataset/kernels/image/resize_with_bbox_op.cc @@ -0,0 +1,53 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "dataset/kernels/image/resize_with_bbox_op.h" +#include +#include +#include "dataset/kernels/image/resize_op.h" +#include "dataset/kernels/image/image_utils.h" +#include "dataset/core/cv_tensor.h" +#include "dataset/core/pybind_support.h" +#include "dataset/core/tensor.h" +#include "dataset/kernels/tensor_op.h" +#include "dataset/util/status.h" + +namespace mindspore { +namespace dataset { + +Status ResizeWithBBoxOp::Compute(const TensorRow &input, TensorRow *output) { + IO_CHECK_VECTOR(input, output); + BOUNDING_BOX_CHECK(input); + + int32_t input_h = input[0]->shape()[0]; + int32_t input_w = input[0]->shape()[1]; + + output->resize(2); + (*output)[1] = std::move(input[1]); // move boxes over to output + + std::shared_ptr input_cv = CVTensor::AsCVTensor(std::move(input[0])); + + RETURN_IF_NOT_OK(ResizeOp::Compute(std::static_pointer_cast(input_cv), &(*output)[0])); + + int32_t output_h = (*output)[0]->shape()[0]; // output height if ResizeWithBBox + int32_t output_w = (*output)[0]->shape()[1]; // output width if ResizeWithBBox + + size_t bboxCount = input[1]->shape()[0]; // number of rows in bbox tensor + RETURN_IF_NOT_OK(UpdateBBoxesForResize((*output)[1], bboxCount, output_w, output_h, input_w, input_h)); + return Status::OK(); +} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/kernels/image/resize_with_bbox_op.h b/mindspore/ccsrc/dataset/kernels/image/resize_with_bbox_op.h new file mode 100644 index 0000000000..17bdd01ef1 --- /dev/null +++ b/mindspore/ccsrc/dataset/kernels/image/resize_with_bbox_op.h @@ -0,0 +1,43 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATASET_KERNELS_IMAGE_RESIZE_WITH_BBOX_OP_H +#define DATASET_KERNELS_IMAGE_RESIZE_WITH_BBOX_OP_H + +#include "dataset/core/tensor.h" +#include "dataset/kernels/image/image_utils.h" +#include "dataset/kernels/tensor_op.h" +#include "dataset/util/status.h" +#include "dataset/kernels/image/resize_op.h" + +namespace mindspore { +namespace dataset { +class ResizeWithBBoxOp : public ResizeOp { + public: + // Constructor for ResizeWithBBoxOp, with default value and passing to base class constructor + explicit ResizeWithBBoxOp(int32_t size_1, int32_t size_2 = kDefWidth, + InterpolationMode mInterpolation = kDefInterpolation) + : ResizeOp(size_1, size_2, mInterpolation) {} + + ~ResizeWithBBoxOp() override = default; + + void Print(std::ostream &out) const override { out << "ResizeWithBBoxOp: " << size1_ << " " << size2_; } + + Status Compute(const TensorRow &input, TensorRow *output) override; +}; +} // namespace dataset +} // namespace mindspore + +#endif // DATASET_KERNELS_IMAGE_RESIZE_WITH_BBOX_OP_H diff --git a/mindspore/dataset/transforms/vision/c_transforms.py b/mindspore/dataset/transforms/vision/c_transforms.py index c2497f9629..aef714953f 100644 --- a/mindspore/dataset/transforms/vision/c_transforms.py +++ b/mindspore/dataset/transforms/vision/c_transforms.py @@ -265,6 +265,7 @@ class BoundingBoxAugment(cde.BoundingBoxAugmentOp): ratio (float, optional): Ratio of bounding boxes to apply augmentation on. Range: [0,1] (default=0.3). """ + @check_bounding_box_augment_cpp def __init__(self, transform, ratio=0.3): self.ratio = ratio @@ -302,6 +303,36 @@ class Resize(cde.ResizeOp): super().__init__(*size, interpoltn) +class ResizeWithBBox(cde.ResizeWithBBoxOp): + """ + Resize the input image to the given size and adjust the bounding boxes accordingly. + + Args: + size (int or sequence): The output size of the resized image. + If size is an int, smaller edge of the image will be resized to this value with + the same image aspect ratio. + If size is a sequence of length 2, it should be (height, width). + interpolation (Inter mode, optional): Image interpolation mode (default=Inter.LINEAR). + It can be any of [Inter.LINEAR, Inter.NEAREST, Inter.BICUBIC]. + + - Inter.LINEAR, means interpolation method is bilinear interpolation. + + - Inter.NEAREST, means interpolation method is nearest-neighbor interpolation. + + - Inter.BICUBIC, means interpolation method is bicubic interpolation. + """ + + @check_resize_interpolation + def __init__(self, size, interpolation=Inter.LINEAR): + self.size = size + self.interpolation = interpolation + interpoltn = DE_C_INTER_MODE[interpolation] + if isinstance(size, int): + super().__init__(size, interpolation=interpoltn) + else: + super().__init__(*size, interpoltn) + + class RandomResizedCropWithBBox(cde.RandomCropAndResizeWithBBoxOp): """ Crop the input image to a random size and aspect ratio and adjust the Bounding Boxes accordingly @@ -326,6 +357,7 @@ class RandomResizedCropWithBBox(cde.RandomCropAndResizeWithBBoxOp): max_attempts (int, optional): The maximum number of attempts to propose a valid crop_area (default=10). If exceeded, fall back to use center_crop instead. """ + @check_random_resize_crop def __init__(self, size, scale=(0.08, 1.0), ratio=(3. / 4., 4. / 3.), interpolation=Inter.BILINEAR, max_attempts=10): @@ -499,6 +531,27 @@ class RandomResize(cde.RandomResizeOp): super().__init__(*size) +class RandomResizeWithBBox(cde.RandomResizeWithBBoxOp): + """ + Tensor operation to resize the input image using a randomly selected interpolation mode and adjust + the bounding boxes accordingly. + + Args: + size (int or sequence): The output size of the resized image. + If size is an int, smaller edge of the image will be resized to this value with + the same image aspect ratio. + If size is a sequence of length 2, it should be (height, width). + """ + + @check_resize + def __init__(self, size): + self.size = size + if isinstance(size, int): + super().__init__(size) + else: + super().__init__(*size) + + class HWC2CHW(cde.ChannelSwapOp): """ Transpose the input image; shape (H, W, C) to shape (C, H, W). diff --git a/tests/ut/python/dataset/test_random_resize_with_bbox.py b/tests/ut/python/dataset/test_random_resize_with_bbox.py new file mode 100644 index 0000000000..66c185d647 --- /dev/null +++ b/tests/ut/python/dataset/test_random_resize_with_bbox.py @@ -0,0 +1,265 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""" +Testing the random resize with bounding boxes op in DE +""" +from enum import Enum +import matplotlib.pyplot as plt +import matplotlib.patches as patches +import numpy as np +import mindspore.dataset as ds +from mindspore import log as logger +import mindspore.dataset.transforms.vision.c_transforms as c_vision + + +GENERATE_GOLDEN = False + +DATA_DIR = "../data/dataset/testVOC2012" + + +def fix_annotate(bboxes): + """ + :param bboxes: in [label, x_min, y_min, w, h, truncate, difficult] format + :return: annotation in [x_min, y_min, w, h, label, truncate, difficult] format + """ + for bbox in bboxes: + tmp = bbox[0] + bbox[0] = bbox[1] + bbox[1] = bbox[2] + bbox[2] = bbox[3] + bbox[3] = bbox[4] + bbox[4] = tmp + return bboxes + + +class BoxType(Enum): + """ + Defines box types for test cases + """ + WidthOverflow = 1 + HeightOverflow = 2 + NegativeXY = 3 + OnEdge = 4 + WrongShape = 5 + + +class AddBadAnnotation: # pylint: disable=too-few-public-methods + """ + Used to add erroneous bounding boxes to object detection pipelines. + Usage: + >>> # Adds a box that covers the whole image. Good for testing edge cases + >>> de = de.map(input_columns=["image", "annotation"], + >>> output_columns=["image", "annotation"], + >>> operations=AddBadAnnotation(BoxType.OnEdge)) + """ + + def __init__(self, box_type): + self.box_type = box_type + + def __call__(self, img, bboxes): + """ + Used to generate erroneous bounding box examples on given img. + :param img: image where the bounding boxes are. + :param bboxes: in [x_min, y_min, w, h, label, truncate, difficult] format + :return: bboxes with bad examples added + """ + height = img.shape[0] + width = img.shape[1] + if self.box_type == BoxType.WidthOverflow: + # use box that overflows on width + return img, np.array([[0, 0, width + 1, height - 1, 0, 0, 0]]).astype(np.uint32) + + if self.box_type == BoxType.HeightOverflow: + # use box that overflows on height + return img, np.array([[0, 0, width - 1, height + 1, 0, 0, 0]]).astype(np.uint32) + + if self.box_type == BoxType.NegativeXY: + # use box with negative xy + return img, np.array([[-10, -10, width - 1, height - 1, 0, 0, 0]]).astype(np.uint32) + + if self.box_type == BoxType.OnEdge: + # use box that covers the whole image + return img, np.array([[0, 0, width - 1, height - 1, 0, 0, 0]]).astype(np.uint32) + + if self.box_type == BoxType.WrongShape: + # use box that covers the whole image + return img, np.array([[0, 0, width - 1]]).astype(np.uint32) + return img, bboxes + + +def check_bad_box(data, box_type, expected_error): + try: + test_op = c_vision.RandomResizeWithBBox(100) # DEFINE TEST OP HERE -- (PROB 1 IN CASE OF RANDOM) + data = data.map(input_columns=["annotation"], + output_columns=["annotation"], + operations=fix_annotate) + # map to use width overflow + data = data.map(input_columns=["image", "annotation"], + output_columns=["image", "annotation"], + columns_order=["image", "annotation"], + operations=AddBadAnnotation(box_type)) # Add column for "annotation" + # map to apply ops + data = data.map(input_columns=["image", "annotation"], + output_columns=["image", "annotation"], + columns_order=["image", "annotation"], + operations=[test_op]) # Add column for "annotation" + for _, _ in enumerate(data.create_dict_iterator()): + break + except RuntimeError as e: + logger.info("Got an exception in DE: {}".format(str(e))) + assert expected_error in str(e) + + +def add_bounding_boxes(axis, bboxes): + """ + :param axis: axis to modify + :param bboxes: bounding boxes to draw on the axis + :return: None + """ + for bbox in bboxes: + rect = patches.Rectangle((bbox[0], bbox[1]), + bbox[2], bbox[3], + linewidth=1, edgecolor='r', facecolor='none') + # Add the patch to the Axes + axis.add_patch(rect) + + +def visualize(unaugmented_data, augment_data): + for idx, (un_aug_item, aug_item) in \ + enumerate(zip(unaugmented_data.create_dict_iterator(), augment_data.create_dict_iterator())): + axis = plt.subplot(141) + plt.imshow(un_aug_item["image"]) + add_bounding_boxes(axis, un_aug_item["annotation"]) # add Orig BBoxes + plt.title("Original" + str(idx + 1)) + logger.info("Original ", str(idx + 1), " :", un_aug_item["annotation"]) + + axis = plt.subplot(142) + plt.imshow(aug_item["image"]) + add_bounding_boxes(axis, aug_item["annotation"]) # add AugBBoxes + plt.title("Augmented" + str(idx + 1)) + logger.info("Augmented ", str(idx + 1), " ", aug_item["annotation"], "\n") + plt.show() + + +def test_random_resize_with_bbox_op(plot=False): + """ + Test random_resize_with_bbox_op + """ + logger.info("Test random resize with bbox") + + # original images + data_original = ds.VOCDataset(DATA_DIR, task="Detection", mode="train", decode=True, shuffle=False) + + # augmented images + data_augmented = ds.VOCDataset(DATA_DIR, task="Detection", mode="train", decode=True, shuffle=False) + + data_original = data_original.map(input_columns=["annotation"], + output_columns=["annotation"], + operations=fix_annotate) + + data_augmented = data_augmented.map(input_columns=["annotation"], + output_columns=["annotation"], + operations=fix_annotate) + + # define map operations + test_op = c_vision.RandomResizeWithBBox(100) # input value being the target size of resizeOp + + data_augmented = data_augmented.map(input_columns=["image", "annotation"], + output_columns=["image", "annotation"], + columns_order=["image", "annotation"], operations=[test_op]) + if plot: + visualize(data_original, data_augmented) + + +def test_random_resize_with_bbox_invalid_bounds(): + data_voc2 = ds.VOCDataset(DATA_DIR, task="Detection", mode="train", decode=True, shuffle=False) + check_bad_box(data_voc2, BoxType.WidthOverflow, "bounding boxes is out of bounds of the image") + data_voc2 = ds.VOCDataset(DATA_DIR, task="Detection", mode="train", decode=True, shuffle=False) + check_bad_box(data_voc2, BoxType.HeightOverflow, "bounding boxes is out of bounds of the image") + data_voc2 = ds.VOCDataset(DATA_DIR, task="Detection", mode="train", decode=True, shuffle=False) + check_bad_box(data_voc2, BoxType.NegativeXY, "min_x") + data_voc2 = ds.VOCDataset(DATA_DIR, task="Detection", mode="train", decode=True, shuffle=False) + check_bad_box(data_voc2, BoxType.WrongShape, "4 features") + + +def test_random_resize_with_bbox_invalid_size(): + """ + Test random_resize_with_bbox_op + """ + logger.info("Test random resize with bbox with invalid target size") + + # original images + data = ds.VOCDataset(DATA_DIR, task="Detection", mode="train", decode=True, shuffle=False) + + data = data.map(input_columns=["annotation"], + output_columns=["annotation"], + operations=fix_annotate) + + # negative target size as input + try: + test_op = c_vision.RandomResizeWithBBox(-10) # DEFINE TEST OP HERE -- (PROB 1 IN CASE OF RANDOM) + + # map to apply ops + data = data.map(input_columns=["image", "annotation"], + output_columns=["image", "annotation"], + columns_order=["image", "annotation"], + operations=[test_op]) # Add column for "annotation" + + for _, _ in enumerate(data.create_dict_iterator()): + break + + except ValueError as e: + logger.info("Got an exception in DE: {}".format(str(e))) + print(e) + assert "Input is not" in str(e) + + # zero target size as input + try: + test_op = c_vision.RandomResizeWithBBox(0) # DEFINE TEST OP HERE -- (PROB 1 IN CASE OF RANDOM) + + # map to apply ops + data = data.map(input_columns=["image", "annotation"], + output_columns=["image", "annotation"], + columns_order=["image", "annotation"], + operations=[test_op]) # Add column for "annotation" + + for _, _ in enumerate(data.create_dict_iterator()): + break + + except ValueError as e: + logger.info("Got an exception in DE: {}".format(str(e))) + assert "Input is not" in str(e) + + # invalid input shape + try: + test_op = c_vision.RandomResizeWithBBox((10, 10, 10)) # DEFINE TEST OP HERE -- (PROB 1 IN CASE OF RANDOM) + + # map to apply ops + data = data.map(input_columns=["image", "annotation"], + output_columns=["image", "annotation"], + columns_order=["image", "annotation"], + operations=[test_op]) # Add column for "annotation" + + for _, _ in enumerate(data.create_dict_iterator()): + break + + except TypeError as e: + logger.info("Got an exception in DE: {}".format(str(e))) + assert "Size should be" in str(e) + +if __name__ == "__main__": + test_random_resize_with_bbox_op(plot=False) + test_random_resize_with_bbox_invalid_bounds() + test_random_resize_with_bbox_invalid_size() diff --git a/tests/ut/python/dataset/test_resize_with_bbox.py b/tests/ut/python/dataset/test_resize_with_bbox.py new file mode 100644 index 0000000000..8b07f17f1a --- /dev/null +++ b/tests/ut/python/dataset/test_resize_with_bbox.py @@ -0,0 +1,295 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""" +Testing the resize with bounding boxes op in DE +""" +from enum import Enum +import numpy as np +import matplotlib.patches as patches +import matplotlib.pyplot as plt +import mindspore.dataset.transforms.vision.c_transforms as c_vision +from mindspore import log as logger +import mindspore.dataset as ds + +GENERATE_GOLDEN = False + +DATA_DIR = "../data/dataset/testVOC2012" + + +def fix_annotate(bboxes): + """ + :param bboxes: in [label, x_min, y_min, w, h, truncate, difficult] format + :return: annotation in [x_min, y_min, w, h, label, truncate, difficult] format + """ + for bbox in bboxes: + tmp = bbox[0] + bbox[0] = bbox[1] + bbox[1] = bbox[2] + bbox[2] = bbox[3] + bbox[3] = bbox[4] + bbox[4] = tmp + return bboxes + + +class BoxType(Enum): + """ + Defines box types for test cases + """ + WidthOverflow = 1 + HeightOverflow = 2 + NegativeXY = 3 + OnEdge = 4 + WrongShape = 5 + + +class AddBadAnnotation: # pylint: disable=too-few-public-methods + """ + Used to add erroneous bounding boxes to object detection pipelines. + Usage: + >>> # Adds a box that covers the whole image. Good for testing edge cases + >>> de = de.map(input_columns=["image", "annotation"], + >>> output_columns=["image", "annotation"], + >>> operations=AddBadAnnotation(BoxType.OnEdge)) + """ + + def __init__(self, box_type): + self.box_type = box_type + + def __call__(self, img, bboxes): + """ + Used to generate erroneous bounding box examples on given img. + :param img: image where the bounding boxes are. + :param bboxes: in [x_min, y_min, w, h, label, truncate, difficult] format + :return: bboxes with bad examples added + """ + height = img.shape[0] + width = img.shape[1] + if self.box_type == BoxType.WidthOverflow: + # use box that overflows on width + return img, np.array([[0, 0, width + 1, height - 1, 0, 0, 0]]).astype(np.uint32) + + if self.box_type == BoxType.HeightOverflow: + # use box that overflows on height + return img, np.array([[0, 0, width - 1, height + 1, 0, 0, 0]]).astype(np.uint32) + + if self.box_type == BoxType.NegativeXY: + # use box with negative xy + return img, np.array([[-10, -10, width - 1, height - 1, 0, 0, 0]]).astype(np.uint32) + + if self.box_type == BoxType.OnEdge: + # use box that covers the whole image + return img, np.array([[0, 0, width - 1, height - 1, 0, 0, 0]]).astype(np.uint32) + + if self.box_type == BoxType.WrongShape: + # use box that covers the whole image + return img, np.array([[0, 0, width - 1]]).astype(np.uint32) + return img, bboxes + + +def check_bad_box(data, box_type, expected_error): + try: + test_op = c_vision.ResizeWithBBox(100) + data = data.map(input_columns=["annotation"], + output_columns=["annotation"], + operations=fix_annotate) + # map to use width overflow + data = data.map(input_columns=["image", "annotation"], + output_columns=["image", "annotation"], + columns_order=["image", "annotation"], + operations=AddBadAnnotation(box_type)) # Add column for "annotation" + # map to apply ops + data = data.map(input_columns=["image", "annotation"], + output_columns=["image", "annotation"], + columns_order=["image", "annotation"], + operations=[test_op]) # Add column for "annotation" + for _, _ in enumerate(data.create_dict_iterator()): + break + except RuntimeError as e: + logger.info("Got an exception in DE: {}".format(str(e))) + assert expected_error in str(e) + + +def add_bounding_boxes(axis, bboxes): + """ + :param axis: axis to modify + :param bboxes: bounding boxes to draw on the axis + :return: None + """ + for bbox in bboxes: + rect = patches.Rectangle((bbox[0], bbox[1]), + bbox[2], bbox[3], + linewidth=1, edgecolor='r', facecolor='none') + # Add the patch to the Axes + axis.add_patch(rect) + + +def visualize(unaugmented_data, augment_data): + for idx, (un_aug_item, aug_item) in enumerate( + zip(unaugmented_data.create_dict_iterator(), augment_data.create_dict_iterator())): + axis = plt.subplot(141) + plt.imshow(un_aug_item["image"]) + add_bounding_boxes(axis, un_aug_item["annotation"]) # add Orig BBoxes + plt.title("Original" + str(idx + 1)) + logger.info("Original ", str(idx + 1), " :", un_aug_item["annotation"]) + + axis = plt.subplot(142) + plt.imshow(aug_item["image"]) + add_bounding_boxes(axis, aug_item["annotation"]) # add AugBBoxes + plt.title("Augmented" + str(idx + 1)) + logger.info("Augmented ", str(idx + 1), " ", aug_item["annotation"], "\n") + plt.show() + + +def test_resize_with_bbox_op(plot=False): + """ + Test resize_with_bbox_op + """ + logger.info("Test resize with bbox") + + # original images + data_original = ds.VOCDataset(DATA_DIR, task="Detection", mode="train", decode=True, shuffle=False) + + # augmented images + data_augmented = ds.VOCDataset(DATA_DIR, task="Detection", mode="train", decode=True, shuffle=False) + + data_original = data_original.map(input_columns=["annotation"], + output_columns=["annotation"], + operations=fix_annotate) + + data_augmented = data_augmented.map(input_columns=["annotation"], + output_columns=["annotation"], + operations=fix_annotate) + + # define map operations + test_op = c_vision.ResizeWithBBox(100) # input value being the target size of resizeOp + + data_augmented = data_augmented.map(input_columns=["image", "annotation"], + output_columns=["image", "annotation"], + columns_order=["image", "annotation"], operations=[test_op]) + if plot: + visualize(data_original, data_augmented) + + +def test_resize_with_bbox_invalid_bounds(): + data_voc2 = ds.VOCDataset(DATA_DIR, task="Detection", mode="train", decode=True, shuffle=False) + check_bad_box(data_voc2, BoxType.WidthOverflow, "bounding boxes is out of bounds of the image") + data_voc2 = ds.VOCDataset(DATA_DIR, task="Detection", mode="train", decode=True, shuffle=False) + check_bad_box(data_voc2, BoxType.HeightOverflow, "bounding boxes is out of bounds of the image") + data_voc2 = ds.VOCDataset(DATA_DIR, task="Detection", mode="train", decode=True, shuffle=False) + check_bad_box(data_voc2, BoxType.NegativeXY, "min_x") + data_voc2 = ds.VOCDataset(DATA_DIR, task="Detection", mode="train", decode=True, shuffle=False) + check_bad_box(data_voc2, BoxType.WrongShape, "4 features") + + +def test_resize_with_bbox_invalid_size(): + """ + Test resize_with_bbox_op + """ + logger.info("Test resize with bbox with invalid target size") + + # original images + data = ds.VOCDataset(DATA_DIR, task="Detection", mode="train", decode=True, shuffle=False) + + data = data.map(input_columns=["annotation"], + output_columns=["annotation"], + operations=fix_annotate) + + # negative target size as input + try: + test_op = c_vision.ResizeWithBBox(-10) + + # map to apply ops + data = data.map(input_columns=["image", "annotation"], + output_columns=["image", "annotation"], + columns_order=["image", "annotation"], + operations=[test_op]) # Add column for "annotation" + + for _, _ in enumerate(data.create_dict_iterator()): + break + + except ValueError as e: + logger.info("Got an exception in DE: {}".format(str(e))) + assert "Input is not" in str(e) + + # zero target size as input + try: + test_op = c_vision.ResizeWithBBox(0) + + # map to apply ops + data = data.map(input_columns=["image", "annotation"], + output_columns=["image", "annotation"], + columns_order=["image", "annotation"], + operations=[test_op]) # Add column for "annotation" + + for _, _ in enumerate(data.create_dict_iterator()): + break + + except ValueError as e: + logger.info("Got an exception in DE: {}".format(str(e))) + assert "Input is not" in str(e) + + # invalid input shape + try: + test_op = c_vision.ResizeWithBBox((10, 10, 10)) + + # map to apply ops + data = data.map(input_columns=["image", "annotation"], + output_columns=["image", "annotation"], + columns_order=["image", "annotation"], + operations=[test_op]) # Add column for "annotation" + + for _, _ in enumerate(data.create_dict_iterator()): + break + + except TypeError as e: + logger.info("Got an exception in DE: {}".format(str(e))) + assert "Size should be" in str(e) + + +def test_resize_with_bbox_invalid_interpolation(): + """ + Test resize_with_bbox_op + """ + logger.info("Test resize with bbox with invalid interpolation size") + + # original images + data = ds.VOCDataset(DATA_DIR, task="Detection", mode="train", decode=True, shuffle=False) + + data = data.map(input_columns=["annotation"], + output_columns=["annotation"], + operations=fix_annotate) + + # invalid interpolation + try: + test_op = c_vision.ResizeWithBBox(100, interpolation="invalid") + + # map to apply ops + data = data.map(input_columns=["image", "annotation"], + output_columns=["image", "annotation"], + columns_order=["image", "annotation"], + operations=[test_op]) # Add column for "annotation" + + for _, _ in enumerate(data.create_dict_iterator()): + break + + except ValueError as e: + logger.info("Got an exception in DE: {}".format(str(e))) + assert "interpolation" in str(e) + +if __name__ == "__main__": + test_resize_with_bbox_op(plot=False) + test_resize_with_bbox_invalid_bounds() + test_resize_with_bbox_invalid_size() + test_resize_with_bbox_invalid_interpolation() From 694a1c8067e94099619f2f2e6554532ea631ec1e Mon Sep 17 00:00:00 2001 From: chenzomi Date: Thu, 25 Jun 2020 09:52:37 +0800 Subject: [PATCH 068/254] fix checkpoint evaliaction. --- mindspore/train/serialization.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/mindspore/train/serialization.py b/mindspore/train/serialization.py index c4fba9ba8a..ff1b8c3122 100644 --- a/mindspore/train/serialization.py +++ b/mindspore/train/serialization.py @@ -186,9 +186,10 @@ def load_checkpoint(ckpt_file_name, model_type="normal", net=None): raise ValueError(e.__str__()) parameter_dict = {} - if model_type != checkpoint_list.model_type: - raise KeyError("Checkpoint file model type({}) is not equal to input model type({}).".format( - checkpoint_list.model_type, model_type)) + if checkpoint_list.model_type: + if model_type != checkpoint_list.model_type: + raise KeyError("Checkpoint file model type({}) is not equal to input model type({}).".format( + checkpoint_list.model_type, model_type)) try: for element in checkpoint_list.value: data = element.tensor.tensor_content From 3d1ecaaeb58f18bdfebd56248f0d69c250cb1091 Mon Sep 17 00:00:00 2001 From: Danish Farid Date: Thu, 25 Jun 2020 03:19:14 -0400 Subject: [PATCH 069/254] updated UT test for Python (3) AugOps with BBox - MD5 checks + imrpv comments --- .../random_crop_and_resize_with_bbox_op.cc | 3 +- .../kernels/image/random_crop_with_bbox_op.cc | 3 +- .../random_vertical_flip_with_bbox_op.cc | 3 +- .../random_crop_with_bbox_01_c_result.npz | Bin 0 -> 1654 bytes ...dom_resized_crop_with_bbox_01_c_result.npz | Bin 0 -> 1654 bytes ...om_vertical_flip_with_bbox_01_c_result.npz | Bin 0 -> 1654 bytes .../test_random_crop_and_resize_with_bbox.py | 216 +++++------------- .../dataset/test_random_crop_with_bbox.py | 211 +++++------------ .../test_random_vertical_flip_with_bbox.py | 95 ++++++-- tests/ut/python/dataset/util.py | 31 +-- 10 files changed, 211 insertions(+), 351 deletions(-) create mode 100644 tests/ut/data/dataset/golden/random_crop_with_bbox_01_c_result.npz create mode 100644 tests/ut/data/dataset/golden/random_resized_crop_with_bbox_01_c_result.npz create mode 100644 tests/ut/data/dataset/golden/random_vertical_flip_with_bbox_01_c_result.npz diff --git a/mindspore/ccsrc/dataset/kernels/image/random_crop_and_resize_with_bbox_op.cc b/mindspore/ccsrc/dataset/kernels/image/random_crop_and_resize_with_bbox_op.cc index b820779ed1..fbaf2c9326 100644 --- a/mindspore/ccsrc/dataset/kernels/image/random_crop_and_resize_with_bbox_op.cc +++ b/mindspore/ccsrc/dataset/kernels/image/random_crop_and_resize_with_bbox_op.cc @@ -30,8 +30,7 @@ Status RandomCropAndResizeWithBBoxOp::Compute(const TensorRow &input, TensorRow BOUNDING_BOX_CHECK(input); CHECK_FAIL_RETURN_UNEXPECTED(input[0]->shape().Size() >= 2, "The shape of input is abnormal"); - (*output).push_back(nullptr); // init memory for return vector - (*output).push_back(nullptr); + output->resize(2); (*output)[1] = std::move(input[1]); // move boxes over to output size_t bboxCount = input[1]->shape()[0]; // number of rows in bbox tensor diff --git a/mindspore/ccsrc/dataset/kernels/image/random_crop_with_bbox_op.cc b/mindspore/ccsrc/dataset/kernels/image/random_crop_with_bbox_op.cc index 2be37f1da3..c873307afd 100644 --- a/mindspore/ccsrc/dataset/kernels/image/random_crop_with_bbox_op.cc +++ b/mindspore/ccsrc/dataset/kernels/image/random_crop_with_bbox_op.cc @@ -36,8 +36,7 @@ Status RandomCropWithBBoxOp::Compute(const TensorRow &input, TensorRow *output) int32_t padded_image_h; int32_t padded_image_w; - (*output).push_back(nullptr); - (*output).push_back(nullptr); + output->resize(2); (*output)[1] = std::move(input[1]); // since some boxes may be removed bool crop_further = true; // Whether further cropping will be required or not, true unless required size matches diff --git a/mindspore/ccsrc/dataset/kernels/image/random_vertical_flip_with_bbox_op.cc b/mindspore/ccsrc/dataset/kernels/image/random_vertical_flip_with_bbox_op.cc index c6aa8450a8..ffea851eac 100644 --- a/mindspore/ccsrc/dataset/kernels/image/random_vertical_flip_with_bbox_op.cc +++ b/mindspore/ccsrc/dataset/kernels/image/random_vertical_flip_with_bbox_op.cc @@ -45,8 +45,7 @@ Status RandomVerticalFlipWithBBoxOp::Compute(const TensorRow &input, TensorRow * RETURN_IF_NOT_OK(input[1]->SetItemAt({i, 1}, newBoxCorner_y)); } - (*output).push_back(nullptr); - (*output).push_back(nullptr); + output->resize(2); (*output)[1] = std::move(input[1]); return VerticalFlip(input[0], &(*output)[0]); diff --git a/tests/ut/data/dataset/golden/random_crop_with_bbox_01_c_result.npz b/tests/ut/data/dataset/golden/random_crop_with_bbox_01_c_result.npz new file mode 100644 index 0000000000000000000000000000000000000000..0c220fd09d2f82888b93437e370325ab758da34d GIT binary patch literal 1654 zcmbW&dr(wW90%}w7Z6wzP*-^+uClpHZVRIF5=8>hys<0?$o0Wv9~YO$K7Mx(;tJ@Z zJS3nXKtv^JYMRoaQmo*gv@x8VCJGuSGwrV?njTixRKwWVJ%{UU`lEmP-PwER{_gpl z@9xf-drVwFpo*EVRm`o_!@j#g3`0JljEb=cLV1$GpjS7k|9>g{ z0XA2i)8jMP+yZZK*45%F@%vhQ%5s6Xxm{i?)Y-5SnE6pkmwo;zupHA0iX7Go0qqco zAV|ltoPtxd!2)Th9s4}I5`xVN@n;sKK?uiiY&(P^Q0lk}L3v7j@j`@Jf4WOD2ScYKv!+=bP z{C5C@AURa4iEt^%utVR$o84Wd_vp3dQPQ~~?7<_5wlibcj@EC?Qve6QwLy#bu z$EgX9+IRcYvZ5_k?MU9unk7lF7K2_kPY`23kSLiXYGU6FENbR1s5HlV-}KyAx(U`{ zNS4idVm2VyD47&$ULU)ZRq&2_a(GMwrN76OK`Mqc*`yPbfnbwlGO4kz)O=}pEo5d} z#_`;S;TBbkS&=fsd>3~{d*-D?p7nWmPKc#e}fzhxw6?tOdf*mlG#B` zx_PPhr?88;+Q#5#&q8NX=l_MB~wr3quW2_bzHSj^2X;3?|tW z5`z#FNv4>ZZ<1nXXM=rs@RuLnO21Zbf)WhRSb4KjTzwXo%Z)9#%I$((rC^qq;BGS0 z9t3-(nfB3{BJYGoj>kq!ef!ajFy3&<2BjFvStZIi!-4CL!)e(h&hblh-CcK>~Ow+a&){p;f=TL`Q4hvBxuHP zR5r(mX+h8`nKo)tGE)W$W2a^>{>hA}YNAg-JBH)3=^*A=1f7!UqQ>j^@YdPxw#_}} zR|X8)`y6y*cuqFY6VrpBS28b9qi>#^IM?sLj)$&QU;V=GhCU4avKb)e1cE`yoTR4a z{P*WS&xm)Xt#0Zci@DweLl{n3c^<=QtHH#v3RS>=Z5860lgdxtj{j^NCN5|JX|Upv JDqa)l-QR!@8nyrc literal 0 HcmV?d00001 diff --git a/tests/ut/data/dataset/golden/random_resized_crop_with_bbox_01_c_result.npz b/tests/ut/data/dataset/golden/random_resized_crop_with_bbox_01_c_result.npz new file mode 100644 index 0000000000000000000000000000000000000000..a909cbe88c5719a72a75f24017e303be51301f54 GIT binary patch literal 1654 zcmbW&dr(wW90%}wdCG?I2vK?HqQEYS2q;NN0fLs?faSsfg4x)ucgn&*6BRX6&DSclO@7zk5FC zySsDd9)sZR6UHr849A~6t(o)XIPwYL!Z@=mS0*Rf8$3K0Oq*mNc}VJbVQGP3H_y4b zMx9M`T4h~^R@YRdOVsMB9Ws{9_DY9r6G?lAxy~sneWzq@5S4pRolXM)|&GkET)0tNlG#i6O#rKO~ZbL?MV~A*-bD2bt9vw5o|A zW(|V1%*0X?m^U3aGF~-Vf8p$Fx8@d15QiaNH9BJS2ojiCM@Gnv$+84hzDp1`qBOMY*;n!WxiJdWWB)npNq zjUb1aZPZkJ5Hr|u=gY3~+H00kbKBRDiy==n+lkqM;7MklqUO$Q&}>!78QY}3ZCB+? zF@l{K@>L@c^E842W(ujv?zsL7dW|5;3r zEd4HrpI-aZfZ% zwdd}>JaYQ2w`;+KVV{~_NlX<2Gc%S48!HBzYD8kH5$tD1qUQYNerI&arq~j@3@_Z%9or+#N4;+S_~wnJqu{{Mpc)`XM&M)yQ*$XlGCu!M z>tLf)KH^SU*$OTUZq*zh=2-+DW}c%auJ3x;k%eSONl&~hFT@=SjTo9#(@abYf`iPo zQqw-*xiDsuLbgXVkA6GdSq^O&4ymS{m<|M;%ydz6vFBve-uzbsJ_`yx@0q?g3Edca zRMSh$^9cHwIZVyOo5s5Fb7?;Aj_Fs%fBDe_M=-Ioc+F`Q7%5HTkayui$h)TB*bLCZo`Uw%Jg`1W#XHV}p}oU(`_hL)ZZomfZdfN--)!SctfOUXl zFbX(uDg$9APG*SFKb$%DqQsc&56{b3# z>*Tt1cFALN>vA-@o=RPcM%O62vD@mj$Zoqt+C|njkL2%r99Ea)zt^XyXwnl?G`*Vt zU#h-9o3o?cYdRF_#8!v@_ywE1tU%t$P7Q+LI(GjCZkjP9DH6Iy$R-f-?X4=N; z(1Z+gE+k_}QOssy3>2@SGS~l=gd4YyR_ED8R5&F@?kwAt+|1gqnQs>T^Zst76}WnUhgx zzIH(=hBCzn#FQhbV8%#|7JomW9Vq)Q#;!fJGXDBkP>I2$m?~lrf@)?SrDh>K?u*Sw zFGTJTnkI_PubzNi78j)=ll702zZzF+hwPfrlAPP2u6W{2PWT zIqb_i-)m8ebI^gIQ!!n{>_^~b<^VMbt-qS(Okdx7^Hnp^B@>I#jiE;|y~I3*;2<-7 z)Wp>+sNz59ujroc>I~g^WCI++(65*QVh$r1WX4Ag-#>i)himo6jhA1UyIpgs9)>Us zD`te4#}OQ1W|W!>Cnm3KnYYijl+0%?yjgS+#xOjgn4`oziQp+_o~C9`!P`}Q%jc(L z=jm@tlNYbSIEG`2nIL8o!86QEQIk@gI(B<&i1^!|r432oIs?-fj*F6n;aSmO68Hdh g;D2ou{x!#{kGviK**Hu>a0qGe{v*qOO{8~!10tFi!T plot_rows: - orig = np.split(orig[:comp_set*plot_rows], comp_set) + [orig[comp_set*plot_rows:]] - aug = np.split(aug[:comp_set*plot_rows], comp_set) + [aug[comp_set*plot_rows:]] + # Create batches of required size and add remainder to last batch + orig = np.split(orig[:split_point], batch_size) + ([orig[split_point:]] if (split_point < orig.shape[0]) else []) # check to avoid empty arrays being added + aug = np.split(aug[:split_point], batch_size) + ([aug[split_point:]] if (split_point < aug.shape[0]) else []) else: orig = [orig] aug = [aug] for ix, allData in enumerate(zip(orig, aug)): - base_ix = ix * plot_rows # will signal what base level we're on + base_ix = ix * plot_rows # current batch starting index + curPlot = len(allData[0]) - sub_plot_count = 2 if (len(allData[0]) < 2) else len(allData[0]) # if 1 image remains, create subplot for 2 to simplify axis selection - fig, axs = plt.subplots(sub_plot_count, 2) + fig, axs = plt.subplots(curPlot, 2) fig.tight_layout(pad=1.5) for x, (dataA, dataB) in enumerate(zip(allData[0], allData[1])): cur_ix = base_ix + x + (axA, axB) = (axs[x, 0], axs[x, 1]) if (curPlot > 1) else (axs[0], axs[1]) # select plotting axes based on number of image rows on plot - else case when 1 row - axs[x, 0].imshow(dataA["image"]) - add_bounding_boxes(axs[x, 0], dataA["annotation"]) - axs[x, 0].title.set_text("Original" + str(cur_ix+1)) - logger.info("Original **\n{} : {}".format(str(cur_ix+1), dataA["annotation"])) + axA.imshow(dataA["image"]) + add_bounding_boxes(axA, dataA["annotation"]) + axA.title.set_text("Original" + str(cur_ix+1)) - axs[x, 1].imshow(dataB["image"]) - add_bounding_boxes(axs[x, 1], dataB["annotation"]) - axs[x, 1].title.set_text("Augmented" + str(cur_ix+1)) + axB.imshow(dataB["image"]) + add_bounding_boxes(axB, dataB["annotation"]) + axB.title.set_text("Augmented" + str(cur_ix+1)) + + logger.info("Original **\n{} : {}".format(str(cur_ix+1), dataA["annotation"])) logger.info("Augmented **\n{} : {}\n".format(str(cur_ix+1), dataB["annotation"])) plt.show() From 0f58f0338e61469b7ef2c08be87847a5ce40eb82 Mon Sep 17 00:00:00 2001 From: islam_amin Date: Wed, 24 Jun 2020 15:36:12 -0400 Subject: [PATCH 070/254] updating ut for RandomHorizontalFlipWithBBox and BBoxAugment --- .../bounding_box_augment_crop_c_result.npz | Bin 0 -> 1654 bytes ...bounding_box_augment_rotation_c_result.npz | Bin 0 -> 1654 bytes ...unding_box_augment_valid_edge_c_result.npz | Bin 0 -> 1654 bytes ...nding_box_augment_valid_ratio_c_result.npz | Bin 0 -> 1654 bytes ..._horizontal_flip_with_bbox_01_c_result.npz | Bin 0 -> 1654 bytes .../dataset/test_bounding_box_augment.py | 370 +++++++++--------- .../test_random_horizontal_flip_bbox.py | 266 ------------- .../test_random_horizontal_flip_with_bbox.py | 229 +++++++++++ 8 files changed, 411 insertions(+), 454 deletions(-) create mode 100644 tests/ut/data/dataset/golden/bounding_box_augment_crop_c_result.npz create mode 100644 tests/ut/data/dataset/golden/bounding_box_augment_rotation_c_result.npz create mode 100644 tests/ut/data/dataset/golden/bounding_box_augment_valid_edge_c_result.npz create mode 100644 tests/ut/data/dataset/golden/bounding_box_augment_valid_ratio_c_result.npz create mode 100644 tests/ut/data/dataset/golden/random_horizontal_flip_with_bbox_01_c_result.npz delete mode 100644 tests/ut/python/dataset/test_random_horizontal_flip_bbox.py create mode 100644 tests/ut/python/dataset/test_random_horizontal_flip_with_bbox.py diff --git a/tests/ut/data/dataset/golden/bounding_box_augment_crop_c_result.npz b/tests/ut/data/dataset/golden/bounding_box_augment_crop_c_result.npz new file mode 100644 index 0000000000000000000000000000000000000000..e4e92210d7a4b8d01498673fad591c90deed965c GIT binary patch literal 1654 zcmbW&eM}Q)90%~bmZB7=C@;c>vusFEPu9F>B| z1Vo&3<7L7ezM{zzmk3!lqR2wbW^looEiiQR4>RT%!zGgt6SKK{uIDXFwtx1!^e*?i z=kt8q=E*$DqQl0vOJVOah z?V;+`7S?5Us?$_zPrf=JA6RfFpP zm$FH2=4xvkZmrqwWVJQ5)!4>=U!z-D=48!w&V`+|W~>CtJ4(q}yib8PD>PQhDntfY`Mi|~R*^iP76`O@1Im*i$QuxhAmmBF zCRqK?0188>sXZfLBxRz0ZONrRL+90#n@5WRAq>OQrY9lKkdSZ$Ys8Rets%VW8n_p0 z%JKMoHk_yFdXyasYcZ%K6G6;#2%Z;Bq+qr^X#8?<;oP#o9px#^L{|x{!w@AIH8C0l z(V|%|nB$6uuOjK(jk*C%gs1cR6vSYNl}sElS_JW;Nf69l`=IO(V}FtEc3XVZ@TGut;1w&KWx zx^)?zdwxf+CBbG4TO^ZC%*zP2ie{T&A_9-o7aGD(XI-lpJb96dhYSp_NG6jQ9fIwm z$r8-+{bRWSm5RAM|HAc#q2?~g#*iZ!hL~Igd7{w^W^s1+jj5gHkA5Hisi4&{G6VS- z43c@37=++8(d-b+1;1hU{mW-V;>L!8Lb0?2 zOUO(+5$qCYdR>^QYRn$PT~Vak#;UH2gnUOsDTX(s^fF?0BQT1_^zX)u!6F%!m~sRa zqOl6*^TLI}+``EI%(Q{%mhV8c);nJQwc5!8r=6U?yweEls&-G+t-%qOU_=hfC zm`*q|YWNZAF?b}?KujZoeWGddnxe*2H@2L+JM}}>&z;wU6JbAwX34xo%mD-~qG=UO zGxx>(_Op&(`+FV-cN9&ZhBgcbCDTsKAp{+wIV_l)CwlHCPg({lvnG>neX5&=w=o=% z%sa#!Mewd@-V@A+`BUg{$K0bOx2OMHbExbgbYkd|OgAw-2;LXXF~Jo6(tFkY?aHtj z<>b)Khf$-@i=ofNvKT%vX$=f53zGlWR>7}1v6{%+@$Zepz$ko4gXWJE{u(XZ{RNrK BCVBt> literal 0 HcmV?d00001 diff --git a/tests/ut/data/dataset/golden/bounding_box_augment_rotation_c_result.npz b/tests/ut/data/dataset/golden/bounding_box_augment_rotation_c_result.npz new file mode 100644 index 0000000000000000000000000000000000000000..8cc7e15e31b116565a79f65dbef223ce5d920953 GIT binary patch literal 1654 zcmbW&dr(wW90%~byF3>mz+D7NSw-DNa0LQ+BoZNyyt=HXfTcv*$Hhh9?*5ju#0DxU z4|(*&NNf@J}8_Kn<27+u*Wng6I0pv2Q^eP*gbcTw`r#S>33)Eo%_4z zbH2MfXYMhw3Lh2Yt(A=WRC(|(z6?V?fsBeVyId8Cdfw@gF`;6U3?vUp^%#n`8_Q%& zJ=3VQadlRgHdCW*D$*usw3QAQcA5DKhs(y1cD}i$juZNIcC(Wc?sdrtn&h|y&0fv_ zFJ-gB%DZcw9=+A!;`Fud8eAoOU$aM9;o__gz7D(GR;&cZJ4(shyiY;KGOZxbm03X1 z4n7Ecb*zk)v+_3ZqeE@jrCYTsj4;a<{+dT9@5zvyBHa27xKaogp)jj z6D<2@0D~dY(vcgwWua5g8y8y!`-}PywGJJIC=Ah-2O$rWkVg*P zd`8g3RQS6sAJyN37z`T8tR!X?f>>%+izZ_*W<%`h56-pBEx9v%bgT~6U|1^|EipO- zan!66P1^N{tC1(O0&KUxh;k?fVjvzvf@IbcqerlTnncmerGz&5hs`_lW>xu%eHV^H z5{6{SJW5Oof>dfYie}Li*}8Bs5|DHCKJJ9)NB&XuWqhk4wS3L ztqp5phpx#X3&Uo~WE1l^f*fj|5Y6xHE57f@`hy+riQC9!caK9ZhAooGBPJg~0X2o9 zX^7^lYmz=&$L8hcAD{Rn9G=9mRWd9w+YoH0#vq!R(Ovgqi-$UXuyux94^!=cA`C{! z>>vgqc#4{*Me})b(v9GEZVpGkcQxnW?Z7iojG@HBnUuonGxNLyn{kzchthH|NlUPd z%=8R`XX#8k#hI!{9P#`m-}I`H>PsI-d?kZ&47;TC3Syo^V5Y|M@5YM3CK-;HN(8&9 zv5V$P$^84(4sTc0|6Vyy2ssKT&EGS$S?AgHB=7tM!;vyHcW8L5Yvm`O=^yarsT)IPd#6TMa?QF z&rd9@y$pRAj#@Yl!&?@;k(J3+ivQXwgf&O2nY(Lff7T+P_UrInBbj6eid$M#iRz#s@1NvU@Ja+ceQjzn!}~_jk|d zeD9w#d#fdLe6lcGD}=Oh=SROJ2m;#{2+4w1RU2}gN=vIrNYa~ZAUnwFxVvUsb)8Ar zCmgUf%2B^+TW__sSJ|?ywx)s>b|OgfKfwR2pZG!Xybjh$}PsVD3dc zL4sW}NpVu#AtdsljkL8z7BR0jPP@#i0?d~L$<&Jl1fpFs*P66$JMM71-Icer(!Zn< ziaULM9^7TGW%nl-_BP8{=)od;KnUoONdYZyF~Jg+7vF=s1G525c}SiONFi8yJD>=u z{|pc)EWW<7q{O3FLdBMXuP1gcyk0i)=`EyDEc4wCNoOJV5M=O>djm`7GRrBfhFQVP zN`m{iS*4qz=+yV#0n5a&=ve+%PFf=Fr^qynjTt*Z7B{PPGhDQxxxVv~rL1I)F#64~ z3l55G!#u!@lVA-uIl7rX^Ht>+RT*`Ee%yI3;cDAfJV=pin6=F05j@1rI^8T@*+5Xr z%|_k0PIWIiEP6gXt3C4Hf9`5S8AZ8aDwuH*Y~p6KZnm6THt_c8&Y80o<=XyW=4EW5 zs5Fek%vOSJ+_-f^rxn+=-cP1>*W9>RL64n86-BjSwlhNrc5t&(Hy3_eH?`Nk=gs&2 zNGZN~KGTmHibs8Nt*EU&ucAb#mj)vW9`m5qSb}wIrd@&=h%WMqPJ*Yn>C#Q; zAXQJT7VFD5y`Hpk?8sFdq8xo;NdM`>C|tg(DO%7^a_@0fM949MjFnyRV(;C_1lLP7Dr~ z554~Zj#CU8<^(e@5}f4bCEfgM9lT62ElAn^`3@#lU<03MN8ol7ZwQsX}6VYF0R?>(`9RQZDw7PR#z+du+Prh1fPo|?E-saGbi?&-F7c0-s@A7w5bV6+JoBv zU#fPclW%GA`VCIO#~GSh8nH+GzIMOL=Hr|K-;8}NPOJhZFiOR{0#8A~GTjiAr*MF> z7lILl=vf6D#0K?1C>`p;elMqjHP#^UXPUEMEz7WqUI;^=(z8mdLhQCdxXENHUKL9^ zu|^CJIr^&MVZD|7Kb|Ig9S4t8!=rjPL++yB31d${!xErF}fq>dHPzM4u2%_%? z;r0 zHgUvkM(`vxTO`w&r0W1j-L6REf!k-yv!k#TL%eKs#OM(uQ1g^zZVhfKEK|qlRHnyP zEi9XtAQ3~7Y?6sFAlODtie#n}2IfjX2|KGE)P#ElCKI+}NR`dg#H1lerzS%(6Z-m# zN8FP&)~}X|zDeFZ0ht)GWb+I$*$AGcCPy;$Q|<7_FB!L*R?@ncqN6Xs4h%bGVFAvHyk$@NEFEtoFdFa_hMH5cc- z@B)Tn*|5ayMo>bHNitQo!aaFxUDxFKyh|m+kw!3Mu*jyA7=++OYF?7eU6u)dZ|e@b za`ZxIW@Pw2D8o?h;H)Zf_1Sq|z;^5rc&MlbtGom&$xJUJ*h6Qkl4feSCM5DxAz7Yl z4O5q+zg0jrhF9ct8!~owsTr0``EMiB{;$`qzp0x0@cO;@D=>oLZ3o9;7=f^_#cz-b>n literal 0 HcmV?d00001 diff --git a/tests/ut/data/dataset/golden/random_horizontal_flip_with_bbox_01_c_result.npz b/tests/ut/data/dataset/golden/random_horizontal_flip_with_bbox_01_c_result.npz new file mode 100644 index 0000000000000000000000000000000000000000..d360bb98ec7b551d207f73903b519123b5244939 GIT binary patch literal 1654 zcmbW&drVtZ90%~*mNM7r#>#Udql}Ff!44R&F$PW;OfIzDnFu&o>ElAn!{l&ynJ z9OE@mm^g-yIVCLM_J>9j4Piqw$&6VvF(#Wpe2l1s=r-dYKC(FMIh+(|dD& z_k7N`ZBFhn@rs}@Zl&V5{vQhUx0D=5KC8Jf&f@WyQ}m+SFXKXJlMEyeNevmw%S=@= zu7zvW+6Ax8qs`H1+l<;|jkeC^!5)ifc6saqX%|_Vyh5Pwby(a&;9i%OtVv5u)*R9N z|5CLpY@)B(?bq8}9zoyiYr@XJ_qF&{W{+TViC*mS*{}+@l~F3uzVZ}gJl6^G0+|&Q z-4KL8spDn5oR@b&FdJ&eez%~4Rh9C<&opMk13bsex?wc}m5x_b$^zXsSYt34N|pmf zPkcUx2d%v|@Q|*O{6EfSOT7SVYhazu!8vGTh$E1<9)X(V1$9EGV-W-~Rp-oToR{oZFKSB)`Mcc+d{D(4TELMnzd$!sGg9YF>&nbfRb zm$duiE741ZzhrG&n1LIRg&|uq+lk3Ru!EUgYMQ@}UifvNV<2t&=E!e4ZimB840)2t zC+2YkyO?=`noB)j&Y6BxeYbUf*_j{LkOBo53MI3fm?8wl%sKIbRN;eZzi@?H+_5Q|&!7dqrm^uXY%s8kKK7q)!E%uC3JbC8%r2>i?(rbc_QZm}hG%ekv*UJcJ-I3}46VvZx|WTuOnW8#&Y#bfTDUmso$@2i*{g>DQz zlIbPp1cE+h`lnE_%35u9Y^MQY9%ucQ9HpYE)h zpS>G%!u$seVR%V0r-&IwaGIGBYHaVQ8`WcfmF~PSkkNDU)LD2L!z)%n!0@V7Z{lV0 hFvWju6@fL!s-3(Y|JgW9yfT Date: Fri, 26 Jun 2020 11:47:40 +0800 Subject: [PATCH 071/254] Make assign-node to be before jump-node, ensure child graph can get its input Signed-off-by: zhoufeng --- .../ccsrc/session/ascend_control_parser.cc | 38 +++++++++++++++---- .../ccsrc/session/ascend_control_parser.h | 5 ++- 2 files changed, 34 insertions(+), 9 deletions(-) diff --git a/mindspore/ccsrc/session/ascend_control_parser.cc b/mindspore/ccsrc/session/ascend_control_parser.cc index 868b968d9e..573c1c1d35 100644 --- a/mindspore/ccsrc/session/ascend_control_parser.cc +++ b/mindspore/ccsrc/session/ascend_control_parser.cc @@ -33,6 +33,21 @@ static constexpr size_t kCNodeSwitchLayerLength = 3; namespace mindspore { namespace session { +static CNodePtr GetJumpNode(NotNull parent_graph, NotNull child_graph) { + auto &nodes = parent_graph->execution_order(); + for (auto &node : nodes) { + if (IsPrimitiveCNode(node, prim::kPrimLabelGoto) && child_graph->get_start_label() == node->input(kCNodeCallArg)) { + return node; + } else if (IsPrimitiveCNode(node, prim::kPrimLabelSwitch) && + (child_graph->get_start_label() == node->input(kCNodeSwitchFalse) || + child_graph->get_start_label() == node->input(kCNodeSwitchTrue))) { + return node; + } + } + MS_LOG(INFO) << "Cannot find jump node from " << parent_graph->ToString() << " to " << child_graph->ToString(); + return nullptr; +} + static void InitUnionFindSet(NotNull kg, const NotNull *> union_find_set, const NotNull *> memo) { if (memo->find(kg.get()) != memo->end()) { @@ -200,7 +215,8 @@ void AscendControlParser::ChildGraphDataAssign(const std::mapsecond), NOT_NULL(arg), NOT_NULL(parameter)); + InsertMultipleAssignToGraph(NOT_NULL(target_graph_iter->second), NOT_NULL(kg), NOT_NULL(arg), + NOT_NULL(parameter)); } } } @@ -433,7 +449,8 @@ std::tuple AscendControlParser::ParsePartial(NotNull kg, NotNull from, +void AscendControlParser::InsertMultipleAssignToGraph(NotNull from_graph, + NotNull to_graph, NotNull from, NotNull to) { std::vector from_outputs = AnfAlgo::GetAllOutput(from, {prim::kPrimTupleGetItem}); std::vector to_outputs = AnfAlgo::GetAllOutput(to, {prim::kPrimTupleGetItem}); @@ -443,18 +460,24 @@ void AscendControlParser::InsertMultipleAssignToGraph(NotNull kg << to_outputs.size() << "]"; } for (size_t i = 0; i < from_outputs.size(); i++) { - InsertAssignToGraph(kg, NOT_NULL(from_outputs[i]), NOT_NULL(to_outputs[i])); + auto assign_node = InsertAssignToGraph(from_graph, NOT_NULL(from_outputs[i]), NOT_NULL(to_outputs[i])); + if (assign_node != nullptr) { + auto jump_node = GetJumpNode(from_graph, to_graph); + if (jump_node != nullptr) { + InsertControlDependToGraph(from_graph, NOT_NULL(assign_node), NOT_NULL(jump_node)); + } + } } } -void AscendControlParser::InsertAssignToGraph(NotNull kg, NotNull from, - NotNull to) { +AnfNodePtr AscendControlParser::InsertAssignToGraph(NotNull kg, NotNull from, + NotNull to) { if (AnfAlgo::OutputAddrExist(from, 0) && AnfAlgo::OutputAddrExist(to, 0) && AnfAlgo::GetOutputAddr(from, 0) == AnfAlgo::GetOutputAddr(to, 0)) { - return; + return nullptr; } if (from.get() == to.get()) { - return; + return nullptr; } MS_LOG(INFO) << "Insert assign to graph " << kg->ToString() << " from " << from->DebugString() << " to " << to->DebugString(); @@ -466,6 +489,7 @@ void AscendControlParser::InsertAssignToGraph(NotNull kg, NotNul assign_node->set_abstract(to->abstract()); // append the assign at the end of from graph InsertDependToGraph(kg, NOT_NULL(assign_node)); + return assign_node; } std::vector AscendControlParser::RecurseGraph(NotNull graph, diff --git a/mindspore/ccsrc/session/ascend_control_parser.h b/mindspore/ccsrc/session/ascend_control_parser.h index 73d68449b3..0cf7069046 100644 --- a/mindspore/ccsrc/session/ascend_control_parser.h +++ b/mindspore/ccsrc/session/ascend_control_parser.h @@ -52,8 +52,9 @@ class AscendControlParser { const CNodePtr &last_label); static std::tuple ParsePartial(NotNull node); - static void InsertMultipleAssignToGraph(NotNull kg, NotNull from, NotNull to); - static void InsertAssignToGraph(NotNull kg, NotNull from, NotNull to); + static void InsertMultipleAssignToGraph(NotNull from_graph, NotNull to_graph, + NotNull from, NotNull to); + static AnfNodePtr InsertAssignToGraph(NotNull kg, NotNull from, NotNull to); // root graph order static bool CheckLabelIndex(uint32_t order_index, uint32_t label_index, const CNodePtr &cnode, From 617eb5510a426d62367278e8a28a55606421f83d Mon Sep 17 00:00:00 2001 From: dayschan <6573942+dayschan@user.noreply.gitee.com> Date: Tue, 23 Jun 2020 19:20:58 +0800 Subject: [PATCH 072/254] GraphKernel support akg batchmatmul --- akg | 2 +- mindspore/ccsrc/kernel/kernel_query.cc | 7 +++ mindspore/ops/_op_impl/akg/__init__.py | 1 + mindspore/ops/_op_impl/akg/batchmatmul.py | 73 +++++++++++++++++++++++ 4 files changed, 82 insertions(+), 1 deletion(-) create mode 100644 mindspore/ops/_op_impl/akg/batchmatmul.py diff --git a/akg b/akg index c460176523..df57a6cf94 160000 --- a/akg +++ b/akg @@ -1 +1 @@ -Subproject commit c460176523d039c8995f1d71089753725ebc0792 +Subproject commit df57a6cf9450e347d1854687d1fe66a420ee3b35 diff --git a/mindspore/ccsrc/kernel/kernel_query.cc b/mindspore/ccsrc/kernel/kernel_query.cc index 5eda847917..4a8ae81afa 100755 --- a/mindspore/ccsrc/kernel/kernel_query.cc +++ b/mindspore/ccsrc/kernel/kernel_query.cc @@ -23,6 +23,7 @@ #include "kernel/tbe/tbe_kernel_select/tbe_kernel_select.h" #include "kernel/akg/akg_kernel_metadata.h" #include "session/anf_runtime_algorithm.h" +#include "utils/context/ms_context.h" namespace mindspore { namespace kernel { @@ -97,6 +98,12 @@ void KernelQuery(const CNodePtr &kernel_node, std::vectorenable_graph_kernel() && IsPrimitiveCNode(kernel_node, prim::kPrimBatchMatMul)) { + kernel_type = KernelType::AKG_KERNEL; + } + switch (kernel_type) { case KernelType::AKG_KERNEL: AkgMetadataInfo(kernel_node, kernel_info_list); diff --git a/mindspore/ops/_op_impl/akg/__init__.py b/mindspore/ops/_op_impl/akg/__init__.py index f38b99f5e4..fd86dbf999 100644 --- a/mindspore/ops/_op_impl/akg/__init__.py +++ b/mindspore/ops/_op_impl/akg/__init__.py @@ -47,6 +47,7 @@ from .gather_v2 import _gather_v2_akg from .less import _less_akg from .log import _log_akg from .matmul import _matmul_akg +from .batchmatmul import _batchmatmul_akg from .max_pool_grad_with_argmax import _max_pool_grad_with_argmax_akg from .max_pool_with_argmax import _max_pool_with_argmax_akg from .max import _max_akg diff --git a/mindspore/ops/_op_impl/akg/batchmatmul.py b/mindspore/ops/_op_impl/akg/batchmatmul.py new file mode 100644 index 0000000000..f5da71aa25 --- /dev/null +++ b/mindspore/ops/_op_impl/akg/batchmatmul.py @@ -0,0 +1,73 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""BatchMatMul op""" +from mindspore.ops.op_info_register import op_info_register + + +@op_info_register("""{ + "op_name": "BatchMatMul", + "imply_type": "AutoDiff", + "fusion_type": "OPAQUE", + "attr": [ + { + "name": "transpose_a", + "param_type": "optional", + "type": "bool" + }, + { + "name": "transpose_b", + "param_type": "optional", + "type": "bool" + } + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float16" + ], + "format": [ + "FRACTAL_NZ" + ], + "name": "x1" + }, + { + "index": 1, + "dtype": [ + "float16" + ], + "format": [ + "FRACTAL_NZ" + ], + "name": "x2" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float16" + ], + "format": [ + "FRACTAL_NZ" + ], + "name": "output" + } + ] +}""") +def _batchmatmul_akg(): + """BatchMatMul AKG register""" + return From 81bf4bde1d09d13463efe499f4fa81a495350ff6 Mon Sep 17 00:00:00 2001 From: Jesse Lee Date: Fri, 26 Jun 2020 12:22:44 -0400 Subject: [PATCH 073/254] AutoIndexObj primary should start with 0 --- mindspore/ccsrc/dataset/util/auto_index.h | 2 +- tests/ut/cpp/dataset/btree_test.cc | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/mindspore/ccsrc/dataset/util/auto_index.h b/mindspore/ccsrc/dataset/util/auto_index.h index 11a2e90b00..5c43ecfd80 100644 --- a/mindspore/ccsrc/dataset/util/auto_index.h +++ b/mindspore/ccsrc/dataset/util/auto_index.h @@ -91,7 +91,7 @@ class AutoIndexObj : public BPlusTree { } private: - static constexpr key_type kMinKey = 1; + static constexpr key_type kMinKey = 0; std::atomic inx_; }; } // namespace dataset diff --git a/tests/ut/cpp/dataset/btree_test.cc b/tests/ut/cpp/dataset/btree_test.cc index 168f550f34..75d5133e58 100644 --- a/tests/ut/cpp/dataset/btree_test.cc +++ b/tests/ut/cpp/dataset/btree_test.cc @@ -190,9 +190,9 @@ TEST_F(MindDataTestBPlusTree, Test3) { EXPECT_TRUE(rc.IsOk()); uint64_t min = ai.min_key(); uint64_t max = ai.max_key(); - EXPECT_EQ(min, 1); - EXPECT_EQ(max, 4); - auto r = ai.Search(3); + EXPECT_EQ(min, 0); + EXPECT_EQ(max, 3); + auto r = ai.Search(2); auto &it = r.first; EXPECT_EQ(it.value(), "b"); MS_LOG(INFO) << "Dump all the values using [] operator."; From 277aba5326b03763579996ac3629cdcfb21be62b Mon Sep 17 00:00:00 2001 From: Cathy Wong Date: Thu, 25 Jun 2020 21:41:42 -0400 Subject: [PATCH 074/254] dataset: Fixup docs; remove pylint disabled messages in UT --- mindspore/dataset/engine/datasets.py | 6 +- .../dataset/transforms/vision/c_transforms.py | 6 +- tests/ut/data/dataset/declient.cfg | 3 +- tests/ut/python/dataset/test_batch.py | 8 +-- tests/ut/python/dataset/test_center_crop.py | 11 +--- tests/ut/python/dataset/test_config.py | 7 ++- tests/ut/python/dataset/test_filterop.py | 57 +++++-------------- tests/ut/python/dataset/test_pad.py | 14 ++--- 8 files changed, 39 insertions(+), 73 deletions(-) diff --git a/mindspore/dataset/engine/datasets.py b/mindspore/dataset/engine/datasets.py index ca6f7ca33e..360cdb1860 100644 --- a/mindspore/dataset/engine/datasets.py +++ b/mindspore/dataset/engine/datasets.py @@ -1040,7 +1040,7 @@ class Dataset: Args: columns (list[str], optional): List of columns to be used to specify the order of columns - (defaults=None, means all columns). + (default=None, means all columns). Returns: Iterator, list of ndarray. @@ -3382,7 +3382,7 @@ class ManifestDataset(MappableDataset): class_indexing (dict, optional): A str-to-int mapping from label name to index (default=None, the folder names will be sorted alphabetically and each class will be given a unique index starting from 0). - decode (bool, optional): decode the images after reading (defaults=False). + decode (bool, optional): decode the images after reading (default=False). num_shards (int, optional): Number of shards that the dataset should be divided into (default=None). shard_id (int, optional): The shard ID within num_shards (default=None). This @@ -4760,7 +4760,7 @@ class _NumpySlicesDataset: def process_dict(self, input_data): """ - Convert the dict like data into tuple format, when input is a tuple of dict then compose it into a dict first. + Convert the dict like data into tuple format, when input is a tuple of dicts then compose it into a dict first. """ # Convert pandas like dict(has "values" column) into General dict data_keys = list(input_data.keys()) diff --git a/mindspore/dataset/transforms/vision/c_transforms.py b/mindspore/dataset/transforms/vision/c_transforms.py index aef714953f..3fdf7795d0 100644 --- a/mindspore/dataset/transforms/vision/c_transforms.py +++ b/mindspore/dataset/transforms/vision/c_transforms.py @@ -202,7 +202,7 @@ class RandomHorizontalFlip(cde.RandomHorizontalFlipOp): Flip the input image horizontally, randomly with a given probability. Args: - prob (float): Probability of the image being flipped (default=0.5). + prob (float, optional): Probability of the image being flipped (default=0.5). """ @check_prob @@ -217,7 +217,7 @@ class RandomHorizontalFlipWithBBox(cde.RandomHorizontalFlipWithBBoxOp): Maintains data integrity by also flipping bounding boxes in an object detection pipeline. Args: - prob (float): Probability of the image being flipped (default=0.5). + prob (float, optional): Probability of the image being flipped (default=0.5). """ @check_prob @@ -231,7 +231,7 @@ class RandomVerticalFlip(cde.RandomVerticalFlipOp): Flip the input image vertically, randomly with a given probability. Args: - prob (float): Probability of the image being flipped (default=0.5). + prob (float, optional): Probability of the image being flipped (default=0.5). """ @check_prob diff --git a/tests/ut/data/dataset/declient.cfg b/tests/ut/data/dataset/declient.cfg index b657ead6d5..e09b24812a 100644 --- a/tests/ut/data/dataset/declient.cfg +++ b/tests/ut/data/dataset/declient.cfg @@ -4,6 +4,7 @@ "numParallelWorkers": 4, "workerConnectorSize": 16, "opConnectorSize": 16, - "seed": 5489 + "seed": 5489, + "monitor_sampling_interval": 15 } diff --git a/tests/ut/python/dataset/test_batch.py b/tests/ut/python/dataset/test_batch.py index 07eba394f1..9b9baeec33 100644 --- a/tests/ut/python/dataset/test_batch.py +++ b/tests/ut/python/dataset/test_batch.py @@ -12,10 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -from util import save_and_check - import mindspore.dataset as ds from mindspore import log as logger +from util import save_and_check # Note: Number of rows in test.data dataset: 12 DATA_DIR = ["../data/dataset/testTFTestAllTypes/test.data"] @@ -434,7 +433,6 @@ def test_batch_exception_11(): assert "drop_remainder" in str(e) -# pylint: disable=redundant-keyword-arg def test_batch_exception_12(): """ Test batch exception: wrong input order, drop_remainder wrongly used as batch_size @@ -447,12 +445,12 @@ def test_batch_exception_12(): # apply dataset operations data1 = ds.TFRecordDataset(DATA_DIR) try: - data1 = data1.batch(drop_remainder, batch_size=batch_size) + data1 = data1.batch(drop_remainder, batch_size) sum([1 for _ in data1]) except Exception as e: logger.info("Got an exception in DE: {}".format(str(e))) - assert "batch_size" in str(e) + assert "drop_remainder" in str(e) def test_batch_exception_13(): diff --git a/tests/ut/python/dataset/test_center_crop.py b/tests/ut/python/dataset/test_center_crop.py index d4f8735fb0..6dfa9fc7c3 100644 --- a/tests/ut/python/dataset/test_center_crop.py +++ b/tests/ut/python/dataset/test_center_crop.py @@ -109,23 +109,18 @@ def test_center_crop_comp(height=375, width=375, plot=False): visualize_list(image_c_cropped, image_py_cropped, visualize_mode=2) -# pylint: disable=unnecessary-lambda def test_crop_grayscale(height=375, width=375): """ Test that centercrop works with pad and grayscale images """ - def channel_swap(image): - """ - Py func hack for our pytransforms to work with c transforms - """ - return (image.transpose(1, 2, 0) * 255).astype(np.uint8) - + # Note: image.transpose performs channel swap to allow py transforms to + # work with c transforms transforms = [ py_vision.Decode(), py_vision.Grayscale(1), py_vision.ToTensor(), - (lambda image: channel_swap(image)) + (lambda image: (image.transpose(1, 2, 0) * 255).astype(np.uint8)) ] transform = py_vision.ComposeOp(transforms) diff --git a/tests/ut/python/dataset/test_config.py b/tests/ut/python/dataset/test_config.py index c4d665b391..59be886c23 100644 --- a/tests/ut/python/dataset/test_config.py +++ b/tests/ut/python/dataset/test_config.py @@ -37,6 +37,7 @@ def test_basic(): num_parallel_workers_original = ds.config.get_num_parallel_workers() prefetch_size_original = ds.config.get_prefetch_size() seed_original = ds.config.get_seed() + monitor_sampling_interval_original = ds.config.get_monitor_sampling_interval() ds.config.load('../data/dataset/declient.cfg') @@ -45,23 +46,27 @@ def test_basic(): # assert ds.config.get_worker_connector_size() == 16 assert ds.config.get_prefetch_size() == 16 assert ds.config.get_seed() == 5489 + # assert ds.config.get_monitor_sampling_interval() == 15 # ds.config.set_rows_per_buffer(1) ds.config.set_num_parallel_workers(2) # ds.config.set_worker_connector_size(3) ds.config.set_prefetch_size(4) ds.config.set_seed(5) + ds.config.set_monitor_sampling_interval(45) # assert ds.config.get_rows_per_buffer() == 1 assert ds.config.get_num_parallel_workers() == 2 # assert ds.config.get_worker_connector_size() == 3 assert ds.config.get_prefetch_size() == 4 assert ds.config.get_seed() == 5 + assert ds.config.get_monitor_sampling_interval() == 45 # Restore original configuration values ds.config.set_num_parallel_workers(num_parallel_workers_original) ds.config.set_prefetch_size(prefetch_size_original) ds.config.set_seed(seed_original) + ds.config.set_monitor_sampling_interval(monitor_sampling_interval_original) def test_get_seed(): @@ -150,7 +155,7 @@ def test_deterministic_run_fail(): def test_deterministic_run_pass(): """ - Test deterministic run with with setting the seed + Test deterministic run with setting the seed """ logger.info("test_deterministic_run_pass") diff --git a/tests/ut/python/dataset/test_filterop.py b/tests/ut/python/dataset/test_filterop.py index 015d580379..876278571d 100644 --- a/tests/ut/python/dataset/test_filterop.py +++ b/tests/ut/python/dataset/test_filterop.py @@ -50,9 +50,7 @@ def test_diff_predicate_func(): def filter_func_ge(data): - if data > 10: - return False - return True + return data <= 10 def generator_1d(): @@ -108,15 +106,11 @@ def test_filter_by_generator_with_repeat_after(): def filter_func_batch(data): - if data[0] > 8: - return False - return True + return data[0] <= 8 def filter_func_batch_after(data): - if data > 20: - return False - return True + return data <= 20 # test with batchOp before @@ -152,9 +146,7 @@ def test_filter_by_generator_with_batch_after(): def filter_func_shuffle(data): - if data > 20: - return False - return True + return data <= 20 # test with batchOp before @@ -169,9 +161,7 @@ def test_filter_by_generator_with_shuffle(): def filter_func_shuffle_after(data): - if data > 20: - return False - return True + return data <= 20 # test with batchOp after @@ -197,15 +187,11 @@ def generator_1d_zip2(): def filter_func_zip(data1, data2): _ = data2 - if data1 > 20: - return False - return True + return data1 <= 20 def filter_func_zip_after(data1): - if data1 > 20: - return False - return True + return data1 <= 20 # test with zipOp before @@ -247,16 +233,11 @@ def test_filter_by_generator_with_zip_after(): def filter_func_map(col1, col2): _ = col2 - if col1[0] > 8: - return True - return False + return col1[0] > 8 -# pylint: disable=simplifiable-if-statement def filter_func_map_part(col1): - if col1 < 3: - return True - return False + return col1 < 3 def filter_func_map_all(col1, col2): @@ -311,9 +292,7 @@ def test_filter_by_generator_with_map_part_col(): def filter_func_rename(data): - if data > 8: - return True - return False + return data > 8 # test with rename before @@ -334,15 +313,11 @@ def test_filter_by_generator_with_rename(): # test input_column def filter_func_input_column1(col1, col2): _ = col2 - if col1[0] < 8: - return True - return False + return col1[0] < 8 def filter_func_input_column2(col1): - if col1[0] < 8: - return True - return False + return col1[0] < 8 def filter_func_input_column3(col1): @@ -439,9 +414,7 @@ def test_filter_by_generator_Partial2(): def filter_func_Partial(col1, col2): _ = col2 - if col1[0] % 3 == 0: - return True - return False + return col1[0] % 3 == 0 def generator_big(maxid=20): @@ -461,9 +434,7 @@ def test_filter_by_generator_Partial(): def filter_func_cifar(col1, col2): _ = col1 - if col2 % 3 == 0: - return True - return False + return col2 % 3 == 0 # test with cifar10 diff --git a/tests/ut/python/dataset/test_pad.py b/tests/ut/python/dataset/test_pad.py index 1b3882cd54..7b66b6b36b 100644 --- a/tests/ut/python/dataset/test_pad.py +++ b/tests/ut/python/dataset/test_pad.py @@ -16,12 +16,12 @@ Testing Pad op in DE """ import numpy as np -from util import diff_mse import mindspore.dataset as ds import mindspore.dataset.transforms.vision.c_transforms as c_vision import mindspore.dataset.transforms.vision.py_transforms as py_vision from mindspore import log as logger +from util import diff_mse DATA_DIR = ["../data/dataset/test_tf_file_3_images/train-0000-of-0001.data"] SCHEMA_DIR = "../data/dataset/test_tf_file_3_images/datasetSchema.json" @@ -69,23 +69,19 @@ def test_pad_op(): assert mse < 0.01 -# pylint: disable=unnecessary-lambda + def test_pad_grayscale(): """ Tests that the pad works for grayscale images """ - def channel_swap(image): - """ - Py func hack for our pytransforms to work with c transforms - """ - return (image.transpose(1, 2, 0) * 255).astype(np.uint8) - + # Note: image.transpose performs channel swap to allow py transforms to + # work with c transforms transforms = [ py_vision.Decode(), py_vision.Grayscale(1), py_vision.ToTensor(), - (lambda image: channel_swap(image)) + (lambda image: (image.transpose(1, 2, 0) * 255).astype(np.uint8)) ] transform = py_vision.ComposeOp(transforms) From 1e869146e992e2f9ee3408dff09c4e1a85929b2e Mon Sep 17 00:00:00 2001 From: avakh Date: Thu, 25 Jun 2020 18:23:12 -0400 Subject: [PATCH 075/254] applying comments removing VOC --- ...random_resize_with_bbox_op_01_c_result.npz | Bin 0 -> 1654 bytes .../resize_with_bbox_op_01_c_result.npz | Bin 0 -> 1654 bytes .../dataset/test_random_resize_with_bbox.py | 323 +++++++---------- .../python/dataset/test_resize_with_bbox.py | 328 ++++++------------ 4 files changed, 225 insertions(+), 426 deletions(-) create mode 100644 tests/ut/data/dataset/golden/random_resize_with_bbox_op_01_c_result.npz create mode 100644 tests/ut/data/dataset/golden/resize_with_bbox_op_01_c_result.npz diff --git a/tests/ut/data/dataset/golden/random_resize_with_bbox_op_01_c_result.npz b/tests/ut/data/dataset/golden/random_resize_with_bbox_op_01_c_result.npz new file mode 100644 index 0000000000000000000000000000000000000000..a5623304ba43eddb4656aafa663157636dae5528 GIT binary patch literal 1654 zcmbW&drVVT90%~*2P&NpaG>L%76jVPQV^jaAc!!WTxj7aP;iy?aj`gP`L(C84wwu? zKqrc*h(wl!Fat)mnEkO0wrtC?5Efr7vKX?FusB@ih{kN$!pL^d<#?AR+duo=^xoXx zJ)iS!o0EGAS(!YDS*{3XAoT5_1wV!%AAcr@vAEq8@j7R%Pr?KUO)`)?B-LXmE+{OM zFm+6W%FcOgZdIyM)o4^{m8wdY8@nye3YXi?k#?@7+QadEkHb>S@%QQktujHQRW>RA ze<_+}HmA3y)~B<%+?=k)TaByu?`!cXD%_mS<@8{;*M=3qERRw+?aNO=!ZK|j&6Zd} z)(&z6eri_2N?GY~SV4!{v9Fd>z)F*p|Cz=VSj94|q#gVbDAcUXB;mV_@QlG=$orEo z;lyTQSZ(bz!y2`T{6AGSmctyZHN!f!gK-Fv0S-Pd5W#wqCvSsi9m@fgi4e3L5R72M z(|~l?_|E_a!*kZo?10>mBOlKVeRFF(_o`YGqF-oO-RjsniDz|=>)VVMV(HLSxqasF) zKts**f?1mq>~dWFJuPb_B64oJU=X%p(26FO7#)H*YT^Z>-!wFG-Fn*aR(IETlGy1< z*p4AVG&_h%M36*{UNA*#x^Hj)tfqe{ZOei0OSzMfj3GrdJBdj}kVZ|qVDft}RmsDP zG>eBcGfDc*6R-hCI=*#N;C=pvEAWg}NMD^p2lyW;_b{R{nY4C>Svm zil&GdgkV242Lz+-nGNkS`x_s3&P~MJ&mVwd3?)|1q~KSd#p!fm3$AiGq0|f}aS4`@ znaU9yq%*xF%yjGPP~{z4YS{W;TYreV-%$o;42Q(@3SwSHV4=qP@5Y9~E*g%QN(6_g zaR_FmB=Lcy=%k<#F zP%9cB#*M&34His`D*ayM7p|$S8xKw4KaP9Bi=j?5^~Ai2z(>t%g1Of8WsK(9=r75o z^P?+c>m|^Dp;0tV#JrB+C^gN389>2&=G*fPx~VxUn5B-XosX09S4Sn72Xjlu9zqX>bD}v<%moC!)LazIUCY6~8!0Bsr|V)X yM}`{%p%23)E5~8zx9SR6i8M&|Ut0yg=4dsOx8vU%MTM29i~wII@VmLiD!o~R&G4qJRDP;ec)mcz0^+o!w*D>#wg zOhp(|VPq@|Ap{d}AtuYj8C{mek+?aGCd6zmE)JJDrZMgxWZS(jueU65|LnQ+F8AE~ zdB1J*=H7h4&p(7)s7UVE`A?3_25=nt1aTpp=<$?pG&=B)%E>D?6+BsrnwIuhe9im&3?{x_VO@iK_Ii&gj zrEc)EJH1tIpV98}NX9B}C9aUaujo^kdL+BcS&cnjJ5~d?FiPz#TX+h*z_oxXi?@Ma z8~7s#&%}a$cC0-pIcJHB8y7?ndTIDPT&N-4T2D;b%LLTm%H`wyvbzB{Yx%- z5;8F?wslxxiOxd)pNcARzXUH>VX4l+IcQ|CL(W@R3s z0T~eb?*IpNKnisViFN-W+sW6JxjW; z#b2)KpHE-A_opK1C?sP@QA{c^X$aDp$)Lu#X5ipS_hj$H$e$6l($F}(f?h^rso60U^!3Gx-+v5fy{YSXGCBuY7_t?!gP0ryuQIcfnv99KH0$JpzRujxx!Kd} z>R=a!T*U~)}O}~D#<-w}&{lDlN12cwv#q1#lA$X0Mz0??bZijVR zgUk;*?p%qvn>PRj7z%BYMJ=yB(dl$y5m&gJP-F#*vIL9COeF~Rv6)_{GktqKO!J*R zEqwVOO+UxoZ7&8ZhBuV-QexgjATncnwy|R&TO{D~uN(85+#Pr%cd> zp>> # Adds a box that covers the whole image. Good for testing edge cases - >>> de = de.map(input_columns=["image", "annotation"], - >>> output_columns=["image", "annotation"], - >>> operations=AddBadAnnotation(BoxType.OnEdge)) - """ + dataVoc2 = ds.VOCDataset(DATA_DIR, task="Detection", mode="train", + decode=True, shuffle=False) - def __init__(self, box_type): - self.box_type = box_type - - def __call__(self, img, bboxes): - """ - Used to generate erroneous bounding box examples on given img. - :param img: image where the bounding boxes are. - :param bboxes: in [x_min, y_min, w, h, label, truncate, difficult] format - :return: bboxes with bad examples added - """ - height = img.shape[0] - width = img.shape[1] - if self.box_type == BoxType.WidthOverflow: - # use box that overflows on width - return img, np.array([[0, 0, width + 1, height - 1, 0, 0, 0]]).astype(np.uint32) - - if self.box_type == BoxType.HeightOverflow: - # use box that overflows on height - return img, np.array([[0, 0, width - 1, height + 1, 0, 0, 0]]).astype(np.uint32) - - if self.box_type == BoxType.NegativeXY: - # use box with negative xy - return img, np.array([[-10, -10, width - 1, height - 1, 0, 0, 0]]).astype(np.uint32) - - if self.box_type == BoxType.OnEdge: - # use box that covers the whole image - return img, np.array([[0, 0, width - 1, height - 1, 0, 0, 0]]).astype(np.uint32) - - if self.box_type == BoxType.WrongShape: - # use box that covers the whole image - return img, np.array([[0, 0, width - 1]]).astype(np.uint32) - return img, bboxes - - -def check_bad_box(data, box_type, expected_error): - try: - test_op = c_vision.RandomResizeWithBBox(100) # DEFINE TEST OP HERE -- (PROB 1 IN CASE OF RANDOM) - data = data.map(input_columns=["annotation"], - output_columns=["annotation"], - operations=fix_annotate) - # map to use width overflow - data = data.map(input_columns=["image", "annotation"], - output_columns=["image", "annotation"], - columns_order=["image", "annotation"], - operations=AddBadAnnotation(box_type)) # Add column for "annotation" - # map to apply ops - data = data.map(input_columns=["image", "annotation"], - output_columns=["image", "annotation"], - columns_order=["image", "annotation"], - operations=[test_op]) # Add column for "annotation" - for _, _ in enumerate(data.create_dict_iterator()): - break - except RuntimeError as e: - logger.info("Got an exception in DE: {}".format(str(e))) - assert expected_error in str(e) - - -def add_bounding_boxes(axis, bboxes): - """ - :param axis: axis to modify - :param bboxes: bounding boxes to draw on the axis - :return: None - """ - for bbox in bboxes: - rect = patches.Rectangle((bbox[0], bbox[1]), - bbox[2], bbox[3], - linewidth=1, edgecolor='r', facecolor='none') - # Add the patch to the Axes - axis.add_patch(rect) - - -def visualize(unaugmented_data, augment_data): - for idx, (un_aug_item, aug_item) in \ - enumerate(zip(unaugmented_data.create_dict_iterator(), augment_data.create_dict_iterator())): - axis = plt.subplot(141) - plt.imshow(un_aug_item["image"]) - add_bounding_boxes(axis, un_aug_item["annotation"]) # add Orig BBoxes - plt.title("Original" + str(idx + 1)) - logger.info("Original ", str(idx + 1), " :", un_aug_item["annotation"]) - - axis = plt.subplot(142) - plt.imshow(aug_item["image"]) - add_bounding_boxes(axis, aug_item["annotation"]) # add AugBBoxes - plt.title("Augmented" + str(idx + 1)) - logger.info("Augmented ", str(idx + 1), " ", aug_item["annotation"], "\n") - plt.show() - - -def test_random_resize_with_bbox_op(plot=False): + test_op = c_vision.RandomResizeWithBBox(200) + + dataVoc1 = dataVoc1.map(input_columns=["annotation"], + output_columns=["annotation"], + operations=fix_annotate) + dataVoc2 = dataVoc2.map(input_columns=["annotation"], + output_columns=["annotation"], + operations=fix_annotate) + # map to apply ops + dataVoc2 = dataVoc2.map(input_columns=["image", "annotation"], + output_columns=["image", "annotation"], + columns_order=["image", "annotation"], + operations=[test_op]) + + filename = "random_resize_with_bbox_op_01_c_result.npz" + save_and_check_md5(dataVoc2, filename, generate_golden=GENERATE_GOLDEN) + + unaugSamp, augSamp = [], [] + + for unAug, Aug in zip(dataVoc1.create_dict_iterator(), dataVoc2.create_dict_iterator()): + unaugSamp.append(unAug) + augSamp.append(Aug) + + if plot_vis: + visualize_with_bounding_boxes(unaugSamp, augSamp) + + # Restore config setting + ds.config.set_seed(original_seed) + ds.config.set_num_parallel_workers(original_num_parallel_workers) + + +def test_random_resize_with_bbox_op_edge_c(plot_vis=False): """ - Test random_resize_with_bbox_op + Prints images and bboxes side by side with and without RandomresizeWithBBox Op applied, + applied on dynamically generated edge case, expected to pass. edge case is when bounding + box has dimensions as the image itself. """ - logger.info("Test random resize with bbox") + logger.info("test_random_resize_with_bbox_op_edge_c") + dataVoc1 = ds.VOCDataset(DATA_DIR, task="Detection", mode="train", + decode=True, shuffle=False) - # original images - data_original = ds.VOCDataset(DATA_DIR, task="Detection", mode="train", decode=True, shuffle=False) + dataVoc2 = ds.VOCDataset(DATA_DIR, task="Detection", mode="train", + decode=True, shuffle=False) - # augmented images - data_augmented = ds.VOCDataset(DATA_DIR, task="Detection", mode="train", decode=True, shuffle=False) + test_op = c_vision.RandomResizeWithBBox(500) - data_original = data_original.map(input_columns=["annotation"], - output_columns=["annotation"], - operations=fix_annotate) + dataVoc1 = dataVoc1.map(input_columns=["annotation"], + output_columns=["annotation"], + operations=fix_annotate) + dataVoc2 = dataVoc2.map(input_columns=["annotation"], + output_columns=["annotation"], + operations=fix_annotate) - data_augmented = data_augmented.map(input_columns=["annotation"], - output_columns=["annotation"], - operations=fix_annotate) + # maps to convert data into valid edge case data + dataVoc1 = dataVoc1.map(input_columns=["image", "annotation"], + output_columns=["image", "annotation"], + columns_order=["image", "annotation"], + operations=[lambda img, bboxes: ( + img, np.array([[0, 0, img.shape[1], img.shape[0]]]).astype(bboxes.dtype))]) - # define map operations - test_op = c_vision.RandomResizeWithBBox(100) # input value being the target size of resizeOp + dataVoc2 = dataVoc2.map(input_columns=["image", "annotation"], + output_columns=["image", "annotation"], + columns_order=["image", "annotation"], + operations=[lambda img, bboxes: ( + img, np.array([[0, 0, img.shape[1], img.shape[0]]]).astype(bboxes.dtype)), test_op]) - data_augmented = data_augmented.map(input_columns=["image", "annotation"], - output_columns=["image", "annotation"], - columns_order=["image", "annotation"], operations=[test_op]) - if plot: - visualize(data_original, data_augmented) + unaugSamp, augSamp = [], [] + for unAug, Aug in zip(dataVoc1.create_dict_iterator(), dataVoc2.create_dict_iterator()): + unaugSamp.append(unAug) + augSamp.append(Aug) -def test_random_resize_with_bbox_invalid_bounds(): - data_voc2 = ds.VOCDataset(DATA_DIR, task="Detection", mode="train", decode=True, shuffle=False) - check_bad_box(data_voc2, BoxType.WidthOverflow, "bounding boxes is out of bounds of the image") - data_voc2 = ds.VOCDataset(DATA_DIR, task="Detection", mode="train", decode=True, shuffle=False) - check_bad_box(data_voc2, BoxType.HeightOverflow, "bounding boxes is out of bounds of the image") - data_voc2 = ds.VOCDataset(DATA_DIR, task="Detection", mode="train", decode=True, shuffle=False) - check_bad_box(data_voc2, BoxType.NegativeXY, "min_x") - data_voc2 = ds.VOCDataset(DATA_DIR, task="Detection", mode="train", decode=True, shuffle=False) - check_bad_box(data_voc2, BoxType.WrongShape, "4 features") + if plot_vis: + visualize_with_bounding_boxes(unaugSamp, augSamp) -def test_random_resize_with_bbox_invalid_size(): +def test_random_resize_with_bbox_op_invalid_c(): + """ + Test RandomResizeWithBBox Op on invalid constructor parameters, expected to raise ValueError """ - Test random_resize_with_bbox_op - """ - logger.info("Test random resize with bbox with invalid target size") + logger.info("test_random_resize_with_bbox_op_invalid_c") - # original images - data = ds.VOCDataset(DATA_DIR, task="Detection", mode="train", decode=True, shuffle=False) + try: + # zero value for resize + c_vision.RandomResizeWithBBox(0) - data = data.map(input_columns=["annotation"], - output_columns=["annotation"], - operations=fix_annotate) + except ValueError as err: + logger.info("Got an exception in DE: {}".format(str(err))) + assert "Input is not" in str(err) - # negative target size as input try: - test_op = c_vision.RandomResizeWithBBox(-10) # DEFINE TEST OP HERE -- (PROB 1 IN CASE OF RANDOM) + # one of the size values is zero + c_vision.RandomResizeWithBBox((0, 100)) - # map to apply ops - data = data.map(input_columns=["image", "annotation"], - output_columns=["image", "annotation"], - columns_order=["image", "annotation"], - operations=[test_op]) # Add column for "annotation" + except ValueError as err: + logger.info("Got an exception in DE: {}".format(str(err))) + assert "Input is not" in str(err) - for _, _ in enumerate(data.create_dict_iterator()): - break + try: + # negative value for resize + c_vision.RandomResizeWithBBox(-10) - except ValueError as e: - logger.info("Got an exception in DE: {}".format(str(e))) - print(e) - assert "Input is not" in str(e) + except ValueError as err: + logger.info("Got an exception in DE: {}".format(str(err))) + assert "Input is not" in str(err) - # zero target size as input try: - test_op = c_vision.RandomResizeWithBBox(0) # DEFINE TEST OP HERE -- (PROB 1 IN CASE OF RANDOM) + # invalid input shape + c_vision.RandomResizeWithBBox((100, 100, 100)) - # map to apply ops - data = data.map(input_columns=["image", "annotation"], - output_columns=["image", "annotation"], - columns_order=["image", "annotation"], - operations=[test_op]) # Add column for "annotation" + except TypeError as err: + logger.info("Got an exception in DE: {}".format(str(err))) + assert "Size should be" in str(err) - for _, _ in enumerate(data.create_dict_iterator()): - break - except ValueError as e: - logger.info("Got an exception in DE: {}".format(str(e))) - assert "Input is not" in str(e) - - # invalid input shape - try: - test_op = c_vision.RandomResizeWithBBox((10, 10, 10)) # DEFINE TEST OP HERE -- (PROB 1 IN CASE OF RANDOM) - - # map to apply ops - data = data.map(input_columns=["image", "annotation"], - output_columns=["image", "annotation"], - columns_order=["image", "annotation"], - operations=[test_op]) # Add column for "annotation" +def test_random_resize_with_bbox_op_bad_c(): + """ + Tests RandomResizeWithBBox Op with invalid bounding boxes, expected to catch multiple errors + """ + logger.info("test_random_resize_with_bbox_op_bad_c") + test_op = c_vision.RandomResizeWithBBox((400, 300)) - for _, _ in enumerate(data.create_dict_iterator()): - break + data_voc2 = ds.VOCDataset(DATA_DIR, task="Detection", mode="train", decode=True, shuffle=False) + check_bad_bbox(data_voc2, test_op, InvalidBBoxType.WidthOverflow, "bounding boxes is out of bounds of the image") + data_voc2 = ds.VOCDataset(DATA_DIR, task="Detection", mode="train", decode=True, shuffle=False) + check_bad_bbox(data_voc2, test_op, InvalidBBoxType.HeightOverflow, "bounding boxes is out of bounds of the image") + data_voc2 = ds.VOCDataset(DATA_DIR, task="Detection", mode="train", decode=True, shuffle=False) + check_bad_bbox(data_voc2, test_op, InvalidBBoxType.NegativeXY, "min_x") + data_voc2 = ds.VOCDataset(DATA_DIR, task="Detection", mode="train", decode=True, shuffle=False) + check_bad_bbox(data_voc2, test_op, InvalidBBoxType.WrongShape, "4 features") - except TypeError as e: - logger.info("Got an exception in DE: {}".format(str(e))) - assert "Size should be" in str(e) if __name__ == "__main__": - test_random_resize_with_bbox_op(plot=False) - test_random_resize_with_bbox_invalid_bounds() - test_random_resize_with_bbox_invalid_size() + test_random_resize_with_bbox_op_rand_c(plot_vis=False) + test_random_resize_with_bbox_op_edge_c(plot_vis=False) + test_random_resize_with_bbox_op_invalid_c() + test_random_resize_with_bbox_op_bad_c() diff --git a/tests/ut/python/dataset/test_resize_with_bbox.py b/tests/ut/python/dataset/test_resize_with_bbox.py index 8b07f17f1a..75500de653 100644 --- a/tests/ut/python/dataset/test_resize_with_bbox.py +++ b/tests/ut/python/dataset/test_resize_with_bbox.py @@ -15,281 +15,151 @@ """ Testing the resize with bounding boxes op in DE """ -from enum import Enum import numpy as np -import matplotlib.patches as patches -import matplotlib.pyplot as plt +import mindspore.dataset as ds import mindspore.dataset.transforms.vision.c_transforms as c_vision + from mindspore import log as logger -import mindspore.dataset as ds +from util import visualize_with_bounding_boxes, InvalidBBoxType, check_bad_bbox, \ + save_and_check_md5 GENERATE_GOLDEN = False -DATA_DIR = "../data/dataset/testVOC2012" +DATA_DIR = "../data/dataset/testVOC2012_2" def fix_annotate(bboxes): """ + Fix annotations to format followed by mindspore. :param bboxes: in [label, x_min, y_min, w, h, truncate, difficult] format :return: annotation in [x_min, y_min, w, h, label, truncate, difficult] format """ - for bbox in bboxes: - tmp = bbox[0] - bbox[0] = bbox[1] - bbox[1] = bbox[2] - bbox[2] = bbox[3] - bbox[3] = bbox[4] - bbox[4] = tmp + for (i, box) in enumerate(bboxes): + bboxes[i] = np.roll(box, -1) return bboxes -class BoxType(Enum): +def test_resize_with_bbox_op_c(plot_vis=False): """ - Defines box types for test cases + Prints images and bboxes side by side with and without ResizeWithBBox Op applied, + tests with MD5 check, expected to pass """ - WidthOverflow = 1 - HeightOverflow = 2 - NegativeXY = 3 - OnEdge = 4 - WrongShape = 5 + logger.info("test_resize_with_bbox_op_c") + # Load dataset + dataVoc1 = ds.VOCDataset(DATA_DIR, task="Detection", mode="train", + decode=True, shuffle=False) -class AddBadAnnotation: # pylint: disable=too-few-public-methods - """ - Used to add erroneous bounding boxes to object detection pipelines. - Usage: - >>> # Adds a box that covers the whole image. Good for testing edge cases - >>> de = de.map(input_columns=["image", "annotation"], - >>> output_columns=["image", "annotation"], - >>> operations=AddBadAnnotation(BoxType.OnEdge)) - """ + dataVoc2 = ds.VOCDataset(DATA_DIR, task="Detection", mode="train", + decode=True, shuffle=False) - def __init__(self, box_type): - self.box_type = box_type - - def __call__(self, img, bboxes): - """ - Used to generate erroneous bounding box examples on given img. - :param img: image where the bounding boxes are. - :param bboxes: in [x_min, y_min, w, h, label, truncate, difficult] format - :return: bboxes with bad examples added - """ - height = img.shape[0] - width = img.shape[1] - if self.box_type == BoxType.WidthOverflow: - # use box that overflows on width - return img, np.array([[0, 0, width + 1, height - 1, 0, 0, 0]]).astype(np.uint32) - - if self.box_type == BoxType.HeightOverflow: - # use box that overflows on height - return img, np.array([[0, 0, width - 1, height + 1, 0, 0, 0]]).astype(np.uint32) - - if self.box_type == BoxType.NegativeXY: - # use box with negative xy - return img, np.array([[-10, -10, width - 1, height - 1, 0, 0, 0]]).astype(np.uint32) - - if self.box_type == BoxType.OnEdge: - # use box that covers the whole image - return img, np.array([[0, 0, width - 1, height - 1, 0, 0, 0]]).astype(np.uint32) - - if self.box_type == BoxType.WrongShape: - # use box that covers the whole image - return img, np.array([[0, 0, width - 1]]).astype(np.uint32) - return img, bboxes - - -def check_bad_box(data, box_type, expected_error): - try: - test_op = c_vision.ResizeWithBBox(100) - data = data.map(input_columns=["annotation"], - output_columns=["annotation"], - operations=fix_annotate) - # map to use width overflow - data = data.map(input_columns=["image", "annotation"], - output_columns=["image", "annotation"], - columns_order=["image", "annotation"], - operations=AddBadAnnotation(box_type)) # Add column for "annotation" - # map to apply ops - data = data.map(input_columns=["image", "annotation"], - output_columns=["image", "annotation"], - columns_order=["image", "annotation"], - operations=[test_op]) # Add column for "annotation" - for _, _ in enumerate(data.create_dict_iterator()): - break - except RuntimeError as e: - logger.info("Got an exception in DE: {}".format(str(e))) - assert expected_error in str(e) - - -def add_bounding_boxes(axis, bboxes): - """ - :param axis: axis to modify - :param bboxes: bounding boxes to draw on the axis - :return: None - """ - for bbox in bboxes: - rect = patches.Rectangle((bbox[0], bbox[1]), - bbox[2], bbox[3], - linewidth=1, edgecolor='r', facecolor='none') - # Add the patch to the Axes - axis.add_patch(rect) - - -def visualize(unaugmented_data, augment_data): - for idx, (un_aug_item, aug_item) in enumerate( - zip(unaugmented_data.create_dict_iterator(), augment_data.create_dict_iterator())): - axis = plt.subplot(141) - plt.imshow(un_aug_item["image"]) - add_bounding_boxes(axis, un_aug_item["annotation"]) # add Orig BBoxes - plt.title("Original" + str(idx + 1)) - logger.info("Original ", str(idx + 1), " :", un_aug_item["annotation"]) - - axis = plt.subplot(142) - plt.imshow(aug_item["image"]) - add_bounding_boxes(axis, aug_item["annotation"]) # add AugBBoxes - plt.title("Augmented" + str(idx + 1)) - logger.info("Augmented ", str(idx + 1), " ", aug_item["annotation"], "\n") - plt.show() - - -def test_resize_with_bbox_op(plot=False): - """ - Test resize_with_bbox_op - """ - logger.info("Test resize with bbox") - - # original images - data_original = ds.VOCDataset(DATA_DIR, task="Detection", mode="train", decode=True, shuffle=False) - - # augmented images - data_augmented = ds.VOCDataset(DATA_DIR, task="Detection", mode="train", decode=True, shuffle=False) - - data_original = data_original.map(input_columns=["annotation"], - output_columns=["annotation"], - operations=fix_annotate) + test_op = c_vision.ResizeWithBBox(200) - data_augmented = data_augmented.map(input_columns=["annotation"], - output_columns=["annotation"], - operations=fix_annotate) + dataVoc1 = dataVoc1.map(input_columns=["annotation"], + output_columns=["annotation"], + operations=fix_annotate) + dataVoc2 = dataVoc2.map(input_columns=["annotation"], + output_columns=["annotation"], + operations=fix_annotate) + # map to apply ops + dataVoc2 = dataVoc2.map(input_columns=["image", "annotation"], + output_columns=["image", "annotation"], + columns_order=["image", "annotation"], + operations=[test_op]) - # define map operations - test_op = c_vision.ResizeWithBBox(100) # input value being the target size of resizeOp + filename = "resize_with_bbox_op_01_c_result.npz" + save_and_check_md5(dataVoc2, filename, generate_golden=GENERATE_GOLDEN) - data_augmented = data_augmented.map(input_columns=["image", "annotation"], - output_columns=["image", "annotation"], - columns_order=["image", "annotation"], operations=[test_op]) - if plot: - visualize(data_original, data_augmented) + unaugSamp, augSamp = [], [] + for unAug, Aug in zip(dataVoc1.create_dict_iterator(), dataVoc2.create_dict_iterator()): + unaugSamp.append(unAug) + augSamp.append(Aug) -def test_resize_with_bbox_invalid_bounds(): - data_voc2 = ds.VOCDataset(DATA_DIR, task="Detection", mode="train", decode=True, shuffle=False) - check_bad_box(data_voc2, BoxType.WidthOverflow, "bounding boxes is out of bounds of the image") - data_voc2 = ds.VOCDataset(DATA_DIR, task="Detection", mode="train", decode=True, shuffle=False) - check_bad_box(data_voc2, BoxType.HeightOverflow, "bounding boxes is out of bounds of the image") - data_voc2 = ds.VOCDataset(DATA_DIR, task="Detection", mode="train", decode=True, shuffle=False) - check_bad_box(data_voc2, BoxType.NegativeXY, "min_x") - data_voc2 = ds.VOCDataset(DATA_DIR, task="Detection", mode="train", decode=True, shuffle=False) - check_bad_box(data_voc2, BoxType.WrongShape, "4 features") + if plot_vis: + visualize_with_bounding_boxes(unaugSamp, augSamp) -def test_resize_with_bbox_invalid_size(): +def test_resize_with_bbox_op_edge_c(plot_vis=False): """ - Test resize_with_bbox_op - """ - logger.info("Test resize with bbox with invalid target size") + Prints images and bboxes side by side with and without ResizeWithBBox Op applied, + applied on dynamically generated edge case, expected to pass. edge case is when bounding + box has dimensions as the image itself. + """ + logger.info("test_resize_with_bbox_op_edge_c") + dataVoc1 = ds.VOCDataset(DATA_DIR, task="Detection", mode="train", + decode=True, shuffle=False) - # original images - data = ds.VOCDataset(DATA_DIR, task="Detection", mode="train", decode=True, shuffle=False) + dataVoc2 = ds.VOCDataset(DATA_DIR, task="Detection", mode="train", + decode=True, shuffle=False) - data = data.map(input_columns=["annotation"], - output_columns=["annotation"], - operations=fix_annotate) + test_op = c_vision.ResizeWithBBox(500) - # negative target size as input - try: - test_op = c_vision.ResizeWithBBox(-10) + dataVoc1 = dataVoc1.map(input_columns=["annotation"], + output_columns=["annotation"], + operations=fix_annotate) + dataVoc2 = dataVoc2.map(input_columns=["annotation"], + output_columns=["annotation"], + operations=fix_annotate) - # map to apply ops - data = data.map(input_columns=["image", "annotation"], - output_columns=["image", "annotation"], - columns_order=["image", "annotation"], - operations=[test_op]) # Add column for "annotation" + # maps to convert data into valid edge case data + dataVoc1 = dataVoc1.map(input_columns=["image", "annotation"], + output_columns=["image", "annotation"], + columns_order=["image", "annotation"], + operations=[lambda img, bboxes: ( + img, np.array([[0, 0, img.shape[1], img.shape[0]]]).astype(bboxes.dtype))]) - for _, _ in enumerate(data.create_dict_iterator()): - break + # Test Op added to list of Operations here + dataVoc2 = dataVoc2.map(input_columns=["image", "annotation"], + output_columns=["image", "annotation"], + columns_order=["image", "annotation"], + operations=[lambda img, bboxes: ( + img, np.array([[0, 0, img.shape[1], img.shape[0]]]).astype(bboxes.dtype)), test_op]) - except ValueError as e: - logger.info("Got an exception in DE: {}".format(str(e))) - assert "Input is not" in str(e) + unaugSamp, augSamp = [], [] - # zero target size as input - try: - test_op = c_vision.ResizeWithBBox(0) + for unAug, Aug in zip(dataVoc1.create_dict_iterator(), dataVoc2.create_dict_iterator()): + unaugSamp.append(unAug) + augSamp.append(Aug) - # map to apply ops - data = data.map(input_columns=["image", "annotation"], - output_columns=["image", "annotation"], - columns_order=["image", "annotation"], - operations=[test_op]) # Add column for "annotation" + if plot_vis: + visualize_with_bounding_boxes(unaugSamp, augSamp) - for _, _ in enumerate(data.create_dict_iterator()): - break - except ValueError as e: - logger.info("Got an exception in DE: {}".format(str(e))) - assert "Input is not" in str(e) +def test_resize_with_bbox_op_invalid_c(): + """ + Test ResizeWithBBox Op on invalid constructor parameters, expected to raise ValueError + """ + logger.info("test_resize_with_bbox_op_invalid_c") - # invalid input shape try: - test_op = c_vision.ResizeWithBBox((10, 10, 10)) + # invalid interpolation value + c_vision.ResizeWithBBox(400, interpolation="invalid") - # map to apply ops - data = data.map(input_columns=["image", "annotation"], - output_columns=["image", "annotation"], - columns_order=["image", "annotation"], - operations=[test_op]) # Add column for "annotation" + except ValueError as err: + logger.info("Got an exception in DE: {}".format(str(err))) + assert "interpolation" in str(err) - for _, _ in enumerate(data.create_dict_iterator()): - break - except TypeError as e: - logger.info("Got an exception in DE: {}".format(str(e))) - assert "Size should be" in str(e) - - -def test_resize_with_bbox_invalid_interpolation(): +def test_resize_with_bbox_op_bad_c(): """ - Test resize_with_bbox_op - """ - logger.info("Test resize with bbox with invalid interpolation size") - - # original images - data = ds.VOCDataset(DATA_DIR, task="Detection", mode="train", decode=True, shuffle=False) - - data = data.map(input_columns=["annotation"], - output_columns=["annotation"], - operations=fix_annotate) - - # invalid interpolation - try: - test_op = c_vision.ResizeWithBBox(100, interpolation="invalid") - - # map to apply ops - data = data.map(input_columns=["image", "annotation"], - output_columns=["image", "annotation"], - columns_order=["image", "annotation"], - operations=[test_op]) # Add column for "annotation" + Tests ResizeWithBBox Op with invalid bounding boxes, expected to catch multiple errors + """ + logger.info("test_resize_with_bbox_op_bad_c") + test_op = c_vision.ResizeWithBBox((200, 300)) - for _, _ in enumerate(data.create_dict_iterator()): - break + data_voc2 = ds.VOCDataset(DATA_DIR, task="Detection", mode="train", decode=True, shuffle=False) + check_bad_bbox(data_voc2, test_op, InvalidBBoxType.WidthOverflow, "bounding boxes is out of bounds of the image") + data_voc2 = ds.VOCDataset(DATA_DIR, task="Detection", mode="train", decode=True, shuffle=False) + check_bad_bbox(data_voc2, test_op, InvalidBBoxType.HeightOverflow, "bounding boxes is out of bounds of the image") + data_voc2 = ds.VOCDataset(DATA_DIR, task="Detection", mode="train", decode=True, shuffle=False) + check_bad_bbox(data_voc2, test_op, InvalidBBoxType.NegativeXY, "min_x") + data_voc2 = ds.VOCDataset(DATA_DIR, task="Detection", mode="train", decode=True, shuffle=False) + check_bad_bbox(data_voc2, test_op, InvalidBBoxType.WrongShape, "4 features") - except ValueError as e: - logger.info("Got an exception in DE: {}".format(str(e))) - assert "interpolation" in str(e) if __name__ == "__main__": - test_resize_with_bbox_op(plot=False) - test_resize_with_bbox_invalid_bounds() - test_resize_with_bbox_invalid_size() - test_resize_with_bbox_invalid_interpolation() + test_resize_with_bbox_op_c(plot_vis=False) + test_resize_with_bbox_op_edge_c(plot_vis=False) + test_resize_with_bbox_op_invalid_c() + test_resize_with_bbox_op_bad_c() From 698abf75fc77b8c48ed6c749ff7a3dd1d495cbd3 Mon Sep 17 00:00:00 2001 From: Alexey Shevlyakov Date: Thu, 25 Jun 2020 11:30:29 -0400 Subject: [PATCH 076/254] Connector throughput + PerfData + CyclicArray --- mindspore/ccsrc/dataset/engine/connector.h | 6 + .../dataset/engine/datasetops/dataset_op.h | 5 + mindspore/ccsrc/dataset/engine/db_connector.h | 1 + .../ccsrc/dataset/engine/execution_tree.h | 4 +- .../ccsrc/dataset/engine/perf/CMakeLists.txt | 4 +- .../dataset/engine/perf/connector_size.cc | 1 - .../dataset/engine/perf/connector_size.h | 10 +- .../engine/perf/connector_throughput.cc | 109 ++++++++++ .../engine/perf/connector_throughput.h | 100 +++++++++ .../ccsrc/dataset/engine/perf/cyclic_array.h | 197 ++++++++++++++++++ .../engine/perf/dataset_iterator_tracing.h | 1 + .../ccsrc/dataset/engine/perf/monitor.cc | 1 - mindspore/ccsrc/dataset/engine/perf/monitor.h | 1 + .../ccsrc/dataset/engine/perf/perf_data.h | 88 ++++++++ .../ccsrc/dataset/engine/perf/profiling.cc | 19 +- .../ccsrc/dataset/engine/perf/profiling.h | 12 +- tests/ut/cpp/dataset/CMakeLists.txt | 2 + tests/ut/cpp/dataset/cyclic_array_test.cc | 128 ++++++++++++ tests/ut/cpp/dataset/perf_data_test.cc | 71 +++++++ tests/ut/python/dataset/test_profiling.py | 21 +- 20 files changed, 754 insertions(+), 27 deletions(-) create mode 100644 mindspore/ccsrc/dataset/engine/perf/connector_throughput.cc create mode 100644 mindspore/ccsrc/dataset/engine/perf/connector_throughput.h create mode 100644 mindspore/ccsrc/dataset/engine/perf/cyclic_array.h create mode 100644 mindspore/ccsrc/dataset/engine/perf/perf_data.h create mode 100644 tests/ut/cpp/dataset/cyclic_array_test.cc create mode 100644 tests/ut/cpp/dataset/perf_data_test.cc diff --git a/mindspore/ccsrc/dataset/engine/connector.h b/mindspore/ccsrc/dataset/engine/connector.h index cdce592c1b..acd92b4145 100644 --- a/mindspore/ccsrc/dataset/engine/connector.h +++ b/mindspore/ccsrc/dataset/engine/connector.h @@ -102,8 +102,10 @@ class Connector { RETURN_IF_NOT_OK(cv_.Wait(&lk, [this, worker_id]() { return expect_consumer_ == worker_id; })); RETURN_IF_NOT_OK(queues_[pop_from_]->PopFront(result)); pop_from_ = (pop_from_ + 1) % num_producers_; + out_buffers_count_++; expect_consumer_ = (expect_consumer_ + 1) % num_consumers_; } + cv_.NotifyAll(); return Status::OK(); } @@ -119,6 +121,8 @@ class Connector { return (queues_[worker_id]->Add(el)); } + auto out_buffers_count() const { return out_buffers_count_.load(); } + // Add an element into the DbConnector without the overhead of synchronization. // It may block when the internal queue is full. // The element passed to this function will be forwarded into the internal queue. @@ -138,6 +142,7 @@ class Connector { } expect_consumer_ = 0; pop_from_ = 0; + out_buffers_count_ = 0; MS_LOG(DEBUG) << "Connector counters reset."; } @@ -198,6 +203,7 @@ class Connector { // Used in the Pop(), when a thread call pop() but it is not the expect_consumer_. std::mutex m_; CondVar cv_; + std::atomic out_buffers_count_ = 0; }; } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/dataset_op.h b/mindspore/ccsrc/dataset/engine/datasetops/dataset_op.h index c444004b79..370bf6a1bd 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/dataset_op.h +++ b/mindspore/ccsrc/dataset/engine/datasetops/dataset_op.h @@ -222,6 +222,7 @@ class DatasetOp : public std::enable_shared_from_this { // Getter function // @return connector size of current op + int32_t ConnectorSize() const { if (!inlined()) { return out_connector_->size(); @@ -230,6 +231,10 @@ class DatasetOp : public std::enable_shared_from_this { return ChildOpConnectorSize(); } + int64_t ConnectorOutBufferCount() const { + return out_connector_ == nullptr ? int64_t(-1) : static_cast(out_connector_->out_buffers_count()); + } + // Getter function // @return connector size of current op int32_t ConnectorCapacity() const { diff --git a/mindspore/ccsrc/dataset/engine/db_connector.h b/mindspore/ccsrc/dataset/engine/db_connector.h index b1fdd14ab6..54909f51ba 100644 --- a/mindspore/ccsrc/dataset/engine/db_connector.h +++ b/mindspore/ccsrc/dataset/engine/db_connector.h @@ -83,6 +83,7 @@ class DbConnector : public Connector> { expect_consumer_ = (expect_consumer_ + 1) % num_consumers_; } } + out_buffers_count_++; cv_.NotifyAll(); return Status::OK(); } diff --git a/mindspore/ccsrc/dataset/engine/execution_tree.h b/mindspore/ccsrc/dataset/engine/execution_tree.h index e1c5e8ff54..b0391bf77b 100644 --- a/mindspore/ccsrc/dataset/engine/execution_tree.h +++ b/mindspore/ccsrc/dataset/engine/execution_tree.h @@ -88,8 +88,10 @@ class ExecutionTree { bool operator!=(const Iterator &rhs) { return nodes_[ind_] != rhs.nodes_[rhs.ind_]; } + int32_t NumNodes() { return nodes_.size(); } + private: - int ind_; // the cur node our Iterator points to + int32_t ind_; // the cur node our Iterator points to std::vector> nodes_; // store the nodes in post order void PostOrderTraverse(const std::shared_ptr &); }; diff --git a/mindspore/ccsrc/dataset/engine/perf/CMakeLists.txt b/mindspore/ccsrc/dataset/engine/perf/CMakeLists.txt index 0b67469d2d..e611add983 100644 --- a/mindspore/ccsrc/dataset/engine/perf/CMakeLists.txt +++ b/mindspore/ccsrc/dataset/engine/perf/CMakeLists.txt @@ -3,4 +3,6 @@ add_library(engine-perf OBJECT monitor.cc device_queue_tracing.cc connector_size.cc - dataset_iterator_tracing.cc) + dataset_iterator_tracing.cc + connector_throughput.cc + ) diff --git a/mindspore/ccsrc/dataset/engine/perf/connector_size.cc b/mindspore/ccsrc/dataset/engine/perf/connector_size.cc index 862ec51c49..0bd2754075 100644 --- a/mindspore/ccsrc/dataset/engine/perf/connector_size.cc +++ b/mindspore/ccsrc/dataset/engine/perf/connector_size.cc @@ -14,7 +14,6 @@ * limitations under the License. */ #include "dataset/engine/perf/connector_size.h" - #include #include #include diff --git a/mindspore/ccsrc/dataset/engine/perf/connector_size.h b/mindspore/ccsrc/dataset/engine/perf/connector_size.h index 6840ffe244..2584289fb4 100644 --- a/mindspore/ccsrc/dataset/engine/perf/connector_size.h +++ b/mindspore/ccsrc/dataset/engine/perf/connector_size.h @@ -13,8 +13,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#ifndef MINDSPORE_QUEUE_DEPTH_H -#define MINDSPORE_QUEUE_DEPTH_H +#ifndef DATASET_CONNECTOR_SIZE_H +#define DATASET_CONNECTOR_SIZE_H #include #include @@ -50,7 +50,7 @@ class ConnectorSize : public Sampling { // This function samples the connector size of every nodes within the ExecutionTree Status Sample() override; - std::string Name() const override { return kDeviceQueueTracingName; }; + std::string Name() const override { return kConnectorSizeSamplingName; } // Save sampling data to file // @return Status - The error code return @@ -65,6 +65,8 @@ class ConnectorSize : public Sampling { ExecutionTree *tree_ = nullptr; // ExecutionTree pointer ConnectorSizeSampleTable sample_table_; // Dataset structure to store all samples of connector size sampling }; + } // namespace dataset } // namespace mindspore -#endif // MINDSPORE_QUEUE_DEPTH_H + +#endif // DATASET_CONNECTOR_SIZE_H diff --git a/mindspore/ccsrc/dataset/engine/perf/connector_throughput.cc b/mindspore/ccsrc/dataset/engine/perf/connector_throughput.cc new file mode 100644 index 0000000000..4fd59de390 --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/perf/connector_throughput.cc @@ -0,0 +1,109 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include "dataset/engine/perf/connector_throughput.h" +#include "dataset/engine/execution_tree.h" +#include "dataset/util/path.h" + +namespace mindspore { +namespace dataset { + +// temporary helper +int ConnectorThroughput::InitNodes() { + auto it = (*tree_).begin(); + return it.NumNodes(); +} +// Sample action +Status ConnectorThroughput::Sample() { + std::vector out_buffer_count_row(n_nodes_); + std::vector throughput_row(n_nodes_); + TimePoint cur_time; // initialised inside the loop, used outside the loop to update prev sample time. + auto col = 0; + for (const auto &node : *tree_) { + auto cur_out_buffer_count = node.ConnectorOutBufferCount(); + out_buffer_count_row[col] = cur_out_buffer_count; + auto sz = timestamps_.size(); + cur_time = std::chrono::steady_clock::now(); + auto _dt = std::chrono::duration_cast(timestamps_[0][sz - 1] - timestamps_[0][sz - 2]); + auto dt = std::chrono::duration(_dt).count(); + auto prev_out_buffer_count = out_buffer_count_table_[col][out_buffer_count_table_.size() - 1]; + if (dt != 0) { + auto thr = (cur_out_buffer_count - prev_out_buffer_count) / (1000 * dt); + throughput_row[col] = thr; + } else { + throughput_row[col] = -1; + } + col++; + } + std::vector v = {cur_time}; // temporary fix + timestamps_.AddSample(v); + // Push new row of sample + out_buffer_count_table_.AddSample(out_buffer_count_row); + throughput_.AddSample(throughput_row); + return Status::OK(); +} + +json ConnectorThroughput::ParseOpInfo(const DatasetOp &node, const std::vector &thr) { + auto children = node.Children(); + std::vector children_id; + std::transform(children.begin(), children.end(), std::back_inserter(children_id), + [](std::shared_ptr op) -> int32_t { return op->id(); }); + json json_node; + json_node["op_id"] = node.id(); + json_node["op_type"] = node.Name(); + json_node["num_workers"] = node.num_workers(); + json metrics; + metrics["output_queue"] = {{"throughput", thr}}; + + json_node["metrics"] = metrics; + if (!children_id.empty()) { + json_node["children"] = children_id; + } + + return json_node; +} + +// Save profiling data to file +Status ConnectorThroughput::SaveToFile() { + std::ofstream os(file_path_); + json output; + output["sampling_interval"] = 10; + // Traverse the ExecutionTree for JSON node generation + int col = 0; + for (auto &node : *tree_) { + std::vector throughput; + for (auto i = 0; i < throughput_.size(); i++) { + throughput.push_back(throughput_[col][i]); + } + json json_node = ParseOpInfo(node, throughput); + output["op_info"].push_back(json_node); + col++; + } + os << output; + return Status::OK(); +} +Status ConnectorThroughput::Init(const std::string &dir_path, const std::string &device_id) { + file_path_ = (Path(dir_path) / Path("pipeline_profiling_" + Name() + "_" + device_id + ".json")).toString(); + return Status::OK(); +} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/perf/connector_throughput.h b/mindspore/ccsrc/dataset/engine/perf/connector_throughput.h new file mode 100644 index 0000000000..e873eb8315 --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/perf/connector_throughput.h @@ -0,0 +1,100 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef DATASET_CONNECTOR_THROUGHPUT_H +#define DATASET_CONNECTOR_THROUGHPUT_H + +#include +#include +#include +#include +#include +#include "dataset/engine/perf/profiling.h" +#include "dataset/engine/perf/perf_data.h" +#include "dataset/engine/perf/cyclic_array.h" +#include "dataset/engine/datasetops/dataset_op.h" + +using json = nlohmann::json; +namespace mindspore { +namespace dataset { +class ExecutionTree; + +// Connector throughput samples the output connector size of each op in the pipeline. +// For the description of the data structure see perf_buffer.h +// It support JSON serialization for external usage. +class ConnectorThroughput : public Sampling { + using OutBufferCount = PerfData>; + using Throughput = PerfData>; + using TimePoint = std::chrono::time_point; + using TimeStamps = PerfData>; + + public: + explicit ConnectorThroughput(ExecutionTree *tree, int64_t max_rows = 1000000) + : tree_(tree), + max_rows_(max_rows), + n_nodes_(InitNodes()), + out_buffer_count_table_(OutBufferCount(max_rows_, n_nodes_)), + throughput_(Throughput(max_rows_, n_nodes_)), + timestamps_(TimeStamps(max_rows_, 1)) { + timestamps_.AddSample(std::vector(1)); + out_buffer_count_table_.AddSample(std::vector(n_nodes_)); + } + // Driver function for connector size sampling. + // This function samples the connector size of every nodes within the ExecutionTree + Status Sample() override; + + /* Status TestPrint() override { + std::ofstream os("performance_monitor.txt"); + if (throughput_.size() == 0) { + os << "data is empty" << std::endl; + return Status::OK(); + } + for (int i = 0; i < throughput_.size(); i++) { + for (int j = 0; j < n_nodes_; j++) { + os << throughput_[j][i] << " "; + } + os << std::endl; + } + return Status::OK(); + };*/ + + // Traverse the tree nodes and count them + int InitNodes(); + + std::string Name() const override { return name_; }; + + // Save sampling data to file + // @return Status - The error code return + Status SaveToFile() override; + + Status Init(const std::string &dir_path, const std::string &device_id); + + json ParseOpInfo(const DatasetOp &node, const std::vector &thr); + + private: + ExecutionTree *tree_ = nullptr; // ExecutionTree pointer + int64_t max_rows_; + int32_t n_nodes_; + OutBufferCount out_buffer_count_table_; + Throughput throughput_; + TimeStamps timestamps_; + std::string name_ = kConnectorThroughputSamplingName; +}; + +} // namespace dataset +} // namespace mindspore + +#endif // DATASET_CONNECTOR_THROUGHPUT_H diff --git a/mindspore/ccsrc/dataset/engine/perf/cyclic_array.h b/mindspore/ccsrc/dataset/engine/perf/cyclic_array.h new file mode 100644 index 0000000000..fa60b401c5 --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/perf/cyclic_array.h @@ -0,0 +1,197 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef DATASET_CYCLIC_ARRAY_H +#define DATASET_CYCLIC_ARRAY_H + +#include +#include +#include +#include +#include "dataset/core/constants.h" + +namespace mindspore { +namespace dataset { + +/// \class CyclicArray "include/cyclic_array.h +/// \brief This is a container with a contiguous memory layout that pnly keeps N last entries, +/// when the number of entries exceeds the capacity +/// Must be preallocated +template +class CyclicArray { + public: + using value_type = T; + class Iterator { + // Add operator[] and make fully compliant with random access iterator + // and add a const iterator + // add resize(), empty() + public: + using iterator_category = std::random_access_iterator_tag; + using value_type = CyclicArray::value_type; + using difference_type = std::ptrdiff_t; + using pointer = CyclicArray::value_type *; + using reference = CyclicArray::value_type &; + + Iterator() = default; + + Iterator(dsize_t idx, pointer ptr, dsize_t capacity, dsize_t head) + : cur_idx_(idx), ptr_(ptr), capacity_(capacity), head_(head) {} + + Iterator(const Iterator &rhs) = default; + + ~Iterator() = default; + + Iterator &operator++() { + cur_idx_ = (cur_idx_ + 1) % (capacity_ + 1); + return *this; + } + + Iterator operator++(int) { + Iterator tmp(*this); + cur_idx_ = (cur_idx_ + 1) % (capacity_ + 1); + return tmp; + } + + Iterator &operator--() { + cur_idx_ = (cur_idx_ + capacity_) % (capacity_ + 1); + return *this; + } + + Iterator operator--(int) { + Iterator tmp(*this); + cur_idx_ = (cur_idx_ + capacity_) % (capacity_ + 1); + return tmp; + } + + Iterator operator+(dsize_t x) { return Iterator((cur_idx_ + x) % (capacity_ + 1), ptr_, capacity_, head_); } + + Iterator operator-(dsize_t x) { + return Iterator((cur_idx_ + (capacity_ + 1 - x)) % (capacity_ + 1), ptr_, capacity_, head_); + } + + bool operator<(const Iterator &rhs) { + return (head_ + cur_idx_) % (capacity_ + 1) < (rhs.head_ + rhs.cur_idx_) % (capacity_ + 1); + } + + bool operator>(const Iterator &rhs) { + return (head_ + cur_idx_) % (capacity_ + 1) > (rhs.head_ + rhs.cur_idx_) % (capacity_ + 1); + } + + bool operator>=(const Iterator &rhs) { + return (head_ + cur_idx_) % (capacity_ + 1) >= (rhs.head_ + rhs.cur_idx_) % (capacity_ + 1); + } + + bool operator<=(const Iterator &rhs) { + return (head_ + cur_idx_) % (capacity_ + 1) <= (rhs.head_ + rhs.cur_idx_) % (capacity_ + 1); + } + + difference_type operator-(const Iterator &rhs) { + return (cur_idx_ - rhs.cur_idx_ + capacity_ + 1) % (capacity_ + 1); + } + + reference operator*() { return ptr_[cur_idx_]; } + + pointer operator->() { return &(ptr_[cur_idx_]); } + + bool operator==(const Iterator &rhs) { return cur_idx_ == rhs.cur_idx_; } + + bool operator!=(const Iterator &rhs) { return cur_idx_ != rhs.cur_idx_; } + + private: + dsize_t cur_idx_; + pointer ptr_; + dsize_t capacity_; + dsize_t head_; + }; + + /// \brief Default constructor + CyclicArray() : buf_(nullptr), head_(0), tail_(0), size_(0), capacity_(0) {} + + /// \brief Constructor + /// \param[in] capacity + explicit CyclicArray(dsize_t capacity) + : buf_(std::make_unique(capacity + 1)), head_(0), tail_(0), size_(0), capacity_(capacity) {} + + CyclicArray(const CyclicArray &rhs) + : buf_(std::make_unique(rhs.capacity_ + 1)), + head_(rhs.head_), + tail_(rhs.tail_), + size_(rhs.size_), + capacity_(rhs.capacity_) { + std::copy(rhs.begin(), rhs.end(), begin()); + } + + CyclicArray(CyclicArray &&rhs) = default; + + ~CyclicArray() = default; + + /// \brief Iterator begin() + Iterator begin() { return Iterator(head_, buf_.get(), capacity_, head_); } + + /// \brief Iterator end() + Iterator end() { return Iterator(tail_, buf_.get(), capacity_, head_); } + + // not really const. + Iterator begin() const { return Iterator(head_, buf_.get(), capacity_, head_); } + + Iterator end() const { return Iterator(tail_, buf_.get(), capacity_, head_); } + + /// \brief clear the array. Does not deallocate memory, capacity remains the same + void clear() { + head_ = 0; + tail_ = 0; + size_ = 0; + } + + /// \brief returns current size + dsize_t size() { return size_; } + + /// \brief returns capacity + dsize_t capacity() { return capacity_; } + + /// \brief pushes a value + /// \param[in] val value + void push_back(T val) { + buf_[tail_] = val; + if (size_ >= capacity_) { + (tail_ != capacity_) ? tail_++ : tail_ = 0; + (head_ != capacity_) ? head_++ : head_ = 0; + } else { + tail_++; + size_++; + } + } + + /// \brief returns const reference to an element of the array + /// \param[in] idx index of the element + /// \param[out] const T& reference to an element of the array + const T &operator[](dsize_t idx) const { return buf_[(head_ + idx) % (capacity_ + 1)]; } + + /// \brief returns non-const reference to an element of the array + /// \param[in] idx index of the element + /// \param[out] T& reference to an element of the array + T &operator[](dsize_t idx) { return buf_[(head_ + idx) % (capacity_ + 1)]; } + + private: + std::unique_ptr buf_; + dsize_t head_; + dsize_t tail_; + dsize_t size_; + dsize_t capacity_; +}; +} // namespace dataset +} // namespace mindspore +#endif // DATASET_CYCLIC_ARRAY_H diff --git a/mindspore/ccsrc/dataset/engine/perf/dataset_iterator_tracing.h b/mindspore/ccsrc/dataset/engine/perf/dataset_iterator_tracing.h index 00264939fc..129863c6d1 100644 --- a/mindspore/ccsrc/dataset/engine/perf/dataset_iterator_tracing.h +++ b/mindspore/ccsrc/dataset/engine/perf/dataset_iterator_tracing.h @@ -13,6 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + #ifndef MINDSPORE_DATASET_ITERATOR_TRACING_H #define MINDSPORE_DATASET_ITERATOR_TRACING_H diff --git a/mindspore/ccsrc/dataset/engine/perf/monitor.cc b/mindspore/ccsrc/dataset/engine/perf/monitor.cc index c9dce004b5..8a0d682b81 100644 --- a/mindspore/ccsrc/dataset/engine/perf/monitor.cc +++ b/mindspore/ccsrc/dataset/engine/perf/monitor.cc @@ -28,7 +28,6 @@ Monitor::Monitor(ExecutionTree *tree) : tree_(tree) { max_samples_ = 0; cur_row_ = 0; } - Status Monitor::operator()() { // Register this thread with TaskManager to receive proper interrupt signal. TaskManager::FindMe()->Post(); diff --git a/mindspore/ccsrc/dataset/engine/perf/monitor.h b/mindspore/ccsrc/dataset/engine/perf/monitor.h index 2a482a6ad7..8b4245db8e 100644 --- a/mindspore/ccsrc/dataset/engine/perf/monitor.h +++ b/mindspore/ccsrc/dataset/engine/perf/monitor.h @@ -29,6 +29,7 @@ class ExecutionTree; class Monitor { public: // Monitor object constructor + explicit Monitor(ExecutionTree *tree); Monitor() = default; diff --git a/mindspore/ccsrc/dataset/engine/perf/perf_data.h b/mindspore/ccsrc/dataset/engine/perf/perf_data.h new file mode 100644 index 0000000000..a201d705ea --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/perf/perf_data.h @@ -0,0 +1,88 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef DATASET_PERF_DATA_H +#define DATASET_PERF_DATA_H + +#include +#include "dataset/core/constants.h" + +namespace mindspore { +namespace dataset { + +// PerfData is a convenience class to record and store the data produced by Monitor +// and represents a 2D column major table with every column storing samples +// for an operator. The number of rows equals to the number of samples, +// the number of columns equals to the number of operators. +// The capacity is determined on construction and cannot be changed. +// ColumnType can be std::vector or CyclicArray. In case of the latter data can be added +// indefinitely without the risk of overflowing otherwise the capacity must not be exceeded. +// Given PerfData pd(n_rows, n_cols) an element in the column i and row j can be accessed as +// pd[i][j] + +template +class PerfData { + public: + PerfData() = default; + ~PerfData() = default; + PerfData(dsize_t max_rows, dsize_t n_cols) : counter_(0), max_rows_(max_rows), n_cols_(n_cols) { + for (auto i = 0; i < n_cols_; i++) { + data_.push_back(ColumnType(max_rows_)); + } + } + PerfData(const PerfData &rhs) = default; + PerfData(PerfData &&rhs) = default; + + // Adds a row of data + // T must be any container working with range based loops + template + void AddSample(const T &row) { + auto i = 0; + for (const auto &e : row) { + data_[i++].push_back(e); + } + counter_++; + } + + // Fetches a row of data by copy + template + auto Row(dsize_t idx) { + std::vector row(n_cols_); + for (auto i = 0; i < n_cols_; i++) { + row[i] = data_[i][idx]; + } + return row; + } + + // returns a column of data + ColumnType &operator[](size_t idx) { return data_[idx]; } + + const ColumnType &operator[](size_t idx) const { return data_[idx]; } + + dsize_t size() { return counter_ < max_rows_ ? counter_ : max_rows_; } + + dsize_t capacity() { return max_rows_; } + + private: + std::vector data_; + dsize_t counter_; + dsize_t max_rows_; + int n_cols_; +}; + +} // namespace dataset +} // namespace mindspore +#endif // DATASET_PERF_DATA_H diff --git a/mindspore/ccsrc/dataset/engine/perf/profiling.cc b/mindspore/ccsrc/dataset/engine/perf/profiling.cc index 4786b8dd69..66f27c46ba 100644 --- a/mindspore/ccsrc/dataset/engine/perf/profiling.cc +++ b/mindspore/ccsrc/dataset/engine/perf/profiling.cc @@ -14,7 +14,6 @@ * limitations under the License. */ #include "dataset/engine/perf/profiling.h" - #include #include #include @@ -23,6 +22,7 @@ #include "dataset/engine/perf/monitor.h" #include "dataset/engine/perf/device_queue_tracing.h" #include "dataset/engine/perf/connector_size.h" +#include "dataset/engine/perf/connector_throughput.h" #include "dataset/engine/perf/dataset_iterator_tracing.h" #include "utils/log_adapter.h" @@ -72,9 +72,11 @@ Status ProfilingManager::Initialize() { std::shared_ptr dataset_iterator_tracing = std::make_shared(); RETURN_IF_NOT_OK(RegisterTracingNode(dataset_iterator_tracing)); - std::shared_ptr monitor_sampling = std::make_shared(tree_); - RETURN_IF_NOT_OK(RegisterSamplingNode(monitor_sampling)); + std::shared_ptr connector_size_sampling = std::make_shared(tree_); + RETURN_IF_NOT_OK(RegisterSamplingNode(connector_size_sampling)); + std::shared_ptr connector_thr_sampling = std::make_shared(tree_); + RETURN_IF_NOT_OK(RegisterSamplingNode(connector_thr_sampling)); return Status::OK(); } @@ -140,14 +142,15 @@ Status ProfilingManager::SaveProfilingData() { RETURN_IF_NOT_OK(node.second->SaveToFile()); } MS_LOG(INFO) << "Save profiling data end."; - return Status::OK(); } -double ProfilingTime::GetCurMilliSecond() { - struct timeval tv = {0, 0}; - (void)gettimeofday(&tv, nullptr); - return tv.tv_sec * 1000 + tv.tv_usec / 1000; +int64_t ProfilingTime::GetCurMilliSecond() { + // because cpplint does not allow using namespace + using std::chrono::duration_cast; + using std::chrono::milliseconds; + using std::chrono::steady_clock; + return duration_cast(steady_clock::now().time_since_epoch()).count(); } } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/perf/profiling.h b/mindspore/ccsrc/dataset/engine/perf/profiling.h index d0ea91d566..e38c2d5e54 100644 --- a/mindspore/ccsrc/dataset/engine/perf/profiling.h +++ b/mindspore/ccsrc/dataset/engine/perf/profiling.h @@ -20,6 +20,7 @@ #include #include #include +#include #include "dataset/util/status.h" namespace mindspore { @@ -28,9 +29,10 @@ namespace dataset { class Monitor; class ExecutionTree; -const char kDeviceQueueTracingName[] = "Device Queue Tracing"; -const char kDatasetIteratorTracingName[] = "Dataset Iterator Tracing"; -const char kConnectorSizeSamplingName[] = "Connector Size Sampling"; +const char kDeviceQueueTracingName[] = "Device_Queue_Tracing"; +const char kDatasetIteratorTracingName[] = "Dataset_Iterator_Tracing"; +const char kConnectorSizeSamplingName[] = "Connector_Size_Sampling"; +const char kConnectorThroughputSamplingName[] = "Connector_Throughput_Sampling"; // Profiling is a class of basic unit of profiling action // This base class encapsulate the serialization output logic @@ -59,6 +61,8 @@ class Sampling : public Profiling { public: // Sampling action function. This function will be invoked by performance monitor thread. virtual Status Sample() = 0; + // virtual Status TestPrint() = 0; + virtual ~Sampling() = default; }; // Tracing is class of profiling which record samples upon request. @@ -132,7 +136,7 @@ enum ProfilingTimeSubType { class ProfilingTime { public: - static double GetCurMilliSecond(); + static int64_t GetCurMilliSecond(); }; } // namespace dataset diff --git a/tests/ut/cpp/dataset/CMakeLists.txt b/tests/ut/cpp/dataset/CMakeLists.txt index 8478c8257e..bfdc2b4cb3 100644 --- a/tests/ut/cpp/dataset/CMakeLists.txt +++ b/tests/ut/cpp/dataset/CMakeLists.txt @@ -79,6 +79,8 @@ SET(DE_UT_SRCS mask_test.cc trucate_pair_test.cc concatenate_op_test.cc + cyclic_array_test.cc + perf_data_test.cc ) add_executable(de_ut_tests ${DE_UT_SRCS}) diff --git a/tests/ut/cpp/dataset/cyclic_array_test.cc b/tests/ut/cpp/dataset/cyclic_array_test.cc new file mode 100644 index 0000000000..746e482439 --- /dev/null +++ b/tests/ut/cpp/dataset/cyclic_array_test.cc @@ -0,0 +1,128 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include "common/common.h" +#include "common/cvop_common.h" +#include "gtest/gtest.h" +#include "securec.h" +#include "dataset/util/de_error.h" +#include "dataset/engine/perf/cyclic_array.h" +#include + +using namespace mindspore::dataset; + +class MindDataTestCyclicArray : public UT::Common { + public: + MindDataTestCyclicArray() {} +}; + +TEST_F(MindDataTestCyclicArray, Test1) { + CyclicArray arr(5); + EXPECT_EQ(5, arr.capacity()); + EXPECT_EQ(0, arr.size()); + arr.push_back(0); + EXPECT_EQ(5, arr.capacity()); + EXPECT_EQ(1, arr.size()); + EXPECT_EQ(arr[0], 0); + arr.push_back(1); + EXPECT_EQ(arr[1], 1); + for (auto i = 2; i < 5; i++) { + arr.push_back(i); + } + EXPECT_EQ(arr.capacity(), arr.size()); + EXPECT_EQ(1, arr[1]); + EXPECT_EQ(4, arr[4]); + arr[4] = 42; + EXPECT_EQ(arr[4], 42); + auto a = arr[4]; + EXPECT_EQ(a, 42); + arr.push_back(5); + EXPECT_EQ(arr[0], 1); + EXPECT_EQ(arr[4], 5); + + CyclicArray arr2 = arr; + EXPECT_EQ(arr2.capacity(), arr.capacity()); + EXPECT_EQ(arr2.size(), arr.size()); + auto last = arr2.end(); + auto first = arr2.begin(); + for (auto i = 0; i < arr.size(); i++) { + EXPECT_EQ(arr2[i], arr[i]); + } + + arr.clear(); + EXPECT_EQ(arr.size(), 0); + arr.push_back(42); + arr.push_back(43); + EXPECT_EQ(arr.size(), 2); + EXPECT_EQ(arr.capacity(), 5); + EXPECT_EQ(arr[0], 42); + EXPECT_EQ(arr[1], 43); + auto arr3 = arr; + EXPECT_EQ(arr3.size(), 2); + EXPECT_EQ(arr3.capacity(), 5); + EXPECT_EQ(arr.size(), 2); + EXPECT_EQ(arr.capacity(), 5); + EXPECT_EQ(arr[0], arr3[0]); + EXPECT_EQ(arr[1], arr3[1]); + + arr.clear(); + arr.push_back(21); + arr.push_back(22); + EXPECT_EQ(arr[arr.size() - 1], 22); + for (auto i = 23; i < 27; i++) { + arr.push_back(i); + } + EXPECT_EQ(arr[0], 22); + EXPECT_EQ(arr[arr.size() - 1], 26); +} + +TEST_F(MindDataTestCyclicArray, TestIterator) { + CyclicArray arr(5); + for (auto i = 0; i < arr.capacity(); i++) { + arr.push_back(i); + } + arr.push_back(6); + arr.push_back(7); + auto i = 0; + for (auto it = arr.begin(); it != arr.end(); ++it) { + EXPECT_EQ(*it, arr[i++]); + } + + std::iota(arr.begin(), arr.end(), -4); + EXPECT_EQ(arr[0], -4); + EXPECT_EQ(arr[4], 0); + const auto sz = 1000000; + CyclicArray arr2(sz); + for (auto i = 0; i < sz - 1; i++) { + arr.push_back(0); + } + const auto val = -500000; + std::iota(arr2.begin(), arr2.end() + sz, val); + EXPECT_EQ(*arr2.begin(), val); + std::random_device rd; + std::mt19937 g(rd()); + std::shuffle(arr2.begin(), arr2.end(), g); + std::sort(arr2.begin(), arr2.end(), [](const auto a, const auto b) { return a > b; }); + EXPECT_EQ(*arr2.begin(), val); + const auto new_val = -600000; + for (auto i = 0; i < 100; i++) { + arr2.push_back(new_val); + } + EXPECT_EQ(*(--arr2.end()), new_val); + std::sort(arr2.begin(), arr2.end(), [](const auto a, const auto b) { return a > b; }); + EXPECT_EQ(*arr2.begin(), new_val); +} diff --git a/tests/ut/cpp/dataset/perf_data_test.cc b/tests/ut/cpp/dataset/perf_data_test.cc new file mode 100644 index 0000000000..eaa5e85fa1 --- /dev/null +++ b/tests/ut/cpp/dataset/perf_data_test.cc @@ -0,0 +1,71 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "common/common.h" +#include "common/cvop_common.h" +#include "gtest/gtest.h" +#include "securec.h" +#include "dataset/util/de_error.h" +#include "dataset/engine/perf/cyclic_array.h" +#include "dataset/engine/perf/perf_data.h" + +using namespace mindspore::dataset; + +class MindDataTestPerfData : public UT::Common { + public: + MindDataTestPerfData() {} +}; + +TEST_F(MindDataTestPerfData, Test1) { + PerfData> p1(2, 3); + PerfData> p2(2, 3); + EXPECT_EQ(p1.capacity(), p2.capacity()); + std::vector row = {1, 2, 3}; + p1.AddSample(row); + p2.AddSample(row); + EXPECT_EQ(p1.size(), p2.size()); + p1.AddSample(row); + p2.AddSample(row); + EXPECT_EQ(p1.size(), p2.size()); + row = {4, 5, 6}; + p2.AddSample(row); + auto r1 = p2.Row(static_cast(0)); + for (auto i = 0; i < 3; i++) { + EXPECT_EQ(r1[i], i + 1); + } + + auto r2 = p2.Row(1); + for (auto i = 0; i < 3; i++) { + EXPECT_EQ(r2[i], i + 4); + } + + EXPECT_EQ(p2[0][1], 4); + EXPECT_EQ(p2[1][1], 5); + EXPECT_EQ(p2[2][1], 6); +} + +TEST_F(MindDataTestPerfData, Test2) { + auto pd = PerfData>(1000000, 3); + auto row = {1, 2, 3}; + pd.AddSample(row); + EXPECT_EQ(pd[0][0], 1); + EXPECT_EQ(pd[1][0], 2); + EXPECT_EQ(pd[2][0], 3); + row = {4, 5, 6}; + pd.AddSample(row); + EXPECT_EQ(pd[0][0], 1); + EXPECT_EQ(pd[1][0], 2); + EXPECT_EQ(pd[2][0], 3); +} \ No newline at end of file diff --git a/tests/ut/python/dataset/test_profiling.py b/tests/ut/python/dataset/test_profiling.py index fca0a4c1dc..a4ee68e435 100644 --- a/tests/ut/python/dataset/test_profiling.py +++ b/tests/ut/python/dataset/test_profiling.py @@ -23,7 +23,8 @@ FILES = ["../data/dataset/testTFTestAllTypes/test.data"] DATASET_ROOT = "../data/dataset/testTFTestAllTypes/" SCHEMA_FILE = "../data/dataset/testTFTestAllTypes/datasetSchema.json" -PIPELINE_FILE = "./pipeline_profiling_1.json" +PIPELINE_FILE_SIZE = "./pipeline_profiling_1.json" +PIPELINE_FILE_THR = "./pipeline_profiling_Connector_Throughput_Sampling_1.json" DATASET_ITERATOR_FILE = "./dataset_iterator_profiling_1.txt" @@ -43,8 +44,10 @@ def test_profiling_simple_pipeline(): for _ in data1: pass - assert os.path.exists(PIPELINE_FILE) is True - os.remove(PIPELINE_FILE) + assert os.path.exists(PIPELINE_FILE_SIZE) is True + os.remove(PIPELINE_FILE_SIZE) + assert os.path.exists(PIPELINE_FILE_THR) is True + os.remove(PIPELINE_FILE_THR) assert os.path.exists(DATASET_ITERATOR_FILE) is True os.remove(DATASET_ITERATOR_FILE) del os.environ['PROFILING_MODE'] @@ -74,8 +77,10 @@ def test_profiling_complex_pipeline(): for _ in data3: pass - assert os.path.exists(PIPELINE_FILE) is True - os.remove(PIPELINE_FILE) + assert os.path.exists(PIPELINE_FILE_SIZE) is True + os.remove(PIPELINE_FILE_SIZE) + assert os.path.exists(PIPELINE_FILE_THR) is True + os.remove(PIPELINE_FILE_THR) assert os.path.exists(DATASET_ITERATOR_FILE) is True os.remove(DATASET_ITERATOR_FILE) del os.environ['PROFILING_MODE'] @@ -103,8 +108,10 @@ def test_profiling_sampling_iterval(): for _ in data1: pass - assert os.path.exists(PIPELINE_FILE) is True - os.remove(PIPELINE_FILE) + assert os.path.exists(PIPELINE_FILE_SIZE) is True + os.remove(PIPELINE_FILE_SIZE) + assert os.path.exists(PIPELINE_FILE_THR) is True + os.remove(PIPELINE_FILE_THR) assert os.path.exists(DATASET_ITERATOR_FILE) is True os.remove(DATASET_ITERATOR_FILE) From 52c58735fc6711169cfba4992578abe2e960e4b4 Mon Sep 17 00:00:00 2001 From: dinghao Date: Sat, 27 Jun 2020 10:01:06 +0800 Subject: [PATCH 077/254] fix serving bugs --- mindspore/ccsrc/CMakeLists.txt | 13 +- mindspore/ccsrc/session/session.cc | 112 ++++++++++++------ mindspore/ccsrc/utils/log_adapter.cc | 32 +++-- serving/core/server.cc | 36 ++++-- serving/core/util/file_system_operation.cc | 5 +- serving/core/util/option_parser.cc | 40 ++++--- serving/core/util/option_parser.h | 3 +- serving/core/version_control/model.cc | 1 - .../version_control/version_controller.cc | 14 +-- .../core/version_control/version_controller.h | 1 - serving/cpp_example/ms_client.cc | 2 +- 11 files changed, 157 insertions(+), 102 deletions(-) diff --git a/mindspore/ccsrc/CMakeLists.txt b/mindspore/ccsrc/CMakeLists.txt index 8109e608c5..cc5845cbf1 100644 --- a/mindspore/ccsrc/CMakeLists.txt +++ b/mindspore/ccsrc/CMakeLists.txt @@ -277,10 +277,11 @@ endif () if (USE_GLOG) target_link_libraries(inference PRIVATE mindspore::glog) -else() - if (CMAKE_SYSTEM_NAME MATCHES "Linux") - target_link_options(inference PRIVATE -Wl,-init,mindspore_log_init) - elseif (CMAKE_SYSTEM_NAME MATCHES "Darwin") - set_target_properties(inference PROPERTIES MACOSX_RPATH ON) - endif () endif() + +if (CMAKE_SYSTEM_NAME MATCHES "Linux") + target_link_options(inference PRIVATE -Wl,-init,common_log_init) +elseif (CMAKE_SYSTEM_NAME MATCHES "Darwin") + set_target_properties(inference PROPERTIES MACOSX_RPATH ON) +endif () + diff --git a/mindspore/ccsrc/session/session.cc b/mindspore/ccsrc/session/session.cc index 90e02b37ff..ae70fc77aa 100644 --- a/mindspore/ccsrc/session/session.cc +++ b/mindspore/ccsrc/session/session.cc @@ -33,9 +33,14 @@ namespace py = pybind11; namespace mindspore::inference { std::shared_ptr LoadModel(const char *model_buf, size_t size, const std::string &device) { - inference::Session::RegAllOp(); - auto anf_graph = lite::AnfConverter::RunAnfConverter(model_buf, size); - return anf_graph; + try { + inference::Session::RegAllOp(); + auto anf_graph = lite::AnfConverter::RunAnfConverter(model_buf, size); + return anf_graph; + } catch (std::exception &e) { + MS_LOG(ERROR) << "Inference LoadModel failed"; + return nullptr; + } } void ExitInference() { @@ -51,12 +56,17 @@ void ExitInference() { } std::shared_ptr MSSession::CreateSession(const std::string &device, uint32_t device_id) { - auto session = std::make_shared(); - auto ret = session->Init(device, device_id); - if (ret != 0) { + try { + auto session = std::make_shared(); + auto ret = session->Init(device, device_id); + if (ret != 0) { + return nullptr; + } + return session; + } catch (std::exception &e) { + MS_LOG(ERROR) << "Inference CreatSession failed"; return nullptr; } - return session; } void Session::RegAllOp() { @@ -113,47 +123,71 @@ void Session::RegAllOp() { uint32_t Session::CompileGraph(std::shared_ptr funcGraphPtr) { MS_ASSERT(session_impl_ != nullptr); - auto graph_id = session_impl_->CompileGraph(NOT_NULL(funcGraphPtr)); - py::gil_scoped_release gil_release; - return graph_id; + try { + auto graph_id = session_impl_->CompileGraph(NOT_NULL(funcGraphPtr)); + py::gil_scoped_release gil_release; + return graph_id; + } catch (std::exception &e) { + MS_LOG(ERROR) << "Inference CompileGraph failed"; + return static_cast(-1); + } } MultiTensor Session::RunGraph(uint32_t graph_id, const std::vector> &inputs) { - std::vector inTensors; - inTensors.resize(inputs.size()); - bool has_error = false; - std::transform(inputs.begin(), inputs.end(), inTensors.begin(), - [&has_error](const std::shared_ptr &tensor_ptr) -> tensor::TensorPtr { - if (tensor_ptr == nullptr) { - MS_LOG(WARNING) << "input MSTensor is nullptr, return nullptr"; - has_error = true; - return nullptr; - } - auto tensor = static_cast(tensor_ptr.get()); - if (tensor == nullptr) { - MS_LOG(ERROR) << "Can not cast input MSTensor to tensor"; - has_error = true; - return nullptr; - } - return tensor->tensor(); - }); - if (has_error) { - MS_LOG(ERROR) << "Init Tensor failed, returning empty result"; - std::vector> multiTensor; - return multiTensor; - } - VectorRef outputs; - session_impl_->RunGraph(graph_id, inTensors, &outputs); + try { + std::vector inTensors; + inTensors.resize(inputs.size()); + bool has_error = false; + std::transform(inputs.begin(), inputs.end(), inTensors.begin(), + [&has_error](const std::shared_ptr &tensor_ptr) -> tensor::TensorPtr { + if (tensor_ptr == nullptr) { + MS_LOG(WARNING) << "input MSTensor is nullptr, return nullptr"; + has_error = true; + return nullptr; + } + auto tensor = static_cast(tensor_ptr.get()); + if (tensor == nullptr) { + MS_LOG(ERROR) << "Can not cast input MSTensor to tensor"; + has_error = true; + return nullptr; + } + return tensor->tensor(); + }); + if (has_error) { + MS_LOG(ERROR) << "Init Tensor failed, returning empty result"; + std::vector> multiTensor; + return multiTensor; + } + VectorRef outputs; + session_impl_->RunGraph(graph_id, inTensors, &outputs); - return TransformVectorRefToMultiTensor(outputs); + return TransformVectorRefToMultiTensor(outputs); + } catch (std::exception &e) { + MS_LOG(ERROR) << "Inference Rungraph failed"; + return MultiTensor(); + } } - +namespace { +string AjustTargetName(const std::string &device) { + if (device == kAscendDevice) { + return std::string(kAscendDevice) + "Inference"; + } else { + MS_LOG(ERROR) << "Only support device Ascend right now"; + return ""; + } +} +} // namespace int Session::Init(const std::string &device, uint32_t device_id) { RegAllOp(); auto ms_context = MsContext::GetInstance(); ms_context->set_execution_mode(kGraphMode); - ms_context->set_device_target(kAscendDevice); - session_impl_ = session::SessionFactory::Get().Create(device); + ms_context->set_device_id(device_id); + auto ajust_device = AjustTargetName(device); + if (ajust_device == "") { + return -1; + } + ms_context->set_device_target(device); + session_impl_ = session::SessionFactory::Get().Create(ajust_device); if (session_impl_ == nullptr) { MS_LOG(ERROR) << "Session create failed!, please make sure target device:" << device << " is available."; return -1; diff --git a/mindspore/ccsrc/utils/log_adapter.cc b/mindspore/ccsrc/utils/log_adapter.cc index d16fbead9b..3588754dae 100644 --- a/mindspore/ccsrc/utils/log_adapter.cc +++ b/mindspore/ccsrc/utils/log_adapter.cc @@ -463,7 +463,7 @@ void InitSubModulesLogLevel() { // set submodule's log level auto submodule = GetEnv("MS_SUBMODULE_LOG_v"); - MS_LOG(INFO) << "MS_SUBMODULE_LOG_v=`" << submodule << "`"; + MS_LOG(DEBUG) << "MS_SUBMODULE_LOG_v=`" << submodule << "`"; LogConfigParser parser(submodule); auto configs = parser.Parse(); for (const auto &cfg : configs) { @@ -489,22 +489,14 @@ void InitSubModulesLogLevel() { } // namespace mindspore extern "C" { -// shared lib init hook #if defined(_WIN32) || defined(_WIN64) -__attribute__((constructor)) void mindspore_log_init(void) { +__attribute__((constructor)) void common_log_init(void) { #else -void mindspore_log_init(void) { +void common_log_init(void) { #endif #ifdef USE_GLOG // do not use glog predefined log prefix FLAGS_log_prefix = false; - static bool is_glog_initialzed = false; - if (!is_glog_initialzed) { -#if !defined(_WIN32) && !defined(_WIN64) - google::InitGoogleLogging("mindspore"); -#endif - is_glog_initialzed = true; - } // set default log level to WARNING if (mindspore::GetEnv("GLOG_v").empty()) { FLAGS_v = mindspore::WARNING; @@ -525,4 +517,22 @@ void mindspore_log_init(void) { #endif mindspore::InitSubModulesLogLevel(); } + +// shared lib init hook +#if defined(_WIN32) || defined(_WIN64) +__attribute__((constructor)) void mindspore_log_init(void) { +#else +void mindspore_log_init(void) { +#endif +#ifdef USE_GLOG + static bool is_glog_initialzed = false; + if (!is_glog_initialzed) { +#if !defined(_WIN32) && !defined(_WIN64) + google::InitGoogleLogging("mindspore"); +#endif + is_glog_initialzed = true; + } +#endif + common_log_init(); +} } diff --git a/serving/core/server.cc b/serving/core/server.cc index add9d16bee..4a3a3b59eb 100644 --- a/serving/core/server.cc +++ b/serving/core/server.cc @@ -22,6 +22,7 @@ #include #include #include +#include #include "mindspore/ccsrc/utils/log_adapter.h" #include "serving/ms_service.grpc.pb.h" @@ -40,7 +41,7 @@ namespace serving { using MSTensorPtr = std::shared_ptr; Status Session::CreatDeviceSession(const std::string &device, uint32_t device_id) { - session_ = inference::MSSession::CreateSession(device + "Inference", device_id); + session_ = inference::MSSession::CreateSession(device, device_id); if (session_ == nullptr) { MS_LOG(ERROR) << "Creat Session Failed"; return FAILED; @@ -67,6 +68,7 @@ Status Session::Predict(const std::vector &inputs, inference::Multi MS_LOG(INFO) << "run Predict"; *outputs = session_->RunGraph(graph_id_, inputs); + MS_LOG(INFO) << "run Predict finished"; return SUCCESS; } @@ -80,12 +82,16 @@ Status Session::Warmup(const MindSporeModelPtr model) { std::string file_name = model->GetModelPath() + '/' + model->GetModelName(); char *graphBuf = ReadFile(file_name.c_str(), &size); if (graphBuf == nullptr) { - MS_LOG(ERROR) << "Load graph model failed, file name is " << file_name.c_str(); + MS_LOG(ERROR) << "Read model file failed, file name is " << file_name.c_str(); return FAILED; } last_graph_ = inference::LoadModel(graphBuf, size, device_type_); + if (last_graph_ == nullptr) { + MS_LOG(ERROR) << "Load graph model failed, file name is " << file_name.c_str(); + return FAILED; + } graph_id_ = session_->CompileGraph(last_graph_); - MS_LOG(INFO) << "Session Warmup"; + MS_LOG(INFO) << "Session Warmup finished"; return SUCCESS; } @@ -95,6 +101,9 @@ Status Session::Clear() { } namespace { +static const uint32_t uint32max = 0x7FFFFFFF; +std::promise exit_requested; + const std::map type2id_map{ {ms_serving::MS_UNKNOWN, TypeId::kNumberTypeBegin}, {ms_serving::MS_BOOL, TypeId::kNumberTypeBool}, {ms_serving::MS_INT8, TypeId::kNumberTypeInt8}, {ms_serving::MS_UINT8, TypeId::kNumberTypeUInt8}, @@ -141,7 +150,7 @@ MSTensorPtr ServingTensor2MSTensor(const ms_serving::Tensor &tensor) { } TypeId type = iter->second; auto ms_tensor = std::shared_ptr(inference::MSTensor::CreateTensor(type, shape)); - memcpy_s(ms_tensor->MutableData(), tensor.data().size(), tensor.data().data(), tensor.data().size()); + memcpy_s(ms_tensor->MutableData(), ms_tensor->Size(), tensor.data().data(), tensor.data().size()); return ms_tensor; } @@ -166,10 +175,7 @@ void ClearEnv() { Session::Instance().Clear(); inference::ExitInference(); } -void HandleSignal(int sig) { - ClearEnv(); - exit(0); -} +void HandleSignal(int sig) { exit_requested.set_value(); } #ifdef ENABLE_D static rtContext_t g_ctx = nullptr; @@ -247,6 +253,7 @@ Status Server::BuildAndStart() { rtError_t rt_ret = rtCtxGetCurrent(&ctx); if (rt_ret != RT_ERROR_NONE || ctx == nullptr) { MS_LOG(ERROR) << "the ascend device context is null"; + ClearEnv(); return FAILED; } g_ctx = ctx; @@ -258,6 +265,7 @@ Status Server::BuildAndStart() { auto option = grpc::MakeChannelArgumentOption(GRPC_ARG_ALLOW_REUSEPORT, 0); grpc::ServerBuilder builder; builder.SetOption(std::move(option)); + builder.SetMaxMessageSize(uint32max); // Listen on the given address without any authentication mechanism. builder.AddListeningPort(server_address, grpc::InsecureServerCredentials()); // Register "service" as the instance through which we'll communicate with @@ -265,13 +273,15 @@ Status Server::BuildAndStart() { builder.RegisterService(&service); // Finally assemble the server. std::unique_ptr server(builder.BuildAndStart()); + auto grpc_server_run = [&server]() { server->Wait(); }; + std::thread serving_thread(grpc_server_run); MS_LOG(INFO) << "Server listening on " << server_address << std::endl; - - // Wait for the server to shutdown. Note that some other thread must be - // responsible for shutting down the server for this call to ever return. - server->Wait(); + auto exit_future = exit_requested.get_future(); + exit_future.wait(); + ClearEnv(); + server->Shutdown(); + serving_thread.join(); return SUCCESS; } - } // namespace serving } // namespace mindspore diff --git a/serving/core/util/file_system_operation.cc b/serving/core/util/file_system_operation.cc index a5143995de..1af512a54c 100644 --- a/serving/core/util/file_system_operation.cc +++ b/serving/core/util/file_system_operation.cc @@ -29,7 +29,6 @@ namespace mindspore { namespace serving { - char *ReadFile(const char *file, size_t *size) { if (file == nullptr) { MS_LOG(ERROR) << "file is nullptr"; @@ -70,8 +69,8 @@ bool DirOrFileExist(const std::string &file_path) { } std::vector GetAllSubDirs(const std::string &dir_path) { - DIR *dir; - struct dirent *ptr; + DIR *dir = nullptr; + struct dirent *ptr = nullptr; std::vector SubDirs; if ((dir = opendir(dir_path.c_str())) == NULL) { diff --git a/serving/core/util/option_parser.cc b/serving/core/util/option_parser.cc index 9cbd7eaee8..c7f00e3733 100644 --- a/serving/core/util/option_parser.cc +++ b/serving/core/util/option_parser.cc @@ -36,17 +36,16 @@ bool RemovePrefix(std::string *str, const std::string &prefix) { bool Option::ParseInt32(std::string *arg) { if (RemovePrefix(arg, "--") && RemovePrefix(arg, name_) && RemovePrefix(arg, "=")) { - char extra; int32_t parsed_value; - if (sscanf(arg->data(), "%d%c", &parsed_value, &extra) != 1) { - std::cout << "Parse " << name_ << "Error for option " << *arg << std::endl; + try { + parsed_value = std::stoi(arg->data()); + } catch (std::invalid_argument) { + std::cout << "Parse " << name_ << " Error for option " << *arg << std::endl; return false; - } else { - *int32_default_ = parsed_value; } + *int32_default_ = parsed_value; return true; } - return false; } @@ -76,17 +75,16 @@ bool Option::ParseString(std::string *arg) { bool Option::ParseFloat(std::string *arg) { if (RemovePrefix(arg, "--") && RemovePrefix(arg, name_) && RemovePrefix(arg, "=")) { - char extra; float parsed_value; - if (sscanf(arg->data(), "%f%c", &parsed_value, &extra) != 1) { - std::cout << "Parse " << name_ << "Error for option " << *arg << std::endl; + try { + parsed_value = std::stof(arg->data()); + } catch (std::invalid_argument) { + std::cout << "Parse " << name_ << " Error for option " << *arg << std::endl; return false; - } else { - *float_default_ = parsed_value; } + *float_default_ = parsed_value; return true; } - return false; } @@ -159,10 +157,11 @@ Options::Options() : args_(nullptr) { CreateOptions(); } void Options::CreateOptions() { args_ = std::make_shared(); std::vector
(); + kernel_addr->addr = addr; + return kernel_addr; + } + + void CreateInputAddress(std::vector &indices) { + inputs_.push_back(CreateKernelAddress(var_.data())); + inputs_.push_back(CreateKernelAddress(m_.data())); + inputs_.push_back(CreateKernelAddress(v_.data())); + inputs_.push_back(CreateKernelAddress(&beta1_power_)); + inputs_.push_back(CreateKernelAddress(&beta2_power_)); + inputs_.push_back(CreateKernelAddress(&lr_)); + inputs_.push_back(CreateKernelAddress(&beta1_)); + inputs_.push_back(CreateKernelAddress(&beta2_)); + inputs_.push_back(CreateKernelAddress(&epsilon_)); + inputs_.push_back(CreateKernelAddress(grad_.data())); + inputs_.push_back(CreateKernelAddress(indices.data())); + } + + void CreateWorkspaceAddress(std::vector &new_grad, std::vector &new_indices, std::vector &m_t) { + workspace_.push_back(CreateKernelAddress(new_grad.data())); + workspace_.push_back(CreateKernelAddress(new_indices.data())); + workspace_.push_back(CreateKernelAddress(m_t.data())); + } + + std::vector var_; + std::vector m_; + std::vector v_; + std::vector grad_; + std::vector inputs_; + std::vector workspace_; + std::vector outputs_; + std::shared_ptr sparse_adam_; + float beta1_power_ = 0.9; + float beta2_power_ = 0.999; + float lr_ = 0.001; + float beta1_ = 0.9; + float beta2_ = 0.999; + float epsilon_ = 1e-8; +}; + +TEST_F(SparseApplyAdamCpuKernelTest, dense_test) { + for (size_t i = 0; i < 3 * 3 * 3; ++i) { + var_.push_back(1.0); + m_.push_back(1.0); + v_.push_back(1.0); + grad_.push_back(1.0); + } + sparse_adam_->indices_size_ = 3; + sparse_adam_->var_first_dim_size_ = 3; + sparse_adam_->var_outer_dim_size_ = 9; + + std::vector indices{0, 1, 2}; + CreateInputAddress(indices); + std::vector new_grad(3 * 3 * 3); + std::vector new_indices(3); + std::vector m_t(3 * 3 * 3); + CreateWorkspaceAddress(new_grad, new_indices, m_t); + sparse_adam_->Launch(inputs_, workspace_, outputs_); + for (size_t i = 0; i < 3 * 3 * 3; ++i) { + EXPECT_TRUE(std::fabs(var_[i] - 0.999684) < 1e-6); + } +} + +TEST_F(SparseApplyAdamCpuKernelTest, sparse_test1) { + for (size_t i = 0; i < 3 * 3 * 3; ++i) { + var_.push_back(1.0); + m_.push_back(1.0); + v_.push_back(1.0); + } + for (size_t i = 0; i < 2 * 3 * 3; ++i) { + grad_.push_back(1.0); + } + sparse_adam_->indices_size_ = 2; + sparse_adam_->var_first_dim_size_ = 3; + sparse_adam_->var_outer_dim_size_ = 9; + + std::vector indices{0, 2}; + CreateInputAddress(indices); + std::vector new_grad(3 * 3 * 3); + std::vector new_indices(3); + std::vector m_t(3 * 3 * 3); + CreateWorkspaceAddress(new_grad, new_indices, m_t); + sparse_adam_->Launch(inputs_, workspace_, outputs_); + for (size_t i = 0; i < 3 * 3; ++i) { + EXPECT_TRUE(std::fabs(var_[i] - 0.999684) < 1e-6); + } + for (size_t i = 3 * 3; i < 2 * 3 * 3; ++i) { + EXPECT_TRUE(std::fabs(var_[i] - 0.999715) < 1e-6); + } + for (size_t i = 2 * 3 * 3; i < 3 * 3 * 3; ++i) { + EXPECT_TRUE(std::fabs(var_[i] - 0.999684) < 1e-6); + } +} + +TEST_F(SparseApplyAdamCpuKernelTest, sparse_test2) { + for (size_t i = 0; i < 3 * 3 * 3; ++i) { + var_.push_back(1.0); + m_.push_back(1.0); + v_.push_back(1.0); + grad_.push_back(1.0); + } + sparse_adam_->indices_size_ = 3; + sparse_adam_->var_first_dim_size_ = 3; + sparse_adam_->var_outer_dim_size_ = 9; + + std::vector indices{2, 2, 1}; + CreateInputAddress(indices); + std::vector new_grad(3 * 3 * 3); + std::vector new_indices(3); + std::vector m_t(3 * 3 * 3); + CreateWorkspaceAddress(new_grad, new_indices, m_t); + sparse_adam_->Launch(inputs_, workspace_, outputs_); + for (size_t i = 0; i < 3 * 3; ++i) { + EXPECT_TRUE(std::fabs(var_[i] - 0.999715) < 1e-6); + } + for (size_t i = 3 * 3; i < 2 * 3 * 3; ++i) { + EXPECT_TRUE(std::fabs(var_[i] - 0.999684) < 1e-6); + } + for (size_t i = 2 * 3 * 3; i < 3 * 3 * 3; ++i) { + EXPECT_TRUE(std::fabs(var_[i] - 0.999653) < 1e-6); + } +} +} // namespace kernel +} // namespace mindspore diff --git a/tests/ut/cpp/kernel/cpu/sparse_apply_ftrl_cpu_kernel_test.cc b/tests/ut/cpp/kernel/cpu/sparse_apply_ftrl_cpu_kernel_test.cc new file mode 100644 index 0000000000..c5c2394538 --- /dev/null +++ b/tests/ut/cpp/kernel/cpu/sparse_apply_ftrl_cpu_kernel_test.cc @@ -0,0 +1,154 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include "common/common_test.h" +#define private public +#define protected public +#include "kernel/cpu/sparse_apply_ftrl_cpu_kernel.h" +#undef private +#undef protected + +namespace mindspore { +namespace kernel { +class SparseApplyFtrlCpuKernelTest : public UT::Common { + public: + SparseApplyFtrlCpuKernelTest() : sparse_ftrl_(std::make_shared()) {} + + void SetUp() override { + sparse_ftrl_->lr_ = 0.001; + sparse_ftrl_->l1_ = 0.0; + sparse_ftrl_->l2_ = 0.0; + sparse_ftrl_->lr_power_ = -0.5; + var_.clear(); + accum_.clear(); + linear_.clear(); + grad_.clear(); + inputs_.clear(); + workspace_.clear(); + outputs_.clear(); + } + + AddressPtr CreateKernelAddress(void *addr) { + auto kernel_addr = std::make_shared
(); + kernel_addr->addr = addr; + return kernel_addr; + } + + void CreateInputAddress(std::vector &indices) { + inputs_.push_back(CreateKernelAddress(var_.data())); + inputs_.push_back(CreateKernelAddress(accum_.data())); + inputs_.push_back(CreateKernelAddress(linear_.data())); + inputs_.push_back(CreateKernelAddress(grad_.data())); + inputs_.push_back(CreateKernelAddress(indices.data())); + } + + void CreateWorkspaceAddress(std::vector &new_grad, std::vector &new_indices) { + workspace_.push_back(CreateKernelAddress(new_grad.data())); + workspace_.push_back(CreateKernelAddress(new_indices.data())); + } + + std::vector var_; + std::vector accum_; + std::vector linear_; + std::vector grad_; + std::vector inputs_; + std::vector workspace_; + std::vector outputs_; + std::shared_ptr sparse_ftrl_; +}; + +TEST_F(SparseApplyFtrlCpuKernelTest, dense_test) { + for (size_t i = 0; i < 3 * 3 * 3; ++i) { + var_.push_back(1.0); + accum_.push_back(1.0); + linear_.push_back(1.0); + grad_.push_back(1.0); + } + sparse_ftrl_->indices_size_ = 3; + sparse_ftrl_->var_first_dim_size_ = 3; + sparse_ftrl_->var_outer_dim_size_ = 9; + + std::vector indices{0, 1, 2}; + CreateInputAddress(indices); + std::vector new_grad(3 * 3 * 3); + std::vector new_indices(3); + CreateWorkspaceAddress(new_grad, new_indices); + sparse_ftrl_->Launch(inputs_, workspace_, outputs_); + for (size_t i = 0; i < 3 * 3 * 3; ++i) { + EXPECT_TRUE(std::fabs(var_[i] - 0.291479) < 1e-6); + } +} + +TEST_F(SparseApplyFtrlCpuKernelTest, sparse_test1) { + for (size_t i = 0; i < 3 * 3 * 3; ++i) { + var_.push_back(1.0); + accum_.push_back(1.0); + linear_.push_back(1.0); + } + for (size_t i = 0; i < 2 * 3 * 3; ++i) { + grad_.push_back(1.0); + } + sparse_ftrl_->indices_size_ = 2; + sparse_ftrl_->var_first_dim_size_ = 3; + sparse_ftrl_->var_outer_dim_size_ = 9; + + std::vector indices{0, 2}; + CreateInputAddress(indices); + std::vector new_grad(3 * 3 * 3); + std::vector new_indices(3); + CreateWorkspaceAddress(new_grad, new_indices); + sparse_ftrl_->Launch(inputs_, workspace_, outputs_); + for (size_t i = 0; i < 3 * 3; ++i) { + EXPECT_TRUE(std::fabs(var_[i] - 0.291479) < 1e-6); + } + for (size_t i = 3 * 3; i < 2 * 3 * 3; ++i) { + EXPECT_EQ(var_[i], 1.0); + } + for (size_t i = 2 * 3 * 3; i < 3 * 3 * 3; ++i) { + EXPECT_TRUE(std::fabs(var_[i] - 0.291479) < 1e-6); + } +} + +TEST_F(SparseApplyFtrlCpuKernelTest, sparse_test2) { + for (size_t i = 0; i < 3 * 3 * 3; ++i) { + var_.push_back(1.0); + accum_.push_back(1.0); + linear_.push_back(1.0); + grad_.push_back(1.0); + } + sparse_ftrl_->indices_size_ = 3; + sparse_ftrl_->var_first_dim_size_ = 3; + sparse_ftrl_->var_outer_dim_size_ = 9; + + std::vector indices{2, 2, 1}; + CreateInputAddress(indices); + std::vector new_grad(3 * 3 * 3); + std::vector new_indices(3); + CreateWorkspaceAddress(new_grad, new_indices); + sparse_ftrl_->Launch(inputs_, workspace_, outputs_); + for (size_t i = 0; i < 3 * 3; ++i) { + EXPECT_EQ(var_[i], 1.0); + } + for (size_t i = 3 * 3; i < 2 * 3 * 3; ++i) { + EXPECT_TRUE(std::fabs(var_[i] - 0.291479) < 1e-6); + } + for (size_t i = 2 * 3 * 3; i < 3 * 3 * 3; ++i) { + EXPECT_TRUE(std::fabs(var_[i] - 0.551445) < 1e-6); + } +} +} // namespace kernel +} // namespace mindspore diff --git a/tests/ut/cpp/kernel/cpu/sparse_apply_lazy_adam_cpu_kernel_test.cc b/tests/ut/cpp/kernel/cpu/sparse_apply_lazy_adam_cpu_kernel_test.cc new file mode 100644 index 0000000000..1765ed896f --- /dev/null +++ b/tests/ut/cpp/kernel/cpu/sparse_apply_lazy_adam_cpu_kernel_test.cc @@ -0,0 +1,162 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include "common/common_test.h" +#define private public +#define protected public +#include "kernel/cpu/sparse_apply_lazy_adam_cpu_kernel.h" +#undef private +#undef protected + +namespace mindspore { +namespace kernel { +class SparseApplyLazyAdamCpuKernelTest : public UT::Common { + public: + SparseApplyLazyAdamCpuKernelTest() : sparse_lazy_adam_(std::make_shared()) {} + + void SetUp() override { + var_.clear(); + m_.clear(); + v_.clear(); + grad_.clear(); + inputs_.clear(); + workspace_.clear(); + outputs_.clear(); + } + + AddressPtr CreateKernelAddress(void *addr) { + auto kernel_addr = std::make_shared
(); + kernel_addr->addr = addr; + return kernel_addr; + } + + void CreateInputAddress(std::vector &indices) { + inputs_.push_back(CreateKernelAddress(var_.data())); + inputs_.push_back(CreateKernelAddress(m_.data())); + inputs_.push_back(CreateKernelAddress(v_.data())); + inputs_.push_back(CreateKernelAddress(&beta1_power_)); + inputs_.push_back(CreateKernelAddress(&beta2_power_)); + inputs_.push_back(CreateKernelAddress(&lr_)); + inputs_.push_back(CreateKernelAddress(&beta1_)); + inputs_.push_back(CreateKernelAddress(&beta2_)); + inputs_.push_back(CreateKernelAddress(&epsilon_)); + inputs_.push_back(CreateKernelAddress(grad_.data())); + inputs_.push_back(CreateKernelAddress(indices.data())); + } + + void CreateWorkspaceAddress(std::vector &new_grad, std::vector &new_indices) { + workspace_.push_back(CreateKernelAddress(new_grad.data())); + workspace_.push_back(CreateKernelAddress(new_indices.data())); + } + + std::vector var_; + std::vector m_; + std::vector v_; + std::vector grad_; + std::vector inputs_; + std::vector workspace_; + std::vector outputs_; + std::shared_ptr sparse_lazy_adam_; + float beta1_power_ = 0.9; + float beta2_power_ = 0.999; + float lr_ = 0.001; + float beta1_ = 0.9; + float beta2_ = 0.999; + float epsilon_ = 1e-8; +}; + +TEST_F(SparseApplyLazyAdamCpuKernelTest, dense_test) { + for (size_t i = 0; i < 3 * 3 * 3; ++i) { + var_.push_back(1.0); + m_.push_back(1.0); + v_.push_back(1.0); + grad_.push_back(1.0); + } + sparse_lazy_adam_->indices_size_ = 3; + sparse_lazy_adam_->var_first_dim_size_ = 3; + sparse_lazy_adam_->var_outer_dim_size_ = 9; + + std::vector indices{0, 1, 2}; + CreateInputAddress(indices); + std::vector new_grad(3 * 3 * 3); + std::vector new_indices(3); + CreateWorkspaceAddress(new_grad, new_indices); + sparse_lazy_adam_->Launch(inputs_, workspace_, outputs_); + for (size_t i = 0; i < 3 * 3 * 3; ++i) { + EXPECT_TRUE(std::fabs(var_[i] - 0.999684) < 1e-6); + } +} + +TEST_F(SparseApplyLazyAdamCpuKernelTest, sparse_test1) { + for (size_t i = 0; i < 3 * 3 * 3; ++i) { + var_.push_back(1.0); + m_.push_back(1.0); + v_.push_back(1.0); + } + for (size_t i = 0; i < 2 * 3 * 3; ++i) { + grad_.push_back(1.0); + } + sparse_lazy_adam_->indices_size_ = 2; + sparse_lazy_adam_->var_first_dim_size_ = 3; + sparse_lazy_adam_->var_outer_dim_size_ = 9; + + std::vector indices{0, 2}; + CreateInputAddress(indices); + std::vector new_grad(3 * 3 * 3); + std::vector new_indices(3); + CreateWorkspaceAddress(new_grad, new_indices); + sparse_lazy_adam_->Launch(inputs_, workspace_, outputs_); + for (size_t i = 0; i < 3 * 3; ++i) { + EXPECT_TRUE(std::fabs(var_[i] - 0.999684) < 1e-6); + } + for (size_t i = 3 * 3; i < 2 * 3 * 3; ++i) { + EXPECT_EQ(var_[i], 1.0); + } + for (size_t i = 2 * 3 * 3; i < 3 * 3 * 3; ++i) { + EXPECT_TRUE(std::fabs(var_[i] - 0.999684) < 1e-6); + } +} + +TEST_F(SparseApplyLazyAdamCpuKernelTest, sparse_test2) { + for (size_t i = 0; i < 3 * 3 * 3; ++i) { + var_.push_back(1.0); + m_.push_back(1.0); + v_.push_back(1.0); + grad_.push_back(1.0); + } + sparse_lazy_adam_->indices_size_ = 3; + sparse_lazy_adam_->var_first_dim_size_ = 3; + sparse_lazy_adam_->var_outer_dim_size_ = 9; + + std::vector indices{2, 2, 1}; + CreateInputAddress(indices); + std::vector new_grad(3 * 3 * 3); + std::vector new_indices(3); + CreateWorkspaceAddress(new_grad, new_indices); + sparse_lazy_adam_->Launch(inputs_, workspace_, outputs_); + for (size_t i = 0; i < 3 * 3; ++i) { + EXPECT_EQ(var_[i], 1.0); + } + for (size_t i = 3 * 3; i < 2 * 3 * 3; ++i) { + EXPECT_TRUE(std::fabs(var_[i] - 0.999684) < 1e-6); + } + for (size_t i = 2 * 3 * 3; i < 3 * 3 * 3; ++i) { + EXPECT_TRUE(std::fabs(var_[i] - 0.999653) < 1e-6); + } +} +} // namespace kernel +} // namespace mindspore diff --git a/tests/ut/cpp/kernel/cpu/sparse_apply_proximal_adagrad_cpu_kernel_test.cc b/tests/ut/cpp/kernel/cpu/sparse_apply_proximal_adagrad_cpu_kernel_test.cc new file mode 100644 index 0000000000..23f66db58c --- /dev/null +++ b/tests/ut/cpp/kernel/cpu/sparse_apply_proximal_adagrad_cpu_kernel_test.cc @@ -0,0 +1,151 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include "common/common_test.h" +#define private public +#define protected public +#include "kernel/cpu/sparse_apply_proximal_adagrad_cpu_kernel.h" +#undef private +#undef protected + +namespace mindspore { +namespace kernel { +class SparseApplyProximalAdagradCpuKernelTest : public UT::Common { + public: + SparseApplyProximalAdagradCpuKernelTest() + : sparse_proximal_adagrad_(std::make_shared()) {} + + void SetUp() override { + var_.clear(); + accum_.clear(); + grad_.clear(); + inputs_.clear(); + workspace_.clear(); + outputs_.clear(); + } + + AddressPtr CreateKernelAddress(void *addr) { + auto kernel_addr = std::make_shared
(); + kernel_addr->addr = addr; + return kernel_addr; + } + + void CreateInputAddress(std::vector &indices) { + inputs_.push_back(CreateKernelAddress(var_.data())); + inputs_.push_back(CreateKernelAddress(accum_.data())); + inputs_.push_back(CreateKernelAddress(&lr_)); + inputs_.push_back(CreateKernelAddress(&l1_)); + inputs_.push_back(CreateKernelAddress(&l2_)); + inputs_.push_back(CreateKernelAddress(grad_.data())); + inputs_.push_back(CreateKernelAddress(indices.data())); + } + + void CreateWorkspaceAddress(std::vector &new_grad, std::vector &new_indices) { + workspace_.push_back(CreateKernelAddress(new_grad.data())); + workspace_.push_back(CreateKernelAddress(new_indices.data())); + } + + std::vector var_; + std::vector accum_; + std::vector grad_; + std::vector inputs_; + std::vector workspace_; + std::vector outputs_; + std::shared_ptr sparse_proximal_adagrad_; + float lr_ = 0.01; + float l1_ = 0.0; + float l2_ = 0.0; +}; + +TEST_F(SparseApplyProximalAdagradCpuKernelTest, dense_test) { + for (size_t i = 0; i < 3 * 3 * 3; ++i) { + var_.push_back(1.0); + accum_.push_back(1.0); + grad_.push_back(1.0); + } + sparse_proximal_adagrad_->indices_size_ = 3; + sparse_proximal_adagrad_->var_first_dim_size_ = 3; + sparse_proximal_adagrad_->var_outer_dim_size_ = 9; + + std::vector indices{0, 1, 2}; + CreateInputAddress(indices); + std::vector new_grad(3 * 3 * 3); + std::vector new_indices(3); + CreateWorkspaceAddress(new_grad, new_indices); + sparse_proximal_adagrad_->Launch(inputs_, workspace_, outputs_); + for (size_t i = 0; i < 3 * 3 * 3; ++i) { + EXPECT_TRUE(std::fabs(var_[i] - 0.9929289) < 1e-6); + } +} + +TEST_F(SparseApplyProximalAdagradCpuKernelTest, sparse_test1) { + for (size_t i = 0; i < 3 * 3 * 3; ++i) { + var_.push_back(1.0); + accum_.push_back(1.0); + } + for (size_t i = 0; i < 2 * 3 * 3; ++i) { + grad_.push_back(1.0); + } + sparse_proximal_adagrad_->indices_size_ = 2; + sparse_proximal_adagrad_->var_first_dim_size_ = 3; + sparse_proximal_adagrad_->var_outer_dim_size_ = 9; + + std::vector indices{0, 2}; + CreateInputAddress(indices); + std::vector new_grad(3 * 3 * 3); + std::vector new_indices(3); + CreateWorkspaceAddress(new_grad, new_indices); + sparse_proximal_adagrad_->Launch(inputs_, workspace_, outputs_); + for (size_t i = 0; i < 3 * 3; ++i) { + EXPECT_TRUE(std::fabs(var_[i] - 0.9929289) < 1e-6); + } + for (size_t i = 3 * 3; i < 2 * 3 * 3; ++i) { + EXPECT_EQ(var_[i], 1.0); + } + for (size_t i = 2 * 3 * 3; i < 3 * 3 * 3; ++i) { + EXPECT_TRUE(std::fabs(var_[i] - 0.9929289) < 1e-6); + } +} + +TEST_F(SparseApplyProximalAdagradCpuKernelTest, sparse_test2) { + for (size_t i = 0; i < 3 * 3 * 3; ++i) { + var_.push_back(1.0); + accum_.push_back(1.0); + grad_.push_back(1.0); + } + sparse_proximal_adagrad_->indices_size_ = 3; + sparse_proximal_adagrad_->var_first_dim_size_ = 3; + sparse_proximal_adagrad_->var_outer_dim_size_ = 9; + + std::vector indices{2, 2, 1}; + CreateInputAddress(indices); + std::vector new_grad(3 * 3 * 3); + std::vector new_indices(3); + CreateWorkspaceAddress(new_grad, new_indices); + sparse_proximal_adagrad_->Launch(inputs_, workspace_, outputs_); + for (size_t i = 0; i < 3 * 3; ++i) { + EXPECT_EQ(var_[i], 1.0); + } + for (size_t i = 3 * 3; i < 2 * 3 * 3; ++i) { + EXPECT_TRUE(std::fabs(var_[i] - 0.9929289) < 1e-6); + } + for (size_t i = 2 * 3 * 3; i < 3 * 3 * 3; ++i) { + EXPECT_TRUE(std::fabs(var_[i] - 0.9910557) < 1e-6); + } +} +} // namespace kernel +} // namespace mindspore From 16976b5f68d4d4b85f9f1e09c5d297a9a4f71d5b Mon Sep 17 00:00:00 2001 From: yanghaoran Date: Mon, 22 Jun 2020 17:26:39 +0800 Subject: [PATCH 124/254] update version to 0.5 --- README.md | 15 ++-- RELEASE.md | 71 ++++++++++++++++++ build.sh | 4 +- docker/mindspore-cpu/0.5.0-beta/Dockerfile | 67 +++++++++++++++++ docker/mindspore-gpu/0.5.0-beta/Dockerfile | 83 ++++++++++++++++++++++ setup.py | 2 +- 6 files changed, 231 insertions(+), 11 deletions(-) create mode 100644 docker/mindspore-cpu/0.5.0-beta/Dockerfile create mode 100644 docker/mindspore-gpu/0.5.0-beta/Dockerfile diff --git a/README.md b/README.md index a6bfd1ebbb..a7226b0f91 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ enrichment of the AI software/hardware application ecosystem. MindSpore Architecture -For more details please check out our [Architecture Guide](https://www.mindspore.cn/docs/en/0.3.0-alpha/architecture.html). +For more details please check out our [Architecture Guide](https://www.mindspore.cn/docs/en/0.5.0-beta/architecture.html). ### Automatic Differentiation @@ -66,7 +66,6 @@ MindSpore offers build options across multiple backends: | Ascend910 | Ubuntu-x86 | ✔️ | | | EulerOS-x86 | ✔️ | | | EulerOS-aarch64 | ✔️ | -| GPU CUDA 9.2 | Ubuntu-x86 | ✔️ | | GPU CUDA 10.1 | Ubuntu-x86 | ✔️ | | CPU | Ubuntu-x86 | ✔️ | | | Windows-x86 | ✔️ | @@ -76,7 +75,7 @@ For installation using `pip`, take `CPU` and `Ubuntu-x86` build version as an ex 1. Download whl from [MindSpore download page](https://www.mindspore.cn/versions/en), and install the package. ``` - pip install https://ms-release.obs.cn-north-4.myhuaweicloud.com/0.3.0-alpha/MindSpore/cpu/ubuntu_x86/mindspore-0.3.0-cp37-cp37m-linux_x86_64.whl + pip install https://ms-release.obs.cn-north-4.myhuaweicloud.com/0.5.0-beta/MindSpore/cpu/ubuntu_x86/mindspore-0.5.0-cp37-cp37m-linux_x86_64.whl ``` 2. Run the following command to verify the install. @@ -133,8 +132,8 @@ currently the containerized build options are supported as follows: For `CPU` backend, you can directly pull and run the latest stable image using the below command: ``` - docker pull mindspore/mindspore-cpu:0.3.0-alpha - docker run -it mindspore/mindspore-cpu:0.3.0-alpha /bin/bash + docker pull mindspore/mindspore-cpu:0.5.0-beta + docker run -it mindspore/mindspore-cpu:0.5.0-beta /bin/bash ``` * GPU @@ -151,8 +150,8 @@ currently the containerized build options are supported as follows: Then you can pull and run the latest stable image using the below command: ``` - docker pull mindspore/mindspore-gpu:0.3.0-alpha - docker run -it --runtime=nvidia --privileged=true mindspore/mindspore-gpu:0.3.0-alpha /bin/bash + docker pull mindspore/mindspore-gpu:0.5.0-beta + docker run -it --runtime=nvidia --privileged=true mindspore/mindspore-gpu:0.5.0-beta /bin/bash ``` To test if the docker image works, please execute the python code below and check the output: @@ -187,7 +186,7 @@ please check out [docker](docker/README.md) repo for the details. ## Quickstart -See the [Quick Start](https://www.mindspore.cn/tutorial/en/0.3.0-alpha/quick_start/quick_start.html) +See the [Quick Start](https://www.mindspore.cn/tutorial/en/0.5.0-beta/quick_start/quick_start.html) to implement the image classification. ## Docs diff --git a/RELEASE.md b/RELEASE.md index 9824f803f0..eb390d3624 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -1,3 +1,74 @@ +# Release 0.5.0-beta + +## Major Features and Improvements + +### Ascend 910 Training and Inference Framework +* New models + * ResNext50: a simple, highly modularized network architecture using aggregated resdiual transformations for image classification on ImageNet 2012 dataset. + * MASS: a pre-training method for sequence to sequence based language generation tasks on Text Summarization and Conversational Response Generation using News Crawls 2007-2017 dataset, Gigaword corpus and Cornell movie dialog corpus. + * Transformer: a neural network architecture for language understanding on WMT 2014 English-German dataset. + * GCN：Graph Convolutional Networks for the task of classification of nodes in a graph on Cora and Citeseer datasets. + * GAT：an attention-based graph neural network for node classification on Cora and CiteSeer dataset. +* Frontend and user interface + * Support tensor value and assignment of mixed tensor index in graph mode. + * Support tensor comparison, len operator, constexpr syntax, value and assignment of tensor index in pynative mode. + * Support converting MindSpore IR to pb format for infer model. + * Support print operator to write data directly on the hard disk. + * Add the double recursive programming solution for very high speed parallel strategy search in automatic parallel. + * User interfaces change log + * Allow the learning rate of AdamWeightDecayDynamicLR and Lamb to be 0([!1826](https://gitee.com/mindspore/mindspore/pulls/1826)) + * Restricting the entire network input parameter is Tensor([!1967](https://gitee.com/mindspore/mindspore/pulls/1967)) + * Turn shape and dtype into attributes instead of interfaces([!1919](https://gitee.com/mindspore/mindspore/pulls/1919)) + * Delete multitypefungraph([!2116](https://gitee.com/mindspore/mindspore/pulls/2116)) + * Refactor the callback module in an encapsulated way, use _CallbackManager instead of _build_callbacks([!2236](https://gitee.com/mindspore/mindspore/pulls/2236)) + * Delete EmbeddingLookup([!2163](https://gitee.com/mindspore/mindspore/pulls/2163)) + * Checkpoint add model_type([!2517](https://gitee.com/mindspore/mindspore/pulls/2517)) +* Executor and performance optimization + * Heterogeneous execution on CPU and Ascend devices supported, and is verified in Wide&Deep model. + * Quantitative training of MobileNetV2, Lenet and Resnet50 on Ascend-910 are supported. + * Support new fusion architecture, which can do fusion optimization across graphs and kernels to improve execution speed. +* Data processing, augmentation, and save format + * Support data processing pipeline performance profiling. + * Support public dataset loading, such as CLUE and Coco. + * Support more text processing, such as more tokenizers and vocab data. + * Support MindRecord padded data. +### Other Hardware Support +* GPU platform + * New model supported: Bert / Wide&Deep. + * Support setting max device memory. +* CPU platform + * New model supported: LSTM. + +## Bugfixes +* Models + * Bert, Move Bert from `example` to `model_zoo`, optimize network for better performance. ([!1902](https://gitee.com/mindspore/mindspore/pulls/1902)) + * VGG16, Move VGG16 from `example` to `model_zoo`, optimize network for better accuracy. ([!2645](https://gitee.com/mindspore/mindspore/pulls/2645)) + * Alexnet, modify parameter setting to improve accuracy ([!1364](https://gitee.com/mindspore/mindspore/pulls/2370)) +* Python API + * Fix bug in auto cast([!1766](https://gitee.com/mindspore/mindspore/pulls/1766)) + * Fix bug of register_backward_hook([!2148](https://gitee.com/mindspore/mindspore/pulls/2148)) + * Fix bug of tuple args in pynative mode([!1878](https://gitee.com/mindspore/mindspore/pulls/1878)) + * Fix bug of checking numbers of arguments and graph parameters([!1701](https://gitee.com/mindspore/mindspore/pulls/1701)) +* Executor + * Fix bug of loading input data repeatedly in pynative mode([!1966](https://gitee.com/mindspore/mindspore/pulls/1966)) + * Fix bug of list cannot be used as input in pynative mode([!1765](https://gitee.com/mindspore/mindspore/pulls/1765)) + * Fix bug of kernel select ([!2103](https://gitee.com/mindspore/mindspore/pulls/2103)) + * Fix bug of pattern matching for batchnorm fusion in the case of auto mix precision.([!1851](https://gitee.com/mindspore/mindspore/pulls/1851)) + * Fix bug of generate hccl's kernel info.([!2393](https://gitee.com/mindspore/mindspore/mindspore/pulls/2393)) +* GPU platform + * Fix bug of summary feature invalid([!2173](https://gitee.com/mindspore/mindspore/pulls/2173)) +* Data processing + * Fix bug of Cifar dataset reading([!2096](https://gitee.com/mindspore/mindspore/pulls/2096)) + * Fix bug of C++ behavior in RandomCropAndResize([!2026](https://gitee.com/mindspore/mindspore/pulls/2026)) + * Fix the bug of mindrecord shuffle([!2420](https://gitee.com/mindspore/mindspore/pulls/2420)) + +## Contributors +Thanks goes to these wonderful people: + +Alexey Shevlyakov, avakh, baihuawei, BowenK, buxue, caifubi, caojian05, Cathy Wong, changzherui, chenfei, chengxianbin, chenhaozhe, chenjianping, chentingting, chenzomi, chujinjin, Danish Farid, dayschan, dengwentao, dinghao, etone-chan, fangzehua, fary86, geekun, Giancarlo Colmenares, gong chen, gukecai, guohongzilong, hangangqiang, heleiwang, hesham, He Wei, hexia, hongxing, huangdongrun, huanghui, islam_amin, Jamie Nisbet, Jesse Lee, jiangjinsheng, jiangzhiwen, jinyaohui, jjfeing, jojobugfree, Jonathan Yan, jonyguo, Junhan Hu, Kang, kingfo, kouzhenzhong, kpy, kswang, laiyongqiang, leopz, liangzelang, lichenever, lihongkang, Li Hongzhang, lilei, limingqi107, lirongzhen1, liubuyu, liuchongming74, liuwenhao4, liuxiao, Lixia Chen, liyanliu, liyong, lizhenyu, lvliang, Mahdi, Margaret_wangrui, meixiaowei, ms_yan, nhussain, ougongchang, panfengfeng, panyifeng, peilinwang, Peilin Wang, pkuliuliu, qianlong, rick_sanchez, shibeiji, Shida He, shijianning, simson, sunsuodong, suteng, Tinazhang, Tron Zhang, unknown, VectorSL, wandongdong, wangcong, wangdongxu, wangdongxu6, wanghua, wangnan39, Wei Luning, wenchunjiang, wenkai, wilfChen, WilliamLian, wukesong, Xian Weizhao, Xiaoda Zhang, xiefangqi, xulei2020, xunxue, xutianchun, Yang, yanghaitao, yanghaitao1, yanghaoran, yangjie, yangjie159, YangLuo, Yanjun Peng, yankai, yanzhenxiang2020, yao_yf, Yi Huaijie, yoonlee666, yuchaojie, yujianfeng, zhangzhongpeng, zhangdengcheng, Zhang Qinghua, zhangyinxia, zhangz0911gm, zhaojichen, zhaoting, zhaozhenlong, zhoufeng, zhouneng, zhousiyi, Zirui Wu, Ziyan, zjun, ZPaC, lihongzhang, wangdongxu + +Contributions of any kind are welcome! + # Release 0.3.0-alpha ## Major Features and Improvements diff --git a/build.sh b/build.sh index c6f80b1269..50c85845fc 100755 --- a/build.sh +++ b/build.sh @@ -461,9 +461,9 @@ build_predict() cd "${BASEPATH}/predict/output/" if [[ "$PREDICT_PLATFORM" == "x86_64" ]]; then - tar -cf MSPredict-0.3.0-linux_x86_64.tar.gz include/ lib/ --warning=no-file-changed + tar -cf MSPredict-0.5.0-linux_x86_64.tar.gz include/ lib/ --warning=no-file-changed elif [[ "$PREDICT_PLATFORM" == "arm64" ]]; then - tar -cf MSPredict-0.3.0-linux_aarch64.tar.gz include/ lib/ --warning=no-file-changed + tar -cf MSPredict-0.5.0-linux_aarch64.tar.gz include/ lib/ --warning=no-file-changed fi echo "success to build predict project!" } diff --git a/docker/mindspore-cpu/0.5.0-beta/Dockerfile b/docker/mindspore-cpu/0.5.0-beta/Dockerfile new file mode 100644 index 0000000000..4da6294296 --- /dev/null +++ b/docker/mindspore-cpu/0.5.0-beta/Dockerfile @@ -0,0 +1,67 @@ +FROM ubuntu:18.04 + +MAINTAINER leonwanghui + +# Set env +ENV PYTHON_ROOT_PATH /usr/local/python-3.7.5 +ENV PATH /usr/local/bin:$PATH + +# Install base tools +RUN apt update \ + && DEBIAN_FRONTEND=noninteractive apt install -y \ + vim \ + wget \ + curl \ + xz-utils \ + net-tools \ + openssh-client \ + git \ + ntpdate \ + tzdata \ + tcl \ + sudo \ + bash-completion + +# Install compile tools +RUN DEBIAN_FRONTEND=noninteractive apt install -y \ + gcc \ + g++ \ + zlibc \ + make \ + libgmp-dev \ + patch \ + autoconf \ + libtool \ + automake \ + flex + +# Set bash +RUN echo "dash dash/sh boolean false" | debconf-set-selections +RUN DEBIAN_FRONTEND=noninteractive dpkg-reconfigure dash + +# Install python (v3.7.5) +RUN apt install -y libffi-dev libssl-dev zlib1g-dev libbz2-dev libncurses5-dev \ + libgdbm-dev libgdbm-compat-dev liblzma-dev libreadline-dev libsqlite3-dev \ + && cd /tmp \ + && wget https://github.com/python/cpython/archive/v3.7.5.tar.gz \ + && tar -xvf v3.7.5.tar.gz \ + && cd /tmp/cpython-3.7.5 \ + && mkdir -p ${PYTHON_ROOT_PATH} \ + && ./configure --prefix=${PYTHON_ROOT_PATH} \ + && make -j4 \ + && make install -j4 \ + && rm -f /usr/local/bin/python \ + && rm -f /usr/local/bin/pip \ + && ln -s ${PYTHON_ROOT_PATH}/bin/python3.7 /usr/local/bin/python \ + && ln -s ${PYTHON_ROOT_PATH}/bin/pip3.7 /usr/local/bin/pip \ + && rm -rf /tmp/cpython-3.7.5 \ + && rm -f /tmp/v3.7.5.tar.gz + +# Set pip source +RUN mkdir -pv /root/.pip \ + && echo "[global]" > /root/.pip/pip.conf \ + && echo "trusted-host=mirrors.aliyun.com" >> /root/.pip/pip.conf \ + && echo "index-url=http://mirrors.aliyun.com/pypi/simple/" >> /root/.pip/pip.conf + +# Install MindSpore cpu whl package +RUN pip install --no-cache-dir https://ms-release.obs.cn-north-4.myhuaweicloud.com/0.5.0-beta/MindSpore/cpu/ubuntu_x86/mindspore-0.5.0-cp37-cp37m-linux_x86_64.whl diff --git a/docker/mindspore-gpu/0.5.0-beta/Dockerfile b/docker/mindspore-gpu/0.5.0-beta/Dockerfile new file mode 100644 index 0000000000..dae6d16370 --- /dev/null +++ b/docker/mindspore-gpu/0.5.0-beta/Dockerfile @@ -0,0 +1,83 @@ +FROM nvidia/cuda:10.1-cudnn7-runtime-ubuntu18.04 + +MAINTAINER leonwanghui + +# Set env +ENV PYTHON_ROOT_PATH /usr/local/python-3.7.5 +ENV OMPI_ROOT_PATH /usr/local/openmpi-3.1.5 +ENV PATH ${OMPI_ROOT_PATH}/bin:/usr/local/bin:$PATH +ENV LD_LIBRARY_PATH ${OMPI_ROOT_PATH}/lib:$LD_LIBRARY_PATH + +# Install base tools +RUN apt update \ + && DEBIAN_FRONTEND=noninteractive apt install -y \ + vim \ + wget \ + curl \ + xz-utils \ + net-tools \ + openssh-client \ + git \ + ntpdate \ + tzdata \ + tcl \ + sudo \ + bash-completion + +# Install compile tools +RUN DEBIAN_FRONTEND=noninteractive apt install -y \ + gcc \ + g++ \ + zlibc \ + make \ + libgmp-dev \ + patch \ + autoconf \ + libtool \ + automake \ + flex \ + libnccl2=2.4.8-1+cuda10.1 \ + libnccl-dev=2.4.8-1+cuda10.1 + +# Set bash +RUN echo "dash dash/sh boolean false" | debconf-set-selections +RUN DEBIAN_FRONTEND=noninteractive dpkg-reconfigure dash + +# Install python (v3.7.5) +RUN apt install -y libffi-dev libssl-dev zlib1g-dev libbz2-dev libncurses5-dev \ + libgdbm-dev libgdbm-compat-dev liblzma-dev libreadline-dev libsqlite3-dev \ + && cd /tmp \ + && wget https://github.com/python/cpython/archive/v3.7.5.tar.gz \ + && tar -xvf v3.7.5.tar.gz \ + && cd /tmp/cpython-3.7.5 \ + && mkdir -p ${PYTHON_ROOT_PATH} \ + && ./configure --prefix=${PYTHON_ROOT_PATH} \ + && make -j4 \ + && make install -j4 \ + && rm -f /usr/local/bin/python \ + && rm -f /usr/local/bin/pip \ + && ln -s ${PYTHON_ROOT_PATH}/bin/python3.7 /usr/local/bin/python \ + && ln -s ${PYTHON_ROOT_PATH}/bin/pip3.7 /usr/local/bin/pip \ + && rm -rf /tmp/cpython-3.7.5 \ + && rm -f /tmp/v3.7.5.tar.gz + +# Set pip source +RUN mkdir -pv /root/.pip \ + && echo "[global]" > /root/.pip/pip.conf \ + && echo "trusted-host=mirrors.aliyun.com" >> /root/.pip/pip.conf \ + && echo "index-url=http://mirrors.aliyun.com/pypi/simple/" >> /root/.pip/pip.conf + +# Install openmpi (v3.1.5) +RUN cd /tmp \ + && wget https://download.open-mpi.org/release/open-mpi/v3.1/openmpi-3.1.5.tar.gz \ + && tar -xvf openmpi-3.1.5.tar.gz \ + && cd /tmp/openmpi-3.1.5 \ + && mkdir -p ${OMPI_ROOT_PATH} \ + && ./configure --prefix=${OMPI_ROOT_PATH} \ + && make -j4 \ + && make install -j4 \ + && rm -rf /tmp/openmpi-3.1.5 \ + && rm -f /tmp/openmpi-3.1.5.tar.gz + +# Install MindSpore cuda-10.1 whl package +RUN pip install --no-cache-dir https://ms-release.obs.cn-north-4.myhuaweicloud.com/0.5.0-beta/MindSpore/gpu/ubuntu_x86/cuda-10.1/mindspore_gpu-0.5.0-cp37-cp37m-linux_x86_64.whl diff --git a/setup.py b/setup.py index dbda644059..2840eb3b14 100644 --- a/setup.py +++ b/setup.py @@ -23,7 +23,7 @@ from setuptools import setup, find_packages from setuptools.command.egg_info import egg_info from setuptools.command.build_py import build_py -version = '0.3.0' +version = '0.5.0' backend_policy = os.getenv('BACKEND_POLICY') commit_id = os.getenv('COMMIT_ID').replace("\n", "") From b854ae7ebc3702ab9b779af881d8e9262647e3d5 Mon Sep 17 00:00:00 2001 From: VectorSL Date: Sun, 28 Jun 2020 21:59:03 +0800 Subject: [PATCH 125/254] gpu mul support int --- mindspore/ccsrc/kernel/gpu/math/broadcast_gpu_kernel.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mindspore/ccsrc/kernel/gpu/math/broadcast_gpu_kernel.cc b/mindspore/ccsrc/kernel/gpu/math/broadcast_gpu_kernel.cc index e299946780..96d51b704c 100644 --- a/mindspore/ccsrc/kernel/gpu/math/broadcast_gpu_kernel.cc +++ b/mindspore/ccsrc/kernel/gpu/math/broadcast_gpu_kernel.cc @@ -96,5 +96,8 @@ MS_REG_GPU_KERNEL_TWO( MS_REG_GPU_KERNEL_TWO( Maximum, KernelAttr().AddInputAttr(kNumberTypeInt32).AddInputAttr(kNumberTypeInt32).AddOutputAttr(kNumberTypeInt32), BroadcastOpGpuKernel, int, int) +MS_REG_GPU_KERNEL_TWO( + Mul, KernelAttr().AddInputAttr(kNumberTypeInt32).AddInputAttr(kNumberTypeInt32).AddOutputAttr(kNumberTypeInt32), + BroadcastOpGpuKernel, int, int) } // namespace kernel } // namespace mindspore From 9febf7fdf511ab930cfb9a5f2d2e8c6c13a3d8e8 Mon Sep 17 00:00:00 2001 From: hongxing Date: Sun, 28 Jun 2020 16:21:11 +0200 Subject: [PATCH 126/254] support GatherV2P --- .../rec_core/rec_generate_strategy.cc | 31 +++++++++++++++++-- .../rec_core/rec_generate_strategy.h | 3 +- .../auto_parallel/rec_core/rec_parse_graph.cc | 4 +-- .../auto_parallel/rec_core/rec_parse_graph.h | 4 +-- .../auto_parallel/rec_core/rec_partition.cc | 9 +++--- .../auto_parallel/rec_core/rec_partition.h | 8 ++--- 6 files changed, 43 insertions(+), 16 deletions(-) diff --git a/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_generate_strategy.cc b/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_generate_strategy.cc index b8a57ae997..9de71231c0 100644 --- a/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_generate_strategy.cc +++ b/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_generate_strategy.cc @@ -164,9 +164,34 @@ std::vector> PrepareOneHot(const std::shared_ptr &gr return strategies; } -std::vector> PrepareGatherV2(const std::shared_ptr> &s) { +std::vector> PrepareGatherV2(const std::vector> &ops, + const size_t iter_ops, std::vector s) { std::vector> strategies; - strategies.push_back(*s); + + int32_t axis = 0; + auto axis_input = GetValue(ops[iter_ops]->input_value().at(2)); + if (axis_input < 0) { + axis_input += SizeToInt(ops[iter_ops]->inputs_tensor_info()[0].shape().size()); + } + axis = axis_input; + if (axis >= SizeToInt(s.size())) { + MS_LOG(EXCEPTION) << "Failure: GatherV2' axis out of range."; + } + s[axis] = 1; + strategies.push_back(s); + + auto pos = ops[iter_ops]->name().find("Info"); + auto name = ops[iter_ops]->name().substr(0, pos); + if (name == "GatherV2") { + return strategies; + } + + std::vector s_indices; + for (size_t i = 0; i < ops[iter_ops]->inputs_tensor_info()[1].shape().size(); i++) { + s_indices.push_back(1); + } + strategies.push_back(s_indices); + return strategies; } @@ -607,7 +632,7 @@ std::vector> GenerateStrategiesFromStrategy(const std::vect return PrepareBiasAdd(s_ptr); } if (ops[iter_ops]->type() == GATHERV2) { - return PrepareGatherV2(s_ptr); + return PrepareGatherV2(ops, iter_ops, basic_stra); } if (ops[iter_ops]->type() == L2_NORMALIZE) { return PrepareL2Normalize(ops, iter_ops, basic_stra); diff --git a/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_generate_strategy.h b/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_generate_strategy.h index 1e8080f2b7..e82efe6798 100644 --- a/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_generate_strategy.h +++ b/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_generate_strategy.h @@ -38,7 +38,8 @@ std::vector> PrepareBiasAdd(const std::shared_ptr> PrepareOneHot(const std::shared_ptr &graph, const std::vector> &ops, const size_t iter_graph, const size_t iter_ops); -std::vector> PrepareGatherV2(const std::shared_ptr> &s); +std::vector> PrepareGatherV2(const std::vector> &ops, + const size_t iter_ops, std::vector s); std::vector> PrepareL2Normalize(const std::vector> &ops, const size_t iter_ops, std::vector s); std::vector> MakeRecSearchStrategy(const std::shared_ptr &graph, diff --git a/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_parse_graph.cc b/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_parse_graph.cc index 190a716063..c0412e9108 100644 --- a/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_parse_graph.cc +++ b/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_parse_graph.cc @@ -40,7 +40,7 @@ const TensorParam MakeTensor(int n, int c, int h, int w) { return tensor; } -Graph::NodeType MakeNewOperator(std::vector> ops, size_t iter_ops) { +Graph::NodeType MakeNewOperator(const std::vector> &ops, size_t iter_ops) { Graph::NodeType NewOp; NewOp.name = ops[iter_ops]->name(); NewOp.info = InfoType::kApplication; @@ -140,7 +140,7 @@ std::shared_ptr ParseGraph(const std::vector> &input_tensor_names, std::shared_ptr graph) { +void MakeEdge(const std::vector> &input_tensor_names, const std::shared_ptr &graph) { for (size_t iter_i = 0; iter_i < input_tensor_names.size(); iter_i++) { for (size_t iter_j = 1; iter_j < input_tensor_names[iter_i].size(); iter_j++) { size_t head_node_index = GetIndexInInputTensorNames(input_tensor_names, input_tensor_names[iter_i][iter_j]); diff --git a/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_parse_graph.h b/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_parse_graph.h index a696e88332..53abefd1c8 100644 --- a/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_parse_graph.h +++ b/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_parse_graph.h @@ -110,7 +110,7 @@ const std::map DictOpType{ const TensorParam MakeTensor(int n, int c, int h, int w); -Graph::NodeType MakeNewOperator(std::vector> ops, size_t iter_ops); +Graph::NodeType MakeNewOperator(const std::vector> &ops, size_t iter_ops); OperatorRec CompleteOperatorInputs(const std::vector> &ops, const size_t iter_ops, Graph::NodeType NewTensor); @@ -121,7 +121,7 @@ TensorParam Complete2DInputs(const std::vector> &o std::shared_ptr ParseGraph(const std::vector> &ops, const std::vector> &input_tensor_names); -void MakeEdge(const std::vector> &input_tensor_names, std::shared_ptr graph); +void MakeEdge(const std::vector> &input_tensor_names, const std::shared_ptr &graph); size_t GetIndexInInputTensorNames(const std::vector> &input_tensor_names, const std::string &input_name); diff --git a/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_partition.cc b/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_partition.cc index 0f6e736d52..d5200f54d8 100644 --- a/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_partition.cc +++ b/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_partition.cc @@ -93,7 +93,7 @@ double GetWeights(const Graph::NodeType &node) { } // Sort all the nodes by their weights -std::vector SortByWeight(const std::shared_ptr graph) { +std::vector SortByWeight(const std::shared_ptr &graph) { MS_EXCEPTION_IF_NULL(graph); std::vector> weight_to_node_index; @@ -124,7 +124,7 @@ std::vector SortByWeight(const std::shared_ptr graph) { // Get optimal strategy to partition the target node StrategyRec PartitionNode(const Graph::NodeType &node, const std::vector> &node_name_to_strategy, - std::shared_ptr graph) { + const std::shared_ptr &graph) { bool enable_conv_chw_partition = false; MS_EXCEPTION_IF_NULL(graph); @@ -191,7 +191,8 @@ StrategyRec PartitionNode(const Graph::NodeType &node, } // Parttion graph into all devices. -Status PartitionForAllDevices(const size_t num_device, const double device_memory, std::shared_ptr graph) { +Status PartitionForAllDevices(const size_t num_device, const double device_memory, + const std::shared_ptr &graph) { if (num_device < 1) { MS_LOG(EXCEPTION) << "ERROR: Number of devices can't be " << num_device << "."; } @@ -261,7 +262,7 @@ Graph::NodeType ApplyStrToTensor(Graph::NodeType Node) { return Node; } -Status DevicesMemoryControl(const size_t num_device, const double device_memory, std::shared_ptr graph) { +Status DevicesMemoryControl(const size_t num_device, const double device_memory, const std::shared_ptr &graph) { MS_EXCEPTION_IF_NULL(graph); if (num_device == 0) { MS_LOG(EXCEPTION) << "Failure: device number is 0."; diff --git a/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_partition.h b/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_partition.h index b2fbeddebd..c98f3317f8 100644 --- a/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_partition.h +++ b/mindspore/ccsrc/parallel/auto_parallel/rec_core/rec_partition.h @@ -32,19 +32,19 @@ namespace mindspore { namespace parallel { -std::vector SortByWeight(const std::shared_ptr graph); +std::vector SortByWeight(const std::shared_ptr &graph); double GetWeights(const Graph::NodeType &node); StrategyRec PartitionNode(const Graph::NodeType &node, const std::vector> &node_name_to_strategy, - std::shared_ptr graph); + const std::shared_ptr &graph); -Status PartitionForAllDevices(const size_t num_device, const double device_memory, std::shared_ptr graph); +Status PartitionForAllDevices(const size_t num_device, const double device_memory, const std::shared_ptr &graph); Graph::NodeType ApplyStrToTensor(Graph::NodeType Node); -Status DevicesMemoryControl(const size_t num_device, const double device_memory, std::shared_ptr graph); +Status DevicesMemoryControl(const size_t num_device, const double device_memory, const std::shared_ptr &graph); size_t GetDataTypeSize(const TensorType &type); } // namespace parallel From 7870329204b4130cb831c37264e149e1e5d2728b Mon Sep 17 00:00:00 2001 From: gengdongjie Date: Wed, 24 Jun 2020 20:35:42 +0800 Subject: [PATCH 127/254] move resnet series from example to model_zoo --- example/resnet50_cifar10/README.md | 137 ---- example/resnet50_cifar10/config.py | 39 - example/resnet50_cifar10/dataset.py | 81 -- example/resnet50_cifar10/eval.py | 72 -- example/resnet50_cifar10/lr_generator.py | 77 -- .../resnet50_cifar10/run_distribute_train.sh | 64 -- example/resnet50_cifar10/run_infer.sh | 64 -- .../resnet50_cifar10/run_standalone_train.sh | 55 -- example/resnet50_cifar10/train.py | 97 --- example/resnet50_imagenet2012/README.md | 150 ---- example/resnet50_imagenet2012/crossentropy.py | 39 - example/resnet50_imagenet2012/dataset.py | 85 -- example/resnet50_imagenet2012/eval.py | 62 -- .../run_distribute_train.sh | 80 -- example/resnet50_imagenet2012/run_infer.sh | 64 -- .../run_standalone_train.sh | 70 -- example/resnet50_imagenet2012/train.py | 122 --- example/resnet50_imagenet2012_THOR/README.md | 118 --- example/resnet50_imagenet2012_THOR/config.py | 37 - .../crossentropy.py | 41 - .../dataset_imagenet.py | 80 -- example/resnet50_imagenet2012_THOR/eval.py | 60 -- .../model/dataset_helper.py | 125 --- .../model/grad_reducer_thor.py | 183 ----- .../model/model_thor.py | 725 ------------------ .../model/resnet.py | 359 --------- .../resnet50_imagenet2012_THOR/model/thor.py | 199 ----- .../model/thor_layer.py | 477 ------------ .../run_distribute_train.sh | 55 -- .../resnet50_imagenet2012_THOR/run_infer.sh | 64 -- example/resnet50_imagenet2012_THOR/train.py | 133 ---- model_zoo/resnet/README.md | 251 ++++++ model_zoo/resnet/eval.py | 90 +++ .../scripts/run_distribute_train.sh | 48 +- .../{resnet101 => resnet}/scripts/run_eval.sh | 35 +- .../scripts/run_standalone_train.sh | 45 +- .../resnet/src}/config.py | 47 +- .../{resnet101 => resnet}/src/crossentropy.py | 5 +- model_zoo/resnet/src/dataset.py | 205 +++++ .../resnet/src}/lr_generator.py | 57 +- .../src/resnet101.py => resnet/src/resnet.py} | 24 +- model_zoo/resnet/train.py | 162 ++++ model_zoo/resnet101/README.md | 147 ---- model_zoo/resnet101/eval.py | 75 -- model_zoo/resnet101/src/config.py | 40 - model_zoo/resnet101/src/dataset.py | 89 --- model_zoo/resnet101/src/lr_generator.py | 56 -- model_zoo/resnet101/train.py | 102 --- 48 files changed, 912 insertions(+), 4580 deletions(-) delete mode 100644 example/resnet50_cifar10/README.md delete mode 100755 example/resnet50_cifar10/config.py delete mode 100755 example/resnet50_cifar10/dataset.py delete mode 100755 example/resnet50_cifar10/eval.py delete mode 100755 example/resnet50_cifar10/lr_generator.py delete mode 100755 example/resnet50_cifar10/run_distribute_train.sh delete mode 100755 example/resnet50_cifar10/run_infer.sh delete mode 100755 example/resnet50_cifar10/run_standalone_train.sh delete mode 100755 example/resnet50_cifar10/train.py delete mode 100644 example/resnet50_imagenet2012/README.md delete mode 100644 example/resnet50_imagenet2012/crossentropy.py delete mode 100755 example/resnet50_imagenet2012/dataset.py delete mode 100755 example/resnet50_imagenet2012/eval.py delete mode 100755 example/resnet50_imagenet2012/run_distribute_train.sh delete mode 100755 example/resnet50_imagenet2012/run_infer.sh delete mode 100755 example/resnet50_imagenet2012/run_standalone_train.sh delete mode 100755 example/resnet50_imagenet2012/train.py delete mode 100644 example/resnet50_imagenet2012_THOR/README.md delete mode 100644 example/resnet50_imagenet2012_THOR/config.py delete mode 100644 example/resnet50_imagenet2012_THOR/crossentropy.py delete mode 100644 example/resnet50_imagenet2012_THOR/dataset_imagenet.py delete mode 100755 example/resnet50_imagenet2012_THOR/eval.py delete mode 100644 example/resnet50_imagenet2012_THOR/model/dataset_helper.py delete mode 100644 example/resnet50_imagenet2012_THOR/model/grad_reducer_thor.py delete mode 100644 example/resnet50_imagenet2012_THOR/model/model_thor.py delete mode 100644 example/resnet50_imagenet2012_THOR/model/resnet.py delete mode 100644 example/resnet50_imagenet2012_THOR/model/thor.py delete mode 100644 example/resnet50_imagenet2012_THOR/model/thor_layer.py delete mode 100644 example/resnet50_imagenet2012_THOR/run_distribute_train.sh delete mode 100755 example/resnet50_imagenet2012_THOR/run_infer.sh delete mode 100644 example/resnet50_imagenet2012_THOR/train.py create mode 100644 model_zoo/resnet/README.md create mode 100755 model_zoo/resnet/eval.py rename model_zoo/{resnet101 => resnet}/scripts/run_distribute_train.sh (58%) rename model_zoo/{resnet101 => resnet}/scripts/run_eval.sh (62%) rename model_zoo/{resnet101 => resnet}/scripts/run_standalone_train.sh (56%) rename {example/resnet50_imagenet2012 => model_zoo/resnet/src}/config.py (54%) rename model_zoo/{resnet101 => resnet}/src/crossentropy.py (98%) create mode 100755 model_zoo/resnet/src/dataset.py rename {example/resnet50_imagenet2012 => model_zoo/resnet/src}/lr_generator.py (64%) rename model_zoo/{resnet101/src/resnet101.py => resnet/src/resnet.py} (94%) create mode 100755 model_zoo/resnet/train.py delete mode 100644 model_zoo/resnet101/README.md delete mode 100755 model_zoo/resnet101/eval.py delete mode 100755 model_zoo/resnet101/src/config.py delete mode 100755 model_zoo/resnet101/src/dataset.py delete mode 100755 model_zoo/resnet101/src/lr_generator.py delete mode 100755 model_zoo/resnet101/train.py diff --git a/example/resnet50_cifar10/README.md b/example/resnet50_cifar10/README.md deleted file mode 100644 index abb0ba4090..0000000000 --- a/example/resnet50_cifar10/README.md +++ /dev/null @@ -1,137 +0,0 @@ -# ResNet-50 Example - -## Description - -This is an example of training ResNet-50 with CIFAR-10 dataset in MindSpore. - -## Requirements - -- Install [MindSpore](https://www.mindspore.cn/install/en). - -- Download the dataset CIFAR-10 - -> Unzip the CIFAR-10 dataset to any path you want and the folder structure should include train and eval dataset as follows: -> ``` -> . -> ├── cifar-10-batches-bin # train dataset -> └── cifar-10-verify-bin # infer dataset -> ``` - - -## Example structure - -```shell -. -├── config.py # parameter configuration -├── dataset.py # data preprocessing -├── eval.py # infer script -├── lr_generator.py # generate learning rate for each step -├── run_distribute_train.sh # launch distributed training(8 pcs) -├── run_infer.sh # launch infering -├── run_standalone_train.sh # launch standalone training(1 pcs) -└── train.py # train script -``` - - -## Parameter configuration - -Parameters for both training and inference can be set in config.py. - -``` -"class_num": 10, # dataset class num -"batch_size": 32, # batch size of input tensor -"loss_scale": 1024, # loss scale -"momentum": 0.9, # momentum -"weight_decay": 1e-4, # weight decay -"epoch_size": 90, # only valid for taining, which is always 1 for inference -"buffer_size": 100, # number of queue size in data preprocessing -"image_height": 224, # image height -"image_width": 224, # image width -"save_checkpoint": True, # whether save checkpoint or not -"save_checkpoint_steps": 195, # the step interval between two checkpoints. By default, the last checkpoint will be saved after the last step -"keep_checkpoint_max": 10, # only keep the last keep_checkpoint_max checkpoint -"save_checkpoint_path": "./", # path to save checkpoint -"warmup_epochs": 5, # number of warmup epoch -"lr_decay_mode": "poly" # decay mode can be selected in steps, ploy and default -"lr_init": 0.01, # initial learning rate -"lr_end": 0.00001, # final learning rate -"lr_max": 0.1, # maximum learning rate -``` - -## Running the example - -### Train - -#### Usage - -``` -# distributed training -Usage: sh run_distribute_train.sh [MINDSPORE_HCCL_CONFIG_PATH] [DATASET_PATH] - -# standalone training -Usage: sh run_standalone_train.sh [DATASET_PATH] -``` - - -#### Launch - -``` -# distribute training example -sh run_distribute_train.sh rank_table.json ~/cifar-10-batches-bin - -# standalone training example -sh run_standalone_train.sh ~/cifar-10-batches-bin -``` - -> About rank_table.json, you can refer to the [distributed training tutorial](https://www.mindspore.cn/tutorial/en/master/advanced_use/distributed_training.html). - -#### Result - -Training result will be stored in the example path, whose folder name begins with "train" or "train_parallel". Under this, you can find checkpoint file together with result like the followings in log. - -``` -# distribute training result(8 pcs) -epoch: 1 step: 195, loss is 1.9601055 -epoch: 2 step: 195, loss is 1.8555021 -epoch: 3 step: 195, loss is 1.6707983 -epoch: 4 step: 195, loss is 1.8162166 -epoch: 5 step: 195, loss is 1.393667 -``` - -### Infer - -#### Usage - -``` -# infer -Usage: sh run_infer.sh [DATASET_PATH] [CHECKPOINT_PATH] -``` - -#### Launch - -``` -# infer example -sh run_infer.sh ~/cifar10-10-verify-bin ~/resnet50_cifar10/train_parallel0/resnet-90_195.ckpt -``` - -> checkpoint can be produced in training process. - -#### Result - -Inference result will be stored in the example path, whose folder name is "infer". Under this, you can find result like the followings in log. - -``` -result: {'acc': 0.91446314102564111} ckpt=~/resnet50_cifar10/train_parallel0/resnet-90_195.ckpt -``` - -### Running on GPU -``` -# distributed training example -mpirun -n 8 python train.py --dataset_path=~/cifar-10-batches-bin --device_target="GPU" --run_distribute=True - -# standalone training example -python train.py --dataset_path=~/cifar-10-batches-bin --device_target="GPU" - -# infer example -python eval.py --dataset_path=~/cifar10-10-verify-bin --device_target="GPU" --checkpoint_path=resnet-90_195.ckpt -``` \ No newline at end of file diff --git a/example/resnet50_cifar10/config.py b/example/resnet50_cifar10/config.py deleted file mode 100755 index 3c50a6aaed..0000000000 --- a/example/resnet50_cifar10/config.py +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ -""" -network config setting, will be used in train.py and eval.py -""" -from easydict import EasyDict as ed - -config = ed({ - "class_num": 10, - "batch_size": 32, - "loss_scale": 1024, - "momentum": 0.9, - "weight_decay": 1e-4, - "epoch_size": 90, - "buffer_size": 100, - "image_height": 224, - "image_width": 224, - "save_checkpoint": True, - "save_checkpoint_epochs": 5, - "keep_checkpoint_max": 10, - "save_checkpoint_path": "./", - "warmup_epochs": 5, - "lr_decay_mode": "poly", - "lr_init": 0.01, - "lr_end": 0.00001, - "lr_max": 0.1 -}) diff --git a/example/resnet50_cifar10/dataset.py b/example/resnet50_cifar10/dataset.py deleted file mode 100755 index 8a66ec573a..0000000000 --- a/example/resnet50_cifar10/dataset.py +++ /dev/null @@ -1,81 +0,0 @@ -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ -""" -create train or eval dataset. -""" -import os -import mindspore.common.dtype as mstype -import mindspore.dataset.engine as de -import mindspore.dataset.transforms.vision.c_transforms as C -import mindspore.dataset.transforms.c_transforms as C2 -from mindspore.communication.management import init, get_rank, get_group_size -from config import config - - -def create_dataset(dataset_path, do_train, repeat_num=1, batch_size=32, target="Ascend"): - """ - create a train or eval dataset - - Args: - dataset_path(string): the path of dataset. - do_train(bool): whether dataset is used for train or eval. - repeat_num(int): the repeat times of dataset. Default: 1 - batch_size(int): the batch size of dataset. Default: 32 - target(str): the device target. Default: Ascend - - Returns: - dataset - """ - if target == "Ascend": - device_num = int(os.getenv("DEVICE_NUM")) - rank_id = int(os.getenv("RANK_ID")) - else: - init("nccl") - rank_id = get_rank() - device_num = get_group_size() - - if device_num == 1: - ds = de.Cifar10Dataset(dataset_path, num_parallel_workers=8, shuffle=True) - else: - ds = de.Cifar10Dataset(dataset_path, num_parallel_workers=8, shuffle=True, - num_shards=device_num, shard_id=rank_id) - - # define map operations - trans = [] - if do_train: - trans += [ - C.RandomCrop((32, 32), (4, 4, 4, 4)), - C.RandomHorizontalFlip(prob=0.5) - ] - - trans += [ - C.Resize((config.image_height, config.image_width)), - C.Rescale(1.0 / 255.0, 0.0), - C.Normalize([0.4914, 0.4822, 0.4465], [0.2023, 0.1994, 0.2010]), - C.HWC2CHW() - ] - - type_cast_op = C2.TypeCast(mstype.int32) - - ds = ds.map(input_columns="label", num_parallel_workers=8, operations=type_cast_op) - ds = ds.map(input_columns="image", num_parallel_workers=8, operations=trans) - - # apply batch operations - ds = ds.batch(batch_size, drop_remainder=True) - - # apply dataset repeat operation - ds = ds.repeat(repeat_num) - - return ds diff --git a/example/resnet50_cifar10/eval.py b/example/resnet50_cifar10/eval.py deleted file mode 100755 index f7d71c8d29..0000000000 --- a/example/resnet50_cifar10/eval.py +++ /dev/null @@ -1,72 +0,0 @@ -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ -""" -eval. -""" -import os -import argparse -from dataset import create_dataset -from config import config -from mindspore import context -from mindspore.model_zoo.resnet import resnet50 -from mindspore.parallel._auto_parallel_context import auto_parallel_context -from mindspore.nn.loss import SoftmaxCrossEntropyWithLogits -from mindspore.train.model import Model, ParallelMode -from mindspore.train.serialization import load_checkpoint, load_param_into_net -from mindspore.communication.management import init, get_group_size - -parser = argparse.ArgumentParser(description='Image classification') -parser.add_argument('--run_distribute', type=bool, default=False, help='Run distribute') -parser.add_argument('--device_num', type=int, default=1, help='Device num.') -parser.add_argument('--do_train', type=bool, default=False, help='Do train or not.') -parser.add_argument('--do_eval', type=bool, default=True, help='Do eval or not.') -parser.add_argument('--checkpoint_path', type=str, default=None, help='Checkpoint file path') -parser.add_argument('--dataset_path', type=str, default=None, help='Dataset path') -parser.add_argument('--device_target', type=str, default='Ascend', help='Device target') -args_opt = parser.parse_args() - -if __name__ == '__main__': - target = args_opt.device_target - context.set_context(mode=context.GRAPH_MODE, device_target=target, save_graphs=False) - if not args_opt.do_eval and args_opt.run_distribute: - if target == "Ascend": - device_id = int(os.getenv('DEVICE_ID')) - context.set_context(device_id=device_id) - context.set_auto_parallel_context(device_num=args_opt.device_num, parallel_mode=ParallelMode.DATA_PARALLEL, - mirror_mean=True) - auto_parallel_context().set_all_reduce_fusion_split_indices([140]) - init() - elif target == "GPU": - init("nccl") - context.set_auto_parallel_context(device_num=get_group_size(), parallel_mode=ParallelMode.DATA_PARALLEL, - mirror_mean=True) - - epoch_size = config.epoch_size - net = resnet50(class_num=config.class_num) - loss = SoftmaxCrossEntropyWithLogits(sparse=True) - - if args_opt.do_eval: - dataset = create_dataset(dataset_path=args_opt.dataset_path, do_train=False, batch_size=config.batch_size, - target=target) - step_size = dataset.get_dataset_size() - - if args_opt.checkpoint_path: - param_dict = load_checkpoint(args_opt.checkpoint_path) - load_param_into_net(net, param_dict) - net.set_train(False) - - model = Model(net, loss_fn=loss, metrics={'acc'}) - res = model.eval(dataset) - print("result:", res, "ckpt=", args_opt.checkpoint_path) diff --git a/example/resnet50_cifar10/lr_generator.py b/example/resnet50_cifar10/lr_generator.py deleted file mode 100755 index 37c8e907d7..0000000000 --- a/example/resnet50_cifar10/lr_generator.py +++ /dev/null @@ -1,77 +0,0 @@ -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ -"""learning rate generator""" -import numpy as np - - -def get_lr(global_step, lr_init, lr_end, lr_max, warmup_epochs, total_epochs, steps_per_epoch, lr_decay_mode): - """ - generate learning rate array - - Args: - global_step(int): total steps of the training - lr_init(float): init learning rate - lr_end(float): end learning rate - lr_max(float): max learning rate - warmup_epochs(int): number of warmup epochs - total_epochs(int): total epoch of training - steps_per_epoch(int): steps of one epoch - lr_decay_mode(string): learning rate decay mode, including steps, poly or default - - Returns: - np.array, learning rate array - """ - lr_each_step = [] - total_steps = steps_per_epoch * total_epochs - warmup_steps = steps_per_epoch * warmup_epochs - if lr_decay_mode == 'steps': - decay_epoch_index = [0.3 * total_steps, 0.6 * total_steps, 0.8 * total_steps] - for i in range(total_steps): - if i < decay_epoch_index[0]: - lr = lr_max - elif i < decay_epoch_index[1]: - lr = lr_max * 0.1 - elif i < decay_epoch_index[2]: - lr = lr_max * 0.01 - else: - lr = lr_max * 0.001 - lr_each_step.append(lr) - elif lr_decay_mode == 'poly': - if warmup_steps != 0: - inc_each_step = (float(lr_max) - float(lr_init)) / float(warmup_steps) - else: - inc_each_step = 0 - for i in range(total_steps): - if i < warmup_steps: - lr = float(lr_init) + inc_each_step * float(i) - else: - base = (1.0 - (float(i) - float(warmup_steps)) / (float(total_steps) - float(warmup_steps))) - lr = float(lr_max) * base * base - if lr < 0.0: - lr = 0.0 - lr_each_step.append(lr) - else: - for i in range(total_steps): - if i < warmup_steps: - lr = lr_init + (lr_max - lr_init) * i / warmup_steps - else: - lr = lr_max - (lr_max - lr_end) * (i - warmup_steps) / (total_steps - warmup_steps) - lr_each_step.append(lr) - - current_step = global_step - lr_each_step = np.array(lr_each_step).astype(np.float32) - learning_rate = lr_each_step[current_step:] - - return learning_rate diff --git a/example/resnet50_cifar10/run_distribute_train.sh b/example/resnet50_cifar10/run_distribute_train.sh deleted file mode 100755 index e4bdd775b3..0000000000 --- a/example/resnet50_cifar10/run_distribute_train.sh +++ /dev/null @@ -1,64 +0,0 @@ -#!/bin/bash -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ - -if [ $# != 2 ] -then - echo "Usage: sh run_distribute_train.sh [MINDSPORE_HCCL_CONFIG_PATH] [DATASET_PATH]" -exit 1 -fi - -get_real_path(){ - if [ "${1:0:1}" == "/" ]; then - echo "$1" - else - echo "$(realpath -m $PWD/$1)" - fi -} - -PATH1=$(get_real_path $1) -PATH2=$(get_real_path $2) - -if [ ! -f "$PATH1" ] -then - echo "error: MINDSPORE_HCCL_CONFIG_PATH=$PATH1 is not a file" -exit 1 -fi - -if [ ! -d "$PATH2" ] -then - echo "error: DATASET_PATH=$PATH2 is not a directory" -exit 1 -fi - -ulimit -u unlimited -export DEVICE_NUM=8 -export RANK_SIZE=8 -export MINDSPORE_HCCL_CONFIG_PATH=$PATH1 - -for((i=0; i<${DEVICE_NUM}; i++)) -do - export DEVICE_ID=$i - export RANK_ID=$i - rm -rf ./train_parallel$i - mkdir ./train_parallel$i - cp *.py ./train_parallel$i - cp *.sh ./train_parallel$i - cd ./train_parallel$i || exit - echo "start training for rank $RANK_ID, device $DEVICE_ID" - env > env.log - python train.py --do_train=True --run_distribute=True --device_num=$DEVICE_NUM --dataset_path=$PATH2 &> log & - cd .. -done diff --git a/example/resnet50_cifar10/run_infer.sh b/example/resnet50_cifar10/run_infer.sh deleted file mode 100755 index 14d7faf981..0000000000 --- a/example/resnet50_cifar10/run_infer.sh +++ /dev/null @@ -1,64 +0,0 @@ -#!/bin/bash -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ - -if [ $# != 2 ] -then - echo "Usage: sh run_infer.sh [DATASET_PATH] [CHECKPOINT_PATH]" -exit 1 -fi - -get_real_path(){ - if [ "${1:0:1}" == "/" ]; then - echo "$1" - else - echo "$(realpath -m $PWD/$1)" - fi -} - -PATH1=$(get_real_path $1) -PATH2=$(get_real_path $2) - - -if [ ! -d $PATH1 ] -then - echo "error: DATASET_PATH=$1 is not a directory" -exit 1 -fi - -if [ ! -f $PATH2 ] -then - echo "error: CHECKPOINT_PATH=$2 is not a file" -exit 1 -fi - -ulimit -u unlimited -export DEVICE_NUM=1 -export DEVICE_ID=0 -export RANK_SIZE=$DEVICE_NUM -export RANK_ID=0 - -if [ -d "infer" ]; -then - rm -rf ./infer -fi -mkdir ./infer -cp *.py ./infer -cp *.sh ./infer -cd ./infer || exit -env > env.log -echo "start infering for device $DEVICE_ID" -python eval.py --do_eval=True --dataset_path=$PATH1 --checkpoint_path=$PATH2 &> log & -cd .. diff --git a/example/resnet50_cifar10/run_standalone_train.sh b/example/resnet50_cifar10/run_standalone_train.sh deleted file mode 100755 index cb08cde6c9..0000000000 --- a/example/resnet50_cifar10/run_standalone_train.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/bin/bash -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ - -if [ $# != 1 ] -then - echo "Usage: sh run_standalone_train.sh [DATASET_PATH]" -exit 1 -fi - -get_real_path(){ - if [ "${1:0:1}" == "/" ]; then - echo "$1" - else - echo "$(realpath -m $PWD/$1)" - fi -} - -PATH1=$(get_real_path $1) - -if [ ! -d "$PATH1" ] -then - echo "error: DATASET_PATH=$PATH1 is not a directory" -exit 1 -fi - -ulimit -u unlimited -export DEVICE_NUM=1 -export DEVICE_ID=0 -export RANK_ID=0 - -if [ -d "train" ]; -then - rm -rf ./train -fi -mkdir ./train -cp *.py ./train -cp *.sh ./train -cd ./train || exit -echo "start training for device $DEVICE_ID" -env > env.log -python train.py --do_train=True --dataset_path=$PATH1 &> log & -cd .. diff --git a/example/resnet50_cifar10/train.py b/example/resnet50_cifar10/train.py deleted file mode 100755 index 323695ae29..0000000000 --- a/example/resnet50_cifar10/train.py +++ /dev/null @@ -1,97 +0,0 @@ -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ -"""train_imagenet.""" -import os -import argparse -import numpy as np -from dataset import create_dataset -from lr_generator import get_lr -from config import config -from mindspore import context -from mindspore import Tensor -from mindspore.model_zoo.resnet import resnet50 -from mindspore.parallel._auto_parallel_context import auto_parallel_context -from mindspore.nn.optim.momentum import Momentum -from mindspore.nn.loss import SoftmaxCrossEntropyWithLogits - -from mindspore.train.model import Model, ParallelMode - -from mindspore.train.callback import ModelCheckpoint, CheckpointConfig, LossMonitor, TimeMonitor -from mindspore.train.loss_scale_manager import FixedLossScaleManager -from mindspore.communication.management import init, get_rank, get_group_size - -parser = argparse.ArgumentParser(description='Image classification') -parser.add_argument('--run_distribute', type=bool, default=False, help='Run distribute') -parser.add_argument('--device_num', type=int, default=1, help='Device num.') -parser.add_argument('--do_train', type=bool, default=True, help='Do train or not.') -parser.add_argument('--do_eval', type=bool, default=False, help='Do eval or not.') -parser.add_argument('--dataset_path', type=str, default=None, help='Dataset path') -parser.add_argument('--device_target', type=str, default='Ascend', help='Device target') -args_opt = parser.parse_args() - - -if __name__ == '__main__': - target = args_opt.device_target - ckpt_save_dir = config.save_checkpoint_path - context.set_context(mode=context.GRAPH_MODE, device_target=target, save_graphs=False) - np.random.seed(1) - if not args_opt.do_eval and args_opt.run_distribute: - if target == "Ascend": - device_id = int(os.getenv('DEVICE_ID')) - context.set_context(mode=context.GRAPH_MODE, device_target="Ascend", save_graphs=False, device_id=device_id, - enable_auto_mixed_precision=True) - init() - context.set_auto_parallel_context(device_num=args_opt.device_num, parallel_mode=ParallelMode.DATA_PARALLEL, - mirror_mean=True) - auto_parallel_context().set_all_reduce_fusion_split_indices([107, 160]) - ckpt_save_dir = config.save_checkpoint_path - elif target == "GPU": - context.set_context(mode=context.GRAPH_MODE, device_target="GPU", save_graphs=False) - init("nccl") - context.set_auto_parallel_context(device_num=get_group_size(), parallel_mode=ParallelMode.DATA_PARALLEL, - mirror_mean=True) - ckpt_save_dir = config.save_checkpoint_path + "ckpt_" + str(get_rank()) + "/" - epoch_size = config.epoch_size - net = resnet50(class_num=config.class_num) - - if args_opt.do_train: - dataset = create_dataset(dataset_path=args_opt.dataset_path, do_train=True, - repeat_num=epoch_size, batch_size=config.batch_size, target=target) - step_size = dataset.get_dataset_size() - - loss_scale = FixedLossScaleManager(config.loss_scale, drop_overflow_update=False) - lr = Tensor(get_lr(global_step=0, lr_init=config.lr_init, lr_end=config.lr_end, lr_max=config.lr_max, - warmup_epochs=config.warmup_epochs, total_epochs=epoch_size, steps_per_epoch=step_size, - lr_decay_mode='poly')) - opt = Momentum(filter(lambda x: x.requires_grad, net.get_parameters()), lr, config.momentum, - config.weight_decay, config.loss_scale) - if target == 'GPU': - loss = SoftmaxCrossEntropyWithLogits(sparse=True, is_grad=False, reduction='mean') - opt = Momentum(filter(lambda x: x.requires_grad, net.get_parameters()), lr, config.momentum) - model = Model(net, loss_fn=loss, optimizer=opt, metrics={'acc'}) - else: - loss = SoftmaxCrossEntropyWithLogits(sparse=True, reduction='mean') - model = Model(net, loss_fn=loss, optimizer=opt, loss_scale_manager=loss_scale, metrics={'acc'}, - amp_level="O2", keep_batchnorm_fp32=False) - - time_cb = TimeMonitor(data_size=step_size) - loss_cb = LossMonitor() - cb = [time_cb, loss_cb] - if config.save_checkpoint: - config_ck = CheckpointConfig(save_checkpoint_steps=config.save_checkpoint_epochs*step_size, - keep_checkpoint_max=config.keep_checkpoint_max) - ckpt_cb = ModelCheckpoint(prefix="resnet", directory=ckpt_save_dir, config=config_ck) - cb += [ckpt_cb] - model.train(epoch_size, dataset, callbacks=cb) diff --git a/example/resnet50_imagenet2012/README.md b/example/resnet50_imagenet2012/README.md deleted file mode 100644 index 6baf863544..0000000000 --- a/example/resnet50_imagenet2012/README.md +++ /dev/null @@ -1,150 +0,0 @@ -# ResNet-50 Example - -## Description - -This is an example of training ResNet-50 with ImageNet2012 dataset in MindSpore. - -## Requirements - -- Install [MindSpore](https://www.mindspore.cn/install/en). - -- Download the dataset ImageNet2012 - -> Unzip the ImageNet2012 dataset to any path you want and the folder structure should include train and eval dataset as follows: -> ``` -> . -> ├── ilsvrc # train dataset -> └── ilsvrc_eval # infer dataset -> ``` - - -## Example structure - -```shell -. -├── crossentropy.py # CrossEntropy loss function -├── config.py # parameter configuration -├── dataset.py # data preprocessing -├── eval.py # infer script -├── lr_generator.py # generate learning rate for each step -├── run_distribute_train.sh # launch distributed training(8 pcs) -├── run_infer.sh # launch infering -├── run_standalone_train.sh # launch standalone training(1 pcs) -└── train.py # train script -``` - - -## Parameter configuration - -Parameters for both training and inference can be set in config.py. - -``` -"class_num": 1001, # dataset class number -"batch_size": 32, # batch size of input tensor -"loss_scale": 1024, # loss scale -"momentum": 0.9, # momentum optimizer -"weight_decay": 1e-4, # weight decay -"epoch_size": 90, # only valid for taining, which is always 1 for inference -"pretrained_epoch_size": 1, # epoch size that model has been trained before load pretrained checkpoint -"buffer_size": 1000, # number of queue size in data preprocessing -"image_height": 224, # image height -"image_width": 224, # image width -"save_checkpoint": True, # whether save checkpoint or not -"save_checkpoint_epochs": 1, # the epoch interval between two checkpoints. By default, the last checkpoint will be saved after the last epoch -"keep_checkpoint_max": 10, # only keep the last keep_checkpoint_max checkpoint -"save_checkpoint_path": "./", # path to save checkpoint relative to the executed path -"warmup_epochs": 0, # number of warmup epoch -"lr_decay_mode": "cosine", # decay mode for generating learning rate -"label_smooth": True, # label smooth -"label_smooth_factor": 0.1, # label smooth factor -"lr_init": 0, # initial learning rate -"lr_max": 0.1, # maximum learning rate -``` - -## Running the example - -### Train - -#### Usage - -``` -# distributed training -Usage: sh run_distribute_train.sh [MINDSPORE_HCCL_CONFIG_PATH] [DATASET_PATH] [PRETRAINED_CKPT_PATH](optional) - -# standalone training -Usage: sh run_standalone_train.sh [DATASET_PATH] [PRETRAINED_CKPT_PATH](optional) - -``` - - -#### Launch - -```bash -# distributed training example(8 pcs) -sh run_distribute_train.sh rank_table_8p.json dataset/ilsvrc - -# If you want to load pretrained ckpt file -sh run_distribute_train.sh rank_table_8p.json dataset/ilsvrc ./pretrained.ckpt - -# standalone training example(1 pcs) -sh run_standalone_train.sh dataset/ilsvrc - -# If you want to load pretrained ckpt file -sh run_standalone_train.sh dataset/ilsvrc ./pretrained.ckpt -``` - -> About rank_table.json, you can refer to the [distributed training tutorial](https://www.mindspore.cn/tutorial/en/master/advanced_use/distributed_training.html). - -#### Result - -Training result will be stored in the example path, whose folder name begins with "train" or "train_parallel". Under this, you can find checkpoint file together with result like the followings in log. - -``` -# distribute training result(8 pcs) -epoch: 1 step: 5004, loss is 4.8995576 -epoch: 2 step: 5004, loss is 3.9235563 -epoch: 3 step: 5004, loss is 3.833077 -epoch: 4 step: 5004, loss is 3.2795618 -epoch: 5 step: 5004, loss is 3.1978393 -``` - -### Infer - -#### Usage - -``` -# infer -Usage: sh run_infer.sh [DATASET_PATH] [CHECKPOINT_PATH] -``` - -#### Launch - -```bash -# infer with checkpoint -sh run_infer.sh dataset/ilsvrc_eval train_parallel0/resnet-90_5004.ckpt -``` - -> checkpoint can be produced in training process. - -#### Result - -Inference result will be stored in the example path, whose folder name is "infer". Under this, you can find result like the followings in log. - -``` -result: {'acc': 0.7671054737516005} ckpt=train_parallel0/resnet-90_5004.ckpt -``` - -### Running on GPU -``` -# distributed training example -mpirun -n 8 python train.py --dataset_path=dataset/ilsvrc/train --device_target="GPU" --run_distribute=True - -# standalone training example -python train.py --dataset_path=dataset/ilsvrc/train --device_target="GPU" - -# standalone training example with pretrained checkpoint -python train.py --dataset_path=dataset/ilsvrc/train --device_target="GPU" --pre_trained=pretrained.ckpt - -# infer example -python eval.py --dataset_path=dataset/ilsvrc/val --device_target="GPU" --checkpoint_path=resnet-90_5004ss.ckpt -``` \ No newline at end of file diff --git a/example/resnet50_imagenet2012/crossentropy.py b/example/resnet50_imagenet2012/crossentropy.py deleted file mode 100644 index b078b29f6f..0000000000 --- a/example/resnet50_imagenet2012/crossentropy.py +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ -"""define loss function for network""" -from mindspore.nn.loss.loss import _Loss -from mindspore.ops import operations as P -from mindspore.ops import functional as F -from mindspore import Tensor -from mindspore.common import dtype as mstype -import mindspore.nn as nn - - -class CrossEntropy(_Loss): - """the redefined loss function with SoftmaxCrossEntropyWithLogits""" - - def __init__(self, smooth_factor=0, num_classes=1001): - super(CrossEntropy, self).__init__() - self.onehot = P.OneHot() - self.on_value = Tensor(1.0 - smooth_factor, mstype.float32) - self.off_value = Tensor(1.0 * smooth_factor / (num_classes - 1), mstype.float32) - self.ce = nn.SoftmaxCrossEntropyWithLogits() - self.mean = P.ReduceMean(False) - - def construct(self, logit, label): - one_hot_label = self.onehot(label, F.shape(logit)[1], self.on_value, self.off_value) - loss = self.ce(logit, one_hot_label) - loss = self.mean(loss, 0) - return loss diff --git a/example/resnet50_imagenet2012/dataset.py b/example/resnet50_imagenet2012/dataset.py deleted file mode 100755 index 0691985e0b..0000000000 --- a/example/resnet50_imagenet2012/dataset.py +++ /dev/null @@ -1,85 +0,0 @@ -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ -""" -create train or eval dataset. -""" -import os -import mindspore.common.dtype as mstype -import mindspore.dataset.engine as de -import mindspore.dataset.transforms.vision.c_transforms as C -import mindspore.dataset.transforms.c_transforms as C2 -from mindspore.communication.management import init, get_rank, get_group_size - -def create_dataset(dataset_path, do_train, repeat_num=1, batch_size=32, target="Ascend"): - """ - create a train or eval dataset - - Args: - dataset_path(string): the path of dataset. - do_train(bool): whether dataset is used for train or eval. - repeat_num(int): the repeat times of dataset. Default: 1 - batch_size(int): the batch size of dataset. Default: 32 - target(str): the device target. Default: Ascend - - Returns: - dataset - """ - if target == "Ascend": - device_num = int(os.getenv("DEVICE_NUM")) - rank_id = int(os.getenv("RANK_ID")) - else: - init("nccl") - rank_id = get_rank() - device_num = get_group_size() - - if device_num == 1: - ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True) - else: - ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True, - num_shards=device_num, shard_id=rank_id) - - image_size = 224 - mean = [0.485 * 255, 0.456 * 255, 0.406 * 255] - std = [0.229 * 255, 0.224 * 255, 0.225 * 255] - - # define map operations - if do_train: - trans = [ - C.RandomCropDecodeResize(image_size, scale=(0.08, 1.0), ratio=(0.75, 1.333)), - C.RandomHorizontalFlip(prob=0.5), - C.Normalize(mean=mean, std=std), - C.HWC2CHW() - ] - else: - trans = [ - C.Decode(), - C.Resize((256, 256)), - C.CenterCrop(image_size), - C.Normalize(mean=mean, std=std), - C.HWC2CHW() - ] - - type_cast_op = C2.TypeCast(mstype.int32) - - ds = ds.map(input_columns="image", num_parallel_workers=8, operations=trans) - ds = ds.map(input_columns="label", num_parallel_workers=8, operations=type_cast_op) - - # apply batch operations - ds = ds.batch(batch_size, drop_remainder=True) - - # apply dataset repeat operation - ds = ds.repeat(repeat_num) - - return ds diff --git a/example/resnet50_imagenet2012/eval.py b/example/resnet50_imagenet2012/eval.py deleted file mode 100755 index 3f7961e786..0000000000 --- a/example/resnet50_imagenet2012/eval.py +++ /dev/null @@ -1,62 +0,0 @@ -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ -""" -eval. -""" -import os -import argparse -from dataset import create_dataset -from config import config -from mindspore import context -from mindspore.model_zoo.resnet import resnet50 -from mindspore.train.model import Model -from mindspore.train.serialization import load_checkpoint, load_param_into_net -from crossentropy import CrossEntropy - -parser = argparse.ArgumentParser(description='Image classification') -parser.add_argument('--run_distribute', type=bool, default=False, help='Run distribute') -parser.add_argument('--device_num', type=int, default=1, help='Device num.') -parser.add_argument('--do_train', type=bool, default=False, help='Do train or not.') -parser.add_argument('--do_eval', type=bool, default=True, help='Do eval or not.') -parser.add_argument('--checkpoint_path', type=str, default=None, help='Checkpoint file path') -parser.add_argument('--dataset_path', type=str, default=None, help='Dataset path') -parser.add_argument('--device_target', type=str, default='Ascend', help='Device target') -args_opt = parser.parse_args() -target = args_opt.device_target -context.set_context(mode=context.GRAPH_MODE, device_target=target, save_graphs=False) -if target == "Ascend": - device_id = int(os.getenv('DEVICE_ID')) - context.set_context(device_id=device_id) - -if __name__ == '__main__': - - net = resnet50(class_num=config.class_num) - if not config.use_label_smooth: - config.label_smooth_factor = 0.0 - loss = CrossEntropy(smooth_factor=config.label_smooth_factor, num_classes=config.class_num) - - if args_opt.do_eval: - dataset = create_dataset(dataset_path=args_opt.dataset_path, do_train=False, batch_size=config.batch_size, - target=target) - step_size = dataset.get_dataset_size() - - if args_opt.checkpoint_path: - param_dict = load_checkpoint(args_opt.checkpoint_path) - load_param_into_net(net, param_dict) - net.set_train(False) - - model = Model(net, loss_fn=loss, metrics={'acc'}) - res = model.eval(dataset) - print("result:", res, "ckpt=", args_opt.checkpoint_path) diff --git a/example/resnet50_imagenet2012/run_distribute_train.sh b/example/resnet50_imagenet2012/run_distribute_train.sh deleted file mode 100755 index 22157608e6..0000000000 --- a/example/resnet50_imagenet2012/run_distribute_train.sh +++ /dev/null @@ -1,80 +0,0 @@ -#!/bin/bash -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ - -if [ $# != 2 ] && [ $# != 3 ] -then - echo "Usage: sh run_distribute_train.sh [MINDSPORE_HCCL_CONFIG_PATH] [DATASET_PATH] [PRETRAINED_CKPT_PATH](optional)" -exit 1 -fi - -get_real_path(){ - if [ "${1:0:1}" == "/" ]; then - echo "$1" - else - echo "$(realpath -m $PWD/$1)" - fi -} - -PATH1=$(get_real_path $1) -PATH2=$(get_real_path $2) -if [ $# == 3 ] -then - PATH3=$(get_real_path $3) -fi - -if [ ! -f "$PATH1" ] -then - echo "error: MINDSPORE_HCCL_CONFIG_PATH=$PATH1 is not a file" -exit 1 -fi - -if [ ! -d "$PATH2" ] -then - echo "error: DATASET_PATH=$PATH2 is not a directory" -exit 1 -fi - -if [ $# == 3 ] && [ ! -f "$PATH3" ] -then - echo "error: PRETRAINED_CKPT_PATH=$PATH3 is not a file" -exit 1 -fi - -ulimit -u unlimited -export DEVICE_NUM=8 -export RANK_SIZE=8 -export MINDSPORE_HCCL_CONFIG_PATH=$PATH1 -export RANK_TABLE_FILE=$PATH1 - -for((i=0; i<${DEVICE_NUM}; i++)) -do - export DEVICE_ID=$i - export RANK_ID=$i - rm -rf ./train_parallel$i - mkdir ./train_parallel$i - cp *.py ./train_parallel$i - cp *.sh ./train_parallel$i - cd ./train_parallel$i || exit - echo "start training for rank $RANK_ID, device $DEVICE_ID" - env > env.log - if [ $# == 2 ] - then - python train.py --do_train=True --run_distribute=True --device_num=$DEVICE_NUM --dataset_path=$PATH2 &> log & - else - python train.py --do_train=True --run_distribute=True --device_num=$DEVICE_NUM --dataset_path=$PATH2 --pre_trained=$PATH3 &> log & - fi - cd .. -done diff --git a/example/resnet50_imagenet2012/run_infer.sh b/example/resnet50_imagenet2012/run_infer.sh deleted file mode 100755 index 1482b63f5f..0000000000 --- a/example/resnet50_imagenet2012/run_infer.sh +++ /dev/null @@ -1,64 +0,0 @@ -#!/bin/bash -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ - -if [ $# != 2 ] -then - echo "Usage: sh run_infer.sh [DATASET_PATH] [CHECKPOINT_PATH]" -exit 1 -fi - -get_real_path(){ - if [ "${1:0:1}" == "/" ]; then - echo "$1" - else - echo "$(realpath -m $PWD/$1)" - fi -} - -PATH1=$(get_real_path $1) -PATH2=$(get_real_path $2) - - -if [ ! -d $PATH1 ] -then - echo "error: DATASET_PATH=$PATH1 is not a directory" -exit 1 -fi - -if [ ! -f $PATH2 ] -then - echo "error: CHECKPOINT_PATH=$PATH2 is not a file" -exit 1 -fi - -ulimit -u unlimited -export DEVICE_NUM=1 -export DEVICE_ID=0 -export RANK_SIZE=$DEVICE_NUM -export RANK_ID=0 - -if [ -d "infer" ]; -then - rm -rf ./infer -fi -mkdir ./infer -cp *.py ./infer -cp *.sh ./infer -cd ./infer || exit -env > env.log -echo "start infering for device $DEVICE_ID" -python eval.py --do_eval=True --dataset_path=$PATH1 --checkpoint_path=$PATH2 &> log & -cd .. diff --git a/example/resnet50_imagenet2012/run_standalone_train.sh b/example/resnet50_imagenet2012/run_standalone_train.sh deleted file mode 100755 index e0eb5efaf0..0000000000 --- a/example/resnet50_imagenet2012/run_standalone_train.sh +++ /dev/null @@ -1,70 +0,0 @@ -#!/bin/bash -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ - -if [ $# != 1 ] && [ $# != 2 ] -then - echo "Usage: sh run_standalone_train.sh [DATASET_PATH] [PRETRAINED_CKPT_PATH](optional)" -exit 1 -fi - -get_real_path(){ - if [ "${1:0:1}" == "/" ]; then - echo "$1" - else - echo "$(realpath -m $PWD/$1)" - fi -} - -PATH1=$(get_real_path $1) -if [ $# == 2 ] -then - PATH2=$(get_real_path $2) -fi - -if [ ! -d "$PATH1" ] -then - echo "error: DATASET_PATH=$PATH1 is not a directory" -exit 1 -fi - -if [ $# == 2 ] && [ ! -f "$PATH2" ] -then - echo "error: PRETRAINED_CKPT_PATH=$PATH2 is not a file" -exit 1 -fi - -ulimit -u unlimited -export DEVICE_NUM=1 -export DEVICE_ID=0 -export RANK_ID=0 - -if [ -d "train" ]; -then - rm -rf ./train -fi -mkdir ./train -cp *.py ./train -cp *.sh ./train -cd ./train || exit -echo "start training for device $DEVICE_ID" -env > env.log -if [ $# == 1 ] -then - python train.py --do_train=True --dataset_path=$PATH1 &> log & -else - python train.py --do_train=True --dataset_path=$PATH1 --pre_trained=$PATH2 &> log & -fi -cd .. diff --git a/example/resnet50_imagenet2012/train.py b/example/resnet50_imagenet2012/train.py deleted file mode 100755 index 6896320ece..0000000000 --- a/example/resnet50_imagenet2012/train.py +++ /dev/null @@ -1,122 +0,0 @@ -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ -"""train_imagenet.""" -import os -import argparse -import numpy as np -from dataset import create_dataset -from lr_generator import get_lr -from config import config -from mindspore import context -from mindspore import Tensor -from mindspore.model_zoo.resnet import resnet50 -from mindspore.parallel._auto_parallel_context import auto_parallel_context -from mindspore.nn.optim.momentum import Momentum - -from mindspore.train.model import Model, ParallelMode - -from mindspore.train.callback import ModelCheckpoint, CheckpointConfig, LossMonitor, TimeMonitor -from mindspore.train.loss_scale_manager import FixedLossScaleManager -from mindspore.train.serialization import load_checkpoint, load_param_into_net -from mindspore.communication.management import init, get_rank, get_group_size -import mindspore.nn as nn -import mindspore.common.initializer as weight_init -from crossentropy import CrossEntropy - -parser = argparse.ArgumentParser(description='Image classification') -parser.add_argument('--run_distribute', type=bool, default=False, help='Run distribute') -parser.add_argument('--device_num', type=int, default=1, help='Device num.') -parser.add_argument('--do_train', type=bool, default=True, help='Do train or not.') -parser.add_argument('--do_eval', type=bool, default=False, help='Do eval or not.') -parser.add_argument('--dataset_path', type=str, default=None, help='Dataset path') -parser.add_argument('--device_target', type=str, default='Ascend', help='Device target') -parser.add_argument('--pre_trained', type=str, default=None, help='Pretrained checkpoint path') -args_opt = parser.parse_args() - -if __name__ == '__main__': - target = args_opt.device_target - ckpt_save_dir = config.save_checkpoint_path - context.set_context(mode=context.GRAPH_MODE, device_target=target, save_graphs=False) - np.random.seed(1) - if not args_opt.do_eval and args_opt.run_distribute: - if target == "Ascend": - device_id = int(os.getenv('DEVICE_ID')) - context.set_context(mode=context.GRAPH_MODE, device_target="Ascend", save_graphs=False, device_id=device_id, - enable_auto_mixed_precision=True) - init() - context.set_auto_parallel_context(device_num=args_opt.device_num, parallel_mode=ParallelMode.DATA_PARALLEL, - mirror_mean=True) - auto_parallel_context().set_all_reduce_fusion_split_indices([107, 160]) - ckpt_save_dir = config.save_checkpoint_path - elif target == "GPU": - context.set_context(mode=context.GRAPH_MODE, device_target="GPU", save_graphs=False) - init("nccl") - context.set_auto_parallel_context(device_num=get_group_size(), parallel_mode=ParallelMode.DATA_PARALLEL, - mirror_mean=True) - ckpt_save_dir = config.save_checkpoint_path + "ckpt_" + str(get_rank()) + "/" - - epoch_size = config.epoch_size - net = resnet50(class_num=config.class_num) - - # weight init - if args_opt.pre_trained: - param_dict = load_checkpoint(args_opt.pre_trained) - load_param_into_net(net, param_dict) - epoch_size = config.epoch_size - config.pretrained_epoch_size - else: - for _, cell in net.cells_and_names(): - if isinstance(cell, nn.Conv2d): - cell.weight.default_input = weight_init.initializer(weight_init.XavierUniform(), - cell.weight.default_input.shape, - cell.weight.default_input.dtype).to_tensor() - if isinstance(cell, nn.Dense): - cell.weight.default_input = weight_init.initializer(weight_init.TruncatedNormal(), - cell.weight.default_input.shape, - cell.weight.default_input.dtype).to_tensor() - if not config.use_label_smooth: - config.label_smooth_factor = 0.0 - - loss = CrossEntropy(smooth_factor=config.label_smooth_factor, num_classes=config.class_num) - - if args_opt.do_train: - dataset = create_dataset(dataset_path=args_opt.dataset_path, do_train=True, - repeat_num=epoch_size, batch_size=config.batch_size, target=target) - step_size = dataset.get_dataset_size() - - loss_scale = FixedLossScaleManager(config.loss_scale, drop_overflow_update=False) - lr = get_lr(lr_init=config.lr_init, lr_end=0.0, lr_max=config.lr_max, warmup_epochs=config.warmup_epochs, - total_epochs=config.epoch_size, steps_per_epoch=step_size, lr_decay_mode='cosine') - if args_opt.pre_trained: - lr = lr[config.pretrained_epoch_size * step_size:] - lr = Tensor(lr) - - opt = Momentum(filter(lambda x: x.requires_grad, net.get_parameters()), lr, config.momentum, - config.weight_decay, config.loss_scale) - if target == "Ascend": - model = Model(net, loss_fn=loss, optimizer=opt, loss_scale_manager=loss_scale, metrics={'acc'}, - amp_level="O2", keep_batchnorm_fp32=False) - elif target == "GPU": - model = Model(net, loss_fn=loss, optimizer=opt, loss_scale_manager=loss_scale, metrics={'acc'}) - - - time_cb = TimeMonitor(data_size=step_size) - loss_cb = LossMonitor() - cb = [time_cb, loss_cb] - if config.save_checkpoint: - config_ck = CheckpointConfig(save_checkpoint_steps=config.save_checkpoint_epochs*step_size, - keep_checkpoint_max=config.keep_checkpoint_max) - ckpt_cb = ModelCheckpoint(prefix="resnet", directory=ckpt_save_dir, config=config_ck) - cb += [ckpt_cb] - model.train(epoch_size, dataset, callbacks=cb) diff --git a/example/resnet50_imagenet2012_THOR/README.md b/example/resnet50_imagenet2012_THOR/README.md deleted file mode 100644 index 6003d8d7b7..0000000000 --- a/example/resnet50_imagenet2012_THOR/README.md +++ /dev/null @@ -1,118 +0,0 @@ -# ResNet-50-THOR Example - -## Description - -This is an example of training ResNet-50 V1.5 with ImageNet2012 dataset by second-order optimizer THOR. THOR is a novel approximate seond-order optimization method in MindSpore. With fewer iterations, THOR can finish ResNet-50 V1.5 training in 72 minutes to top-1 accuracy of 75.9% using 8 Ascend 910, which is much faster than SGD with Momentum. - -## Requirements - -- Install [MindSpore](https://www.mindspore.cn/install/en). - -- Download the dataset ImageNet2012 - -> Unzip the ImageNet2012 dataset to any path you want and the folder structure should include train and eval dataset as follows: -> ``` -> . -> ├── ilsvrc # train dataset -> └── ilsvrc_eval # infer dataset -> ``` - - -## Example structure - -```shell -. -├── crossentropy.py # CrossEntropy loss function -├── config.py # parameter configuration -├── dataset_imagenet.py # data preprocessing -├── eval.py # infer script -├── model # include model file of the optimizer -├── run_distribute_train.sh # launch distributed training(8 pcs) -├── run_infer.sh # launch infering -└── train.py # train script -``` - - -## Parameter configuration - -Parameters for both training and inference can be set in config.py. - -``` -"class_num": 1000, # dataset class number -"batch_size": 32, # batch size of input tensor -"loss_scale": 128, # loss scale -"momentum": 0.9, # momentum of THOR optimizer -"weight_decay": 5e-4, # weight decay -"epoch_size": 45, # only valid for taining, which is always 1 for inference -"buffer_size": 1000, # number of queue size in data preprocessing -"image_height": 224, # image height -"image_width": 224, # image width -"save_checkpoint": True, # whether save checkpoint or not -"save_checkpoint_steps": 5004, # the step interval between two checkpoints. By default, the checkpoint will be saved every epoch -"keep_checkpoint_max": 20, # only keep the last keep_checkpoint_max checkpoint -"save_checkpoint_path": "./", # path to save checkpoint relative to the executed path -"label_smooth": True, # label smooth -"label_smooth_factor": 0.1, # label smooth factor -"frequency": 834, # the step interval to update second-order information matrix -``` - -## Running the example - -### Train - -#### Usage - -``` -# distributed training -Usage: sh run_distribute_train.sh [MINDSPORE_HCCL_CONFIG_PATH] [DATASET_PATH] [DEVICE_NUM] -``` - - -#### Launch - -```bash -# distributed training example(8 pcs) -sh run_distribute_train.sh rank_table_8p.json dataset/ilsvrc -``` - -> About rank_table.json, you can refer to the [distributed training tutorial](https://www.mindspore.cn/tutorial/en/master/advanced_use/distributed_training.html). - -#### Result - -Training result will be stored in the example path, whose folder name begins with "train_parallel". Under this, you can find checkpoint file together with result like the followings in log. - -``` -# distribute training result(8 pcs) -epoch: 1 step: 5004, loss is 4.4182425 -epoch: 2 step: 5004, loss is 3.740064 -epoch: 3 step: 5004, loss is 4.0546017 -epoch: 4 step: 5004, loss is 3.7598825 -epoch: 5 step: 5004, loss is 3.3744206 -...... -``` - -### Infer - -#### Usage - -``` -# infer -Usage: sh run_infer.sh [DATASET_PATH] [CHECKPOINT_PATH] -``` - -#### Launch - -```bash -# infer with checkpoint -sh run_infer.sh dataset/ilsvrc_eval train_parallel0/resnet-42_5004.ckpt -``` - -> checkpoint can be produced in training process. - -#### Result - -Inference result will be stored in the example path, whose folder name is "infer". Under this, you can find result like the followings in log. - -``` -result: {'acc': 0.759503041} ckpt=train_parallel0/resnet-42_5004.ckpt -``` diff --git a/example/resnet50_imagenet2012_THOR/config.py b/example/resnet50_imagenet2012_THOR/config.py deleted file mode 100644 index cd0a81d5e6..0000000000 --- a/example/resnet50_imagenet2012_THOR/config.py +++ /dev/null @@ -1,37 +0,0 @@ -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ -""" -network config setting, will be used in train.py and eval.py -""" -from easydict import EasyDict as ed - -config = ed({ - "class_num": 1000, - "batch_size": 32, - "loss_scale": 128, - "momentum": 0.9, - "weight_decay": 5e-4, - "epoch_size": 45, - "buffer_size": 1000, - "image_height": 224, - "image_width": 224, - "save_checkpoint": True, - "save_checkpoint_steps": 5004, - "keep_checkpoint_max": 20, - "save_checkpoint_path": "./", - "label_smooth": 1, - "label_smooth_factor": 0.1, - "frequency": 834 -}) diff --git a/example/resnet50_imagenet2012_THOR/crossentropy.py b/example/resnet50_imagenet2012_THOR/crossentropy.py deleted file mode 100644 index e8681ff497..0000000000 --- a/example/resnet50_imagenet2012_THOR/crossentropy.py +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ -"""CrossEntropy""" -import mindspore.nn as nn -from mindspore import Tensor -from mindspore.common import dtype as mstype -from mindspore.nn.loss.loss import _Loss -from mindspore.ops import functional as F -from mindspore.ops import operations as P - - -class CrossEntropy(_Loss): - """CrossEntropy""" - def __init__(self, smooth_factor=0., num_classes=1000): - super(CrossEntropy, self).__init__() - self.onehot = P.OneHot() - self.on_value = Tensor(1.0 - smooth_factor, mstype.float32) - self.off_value = Tensor(1.0 * smooth_factor / (num_classes - 1), mstype.float32) - # self.cast = P.Cast() - self.ce = nn.SoftmaxCrossEntropyWithLogits() - self.mean = P.ReduceMean(False) - - def construct(self, logit, label): - # one_hot_label = self.onehot(self.cast(label, mstype.int32), - # F.shape(logit)[1], self.on_value, self.off_value)、 - one_hot_label = self.onehot(label, F.shape(logit)[1], self.on_value, self.off_value) - loss = self.ce(logit, one_hot_label) - loss = self.mean(loss, 0) - return loss diff --git a/example/resnet50_imagenet2012_THOR/dataset_imagenet.py b/example/resnet50_imagenet2012_THOR/dataset_imagenet.py deleted file mode 100644 index 296b675136..0000000000 --- a/example/resnet50_imagenet2012_THOR/dataset_imagenet.py +++ /dev/null @@ -1,80 +0,0 @@ -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ -""" -create train or eval dataset. -""" -import os - -import mindspore.common.dtype as mstype -import mindspore.dataset.engine as de -import mindspore.dataset.transforms.c_transforms as C2 -import mindspore.dataset.transforms.vision.c_transforms as V_C - - -def create_dataset(dataset_path, do_train, repeat_num=1, batch_size=32): - """ - create a train or eval dataset - Args: - dataset_path(string): the path of dataset. - do_train(bool): whether dataset is used for train or eval. - repeat_num(int): the repeat times of dataset. Default: 1 - batch_size(int): the batch size of dataset. Default: 32 - Returns: - dataset - """ - - device_num = int(os.getenv("RANK_SIZE")) - rank_id = int(os.getenv("RANK_ID")) - - if device_num == 1: - ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=False) - else: - ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True, - num_shards=device_num, shard_id=rank_id) - - image_size = 224 - mean = [0.485 * 255, 0.456 * 255, 0.406 * 255] - std = [0.229 * 255, 0.224 * 255, 0.225 * 255] - if do_train: - transform_img = [ - V_C.RandomCropDecodeResize(image_size, scale=(0.08, 1.0), ratio=(0.75, 1.333)), - V_C.RandomHorizontalFlip(prob=0.5), - V_C.Normalize(mean=mean, std=std), - V_C.HWC2CHW() - ] - else: - transform_img = [ - V_C.Decode(), - V_C.Resize((256, 256)), - V_C.CenterCrop(image_size), - V_C.Normalize(mean=mean, std=std), - V_C.HWC2CHW() - ] - # type_cast_op = C2.TypeCast(mstype.float16) - type_cast_op = C2.TypeCast(mstype.int32) - - ds = ds.map(input_columns="image", operations=transform_img, num_parallel_workers=8) - ds = ds.map(input_columns="label", operations=type_cast_op, num_parallel_workers=8) - - # apply shuffle operations - # ds = ds.shuffle(buffer_size=config.buffer_size) - - # apply batch operations - ds = ds.batch(batch_size, drop_remainder=True) - - # apply dataset repeat operation - ds = ds.repeat(repeat_num) - - return ds diff --git a/example/resnet50_imagenet2012_THOR/eval.py b/example/resnet50_imagenet2012_THOR/eval.py deleted file mode 100755 index db82b9fcac..0000000000 --- a/example/resnet50_imagenet2012_THOR/eval.py +++ /dev/null @@ -1,60 +0,0 @@ -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ -""" -eval. -""" -import os -import argparse -from dataset_imagenet import create_dataset -from config import config -from mindspore import context -from mindspore.model_zoo.resnet import resnet50 -from mindspore.train.model import Model -from mindspore.train.serialization import load_checkpoint, load_param_into_net -from crossentropy import CrossEntropy - -parser = argparse.ArgumentParser(description='Image classification') -parser.add_argument('--run_distribute', type=bool, default=False, help='Run distribute') -parser.add_argument('--device_num', type=int, default=1, help='Device num.') -parser.add_argument('--do_train', type=bool, default=False, help='Do train or not.') -parser.add_argument('--do_eval', type=bool, default=True, help='Do eval or not.') -parser.add_argument('--checkpoint_path', type=str, default=None, help='Checkpoint file path') -parser.add_argument('--dataset_path', type=str, default=None, help='Dataset path') -args_opt = parser.parse_args() - -device_id = int(os.getenv('DEVICE_ID')) - -context.set_context(mode=context.GRAPH_MODE, device_target="Ascend", save_graphs=False) -context.set_context(device_id=device_id) - -if __name__ == '__main__': - - net = resnet50(class_num=config.class_num) - if not config.label_smooth: - config.label_smooth_factor = 0.0 - loss = CrossEntropy(smooth_factor=config.label_smooth_factor, num_classes=config.class_num) - - if args_opt.do_eval: - dataset = create_dataset(dataset_path=args_opt.dataset_path, do_train=False, batch_size=config.batch_size) - step_size = dataset.get_dataset_size() - - if args_opt.checkpoint_path: - param_dict = load_checkpoint(args_opt.checkpoint_path) - load_param_into_net(net, param_dict) - net.set_train(False) - - model = Model(net, loss_fn=loss, metrics={'acc'}) - res = model.eval(dataset) - print("result:", res, "ckpt=", args_opt.checkpoint_path) diff --git a/example/resnet50_imagenet2012_THOR/model/dataset_helper.py b/example/resnet50_imagenet2012_THOR/model/dataset_helper.py deleted file mode 100644 index 77f67344c2..0000000000 --- a/example/resnet50_imagenet2012_THOR/model/dataset_helper.py +++ /dev/null @@ -1,125 +0,0 @@ -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ -"""Dataset help for minddata dataset""" -from mindspore._checkparam import check_bool -from mindspore.parallel._utils import _get_device_num, _get_parallel_mode -from mindspore.train.dataset_helper import _send_data -from mindspore.train._utils import _exec_datagraph, _get_types_and_shapes, \ - _to_full_shapes -from mindspore.train.parallel_utils import ParallelMode - - -class DatasetHelper: - """ - Help function to use the Minddata dataset. - - According to different context, change the iter of dataset, to use the same for loop in different context. - - Note: - The iter of DatasetHelper will give one epoch data. - - Args: - dataset (DataSet): The dataset. - dataset_sink_mode (bool): If true use GetNext to fetch the data, or else feed the data from host. - Default: True. - - Examples: - >>> dataset_helper = DatasetHelper(dataset) - >>> for inputs in dataset_helper: - >>> outputs = network(*inputs) - """ - - def __init__(self, dataset, dataset_sink_mode=True, iter_first_order=0): - check_bool(dataset_sink_mode) - self.iter = _DatasetIterMSLoopSink(dataset, iter_first_order) - - def __iter__(self): - return self.iter.__iter__() - - # A temp solution for loop sink. Delete later - def types_shapes(self): - """Get the types and shapes from dataset on current config.""" - return self.iter.types_shapes() - - def loop_size(self): - """Get loop_size for every iteration.""" - return self.iter.loop_size - - -class _DatasetIter: - """Base iter for dataset help""" - - def __init__(self, dataset): - self.loop_size = 1 - if not hasattr(dataset, '__ME_INITED__'): - if not hasattr(dataset, '__loop_size__'): - self.loop_size = dataset.get_dataset_size() - else: - self.loop_size = dataset.__loop_size__ - dataset.__TRANSFER_DATASET__ = _exec_datagraph(dataset, self.loop_size) - dataset.__ME_INITED__ = dataset.__TRANSFER_DATASET__.queue_name - - if not hasattr(dataset, '__no_send__'): - _send_data(dataset) - else: - _send_data(dataset) - - self.ind = 0 - self.dataset = dataset - dataset_types, dataset_shapes = _get_types_and_shapes(dataset) - self.dataset_types, self.dataset_shapes = dataset_types, dataset_shapes - - def __iter__(self): - self.ind = 0 - return self - - def __next__(self): - if self.ind >= self.loop_count: - raise StopIteration() - self.ind += 1 - return self.op() - - def types_shapes(self): - return self.dataset_types, self.dataset_shapes - - def get_loop_count(self, dataset): - loop_count = 1 - if hasattr(dataset, '__loop_size__'): - loop_size = dataset.__loop_size__ - if dataset.get_dataset_size() % loop_size != 0: - raise ValueError(f'Dataset size {dataset.get_dataset_size()} and ' - f'loop_size {loop_size} are not matched.') - loop_count = int(dataset.get_dataset_size() / loop_size) - return loop_count - - -class _DatasetIterMSLoopSink(_DatasetIter): - """Iter for context (device_target=Ascend)""" - - def __init__(self, dataset, iter_first_order): - super(_DatasetIterMSLoopSink, self).__init__(dataset) - loop_size = dataset.__loop_size__ + iter_first_order - self.loop_count = int(dataset.get_dataset_size() / loop_size) * 2 - # for self._parallel_mode equal to semi_auto_parallel or auto_parallel, use a complete tensor to - # compile, and slice tensor to run. The batch dimension of tensors for compile is device_number - # times the batch dimension of tensors for run. Now only support LoopSink. - if _get_parallel_mode() in (ParallelMode.SEMI_AUTO_PARALLEL, ParallelMode.AUTO_PARALLEL): - device_num = _get_device_num() - self.dataset_shapes = _to_full_shapes(self.dataset_shapes, device_num) - - def op(): - return tuple() - - self.op = op diff --git a/example/resnet50_imagenet2012_THOR/model/grad_reducer_thor.py b/example/resnet50_imagenet2012_THOR/model/grad_reducer_thor.py deleted file mode 100644 index ad8d8dd8e4..0000000000 --- a/example/resnet50_imagenet2012_THOR/model/grad_reducer_thor.py +++ /dev/null @@ -1,183 +0,0 @@ -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ -"""grad_reducer_thor""" -import mindspore.common.dtype as mstype -from mindspore.communication.management import GlobalComm, get_group_size -from mindspore.nn.cell import Cell -from mindspore.ops import functional as F, composite as C, operations as P -from mindspore.ops.operations.comm_ops import AllReduce, ReduceOp - -reduce_opt = C.MultitypeFuncGraph("reduce_opt") - -_all_reduce_A = AllReduce() - - -def _init_optimizer_allreduce(group): - global _all_reduce_A - _all_reduce_A = AllReduce(ReduceOp.SUM, GlobalComm.WORLD_COMM_GROUP) - _all_reduce_A.add_prim_attr('fusion', group) - - -@reduce_opt.register("Function", "Number", "Tensor") -def _tensors_allreduce_mean(mul, degree, grad): - degree = F.scalar_cast(degree, F.dtype(grad)) - grad = _all_reduce_A(grad) - cast_op = P.Cast() - return mul(grad, cast_op(F.scalar_to_array(1.0 / degree), F.dtype(grad))) - - -@reduce_opt.register("Bool", "Tensor") -def _tensors_allreduce(allreduce_filter, grad): - if allreduce_filter: - return _all_reduce_A(grad) - return grad - - -_get_datatype = C.MultitypeFuncGraph("_get_datatype") - - -@_get_datatype.register("Tensor") -def _tensors_get_datatype(grad): - """ - Acquire gradient datatype. - - Args: - grad (Tensor): The gradient tensor before operation. - - Returns: - mstype, the datatype of gradient. - """ - return F.dtype(grad) - - -_cast_datatype = C.MultitypeFuncGraph("_cast_datatype") - - -@_cast_datatype.register("TypeType", "Tensor") -def _tensors_cast_datatype(datatype, grad): - """ - Cast gradient to datatype. - - Args: - datatype (mstype): the destination datatype of gradient. - grad (Tensor): The gradient tensor before operation. - - Returns: - Tensor, the gradient tensor after operation. - """ - return F.cast(grad, datatype) - - -class DistributedGradReducerThor(Cell): - """ - A distributed optimizer. - - Constructs a gradient reducer Cell, which applies communication and average operations on - single-process gradient values. - - Args: - parameters (list): the parameters to be updated. - mean (bool): When mean is true, the mean coefficient (degree) would apply on gradients. Default: False. - degree (int): The mean coefficient. Usually it equals to device number. Default: None. - - Raises: - ValueError: If degree is not a int or less than 0. - - Examples: - >>> from mindspore.communication import init, get_group_size - >>> from mindspore.ops import composite as C - >>> from mindspore.ops import operations as P - >>> from mindspore.ops import functional as F - >>> from mindspore import context - >>> from mindspore import nn - >>> from mindspore import ParallelMode, ParameterTuple - >>> - >>> device_id = int(os.environ["DEVICE_ID"]) - >>> context.set_context(mode=context.GRAPH_MODE, device_target="Ascend", save_graphs=True, - >>> device_id=int(device_id), enable_hccl=True) - >>> init() - >>> context.reset_auto_parallel_context() - >>> context.set_auto_parallel_context(parallel_mode=ParallelMode.DATA_PARALLEL) - >>> - >>> - >>> class TrainingWrapper(nn.Cell): - >>> def __init__(self, network, optimizer, sens=1.0): - >>> super(TrainingWrapper, self).__init__(auto_prefix=False) - >>> self.network = network - >>> self.network.add_flags(defer_inline=True) - >>> self.weights = ParameterTuple(network.trainable_params()) - >>> self.optimizer = optimizer - >>> self.grad = C.GradOperation('grad', get_by_list=True, sens_param=True) - >>> self.sens = sens - >>> self.reducer_flag = False - >>> self.grad_reducer = None - >>> self.parallel_mode = context.get_auto_parallel_context("parallel_mode") - >>> if self.parallel_mode in [ParallelMode.DATA_PARALLEL, - >>> ParallelMode.HYBRID_PARALLEL]: - >>> self.reducer_flag = True - >>> if self.reducer_flag: - >>> mean = context.get_auto_parallel_context("mirror_mean") - >>> if mean.get_device_num_is_set(): - >>> degree = context.get_auto_parallel_context("device_num") - >>> else: - >>> degree = get_group_size() - >>> self.grad_reducer = nn.DistributedGradReducer(optimizer.parameters, mean, degree) - >>> - >>> def construct(self, *args): - >>> weights = self.weights - >>> loss = self.network(*args) - >>> sens = P.Fill()(P.DType()(loss), P.Shape()(loss), self.sens) - >>> grads = self.grad(self.network, weights)(*args, sens) - >>> if self.reducer_flag: - >>> # apply grad reducer on grads - >>> grads = self.grad_reducer(grads) - >>> return F.depend(loss, self.optimizer(grads)) - >>> - >>> network = Net() - >>> optimizer = nn.Momentum(network.trainable_params(), learning_rate=0.1, momentum=0.9) - >>> train_cell = TrainingWrapper(network, optimizer) - >>> inputs = Tensor(np.ones([16, 16]).astype(np.float32)) - >>> label = Tensor(np.zeros([16, 16]).astype(np.float32)) - >>> grads = train_cell(inputs, label) - """ - - def __init__(self, parameters, group, mean=True, degree=None): - super(DistributedGradReducerThor, self).__init__(auto_prefix=False) - self.hyper_map = C.HyperMap() - self.mul = P.Mul() - if degree is None: - self.degree = get_group_size() - else: - if not isinstance(degree, int) or degree <= 0: - raise ValueError("Parameter 'degree' in DistributedGradReducer should large than 0 and be int") - self.degree = degree - self.mean = mean - self.allreduce_filter = tuple(x.layerwise_parallel is False for x in parameters) - _init_optimizer_allreduce(group) - - def construct(self, grads): - # In some circumstances, the data precision of grads could be mixed with float16 and float32. Thus, the - # result of AllReduce is unreliable. To solve the problem, grads should be cast to float32 before AllReduce, - # and cast back after the operation. - datatypes = self.hyper_map(F.partial(_get_datatype), grads) - grads = self.hyper_map(F.partial(_cast_datatype, mstype.float32), grads) - - if self.mean: - new_grad = self.hyper_map(F.partial(reduce_opt, self.mul, self.degree), grads) - else: - new_grad = self.hyper_map(F.partial(reduce_opt), self.allreduce_filter, grads) - - new_grad = self.hyper_map(F.partial(_cast_datatype), datatypes, new_grad) - return new_grad diff --git a/example/resnet50_imagenet2012_THOR/model/model_thor.py b/example/resnet50_imagenet2012_THOR/model/model_thor.py deleted file mode 100644 index 25e3dd7f82..0000000000 --- a/example/resnet50_imagenet2012_THOR/model/model_thor.py +++ /dev/null @@ -1,725 +0,0 @@ -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ -"""Model.""" - -import numpy as np -from mindspore import context -from mindspore import log as logger -from mindspore import nn -from mindspore._c_expression import init_exec_dataset -from mindspore._checkparam import check_input_data, check_output_data, check_int_positive, check_bool -from mindspore.common import dtype as mstype -from mindspore.common.dtype import pytype_to_dtype -from mindspore.common.tensor import Tensor -from mindspore.nn.metrics import Loss -from mindspore.nn.metrics import get_metrics -from mindspore.nn.wrap.cell_wrapper import _VirtualDatasetCell -from mindspore.parallel._utils import _get_parallel_mode, _get_device_num, _get_global_rank, \ - _get_parameter_broadcast, _device_number_check, _parameter_broadcast_check -from mindspore.train import amp -from mindspore.train.callback import _InternalCallbackParam, RunContext, _CallbackManager -from mindspore.train.parallel_utils import ParallelMode - -from model.dataset_helper import DatasetHelper - - -def _convert_type(types): - """ - Convert from numpy type to tensor type. - - Args: - types (list): Numpy type list of element in dataset. - - Returns: - list, list of element in dataset. - """ - ms_types = [] - for np_type in types: - ms_type = pytype_to_dtype(np_type) - ms_types.append(ms_type) - return ms_types - - -def _get_types_and_shapes(dataset): - """Get dataset types and shapes.""" - dataset_types = _convert_type(dataset.output_types()) - dataset_shapes = dataset.output_shapes() - return dataset_types, dataset_shapes - - -def _exec_datagraph(exec_dataset, dataset_size, phase='dataset'): - """Initialize and execute the dataset graph.""" - batch_size = exec_dataset.get_batch_size() - input_indexs = exec_dataset.input_indexs - - # transform data format - dataset_types, dataset_shapes = _get_types_and_shapes(exec_dataset) - init_exec_dataset(exec_dataset.__ME_INITED__, - dataset_size, - batch_size, - dataset_types, - dataset_shapes, - input_indexs, - phase=phase, - need_run=False) - - -class Model: - """ - High-Level API for Training or Testing. - - `Model` groups layers into an object with training and inference features. - - Args: - network (Cell): The training or testing network. - loss_fn (Cell): Objective function, if loss_fn is None, the - network should contain the logic of loss and grads calculation, and the logic - of parallel if needed. Default: None. - optimizer (Cell): Optimizer for updating the weights. Default: None. - metrics (Union[dict, set]): Dict or set of metrics to be evaluated by the model during - training and testing. eg: {'accuracy', 'recall'}. Default: None. - eval_network (Cell): Network for evaluation. If not defined, `network` and `loss_fn` would be wrapped as - `eval_network`. Default: None. - eval_indexes (list): In case of defining the `eval_network`, if `eval_indexes` is None, all outputs of - `eval_network` would be passed to metrics, otherwise `eval_indexes` must contain three - elements, representing the positions of loss value, predict value and label, the loss - value would be passed to `Loss` metric, predict value and label would be passed to other - metric. Default: None. - amp_level (str): Option for argument `level` in `mindspore.amp.build_train_network`, level for mixed - precision training. Supports [O0, O2]. Default: "O0". - - - O0: Do not change. - - O2: Cast network to float16, keep batchnorm run in float32, using dynamic loss scale. - - loss_scale_manager (Union[None, LossScaleManager]): If None, not scale the loss, or else - scale the loss by LossScaleManager. If it is set, overwrite the level setting. It's a eyword argument. - e.g. Use `loss_scale_manager=None` to set the value. - keep_batchnorm_fp32 (bool): Keep Batchnorm run in `float32`. If set, overwrite the level setting. Default: True. - - Examples: - >>> class Net(nn.Cell): - >>> def __init__(self): - >>> super(Net, self).__init__() - >>> self.conv = nn.Conv2d(3, 64, 3, has_bias=False, weight_init='normal') - >>> self.bn = nn.BatchNorm2d(64) - >>> self.relu = nn.ReLU() - >>> self.flatten = nn.Flatten() - >>> self.fc = nn.Dense(64*224*224, 12) # padding=0 - >>> - >>> def construct(self, x): - >>> x = self.conv(x) - >>> x = self.bn(x) - >>> x = self.relu(x) - >>> x = self.flatten(x) - >>> out = self.fc(x) - >>> return out - >>> - >>> net = Net() - >>> loss = nn.SoftmaxCrossEntropyWithLogits(is_grad=False, sparse=True) - >>> optim = Momentum(params=net.trainable_params(), learning_rate=0.1, momentum=0.9) - >>> model = Model(net, loss_fn=loss, optimizer=optim, metrics=None) - >>> dataset = get_dataset() - >>> model.train(2, dataset) - """ - - def __init__(self, network, loss_fn=None, optimizer=None, metrics=None, eval_network=None, - eval_indexes=None, amp_level="O0", frequency=278, stop_epoch=100, **kwargs): - self._network = network - self._loss_fn = loss_fn - self._optimizer = optimizer - self._loss_scale_manager = None - self._loss_scale_manager_set = False - self._keep_bn_fp32 = True - self._check_kwargs(kwargs) - self._amp_level = amp_level - self._process_amp_args(kwargs) - self._parallel_mode = _get_parallel_mode() - self._device_number = _get_device_num() - self._global_rank = _get_global_rank() - self._parameter_broadcast = _get_parameter_broadcast() - self._frequency = frequency - self._stop_epoch = stop_epoch - - self._train_network = self._build_train_network() - self._build_eval_network(metrics, eval_network, eval_indexes) - self._build_predict_network() - - def _process_amp_args(self, kwargs): - if self._amp_level == "O0": - self._keep_bn_fp32 = False - if 'keep_batchnorm_fp32' in kwargs: - self._keep_bn_fp32 = kwargs['keep_batchnorm_fp32'] - if 'loss_scale_manager' in kwargs: - self._loss_scale_manager = kwargs['loss_scale_manager'] - self._loss_scale_manager_set = True - - def _check_kwargs(self, kwargs): - for arg in kwargs: - if arg not in ['loss_scale_manager', 'keep_batchnorm_fp32']: - raise ValueError(f"Unsupport arg '{arg}'") - - def _build_train_network(self): - """Build train network""" - network = self._network - if self._optimizer: - if self._loss_scale_manager_set: - network = amp.build_train_network(network, - self._optimizer, - self._loss_fn, - level=self._amp_level, - loss_scale_manager=self._loss_scale_manager, - keep_batchnorm_fp32=self._keep_bn_fp32) - else: - network = amp.build_train_network(network, - self._optimizer, - self._loss_fn, - level=self._amp_level, - keep_batchnorm_fp32=self._keep_bn_fp32) - elif self._loss_fn: - network = nn.WithLossCell(network, self._loss_fn) - # If need to check if loss_fn is not None, but optimizer is None - - if self._parallel_mode in (ParallelMode.SEMI_AUTO_PARALLEL, ParallelMode.AUTO_PARALLEL): - network.set_auto_parallel() - return network - - def _build_eval_network(self, metrics, eval_network, eval_indexes): - """Build the network for evaluation.""" - self._metric_fns = get_metrics(metrics) - if not self._metric_fns: - return - - if eval_network is not None: - if eval_indexes is not None and not (isinstance(eval_indexes, list) and len(eval_indexes) == 3): - raise ValueError("Eval_indexes must be a list or None. If eval_indexes is a list, length of it \ - must be three. But got {}".format(eval_indexes)) - - self._eval_network = eval_network - self._eval_indexes = eval_indexes - else: - if self._loss_fn is None: - raise ValueError("loss_fn can not be None.") - self._eval_network = nn.WithEvalCell(self._network, self._loss_fn, self._amp_level == "O2") - self._eval_indexes = [0, 1, 2] - - if self._parallel_mode in (ParallelMode.SEMI_AUTO_PARALLEL, ParallelMode.AUTO_PARALLEL): - self._eval_network.set_auto_parallel() - - def _build_predict_network(self): - """Build the network for prediction.""" - self._predict_network = self._network - if self._parallel_mode in (ParallelMode.SEMI_AUTO_PARALLEL, ParallelMode.AUTO_PARALLEL): - self._predict_network = _VirtualDatasetCell(self._network) - self._predict_network.set_auto_parallel() - - def _clear_metrics(self): - """Clear metrics local values.""" - for metric in self._metric_fns.values(): - metric.clear() - - def _update_metrics(self, outputs): - """Update metrics local values.""" - if not isinstance(outputs, tuple): - raise ValueError("The `outputs` is not tuple.") - - if self._eval_indexes is not None and len(outputs) < 3: - raise ValueError("The length of `outputs` must be greater than or equal to 3, \ - but got {}".format(len(outputs))) - - for metric in self._metric_fns.values(): - if self._eval_indexes is None: - metric.update(*outputs) - else: - if isinstance(metric, Loss): - metric.update(outputs[self._eval_indexes[0]]) - else: - metric.update(outputs[self._eval_indexes[1]], outputs[self._eval_indexes[2]]) - - def _get_metrics(self): - """Get metrics local values.""" - metrics = dict() - for key, value in self._metric_fns.items(): - metrics[key] = value.eval() - return metrics - - def _get_scaling_sens(self): - """get the scaling sens""" - scaling_sens = 1 - if self._loss_scale_manager is not None: - scaling_sens = self._loss_scale_manager.get_loss_scale() - if self._parallel_mode == ParallelMode.DATA_PARALLEL: - scaling_sens /= self._device_number - return scaling_sens - - def _exec_preprocess(self, network, is_train, phase, dataset, dataset_sink_mode, iter_first_order): - """Initializes dataset.""" - need_wrap = False - if dataset_sink_mode: - # remove later to deal with loop sink - if not hasattr(dataset, '__ME_INITED__') and context.get_context("device_target") == "Ascend" \ - and not context.get_context("enable_ge"): - need_wrap = True - - if not is_train: - dataset.__loop_size__ = 1 - - dataset_helper = DatasetHelper(dataset, dataset_sink_mode, iter_first_order) - - # remove later to deal with loop sink - if need_wrap: - network = nn.DataWrapper(network, *(dataset_helper.types_shapes()), dataset.__ME_INITED__) - network.set_train(is_train) - network.phase = phase - - return dataset_helper, network - - def init(self, train_dataset=None, valid_dataset=None): - """ - Initializes compute graphs and data graphs with sink mode. - - Note: - Pre-init process only supports `GRAPH_MODE` and `Ascend` target currently. - - Args: - train_dataset (Dataset): A training dataset iterator. If define `train_dataset`, training graphs will be - initialized. Default: None. - valid_dataset (Dataset): A evaluating dataset iterator. If define `valid_dataset`, evaluation graphs will - be initialized, and `metrics` in `Model` can not be None. Default: None. - - Examples: - >>> train_dataset = get_train_dataset() - >>> valid_dataset = get_valid_dataset() - >>> net = Net() - >>> loss = nn.SoftmaxCrossEntropyWithLogits(is_grad=False, sparse=True) - >>> optim = Momentum(params=net.trainable_params(), learning_rate=0.1, momentum=0.9) - >>> model = Model(net, loss_fn=loss, optimizer=optim, metrics={'acc'}) - >>> model.init(train_dataset, valid_dataset) - >>> model.train(2, train_dataset) - >>> model.eval(valid_dataset) - """ - if context.get_context("mode") != context.GRAPH_MODE or context.get_context("device_target") != "Ascend": - raise RuntimeError('Pre-init process only supports GRAPH MODE and Ascend target currently.') - - if not train_dataset and not valid_dataset: - raise ValueError('Both train_dataset and valid_dataset can not be None or empty.') - - _device_number_check(self._parallel_mode, self._device_number) - - if train_dataset: - _parameter_broadcast_check(self._parallel_mode, self._parameter_broadcast) - self._train_network.set_train() - self._train_network.phase = 'train' - - if self._parameter_broadcast: - self._train_network.set_broadcast_flag() - - train_dataset_helper, train_network = self._exec_preprocess(self._train_network, - is_train=True, - phase='train', - dataset=train_dataset, - dataset_sink_mode=True) - self._train_network = train_network - for inputs in train_dataset_helper: - self._train_network.compile(*inputs) - break - - if valid_dataset: - if not self._metric_fns: - raise RuntimeError('If define `valid_dataset`, metric fn can not be None or empty.') - - self._eval_network.set_train(False) - self._eval_network.phase = 'eval' - valid_dataset_helper, eval_network = self._exec_preprocess(self._eval_network, - is_train=False, - phase='eval', - dataset=valid_dataset, - dataset_sink_mode=True) - self._eval_network = eval_network - for inputs in valid_dataset_helper: - self._eval_network.compile(*inputs) - break - - def _train(self, epoch, train_dataset, callbacks=None, dataset_sink_mode=True): - """ - Training. - - Args: - epoch (int): Total number of iterations on the data. - train_dataset (Dataset): A training dataset iterator. If there is no - loss_fn, a tuple with multiply data (data1, data2, data3, ...) will be - returned and passed to the network. Otherwise, a tuple (data, label) will - be returned, and the data and label are passed to the network and loss - function respectively. - callbacks (list): List of callback object. Callbacks which should be executed while training. Default: None. - dataset_sink_mode (bool): Determines whether to pass the data through dataset channel. Default: True. - Configure pynative mode, the training process will be performed with - dataset not sink. - """ - epoch = check_int_positive(epoch) - self._train_network.set_train() - - if self._parameter_broadcast: - self._train_network.set_broadcast_flag() - - # build callback list - cb_params = _InternalCallbackParam() - cb_params.train_network = self._train_network - cb_params.epoch_num = epoch - cb_params.batch_num = train_dataset.get_dataset_size() - cb_params.mode = "train" - cb_params.loss_fn = self._loss_fn - cb_params.optimizer = self._optimizer - cb_params.parallel_mode = self._parallel_mode - cb_params.device_number = self._device_number - cb_params.train_dataset = train_dataset - cb_params.list_callback = callbacks - - with _CallbackManager(callbacks) as list_callback: - if not dataset_sink_mode: - self._train_process(epoch, train_dataset, list_callback, cb_params) - elif context.get_context("mode") == context.PYNATIVE_MODE: - logger.warning("The pynative mode cannot support dataset sink mode currently." - "So the training process will be performed with dataset not sink.") - self._train_process(epoch, train_dataset, list_callback, cb_params) - else: - self._train_dataset_sink_process(epoch, train_dataset, list_callback, cb_params) - - def _train_dataset_sink_process(self, epoch, train_dataset, list_callback=None, cb_params=None): - """ - Training process. The data would be passed to network through dataset channel. - - Args: - epoch (int): Total number of iterations on the data. - train_dataset (Dataset): A training dataset iterator. If there is no - loss_fn, a tuple with multiply data (data1, data2, data3, ...) should be - returned and passed to the network. Otherwise, a tuple (data, label) should - be returned, and the data and label are passed to the network and loss - function respectively. - list_callback (Callback): Executor of callback list. Default: None. - cb_params (_InternalCallbackParam): Callback parameters. Default: None. - """ - iter_first_order = self._frequency - 1 - iter_second_order = 1 - train_dataset.__loop_size__ = iter_second_order - dataset_helper, train_network = self._exec_preprocess(self._train_network, - is_train=True, - phase='train', - dataset=train_dataset, - dataset_sink_mode=True, - iter_first_order=iter_first_order) - self._train_network = train_network - cb_params.train_network = self._train_network - cb_params.cur_step_num = 0 - - loop_size = dataset_helper.loop_size() - run_context = RunContext(cb_params) - list_callback.begin(run_context) - - # used to stop training for early stop, such as stopAtTIme or stopATStep - should_stop = False - has_do_dataset_init = False - switch_branch_one = True - for i in range(epoch): - cb_params.cur_epoch_num = i + 1 - list_callback.epoch_begin(run_context) - - # for data sink dataset_helper only iter once, other wise iter epoch_size times. - for inputs in dataset_helper: - list_callback.step_begin(run_context) - if switch_branch_one: - cb_params.cur_step_num += loop_size - self._train_network.add_flags_recursive(thor=True) - self._train_network.phase = 'train0' - else: - cb_params.cur_step_num += iter_first_order - self._train_network.add_flags_recursive(thor=False) - self._train_network.phase = 'train1' - if not has_do_dataset_init: - _exec_datagraph(train_dataset, iter_first_order, phase='train1_dataset') - has_do_dataset_init = True - switch_branch_one = not switch_branch_one - outputs = self._train_network(*inputs) - cb_params.net_outputs = outputs - list_callback.step_end(run_context) - - list_callback.epoch_end(run_context) - should_stop = should_stop or run_context.get_stop_requested() - if should_stop: - break - - list_callback.end(run_context) - - def _train_process(self, epoch, train_dataset, list_callback=None, cb_params=None): - """ - Training process. The data would be passed to network directly. - - Args: - epoch (int): Total number of iterations on the data. - train_dataset (Dataset): A training dataset iterator. If there is no - loss_fn, a tuple with multiply data (data1, data2, data3, ...) should be - returned and passed to the network. Otherwise, a tuple (data, label) should - be returned, and the data and label are passed to the network and loss - function respectively. - list_callback (Callback): Executor of callback list. Default: None. - cb_params (_InternalCallbackParam): Callback parameters. Default: None. - """ - dataset_helper, _ = self._exec_preprocess(self._train_network, - is_train=True, - phase='train', - dataset=train_dataset, - dataset_sink_mode=False) - cb_params.cur_step_num = 0 - run_context = RunContext(cb_params) - list_callback.begin(run_context) - # used to stop training for early stop, such as stopAtTIme or stopATStep - should_stop = False - - for i in range(epoch): - cb_params.cur_epoch_num = i + 1 - - list_callback.epoch_begin(run_context) - - for next_element in dataset_helper: - len_element = len(next_element) - if self._loss_fn and len_element != 2: - raise ValueError("when loss_fn is not None, train_dataset should" - "return two elements, but got {}".format(len_element)) - cb_params.cur_step_num += 1 - list_callback.step_begin(run_context) - - overflow = False - if self._loss_scale_manager and self._loss_scale_manager.get_drop_overflow_update(): - scaling_sens = self._get_scaling_sens() - next_element = tuple(next_element) + (Tensor(scaling_sens, mstype.float32),) - - outputs = self._train_network(*next_element) - cb_params.net_outputs = outputs - if self._loss_scale_manager and self._loss_scale_manager.get_drop_overflow_update(): - _, overflow, _ = outputs - overflow = np.all(overflow.asnumpy()) - self._loss_scale_manager.update_loss_scale(overflow) - - list_callback.step_end(run_context) - should_stop = should_stop or run_context.get_stop_requested() - if should_stop: - break - - train_dataset.reset() - - list_callback.epoch_end(run_context) - should_stop = should_stop or run_context.get_stop_requested() - if should_stop: - break - - list_callback.end(run_context) - - def train(self, epoch, train_dataset, callbacks=None, dataset_sink_mode=True): - """ - Training API where the iteration is controlled by python front-end. - - When setting pynative mode, the training process will be performed with dataset not sink. - - Note: - CPU is not supported when dataset_sink_mode is true. - If dataset_sink_mode is True, epoch of training should be equal to the count of repeat - operation in dataset processing. Otherwise, errors could occur since the amount of data - is not the amount training requires. - If dataset_sink_mode is True, data will be sent to device. If device is Ascend, features - of data will be transferred one by one. The limitation of data transmission per time is 256M. - - Args: - epoch (int): Total number of iterations on the data. - train_dataset (Dataset): A training dataset iterator. If there is no - loss_fn, a tuple with multiply data (data1, data2, data3, ...) should be - returned and passed to the network. Otherwise, a tuple (data, label) should - be returned, and the data and label are passed to the network and loss - function respectively. - callbacks (list): List of callback object. Callbacks which should be excuted while training. Default: None. - dataset_sink_mode (bool): Determines whether to pass the data through dataset channel. Default: True. - Configure pynative mode, the training process will be performed with - dataset not sink. - - - Examples: - >>> dataset = get_dataset() - >>> net = Net() - >>> loss = nn.SoftmaxCrossEntropyWithLogits(is_grad=False, sparse=True) - >>> loss_scale_manager = FixedLossScaleManager() - >>> optim = Momentum(params=net.trainable_params(), learning_rate=0.1, momentum=0.9) - >>> model = Model(net, loss_fn=loss, optimizer=optim, metrics=None, loss_scale_manager=loss_scale_manager) - >>> model.train(2, dataset) - """ - repeat_count = train_dataset.get_repeat_count() - if epoch != repeat_count and dataset_sink_mode is True: - logger.warning(f"The epoch_size {epoch} is not the same with dataset repeat_count {repeat_count}") - check_bool(dataset_sink_mode) - _device_number_check(self._parallel_mode, self._device_number) - _parameter_broadcast_check(self._parallel_mode, self._parameter_broadcast) - - self._train(epoch, - train_dataset, - callbacks=callbacks, - dataset_sink_mode=dataset_sink_mode) - - def _eval_dataset_sink_process(self, valid_dataset, list_callback=None, cb_params=None): - """ - Evaluation. The data would be passed to network through dataset channel. - - Args: - valid_dataset (Dataset): Dataset to evaluate the model. - list_callback (Callback): Executor of callback list. Default: None. - cb_params (_InternalCallbackParam): Callback parameters. Default: None. - - Returns: - Dict, returns the loss value & metrics values for the model in test mode. - """ - run_context = RunContext(cb_params) - - dataset_helper, eval_network = self._exec_preprocess(self._eval_network, - is_train=False, - phase='eval', - dataset=valid_dataset, - dataset_sink_mode=True) - self._eval_network = eval_network - cb_params.eval_network = self._eval_network - list_callback.begin(run_context) - - for inputs in dataset_helper: - cb_params.cur_step_num += 1 - list_callback.step_begin(run_context) - - outputs = self._eval_network(*inputs) - - cb_params.net_outputs = outputs - list_callback.step_end(run_context) - self._update_metrics(outputs) - - metrics = self._get_metrics() - cb_params.metrics = metrics - list_callback.end(run_context) - - return metrics - - def _eval_process(self, valid_dataset, list_callback=None, cb_params=None): - """ - Evaluation. The data would be passed to network directly. - - Args: - valid_dataset (Dataset): Dataset to evaluate the model. - list_callback (Callback): Executor of callback list. Default: None. - cb_params (_InternalCallbackParam): Callback parameters. Default: None. - - Returns: - Dict, returns the loss value & metrics values for the model in test mode. - """ - run_context = RunContext(cb_params) - list_callback.begin(run_context) - - dataset_helper, _ = self._exec_preprocess(self._eval_network, - is_train=False, - phase='eval', - dataset=valid_dataset, - dataset_sink_mode=False) - for next_element in dataset_helper: - cb_params.cur_step_num += 1 - list_callback.step_begin(run_context) - outputs = self._eval_network(*next_element) - cb_params.net_outputs = outputs - list_callback.step_end(run_context) - self._update_metrics(outputs) - - metrics = self._get_metrics() - cb_params.metrics = metrics - list_callback.end(run_context) - return metrics - - def eval(self, valid_dataset, callbacks=None, dataset_sink_mode=True): - """ - Evaluation API where the iteration is controlled by python front-end. - - Configure to pynative mode, the evaluation will be performed with dataset non-sink mode. - - Note: - CPU is not supported when dataset_sink_mode is true. - If dataset_sink_mode is True, data will be sent to device. If device is Ascend, features - of data will be transferred one by one. The limitation of data transmission per time is 256M. - - Args: - valid_dataset (Dataset): Dataset to evaluate the model. - callbacks (list): List of callback object. Callbacks which should be excuted - while training. Default: None. - dataset_sink_mode (bool): Determines whether to pass the data through dataset channel. Default: True. - - Returns: - Dict, returns the loss value & metrics values for the model in test mode. - - Examples: - >>> dataset = get_dataset() - >>> net = Net() - >>> loss = nn.SoftmaxCrossEntropyWithLogits(is_grad=False, sparse=True) - >>> model = Model(net, loss_fn=loss, optimizer=None, metrics={'acc'}) - >>> model.eval(dataset) - """ - check_bool(dataset_sink_mode) - _device_number_check(self._parallel_mode, self._device_number) - if not self._metric_fns: - raise ValueError("metric fn can not be None or empty.") - - cb_params = _InternalCallbackParam() - cb_params.eval_network = self._eval_network - cb_params.valid_dataset = valid_dataset - cb_params.batch_num = valid_dataset.get_dataset_size() - cb_params.mode = "eval" - cb_params.cur_step_num = 0 - - self._eval_network.set_train(mode=False) - self._eval_network.phase = 'eval' - - self._clear_metrics() - - with _CallbackManager(callbacks) as list_callback: - if dataset_sink_mode: - return self._eval_dataset_sink_process(valid_dataset, list_callback, cb_params) - return self._eval_process(valid_dataset, list_callback, cb_params) - - def predict(self, *predict_data): - """ - Generates output predictions for the input samples. - - Data could be single tensor, or list of tensor, tuple of tensor. - - Note: - Batch data should be put together in one tensor. - - Args: - predict_data (Tensor): Tensor of predict data. can be array, list or tuple. - - Returns: - Tensor, array(s) of predictions. - - Examples: - >>> input_data = Tensor(np.random.randint(0, 255, [1, 3, 224, 224]), mindspore.float32) - >>> model = Model(Net()) - >>> model.predict(input_data) - """ - self._predict_network.set_train(False) - check_input_data(*predict_data, data_class=Tensor) - result = self._predict_network(*predict_data) - - check_output_data(result) - return result - - -__all__ = ["Model"] diff --git a/example/resnet50_imagenet2012_THOR/model/resnet.py b/example/resnet50_imagenet2012_THOR/model/resnet.py deleted file mode 100644 index f3305022e8..0000000000 --- a/example/resnet50_imagenet2012_THOR/model/resnet.py +++ /dev/null @@ -1,359 +0,0 @@ -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ -"""ResNet.""" -import math -import numpy as np -import mindspore.nn as nn -from mindspore.common.tensor import Tensor -from mindspore.ops import operations as P - -from model.thor_layer import Conv2d_Thor, Dense_Thor - - -def calculate_gain(nonlinearity, param=None): - """calculate_gain""" - linear_fns = ['linear', 'conv1d', 'conv2d', 'conv3d', 'conv_transpose1d', 'conv_transpose2d', 'conv_transpose3d'] - res = 0 - if nonlinearity in linear_fns or nonlinearity == 'sigmoid': - res = 1 - elif nonlinearity == 'tanh': - res = 5.0 / 3 - elif nonlinearity == 'relu': - res = math.sqrt(2.0) - elif nonlinearity == 'leaky_relu': - if param is None: - negative_slope = 0.01 - elif not isinstance(param, bool) and isinstance(param, int) or isinstance(param, float): - # True/False are instances of int, hence check above - negative_slope = param - else: - raise ValueError("negative_slope {} not a valid number".format(param)) - res = math.sqrt(2.0 / (1 + negative_slope ** 2)) - else: - raise ValueError("Unsupported nonlinearity {}".format(nonlinearity)) - return res - - -def _calculate_fan_in_and_fan_out(tensor): - """_calculate_fan_in_and_fan_out""" - dimensions = len(tensor) - if dimensions < 2: - raise ValueError("Fan in and fan out can not be computed for tensor with fewer than 2 dimensions") - if dimensions == 2: # Linear - fan_in = tensor[1] - fan_out = tensor[0] - else: - num_input_fmaps = tensor[1] - num_output_fmaps = tensor[0] - receptive_field_size = 1 - if dimensions > 2: - receptive_field_size = tensor[2] * tensor[3] - fan_in = num_input_fmaps * receptive_field_size - fan_out = num_output_fmaps * receptive_field_size - return fan_in, fan_out - - -def _calculate_correct_fan(tensor, mode): - mode = mode.lower() - valid_modes = ['fan_in', 'fan_out'] - if mode not in valid_modes: - raise ValueError("Mode {} not supported, please use one of {}".format(mode, valid_modes)) - fan_in, fan_out = _calculate_fan_in_and_fan_out(tensor) - return fan_in if mode == 'fan_in' else fan_out - - -def kaiming_normal(inputs_shape, a=0, mode='fan_in', nonlinearity='leaky_relu'): - fan = _calculate_correct_fan(inputs_shape, mode) - gain = calculate_gain(nonlinearity, a) - std = gain / math.sqrt(fan) - return np.random.normal(0, std, size=inputs_shape).astype(np.float32) - - -def kaiming_uniform(inputs_shape, a=0, mode='fan_in', nonlinearity='leaky_relu'): - fan = _calculate_correct_fan(inputs_shape, mode) - gain = calculate_gain(nonlinearity, a) - std = gain / math.sqrt(fan) - bound = math.sqrt(3.0) * std # Calculate uniform bounds from standard deviation - return np.random.uniform(-bound, bound, size=inputs_shape).astype(np.float32) - - -def _conv3x3(in_channel, out_channel, stride=1, damping=0.03, loss_scale=1, frequency=278): - weight_shape = (out_channel, in_channel, 3, 3) - weight = Tensor(kaiming_normal(weight_shape, mode="fan_out", nonlinearity='relu')) - return Conv2d_Thor(in_channel, out_channel, - kernel_size=3, stride=stride, padding=0, pad_mode='same', weight_init=weight, - damping=damping, loss_scale=loss_scale, frequency=frequency) - - -def _conv1x1(in_channel, out_channel, stride=1, damping=0.03, loss_scale=1, frequency=278): - weight_shape = (out_channel, in_channel, 1, 1) - weight = Tensor(kaiming_normal(weight_shape, mode="fan_out", nonlinearity='relu')) - return Conv2d_Thor(in_channel, out_channel, - kernel_size=1, stride=stride, padding=0, pad_mode='same', weight_init=weight, - damping=damping, loss_scale=loss_scale, frequency=frequency) - - -def _conv7x7(in_channel, out_channel, stride=1, damping=0.03, loss_scale=1, frequency=278): - weight_shape = (out_channel, in_channel, 7, 7) - weight = Tensor(kaiming_normal(weight_shape, mode="fan_out", nonlinearity='relu')) - return Conv2d_Thor(in_channel, out_channel, - kernel_size=7, stride=stride, padding=0, pad_mode='same', weight_init=weight, - damping=damping, loss_scale=loss_scale, frequency=frequency) - - -def _bn(channel): - return nn.BatchNorm2d(channel, eps=1e-4, momentum=0.9, - gamma_init=1, beta_init=0, moving_mean_init=0, moving_var_init=1) - - -def _bn_last(channel): - return nn.BatchNorm2d(channel, eps=1e-4, momentum=0.9, - gamma_init=1, beta_init=0, moving_mean_init=0, moving_var_init=1) - - -def _fc(in_channel, out_channel, damping, loss_scale, frequency): - weight_shape = (out_channel, in_channel) - weight = Tensor(kaiming_uniform(weight_shape, a=math.sqrt(5))) - return Dense_Thor(in_channel, out_channel, has_bias=False, weight_init=weight, - bias_init=0, damping=damping, loss_scale=loss_scale, frequency=frequency) - - -class ResidualBlock(nn.Cell): - """ - ResNet V1 residual block definition. - - Args: - in_channel (int): Input channel. - out_channel (int): Output channel. - stride (int): Stride size for the first convolutional layer. Default: 1. - - Returns: - Tensor, output tensor. - - Examples: - >>> ResidualBlock(3, 256, stride=2) - """ - expansion = 4 - - def __init__(self, - in_channel, - out_channel, - stride=1, - damping=0.03, - loss_scale=1, - frequency=278): - super(ResidualBlock, self).__init__() - - channel = out_channel // self.expansion - self.conv1 = _conv1x1(in_channel, channel, stride=1, damping=damping, loss_scale=loss_scale, - frequency=frequency) - self.bn1 = _bn(channel) - - self.conv2 = _conv3x3(channel, channel, stride=stride, damping=damping, loss_scale=loss_scale, - frequency=frequency) - self.bn2 = _bn(channel) - - self.conv3 = _conv1x1(channel, out_channel, stride=1, damping=damping, loss_scale=loss_scale, - frequency=frequency) - self.bn3 = _bn_last(out_channel) - - self.relu = nn.ReLU() - - self.down_sample = False - - if stride != 1 or in_channel != out_channel: - self.down_sample = True - self.down_sample_layer = None - - if self.down_sample: - self.down_sample_layer = nn.SequentialCell([_conv1x1(in_channel, out_channel, stride, - damping=damping, loss_scale=loss_scale, - frequency=frequency), - _bn(out_channel)]) - self.add = P.TensorAdd() - - def construct(self, x): - identity = x - - out = self.conv1(x) - out = self.bn1(out) - out = self.relu(out) - - out = self.conv2(out) - out = self.bn2(out) - out = self.relu(out) - - out = self.conv3(out) - out = self.bn3(out) - - if self.down_sample: - identity = self.down_sample_layer(identity) - - out = self.add(out, identity) - out = self.relu(out) - - return out - - -class ResNet(nn.Cell): - """ - ResNet architecture. - - Args: - block (Cell): Block for network. - layer_nums (list): Numbers of block in different layers. - in_channels (list): Input channel in each layer. - out_channels (list): Output channel in each layer. - strides (list): Stride size in each layer. - num_classes (int): The number of classes that the training images are belonging to. - Returns: - Tensor, output tensor. - - Examples: - >>> ResNet(ResidualBlock, - >>> [3, 4, 6, 3], - >>> [64, 256, 512, 1024], - >>> [256, 512, 1024, 2048], - >>> [1, 2, 2, 2], - >>> 10) - """ - - def __init__(self, - block, - layer_nums, - in_channels, - out_channels, - strides, - num_classes, - damping, - loss_scale, - frequency): - super(ResNet, self).__init__() - - if not len(layer_nums) == len(in_channels) == len(out_channels) == 4: - raise ValueError("the length of layer_num, in_channels, out_channels list must be 4!") - - self.conv1 = _conv7x7(3, 64, stride=2, damping=damping, loss_scale=loss_scale, frequency=frequency) - self.bn1 = _bn(64) - self.relu = P.ReLU() - self.maxpool = P.MaxPoolWithArgmax(padding="same", ksize=3, strides=2) - - self.layer1 = self._make_layer(block, - layer_nums[0], - in_channel=in_channels[0], - out_channel=out_channels[0], - stride=strides[0], - damping=damping, - loss_scale=loss_scale, - frequency=frequency) - self.layer2 = self._make_layer(block, - layer_nums[1], - in_channel=in_channels[1], - out_channel=out_channels[1], - stride=strides[1], - damping=damping, - loss_scale=loss_scale, - frequency=frequency) - self.layer3 = self._make_layer(block, - layer_nums[2], - in_channel=in_channels[2], - out_channel=out_channels[2], - stride=strides[2], damping=damping, - loss_scale=loss_scale, - frequency=frequency) - self.layer4 = self._make_layer(block, - layer_nums[3], - in_channel=in_channels[3], - out_channel=out_channels[3], - stride=strides[3], - damping=damping, - loss_scale=loss_scale, - frequency=frequency) - - self.mean = P.ReduceMean(keep_dims=True) - self.flatten = nn.Flatten() - self.end_point = _fc(out_channels[3], num_classes, damping=damping, loss_scale=loss_scale, frequency=frequency) - - def _make_layer(self, block, layer_num, in_channel, out_channel, stride, - damping, loss_scale, frequency): - """ - Make stage network of ResNet. - - Args: - block (Cell): Resnet block. - layer_num (int): Layer number. - in_channel (int): Input channel. - out_channel (int): Output channel. - stride (int): Stride size for the first convolutional layer. - - Returns: - SequentialCell, the output layer. - - Examples: - >>> _make_layer(ResidualBlock, 3, 128, 256, 2) - """ - layers = [] - - resnet_block = block(in_channel, out_channel, stride=stride, - damping=damping, loss_scale=loss_scale, frequency=frequency) - layers.append(resnet_block) - - for _ in range(1, layer_num): - resnet_block = block(out_channel, out_channel, stride=1, - damping=damping, loss_scale=loss_scale, frequency=frequency) - layers.append(resnet_block) - - return nn.SequentialCell(layers) - - def construct(self, x): - x = self.conv1(x) - x = self.bn1(x) - x = self.relu(x) - c1, _ = self.maxpool(x) - - c2 = self.layer1(c1) - c3 = self.layer2(c2) - c4 = self.layer3(c3) - c5 = self.layer4(c4) - - out = self.mean(c5, (2, 3)) - out = self.flatten(out) - out = self.end_point(out) - - return out - - -def resnet50(class_num=10, damping=0.03, loss_scale=1, frequency=278): - """ - Get ResNet50 neural network. - - Args: - class_num (int): Class number. - - Returns: - Cell, cell instance of ResNet50 neural network. - - Examples: - >>> net = resnet50(10) - """ - return ResNet(ResidualBlock, - [3, 4, 6, 3], - [64, 256, 512, 1024], - [256, 512, 1024, 2048], - [1, 2, 2, 2], - class_num, - damping, - loss_scale, - frequency) diff --git a/example/resnet50_imagenet2012_THOR/model/thor.py b/example/resnet50_imagenet2012_THOR/model/thor.py deleted file mode 100644 index 6786cb7485..0000000000 --- a/example/resnet50_imagenet2012_THOR/model/thor.py +++ /dev/null @@ -1,199 +0,0 @@ -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ -"""momentum""" -import mindspore.common.dtype as mstype -from mindspore.common.initializer import initializer -from mindspore.common.parameter import Parameter -from mindspore.common.parameter import ParameterTuple -from mindspore.common.tensor import Tensor -from mindspore.nn.optim.optimizer import Optimizer -from mindspore.ops import functional as F, composite as C, operations as P -from mindspore.parallel._utils import _get_device_num, _get_mirror_mean -from model.grad_reducer_thor import DistributedGradReducerThor - -momentum_opt = C.MultitypeFuncGraph("momentum_opt") - - -@momentum_opt.register("Function", "Tensor", "Tensor", "Tensor", "Tensor", "Tensor") -def _tensor_run_opt_ext(opt, learning_rate, momentum, gradient, weight, moment): - """Apply momentum optimizer to the weight parameter using Tensor.""" - success = True - success = F.depend(success, opt(weight, moment, learning_rate, gradient, momentum)) - return success - - -op_add = P.AddN() -apply_decay = C.MultitypeFuncGraph("apply_decay") - - -@apply_decay.register("Number", "Bool", "Tensor", "Tensor") -def _tensor_apply_decay(weight_decay, if_apply, weight, gradient): - """Get grad with weight_decay.""" - if if_apply: - return op_add((weight * weight_decay, gradient)) - return gradient - - -class THOR(Optimizer): - """THOR""" - def __init__(self, params, learning_rate, momentum, matrix_A, matrix_G, A_inv_max, G_inv_max, weight_decay=0.0, - loss_scale=1.0, - decay_filter=lambda x: x.name not in []): - super(THOR, self).__init__(learning_rate, params, weight_decay, loss_scale) - if isinstance(momentum, float) and momentum < 0.0: - raise ValueError("momentum should be at least 0.0, but got momentum {}".format(momentum)) - self.momentum = Parameter(Tensor(momentum, mstype.float32), name="momentum") - self.params = self.parameters - self.moments = self.params.clone(prefix="moments", init='zeros') - self.hyper_map = C.HyperMap() - self.opt = P.ApplyMomentum() - self.matrix_A = ParameterTuple(matrix_A) - self.matrix_G = ParameterTuple(matrix_G) - self.A_inv_max = ParameterTuple(A_inv_max) - self.G_inv_max = ParameterTuple(G_inv_max) - self.cube_matmul_left = P.CusMatMulCubeFraczLeftCast() - self.cube_matmul_left_fc = P.CusMatMulCubeDenseLeft() - self.cube_matmul_right_fc = P.CusMatMulCubeDenseRight() - self.cube_matmul_right_mul = P.CusMatMulCubeFraczRightMul() - self.transpose = P.Transpose() - self.shape = P.Shape() - self.reshape = P.Reshape() - self.mul = P.Mul() - self.weight_idx = [] - for i in range(len(self.params)): - if "conv" in self.params[i].name or "end_point" in self.params[i].name: - self.weight_idx.append(i) - self.weight_idx.append(len(self.params)) - self.feature_map = [1.0 / 12544, 1.0 / 3136, 1.0 / 3136, 1.0 / 3136, 1.0 / 3136, 1.0 / 3136, 1.0 / 3136, - 1.0 / 3136, 1.0 / 3136, 1.0 / 3136, 1.0 / 3136, 1.0 / 3136, - 1.0 / 784, 1.0 / 784, 1.0 / 784, 1.0 / 784, 1.0 / 784, 1.0 / 784, 1.0 / 784, 1.0 / 784, - 1.0 / 784, 1.0 / 784, 1.0 / 784, 1.0 / 784, 1.0 / 784, - 1.0 / 196, 1.0 / 196, 1.0 / 196, 1.0 / 196, 1.0 / 196, 1.0 / 196, 1.0 / 196, 1.0 / 196, - 1.0 / 196, 1.0 / 196, 1.0 / 196, 1.0 / 196, 1.0 / 196, 1.0 / 196, 1.0 / 196, 1.0 / 196, - 1.0 / 196, 1.0 / 196, 1.0 / 196, - 1.0 / 49, 1.0 / 49, 1.0 / 49, 1.0 / 49, 1.0 / 49, 1.0 / 49, 1.0 / 49, 1.0 / 49, 1.0 / 49, - 1.0] - mean = _get_mirror_mean() - degree = _get_device_num() - self.grad_reducer_Amax = DistributedGradReducerThor(self.parameters, 2, mean, degree) - self.grad_reducer_Gmax = DistributedGradReducerThor(self.parameters, 5, mean, degree) - self.grad_reducer_A = DistributedGradReducerThor(self.parameters, 3, mean, degree) - self.grad_reducer_G = DistributedGradReducerThor(self.parameters, 4, mean, degree) - self.matrix_A_inv = () - self.matrix_G_inv = () - self.matrix_max_inv = () - - for i in range(54): - self.matrix_max_inv = self.matrix_max_inv + ( - Parameter(initializer(1, [1], mstype.float32), name="matrix_max" + str(i), requires_grad=False),) - self.log = P.Log() - self.exp = P.Exp() - self.sqrt = P.Sqrt() - self.matrix_max_inv = ParameterTuple(self.matrix_max_inv) - self.assign = P.Assign() - self.cast = P.Cast() - self.thor = True - self.weight_decay = weight_decay * loss_scale - self.decay_flags = tuple(decay_filter(x) for x in self.parameters) - - def construct(self, gradients): - params = self.params - moments = self.moments - if self.thor: - matrix_A_allreduce = () - matrix_G_allreduce = () - matrix_A_max_allreduce = () - matrix_G_max_allreduce = () - for i in range(54): - g = gradients[i * 3] - matrix_A = self.matrix_A[i] - matrix_G = self.matrix_G[i] - A_max = self.A_inv_max[i] - G_max = self.G_inv_max[i] - matrix_A = F.depend(matrix_A, g) - matrix_G = F.depend(matrix_G, g) - A_max = F.depend(A_max, g) - G_max = F.depend(G_max, g) - matrix_A_allreduce = matrix_A_allreduce + (matrix_A,) - matrix_G_allreduce = matrix_G_allreduce + (matrix_G,) - matrix_A_max_allreduce = matrix_A_max_allreduce + (A_max,) - matrix_G_max_allreduce = matrix_G_max_allreduce + (G_max,) - matrix_A_allreduce = self.grad_reducer_A(matrix_A_allreduce) - matrix_G_allreduce = self.grad_reducer_G(matrix_G_allreduce) - matrix_A_max_allreduce = self.grad_reducer_Amax(matrix_A_max_allreduce) - matrix_G_max_allreduce = self.grad_reducer_Gmax(matrix_G_max_allreduce) - new_grads = () - for i in range(54): - g = gradients[i * 3] - temp_a = matrix_A_allreduce[i] - temp_g = matrix_G_allreduce[i] - temp_a = self.cast(temp_a, mstype.float32) - temp_g = self.cast(temp_g, mstype.float32) - matrix_A_inv_max = self.log(matrix_A_max_allreduce[i]) - matrix_A_inv_max = self.mul(matrix_A_inv_max, -1) - matrix_A_inv_max = self.exp(matrix_A_inv_max) - temp_a = self.mul(temp_a, matrix_A_inv_max) - matrix_G_inv_max = self.log(matrix_G_max_allreduce[i]) - matrix_G_inv_max = self.mul(matrix_G_inv_max, -1) - matrix_G_inv_max = self.exp(matrix_G_inv_max) - temp_g = self.mul(temp_g, matrix_G_inv_max) - temp_max = self.mul(matrix_A_max_allreduce[i], matrix_G_max_allreduce[i]) - temp_max = self.mul(temp_max, self.feature_map[i]) - temp_a = self.cast(temp_a, mstype.float16) - temp_g = self.cast(temp_g, mstype.float16) - if i == 53: - g = self.cube_matmul_left_fc(temp_g, g) - g = self.cube_matmul_right_fc(g, temp_a, temp_max) - else: - g = self.cube_matmul_left(temp_g, g) - g = self.cube_matmul_right_mul(g, temp_a, temp_max) - fake_A = self.assign(self.matrix_A[i], temp_a) - fake_G = self.assign(self.matrix_G[i], temp_g) - fake_max = self.assign(self.matrix_max_inv[i], temp_max) - g = F.depend(g, fake_A) - g = F.depend(g, fake_G) - g = F.depend(g, fake_max) - if i == 53: - new_grads = new_grads + (g,) - else: - new_grads = new_grads + (g, gradients[i * 3 + 1], gradients[i * 3 + 2]) - gradients = new_grads - else: - new_grads = () - for i in range(54): - g = gradients[i * 3] - matrix_A = self.matrix_A[i] - matrix_G = self.matrix_G[i] - matrix_max = self.matrix_max_inv[i] - matrix_A = F.depend(matrix_A, g) - matrix_G = F.depend(matrix_G, g) - matrix_max = F.depend(matrix_max, g) - if i == 53: - g = self.cube_matmul_left_fc(matrix_G, g) - g = self.cube_matmul_right_fc(g, matrix_A, matrix_max) - new_grads = new_grads + (g,) - else: - g = self.cube_matmul_left(matrix_G, g) - g = self.cube_matmul_right_mul(g, matrix_A, matrix_max) - new_grads = new_grads + (g, gradients[i * 3 + 1], gradients[i * 3 + 2]) - gradients = new_grads - - if self.weight_decay > 0: - gradients = self.hyper_map(F.partial(apply_decay, self.weight_decay), self.decay_flags, - params, gradients) - gradients = self.scale_grad(gradients) - lr = self.get_lr() - success = self.hyper_map(F.partial(momentum_opt, self.opt, lr, self.momentum), gradients, params, moments) - return success diff --git a/example/resnet50_imagenet2012_THOR/model/thor_layer.py b/example/resnet50_imagenet2012_THOR/model/thor_layer.py deleted file mode 100644 index d84cbf7a93..0000000000 --- a/example/resnet50_imagenet2012_THOR/model/thor_layer.py +++ /dev/null @@ -1,477 +0,0 @@ -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ -"""thor_layer""" -import numpy as np - -import mindspore as ms -import mindspore.common.dtype as mstype -from mindspore._checkparam import check_bool, twice, check_int_positive -from mindspore._extends import cell_attr_register -from mindspore.common.initializer import initializer -from mindspore.common.parameter import Parameter -from mindspore.common.tensor import Tensor -from mindspore.nn.cell import Cell -from mindspore.nn.layer.activation import get_activation -from mindspore.ops import operations as P -C0 = 16 - -def caculate_device_shape(matrix_dim, channel, is_A): - ll = (0) - if is_A: - if channel // C0 == 0: - matrix_dim = (matrix_dim / channel) * C0 - ll = (int(matrix_dim // C0), int(matrix_dim // C0), C0, C0), int(matrix_dim) - else: - ll = (int(matrix_dim // C0), int(matrix_dim // C0), C0, C0), int(matrix_dim) - return ll - -class _Conv(Cell): - r"""Applies a N-D convolution over an input signal composed of several input - planes. - """ - - def __init__(self, - in_channels, - out_channels, - kernel_size, - stride, - pad_mode, - padding, - dilation, - group, - data_format, - has_bias, - weight_init, - bias_init, - ): - super(_Conv, self).__init__() - self.in_channels = in_channels - self.out_channels = out_channels - self.kernel_size = kernel_size - self.stride = stride - self.pad_mode = pad_mode - self.padding = padding - self.dilation = dilation - self.group = group - self.data_format = data_format - self.has_bias = has_bias - if not (isinstance(in_channels, int) and in_channels > 0): - raise ValueError('Attr \'in_channels\' of \'Conv2D\' Op passed ' - + str(in_channels) + ', should be a int and greater than 0.') - if (not isinstance(kernel_size, tuple)) or len(kernel_size) != 2 or \ - (not isinstance(kernel_size[0], int)) or (not isinstance(kernel_size[1], int)) or \ - kernel_size[0] < 1 or kernel_size[1] < 1: - raise ValueError('Attr \'kernel_size\' of \'Conv2D\' Op passed ' - + str(self.kernel_size) + ', should be a int or tuple and equal to or greater than 1.') - if in_channels % group != 0: - raise ValueError('Attr \'in_channels\' of \'Conv2D\' Op must be divisible by ' - 'attr \'group\' of \'Conv2D\' Op.') - if out_channels % group != 0: - raise ValueError('Attr \'out_channels\' of \'Conv2D\' Op must be divisible by ' - 'attr \'group\' of \'Conv2D\' Op.') - - self.weight = Parameter(initializer( - weight_init, [out_channels, in_channels // group, *kernel_size]), name='weight') - - if check_bool(has_bias): - self.bias = Parameter(_initializer( - bias_init, [out_channels]), name='bias') - else: - if bias_init != 'zeros': - logger.warning("Value of 'has_bias' is False, value of 'bias_init' will be ignored.") - self.bias = None - - def construct(self, *inputs): - raise NotImplementedError - - -class Conv2d_Thor(_Conv): - """Conv2d_Thor""" - def __init__(self, - in_channels, - out_channels, - kernel_size, - stride=1, - pad_mode='same', - padding=0, - dilation=1, - group=1, - data_format='NCHW', - has_bias=False, - weight_init='normal', - damping=0.03, - loss_scale=1, - frequency=278, - bias_init='zeros'): - self.thor = True - ksizes = (1, kernel_size, kernel_size, 1) - self.hw = kernel_size * kernel_size - strides = (1, stride, stride, 1) - kernel_size = twice(kernel_size) - super(Conv2d_Thor, self).__init__( - in_channels, - out_channels, - kernel_size, - stride, - pad_mode, - padding, - dilation, - group, - data_format, - has_bias, - weight_init, - bias_init, - ) - self.conv2d = P.Conv2D(out_channel=self.out_channels, - kernel_size=self.kernel_size, - mode=1, - pad_mode=self.pad_mode, - pad=self.padding, - stride=self.stride, - dilation=self.dilation, - group=self.group - ) - - self.img2col = P.CusImg2Col(ksizes=ksizes, strides=strides) - self.cube_matmul = P.CusMatMulCube(transpose_a=True) - self.matrix_combine = P.CusMatrixCombine() - self.cholesky = P.CusCholeskyTrsm() - self.transpose02314 = P.CusTranspose02314() - self.matrix_A_dim = self.in_channels * self.kernel_size[0] * self.kernel_size[1] - self.matrix_G_dim = self.out_channels - self.matrix_A_device_shape, self.matrix_A_device_dim = caculate_device_shape(self.matrix_A_dim, - self.in_channels, True) - self.matrix_G_device_shape, self.matrix_G_device_dim = caculate_device_shape(self.matrix_G_dim, - self.in_channels, False) - self.matrix_A_device_temp_shape = ( - self.matrix_A_device_shape[0], self.matrix_A_device_shape[2], self.matrix_A_device_shape[1], - self.matrix_A_device_shape[3]) - self.matrix_G_device_temp_shape = ( - self.matrix_G_device_shape[0], self.matrix_G_device_shape[2], self.matrix_G_device_shape[1], - self.matrix_G_device_shape[3]) - self.matrix_A_inv = Parameter( - Tensor(np.reshape(np.identity(self.matrix_A_device_dim).astype(np.float16), self.matrix_A_device_shape)), - name='matrix_A_inv', requires_grad=False) - self.A_inv_max = Parameter(initializer(0, [1], mstype.float32), name="A_inv_max", requires_grad=False) - self.matrix_G_inv = Parameter( - Tensor(np.reshape(np.identity(self.matrix_G_device_dim).astype(np.float16), self.matrix_G_device_shape)), - name="matrix_G_inv", requires_grad=False) - - self.G_inv_max = Parameter(initializer(0, [1], mstype.float32), name="G_inv_max", requires_grad=False) - self.fake_G = Tensor( - np.reshape(np.identity(self.matrix_G_device_dim).astype(np.float16), self.matrix_G_device_shape)) - - self.shape = P.Shape() - self.reshape = P.Reshape() - self.transpose = P.Transpose() - self.cov_step = Parameter(initializer(0, [1], mstype.int32), name="cov_step", requires_grad=False) - self.mul = P.Mul() - self.cast = P.Cast() - self.damping = Tensor(damping) - self.vector_matmul = P.CusBatchMatMul() - self.diag_block_dim = 128 - self.channels_slice_flag = False - if self.in_channels % C0 != 0: - self.channels_slice_flag = True - - self.padA_flag = False - if (self.matrix_A_dim // self.diag_block_dim) * self.diag_block_dim != self.matrix_A_dim \ - and self.matrix_A_dim > self.diag_block_dim: - self.padA_flag = True - pad_dim = self.diag_block_dim - self.matrix_A_dim % self.diag_block_dim - self.padA = P.Pad(((0, pad_dim), (0, pad_dim))) - self.device_shape_pad_flag = False - if self.matrix_A_dim != self.matrix_A_device_dim: - self.device_shape_pad_flag = True - self.device_shape_pad = P.Pad(((0, 0), (0, C0 - self.in_channels), (0, 0), (0, C0 - self.in_channels))) - self.slice = P.Slice() - self.gather = P.GatherV2() - self.freq = Tensor(frequency, mstype.int32) - self.loss_scale = Tensor(1 / loss_scale, mstype.float16) - self.axis = 0 - - dampingA_dim = self.matrix_A_dim - if (self.matrix_A_dim % self.diag_block_dim) != 0 and self.matrix_A_dim > self.diag_block_dim: - dampingA_dim = (self.matrix_A_dim // self.diag_block_dim + 1) * self.diag_block_dim - dampingG_dim = self.matrix_G_dim - if (self.matrix_G_dim % self.diag_block_dim) != 0 and self.matrix_G_dim > self.diag_block_dim: - dampingG_dim = (self.matrix_G_dim // self.diag_block_dim + 1) * self.diag_block_dim - - self.dampingA = Tensor(np.identity(dampingA_dim), mstype.float32) - self.dampingG = Tensor(np.identity(dampingG_dim), mstype.float32) - self.fused_abs_max1 = P.CusFusedAbsMax1([self.matrix_A_dim, self.matrix_A_dim]) - self.fused_abs_max2 = P.CusFusedAbsMax1() - self.log = P.Log() - self.exp = P.Exp() - self.sqrt = P.Sqrt() - self.getG = P.InsertGradientOf(self.save_gradient) - - def save_gradient(self, dout): - """save_gradient""" - out = dout - dout = self.mul(dout, self.loss_scale) - dout = self.mul(dout, 32.0) - dout = self.transpose02314(dout) - dout_shape = self.shape(dout) - normalizer = dout_shape[0] - - matrix_G = self.cube_matmul(dout, dout) - normalizer = self.cast(normalizer, ms.float32) - matrix_G = self.mul(matrix_G, 1.0 / normalizer) - damping_step = self.gather(self.damping, self.cov_step, 0) - self.cov_step = self.cov_step + self.freq - damping_step = self.cast(damping_step, mstype.float32) - damping = self.mul(damping_step, 32.0 / normalizer) - damping = self.sqrt(damping) - dampingG = self.cast(self.dampingG, mstype.float32) - matrix_G = matrix_G + damping * dampingG - - matrix_G_inv = self.cholesky(matrix_G) - matrix_G_inv = self.vector_matmul(matrix_G_inv, matrix_G_inv) - matrix_G_inv_max = self.fused_abs_max2(matrix_G_inv) - matrix_G_inv_max = self.fused_abs_max2(matrix_G_inv_max) - self.G_inv_max = matrix_G_inv_max - matrix_G_inv = self.matrix_combine(matrix_G_inv) - matrix_G_inv = self.reshape(matrix_G_inv, self.matrix_G_device_temp_shape) - matrix_G_inv = self.transpose(matrix_G_inv, (2, 0, 1, 3)) - matrix_G = self.cast(matrix_G_inv, mstype.float16) - self.matrix_G_inv = matrix_G - return out - - def construct(self, x): - if self.thor: - matrix_A = self.img2col(x) - matrix_A_shape = self.shape(matrix_A) - normalizer = matrix_A_shape[0] - matrix_A = self.cube_matmul(matrix_A, matrix_A) - - if self.channels_slice_flag: - matrix_A = self.reshape(matrix_A, (self.hw, C0, self.hw, C0)) - matrix_A = self.slice(matrix_A, (0, 0, 0, 0), (self.hw, self.in_channels, self.hw, self.in_channels)) - matrix_A = self.reshape(matrix_A, (self.matrix_A_dim, self.matrix_A_dim)) - normalizer = self.cast(normalizer, ms.float32) - matrix_A = self.mul(matrix_A, 1.0 / normalizer) - if self.padA_flag: - matrix_A = self.padA(matrix_A) - damping_step = self.gather(self.damping, self.cov_step, self.axis) - damping_step = self.cast(damping_step, mstype.float32) - damping = self.mul(damping_step, 32.0 / normalizer) - damping = self.sqrt(damping) - damping_A = self.cast(self.dampingA, mstype.float32) - matrix_A = matrix_A + damping * damping_A - matrix_A_inv = self.cholesky(matrix_A) - matrix_A_inv = self.vector_matmul(matrix_A_inv, matrix_A_inv) - matrix_A_inv_max = self.fused_abs_max1(matrix_A_inv) - matrix_A_inv_max = self.fused_abs_max2(matrix_A_inv_max) - self.A_inv_max = matrix_A_inv_max - matrix_A_inv = self.matrix_combine(matrix_A_inv) - matrix_A_inv = self.cast(matrix_A_inv, mstype.float16) - if self.padA_flag: - matrix_A_inv = self.slice(matrix_A_inv, (0, 0), (self.matrix_A_dim, self.matrix_A_dim)) - - if self.device_shape_pad_flag: - matrix_A_inv = self.reshape(matrix_A_inv, (self.hw, self.in_channels, self.hw, self.in_channels)) - matrix_A_inv = self.device_shape_pad(matrix_A_inv) - matrix_A_inv = self.reshape(matrix_A_inv, self.matrix_A_device_temp_shape) - matrix_A_inv = self.transpose(matrix_A_inv, (2, 0, 1, 3)) - self.matrix_A_inv = matrix_A_inv - self.matrix_G_inv = self.fake_G - out = self.conv2d(x, self.weight) - out = self.getG(out) - else: - out = self.conv2d(x, self.weight) - - return out - - def extra_repr(self): - """extra_repr""" - s = 'input_channels={}, output_channels={}, kernel_size={},' \ - 'stride={}, pad_mode={}, padding={}, dilation={}, ' \ - 'group={}, data_format={}, has_bias={},' \ - 'weight_init={}, bias_init={}'.format( - self.in_channels, - self.out_channels, - self.kernel_size, - self.stride, - self.pad_mode, - self.padding, - self.dilation, - self.group, - self.data_format, - self.has_bias, - self.weight, - self.bias) - - if self.has_bias: - s += ', bias={}'.format(self.bias) - return s - - -class Dense_Thor(Cell): - """Dense_Thor""" - @cell_attr_register(attrs=['has_bias', 'activation']) - def __init__(self, - in_channels, - out_channels, - weight_init='normal', - bias_init='zeros', - damping=0.03, - loss_scale=1, - frequency=278, - has_bias=True, - activation=None): - super(Dense_Thor, self).__init__() - self.in_channels = check_int_positive(in_channels) - self.out_channels = check_int_positive(out_channels) - self.has_bias = check_bool(has_bias) - self.thor = True - if isinstance(weight_init, Tensor): - if weight_init.dim() != 2 or weight_init.shape[0] != out_channels or \ - weight_init.shape[1] != in_channels: - raise ValueError("weight_init shape error") - - self.weight = Parameter(initializer(weight_init, [out_channels, in_channels]), name="weight") - - if self.has_bias: - if isinstance(bias_init, Tensor): - if bias_init.dim() != 1 or bias_init.shape[0] != out_channels: - raise ValueError("bias_init shape error") - - self.bias = Parameter(initializer(bias_init, [out_channels]), name="bias") - - self.matmul = P.MatMul(transpose_b=True) - self.bias_add = P.BiasAdd() - - self.activation = get_activation(activation) - self.activation_flag = self.activation is not None - - self.matrix_A_inv = Parameter(Tensor(np.zeros([128, 128, 16, 16]).astype(np.float16)), name='matrix_A_inv', - requires_grad=False) - self.matrix_G_inv = Parameter(Tensor(np.zeros([63, 63, 16, 16]).astype(np.float16)), name="matrix_G_inv", - requires_grad=False) - self.fake_G = Tensor(np.zeros([63, 63, 16, 16]).astype(np.float16)) - - self.matmul = P.MatMul(transpose_b=True) - self.cube_matmul = P.CusMatMulCube(transpose_a=True) - self.matrix_combine = P.CusMatrixCombine() - self.cholesky = P.CusCholeskyTrsm() - self.shape = P.Shape() - self.reshape = P.Reshape() - self.transpose = P.Transpose() - self.cov_step = Parameter(initializer(0, [1], mstype.int32), name="cov_step", requires_grad=False) - self.mul = P.Mul() - self.cast = P.Cast() - self.damping = Tensor(damping) - self.loss_scale = Tensor(1 / loss_scale, mstype.float16) - self.vector_matmul = P.CusBatchMatMul() - self.pad = P.Pad(((0, 24), (0, 24))) - self.pad1 = P.Pad(((0, 8), (0, 8))) - self.slice = P.Slice() - self.gather = P.GatherV2() - self.assignadd = P.AssignAdd() - self.freq = Tensor(frequency, mstype.int32) - self.axis = 0 - self.A_inv_max = Parameter(initializer(0, [1], mstype.float32), name="A_inv_max", requires_grad=False) - self.G_inv_max = Parameter(initializer(0, [1], mstype.float32), name="G_inv_max", requires_grad=False) - self.fused_abs_max1 = P.CusFusedAbsMax1([1000, 1000]) - self.fused_abs_max2 = P.CusFusedAbsMax1() - self.log = P.Log() - self.exp = P.Exp() - self.dampingA = Tensor(np.identity(2048), mstype.float32) - self.dampingG = Tensor(np.identity(1024), mstype.float32) - self.add = P.TensorAdd() - self.sqrt = P.Sqrt() - self.getG = P.InsertGradientOf(self.save_gradient) - - def save_gradient(self, dout): - """save_gradient""" - out = dout - dout = self.mul(dout, self.loss_scale) - dout = self.mul(dout, 32.0) - normalizer = 32 - matrix_G = self.cube_matmul(dout, dout) - normalizer = self.cast(normalizer, ms.float32) - matrix_G = self.mul(matrix_G, 1.0 / normalizer) - matrix_G = self.pad(matrix_G) - damping_step = self.gather(self.damping, self.cov_step, 0) - damping_step = self.cast(damping_step, mstype.float32) - self.cov_step = self.cov_step + self.freq - damping = self.sqrt(damping_step) - dampingG = self.cast(self.dampingG, mstype.float32) - matrix_G = matrix_G + damping * dampingG - matrix_G_inv = self.cholesky(matrix_G) - matrix_G_inv = self.vector_matmul(matrix_G_inv, matrix_G_inv) - matrix_G_inv_max = self.fused_abs_max1(matrix_G_inv) - matrix_G_inv_max = self.fused_abs_max2(matrix_G_inv_max) - self.G_inv_max = matrix_G_inv_max - matrix_G_inv = self.matrix_combine(matrix_G_inv) - matrix_G_inv = self.slice(matrix_G_inv, (0, 0), (1000, 1000)) - matrix_G_inv = self.pad1(matrix_G_inv) - matrix_G_inv_shape = self.shape(matrix_G_inv) - matrix_G_inv = self.reshape(matrix_G_inv, (matrix_G_inv_shape[0] / 16, 16, matrix_G_inv_shape[0] / 16, 16)) - matrix_G_inv = self.transpose(matrix_G_inv, (2, 0, 1, 3)) - matrix_G_inv = self.cast(matrix_G_inv, mstype.float16) - self.matrix_G_inv = matrix_G_inv - return out - - def construct(self, x): - """construct""" - if self.thor: - inputs = self.cube_matmul(x, x) - normalizer = 32 - normalizer = self.cast(normalizer, ms.float32) - matrix_A = self.mul(inputs, 1.0 / normalizer) - - damping_step = self.gather(self.damping, self.cov_step, self.axis) - damping_step = self.cast(damping_step, mstype.float32) - damping = self.sqrt(damping_step) - dampingA = self.cast(self.dampingA, mstype.float32) - matrix_A = matrix_A + damping * dampingA - matrix_A_inv = self.cholesky(matrix_A) - matrix_A_inv = self.vector_matmul(matrix_A_inv, matrix_A_inv) - - matrix_A_inv_max = self.fused_abs_max2(matrix_A_inv) - matrix_A_inv_max = self.fused_abs_max2(matrix_A_inv_max) - self.A_inv_max = matrix_A_inv_max - - matrix_A_inv = self.matrix_combine(matrix_A_inv) - matrix_A_inv_shape = self.shape(matrix_A_inv) - matrix_A_inv = self.reshape(matrix_A_inv, (matrix_A_inv_shape[0] / 16, 16, matrix_A_inv_shape[0] / 16, 16)) - matrix_A_inv = self.transpose(matrix_A_inv, (2, 0, 1, 3)) - matrix_A_inv = self.cast(matrix_A_inv, mstype.float16) - self.matrix_A_inv = matrix_A_inv - self.matrix_G_inv = self.fake_G - output = self.matmul(x, self.weight) - output = self.getG(output) - else: - output = self.matmul(x, self.weight) - - if self.has_bias: - output = self.bias_add(output, self.bias) - if self.activation_flag: - return self.activation(output) - return output - - def extend_repr(self): - """extend_repr""" - str_info = 'in_channels={}, out_channels={}, weight={}, has_bias={}' \ - .format(self.in_channels, self.out_channels, self.weight, self.has_bias) - if self.has_bias: - str_info = str_info + ', bias={}'.format(self.bias) - - if self.activation_flag: - str_info = str_info + ', activation={}'.format(self.activation) - - return str_info diff --git a/example/resnet50_imagenet2012_THOR/run_distribute_train.sh b/example/resnet50_imagenet2012_THOR/run_distribute_train.sh deleted file mode 100644 index e39034a912..0000000000 --- a/example/resnet50_imagenet2012_THOR/run_distribute_train.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/bin/bash -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ - -if [ $# != 3 ] -then - echo "Usage: sh run_distribute_train.sh [MINDSPORE_HCCL_CONFIG_PATH] [DATASET_PATH] [DEVICE_NUM]" -exit 1 -fi - -if [ ! -f $1 ] -then - echo "error: DMINDSPORE_HCCL_CONFIG_PATH=$1 is not a file" -exit 1 -fi - -if [ ! -d $2 ] -then - echo "error: DATASET_PATH=$2 is not a directory" -exit 1 -fi - -ulimit -u unlimited -export DEVICE_NUM=$3 -export RANK_SIZE=$3 -export MINDSPORE_HCCL_CONFIG_PATH=$1 - -for((i=0; i<${DEVICE_NUM}; i++)) -do - export DEVICE_ID=$i - export RANK_ID=$i - rm -rf ./train_parallel$i - mkdir ./train_parallel$i - cp *.py ./train_parallel$i - cp *.sh ./train_parallel$i - cp -r model ./train_parallel$i - cd ./train_parallel$i || exit - echo "start training for rank $RANK_ID, device $DEVICE_ID" - - env > env.log - python train.py --do_train=True --run_distribute=True --device_num=$DEVICE_NUM --dataset_path=$2 > log 2>&1 & - cd .. -done diff --git a/example/resnet50_imagenet2012_THOR/run_infer.sh b/example/resnet50_imagenet2012_THOR/run_infer.sh deleted file mode 100755 index 14d7faf981..0000000000 --- a/example/resnet50_imagenet2012_THOR/run_infer.sh +++ /dev/null @@ -1,64 +0,0 @@ -#!/bin/bash -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ - -if [ $# != 2 ] -then - echo "Usage: sh run_infer.sh [DATASET_PATH] [CHECKPOINT_PATH]" -exit 1 -fi - -get_real_path(){ - if [ "${1:0:1}" == "/" ]; then - echo "$1" - else - echo "$(realpath -m $PWD/$1)" - fi -} - -PATH1=$(get_real_path $1) -PATH2=$(get_real_path $2) - - -if [ ! -d $PATH1 ] -then - echo "error: DATASET_PATH=$1 is not a directory" -exit 1 -fi - -if [ ! -f $PATH2 ] -then - echo "error: CHECKPOINT_PATH=$2 is not a file" -exit 1 -fi - -ulimit -u unlimited -export DEVICE_NUM=1 -export DEVICE_ID=0 -export RANK_SIZE=$DEVICE_NUM -export RANK_ID=0 - -if [ -d "infer" ]; -then - rm -rf ./infer -fi -mkdir ./infer -cp *.py ./infer -cp *.sh ./infer -cd ./infer || exit -env > env.log -echo "start infering for device $DEVICE_ID" -python eval.py --do_eval=True --dataset_path=$PATH1 --checkpoint_path=$PATH2 &> log & -cd .. diff --git a/example/resnet50_imagenet2012_THOR/train.py b/example/resnet50_imagenet2012_THOR/train.py deleted file mode 100644 index 309018da57..0000000000 --- a/example/resnet50_imagenet2012_THOR/train.py +++ /dev/null @@ -1,133 +0,0 @@ -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ -"""train_imagenet.""" -import argparse -import os -import random - -import numpy as np - -from mindspore import Tensor -from mindspore import context -from mindspore.communication.management import init -from mindspore.parallel._auto_parallel_context import auto_parallel_context -from mindspore.train.callback import ModelCheckpoint, CheckpointConfig, LossMonitor, TimeMonitor -from mindspore.train.loss_scale_manager import FixedLossScaleManager -from mindspore.train.model import ParallelMode -from model.model_thor import Model -from model.resnet import resnet50 -from model.thor import THOR - -from config import config -from crossentropy import CrossEntropy -from dataset_imagenet import create_dataset - -random.seed(1) -np.random.seed(1) - -parser = argparse.ArgumentParser(description='Image classification') -parser.add_argument('--run_distribute', type=bool, default=False, help='Run distribute') -parser.add_argument('--device_num', type=int, default=1, help='Device num.') -parser.add_argument('--do_train', type=bool, default=True, help='Do train or not.') -parser.add_argument('--do_eval', type=bool, default=False, help='Do eval or not.') -parser.add_argument('--dataset_path', type=str, default=None, help='Dataset path') - -args_opt = parser.parse_args() -device_id = int(os.getenv('DEVICE_ID')) - -context.set_context(mode=context.GRAPH_MODE, device_target="Ascend", save_graphs=False, device_id=device_id) - - -def get_model_lr(global_step, lr_init, decay, total_epochs, steps_per_epoch): - """get_model_lr""" - lr_each_step = [] - total_steps = steps_per_epoch * total_epochs - for i in range(total_steps): - epoch = (i + 1) / steps_per_epoch - base = (1.0 - float(epoch) / total_epochs) ** decay - lr_local = lr_init * base - if epoch >= 39: - lr_local = lr_local * 0.5 - if epoch >= 40: - lr_local = lr_local * 0.5 - lr_each_step.append(lr_local) - current_step = global_step - lr_each_step = np.array(lr_each_step).astype(np.float32) - learning_rate = lr_each_step[current_step:] - return learning_rate - - -def get_model_damping(global_step, damping_init, decay_rate, total_epochs, steps_per_epoch): - """get_model_damping""" - damping_each_step = [] - total_steps = steps_per_epoch * total_epochs - for step in range(total_steps): - epoch = (step + 1) / steps_per_epoch - damping_here = damping_init * (decay_rate ** (epoch / 10)) - damping_each_step.append(damping_here) - - current_step = global_step - damping_each_step = np.array(damping_each_step).astype(np.float32) - damping_now = damping_each_step[current_step:] - return damping_now - - -if __name__ == '__main__': - if not args_opt.do_eval and args_opt.run_distribute: - context.set_auto_parallel_context(device_num=args_opt.device_num, parallel_mode=ParallelMode.DATA_PARALLEL, - mirror_mean=True, parameter_broadcast=True) - auto_parallel_context().set_all_reduce_fusion_split_indices([107], "hccl_world_groupsum1") - auto_parallel_context().set_all_reduce_fusion_split_indices([27], "hccl_world_groupsum2") - auto_parallel_context().set_all_reduce_fusion_split_indices([27], "hccl_world_groupsum3") - auto_parallel_context().set_all_reduce_fusion_split_indices([27], "hccl_world_groupsum4") - auto_parallel_context().set_all_reduce_fusion_split_indices([27], "hccl_world_groupsum5") - - init() - - epoch_size = config.epoch_size - damping = get_model_damping(0, 0.03, 0.87, 50, 5004) - net = resnet50(class_num=config.class_num, damping=damping, loss_scale=config.loss_scale, - frequency=config.frequency) - - if not config.label_smooth: - config.label_smooth_factor = 0.0 - loss = CrossEntropy(smooth_factor=config.label_smooth_factor, num_classes=config.class_num) - if args_opt.do_train: - dataset = create_dataset(dataset_path=args_opt.dataset_path, do_train=True, - repeat_num=epoch_size, batch_size=config.batch_size) - step_size = dataset.get_dataset_size() - - loss_scale = FixedLossScaleManager(config.loss_scale, drop_overflow_update=False) - lr = Tensor(get_model_lr(0, 0.045, 6, 70, 5004)) - opt = THOR(filter(lambda x: x.requires_grad, net.get_parameters()), lr, config.momentum, - filter(lambda x: 'matrix_A' in x.name, net.get_parameters()), - filter(lambda x: 'matrix_G' in x.name, net.get_parameters()), - filter(lambda x: 'A_inv_max' in x.name, net.get_parameters()), - filter(lambda x: 'G_inv_max' in x.name, net.get_parameters()), - config.weight_decay, config.loss_scale) - - model = Model(net, loss_fn=loss, optimizer=opt, amp_level='O2', loss_scale_manager=loss_scale, - keep_batchnorm_fp32=False, metrics={'acc'}, frequency=config.frequency) - - time_cb = TimeMonitor(data_size=step_size) - loss_cb = LossMonitor() - cb = [time_cb, loss_cb] - if config.save_checkpoint: - config_ck = CheckpointConfig(save_checkpoint_steps=config.save_checkpoint_steps, - keep_checkpoint_max=config.keep_checkpoint_max) - ckpt_cb = ModelCheckpoint(prefix="resnet", directory=config.save_checkpoint_path, config=config_ck) - cb += [ckpt_cb] - - model.train(epoch_size, dataset, callbacks=cb) diff --git a/model_zoo/resnet/README.md b/model_zoo/resnet/README.md new file mode 100644 index 0000000000..ad93453602 --- /dev/null +++ b/model_zoo/resnet/README.md @@ -0,0 +1,251 @@ +# ResNet Example + +## Description + +These are examples of training ResNet-50/ResNet-101 with CIFAR-10/ImageNet2012 dataset in MindSpore. +(Training ResNet-101 with dataset CIFAR-10 is unsupported now.) + +## Requirements + +- Install [MindSpore](https://www.mindspore.cn/install/en). + +- Download the dataset CIFAR-10 or ImageNet2012 + +CIFAR-10 + +> Unzip the CIFAR-10 dataset to any path you want and the folder structure should include train and eval dataset as follows: +> ``` +> . +> └─dataset +> ├─ cifar-10-batches-bin # train dataset +> └─ cifar-10-verify-bin # evaluate dataset +> ``` + +ImageNet2012 + +> Unzip the ImageNet2012 dataset to any path you want and the folder should include train and eval dataset as follows: +> +> ``` +> . +> └─dataset +> ├─ilsvrc # train dataset +> └─validation_preprocess # evaluate dataset +> ``` + + + +## Structure + +```shell +. +└──resnet + ├── README.md + ├── script + ├── run_distribute_train.sh # launch distributed training(8 pcs) + ├── run_eval.sh # launch evaluation + └── run_standalone_train.sh # launch standalone training(1 pcs) + ├── src + ├── config.py # parameter configuration + ├── dataset.py # data preprocessing + ├── crossentropy.py # loss definition for ImageNet2012 dataset + ├── lr_generator.py # generate learning rate for each step + └── resnet.py # resnet backbone, including resnet50 and resnet101 + ├── eval.py # eval net + └── train.py # train net +``` + + +## Parameter configuration + +Parameters for both training and evaluation can be set in config.py. + +- config for ResNet-50, CIFAR-10 dataset + +``` +"class_num": 10, # dataset class num +"batch_size": 32, # batch size of input tensor +"loss_scale": 1024, # loss scale +"momentum": 0.9, # momentum +"weight_decay": 1e-4, # weight decay +"epoch_size": 90, # only valid for taining, which is always 1 for inference +"save_checkpoint": True, # whether save checkpoint or not +"save_checkpoint_steps": 195, # the step interval between two checkpoints. By default, the last checkpoint will be saved after the last step +"keep_checkpoint_max": 10, # only keep the last keep_checkpoint_max checkpoint +"save_checkpoint_path": "./", # path to save checkpoint +"warmup_epochs": 5, # number of warmup epoch +"lr_decay_mode": "poly" # decay mode can be selected in steps, ploy and default +"lr_init": 0.01, # initial learning rate +"lr_end": 0.00001, # final learning rate +"lr_max": 0.1, # maximum learning rate +``` + +- config for ResNet-50, ImageNet2012 dataset + +``` +"class_num": 1001, # dataset class number +"batch_size": 32, # batch size of input tensor +"loss_scale": 1024, # loss scale +"momentum": 0.9, # momentum optimizer +"weight_decay": 1e-4, # weight decay +"epoch_size": 90, # only valid for taining, which is always 1 for inference +"pretrained_epoch_size": 1, # epoch size that model has been trained before load pretrained checkpoint +"save_checkpoint": True, # whether save checkpoint or not +"save_checkpoint_epochs": 1, # the epoch interval between two checkpoints. By default, the last checkpoint will be saved after the last epoch +"keep_checkpoint_max": 10, # only keep the last keep_checkpoint_max checkpoint +"save_checkpoint_path": "./", # path to save checkpoint relative to the executed path +"warmup_epochs": 0, # number of warmup epoch +"lr_decay_mode": "cosine", # decay mode for generating learning rate +"label_smooth": True, # label smooth +"label_smooth_factor": 0.1, # label smooth factor +"lr_init": 0, # initial learning rate +"lr_max": 0.1, # maximum learning rate +``` + +- config for ResNet-101, ImageNet2012 dataset + +``` +"class_num": 1001, # dataset class number +"batch_size": 32, # batch size of input tensor +"loss_scale": 1024, # loss scale +"momentum": 0.9, # momentum optimizer +"weight_decay": 1e-4, # weight decay +"epoch_size": 120, # epoch sizes for training +"pretrain_epoch_size": 0, # epoch size of pretrain checkpoint +"save_checkpoint": True, # whether save checkpoint or not +"save_checkpoint_epochs": 1, # the epoch interval between two checkpoints. By default, the last checkpoint will be saved after the last epoch +"keep_checkpoint_max": 10, # only keep the last keep_checkpoint_max checkpoint +"save_checkpoint_path": "./", # path to save checkpoint relative to the executed path +"warmup_epochs": 0, # number of warmup epoch +"lr_decay_mode": "cosine" # decay mode for generating learning rate +"label_smooth": 1, # label_smooth +"label_smooth_factor": 0.1, # label_smooth_factor +"lr": 0.1 # base learning rate +``` + + + +## Running the example + +### Train + +#### Usage + +``` +# distributed training +Usage: sh run_distribute_train.sh [resnet50|resnet101] [cifar10|imagenet2012] [MINDSPORE_HCCL_CONFIG_PATH] [DATASET_PATH] + [PRETRAINED_CKPT_PATH](optional) + +# standalone training +Usage: sh run_standalone_train.sh [resnet50|resnet101] [cifar10|imagenet2012] [DATASET_PATH] + [PRETRAINED_CKPT_PATH](optional) +``` + + +#### Launch + +``` +# distribute training example +sh run_distribute_train.sh resnet50 cifar10 rank_table.json ~/cifar-10-batches-bin + +# standalone training example +sh run_standalone_train.sh resnet50 cifar10 ~/cifar-10-batches-bin +``` + +> About rank_table.json, you can refer to the [distributed training tutorial](https://www.mindspore.cn/tutorial/en/master/advanced_use/distributed_training.html). + +#### Result + +Training result will be stored in the example path, whose folder name begins with "train" or "train_parallel". Under this, you can find checkpoint file together with result like the followings in log. + +- training ResNet-50 with CIFAR-10 dataset + +``` +# distribute training result(8 pcs) +epoch: 1 step: 195, loss is 1.9601055 +epoch: 2 step: 195, loss is 1.8555021 +epoch: 3 step: 195, loss is 1.6707983 +epoch: 4 step: 195, loss is 1.8162166 +epoch: 5 step: 195, loss is 1.393667 +... +``` + +- training ResNet-50 with ImageNet2012 dataset + +``` +# distribute training result(8 pcs) +epoch: 1 step: 5004, loss is 4.8995576 +epoch: 2 step: 5004, loss is 3.9235563 +epoch: 3 step: 5004, loss is 3.833077 +epoch: 4 step: 5004, loss is 3.2795618 +epoch: 5 step: 5004, loss is 3.1978393 +... +``` + +- training ResNet-101 with ImageNet2012 dataset + +``` +# distribute training result(8p) +epoch: 1 step: 5004, loss is 4.805483 +epoch: 2 step: 5004, loss is 3.2121816 +epoch: 3 step: 5004, loss is 3.429647 +epoch: 4 step: 5004, loss is 3.3667371 +epoch: 5 step: 5004, loss is 3.1718972 +... +epoch: 67 step: 5004, loss is 2.2768745 +epoch: 68 step: 5004, loss is 1.7223864 +epoch: 69 step: 5004, loss is 2.0665488 +epoch: 70 step: 5004, loss is 1.8717369 +... +``` + +### Evaluation + +#### Usage + +``` +# evaluation +Usage: sh run_eval.sh [resnet50|resnet101] [cifar10|imagenet2012] [DATASET_PATH] [CHECKPOINT_PATH] +``` + +#### Launch + +``` +# evaluation example +sh run_eval.sh resnet50 cifar10 ~/cifar10-10-verify-bin ~/resnet50_cifar10/train_parallel0/resnet-90_195.ckpt +``` + +> checkpoint can be produced in training process. + +#### Result + +Evaluation result will be stored in the example path, whose folder name is "eval". Under this, you can find result like the followings in log. + +- evaluating ResNet-50 with CIFAR-10 dataset + +``` +result: {'acc': 0.91446314102564111} ckpt=~/resnet50_cifar10/train_parallel0/resnet-90_195.ckpt +``` + +- evaluating ResNet-50 with ImageNet2012 dataset + +``` +result: {'acc': 0.7671054737516005} ckpt=train_parallel0/resnet-90_5004.ckpt +``` + +- evaluating ResNet-101 with ImageNet2012 dataset + +``` +result: {'top_5_accuracy': 0.9429417413572343, 'top_1_accuracy': 0.7853513124199744} ckpt=train_parallel0/resnet-120_5004.ckpt +``` + +### Running on GPU +``` +# distributed training example +mpirun -n 8 python train.py ---net=resnet50 --dataset=cifar10 -dataset_path=~/cifar-10-batches-bin --device_target="GPU" --run_distribute=True + +# standalone training example +python train.py --net=resnet50 --dataset=cifar10 --dataset_path=~/cifar-10-batches-bin --device_target="GPU" + +# infer example +python eval.py --net=resnet50 --dataset=cifar10 --dataset_path=~/cifar10-10-verify-bin --device_target="GPU" --checkpoint_path=resnet-90_195.ckpt +``` diff --git a/model_zoo/resnet/eval.py b/model_zoo/resnet/eval.py new file mode 100755 index 0000000000..426b8c9f3d --- /dev/null +++ b/model_zoo/resnet/eval.py @@ -0,0 +1,90 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""train resnet.""" +import os +import random +import argparse +import numpy as np +from mindspore import context +from mindspore import dataset as de +from mindspore.nn.loss import SoftmaxCrossEntropyWithLogits +from mindspore.train.model import Model +from mindspore.train.serialization import load_checkpoint, load_param_into_net +from src.crossentropy import CrossEntropy + +parser = argparse.ArgumentParser(description='Image classification') +parser.add_argument('--net', type=str, default=None, help='Resnet Model, either resnet50 or resnet101') +parser.add_argument('--dataset', type=str, default=None, help='Dataset, either cifar10 or imagenet2012') + +parser.add_argument('--checkpoint_path', type=str, default=None, help='Checkpoint file path') +parser.add_argument('--dataset_path', type=str, default=None, help='Dataset path') +parser.add_argument('--device_target', type=str, default='Ascend', help='Device target') +args_opt = parser.parse_args() + +random.seed(1) +np.random.seed(1) +de.config.set_seed(1) + +if args_opt.net == "resnet50": + from src.resnet import resnet50 as resnet + + if args_opt.dataset == "cifar10": + from src.config import config1 as config + from src.dataset import create_dataset1 as create_dataset + else: + from src.config import config2 as config + from src.dataset import create_dataset2 as create_dataset +else: + from src.resnet import resnet101 as resnet + from src.config import config3 as config + from src.dataset import create_dataset3 as create_dataset + +if __name__ == '__main__': + target = args_opt.device_target + + # init context + device_id = int(os.getenv('DEVICE_ID')) + context.set_context(mode=context.GRAPH_MODE, device_target=target, save_graphs=False, device_id=device_id) + + # create dataset + if args_opt.net == "resnet50": + dataset = create_dataset(dataset_path=args_opt.dataset_path, do_train=False, batch_size=config.batch_size, + target=target) + else: + dataset = create_dataset(dataset_path=args_opt.dataset_path, do_train=False, batch_size=config.batch_size) + step_size = dataset.get_dataset_size() + + # define net + net = resnet(class_num=config.class_num) + + # load checkpoint + param_dict = load_checkpoint(args_opt.checkpoint_path) + load_param_into_net(net, param_dict) + net.set_train(False) + + # define loss, model + if args_opt.dataset == "imagenet2012": + if not config.use_label_smooth: + config.label_smooth_factor = 0.0 + loss = CrossEntropy(smooth_factor=config.label_smooth_factor, num_classes=config.class_num) + else: + loss = SoftmaxCrossEntropyWithLogits(sparse=True, reduction='mean') + + # define model + model = Model(net, loss_fn=loss, metrics={'top_1_accuracy', 'top_5_accuracy'}) + + # eval model + res = model.eval(dataset) + print("result:", res, "ckpt=", args_opt.checkpoint_path) diff --git a/model_zoo/resnet101/scripts/run_distribute_train.sh b/model_zoo/resnet/scripts/run_distribute_train.sh similarity index 58% rename from model_zoo/resnet101/scripts/run_distribute_train.sh rename to model_zoo/resnet/scripts/run_distribute_train.sh index 65790b88c1..efcb620cd8 100755 --- a/model_zoo/resnet101/scripts/run_distribute_train.sh +++ b/model_zoo/resnet/scripts/run_distribute_train.sh @@ -14,12 +14,31 @@ # limitations under the License. # ============================================================================ -if [ $# != 2 ] && [ $# != 3 ] +if [ $# != 4 ] && [ $# != 5 ] then - echo "Usage: sh run_distribute_train.sh [MINDSPORE_HCCL_CONFIG_PATH] [DATASET_PATH] [PRETRAINED_PATH](optional)" + echo "Usage: sh run_distribute_train.sh [resnet50|resnet101] [cifar10|imagenet2012] [MINDSPORE_HCCL_CONFIG_PATH] [DATASET_PATH] [PRETRAINED_CKPT_PATH](optional)" exit 1 fi +if [ $1 != "resnet50" ] && [ $1 != "resnet101" ] +then + echo "error: the selected net is neither resnet50 nor resnet101" +exit 1 +fi + +if [ $2 != "cifar10" ] && [ $2 != "imagenet2012" ] +then + echo "error: the selected dataset is neither cifar10 nor imagenet2012" +exit 1 +fi + +if [ $1 == "resnet101" ] && [ $2 == "cifar10" ] +then + echo "error: training resnet101 with cifar10 dataset is unsupported now!" +exit 1 +fi + + get_real_path(){ if [ "${1:0:1}" == "/" ]; then echo "$1" @@ -27,14 +46,13 @@ get_real_path(){ echo "$(realpath -m $PWD/$1)" fi } -PATH1=$(get_real_path $1) -PATH2=$(get_real_path $2) -echo $PATH1 -echo $PATH2 -if [ $# == 3 ] + +PATH1=$(get_real_path $3) +PATH2=$(get_real_path $4) + +if [ $# == 5 ] then - PATH3=$(get_real_path $3) - echo $PATH3 + PATH3=$(get_real_path $5) fi if [ ! -f $PATH1 ] @@ -49,9 +67,9 @@ then exit 1 fi -if [ $# == 3 ] && [ ! -f $PATH3 ] +if [ $# == 5 ] && [ ! -f $PATH3 ] then - echo "error: PRETRAINED_PATH=$PATH3 is not a file" + echo "error: PRETRAINED_CKPT_PATH=$PATH3 is not a file" exit 1 fi @@ -73,14 +91,14 @@ do cd ./train_parallel$i || exit echo "start training for rank $RANK_ID, device $DEVICE_ID" env > env.log - if [ $# == 2 ] + if [ $# == 4 ] then - python train.py --do_train=True --run_distribute=True --device_num=$DEVICE_NUM --dataset_path=$PATH2 &> log & + python train.py --net=$1 --dataset=$2 --run_distribute=True --device_num=$DEVICE_NUM --dataset_path=$PATH2 &> log & fi - if [ $# == 3 ] + if [ $# == 5 ] then - python train.py --do_train=True --run_distribute=True --device_num=$DEVICE_NUM --dataset_path=$PATH2 --pre_trained=$PATH3 &> log & + python train.py --net=$1 --dataset=$2 --run_distribute=True --device_num=$DEVICE_NUM --dataset_path=$PATH2 --pre_trained=$PATH3 &> log & fi cd .. diff --git a/model_zoo/resnet101/scripts/run_eval.sh b/model_zoo/resnet/scripts/run_eval.sh similarity index 62% rename from model_zoo/resnet101/scripts/run_eval.sh rename to model_zoo/resnet/scripts/run_eval.sh index 88f5d364ce..496b3c1e2b 100755 --- a/model_zoo/resnet101/scripts/run_eval.sh +++ b/model_zoo/resnet/scripts/run_eval.sh @@ -14,12 +14,31 @@ # limitations under the License. # ============================================================================ -if [ $# != 2 ] +if [ $# != 4 ] then - echo "Usage: sh run_eval.sh [DATASET_PATH] [CHECKPOINT_PATH]" + echo "Usage: sh run_eval.sh [resnet50|resnet101] [cifar10|imagenet2012] [DATASET_PATH] [CHECKPOINT_PATH]" exit 1 fi +if [ $1 != "resnet50" ] && [ $1 != "resnet101" ] +then + echo "error: the selected net is neither resnet50 nor resnet101" +exit 1 +fi + +if [ $2 != "cifar10" ] && [ $2 != "imagenet2012" ] +then + echo "error: the selected dataset is neither cifar10 nor imagenet2012" +exit 1 +fi + +if [ $1 == "resnet101" ] && [ $2 == "cifar10" ] +then + echo "error: evaluating resnet101 with cifar10 dataset is unsupported now!" +exit 1 +fi + + get_real_path(){ if [ "${1:0:1}" == "/" ]; then echo "$1" @@ -27,10 +46,10 @@ get_real_path(){ echo "$(realpath -m $PWD/$1)" fi } -PATH1=$(get_real_path $1) -PATH2=$(get_real_path $2) -echo $PATH1 -echo $PATH2 + +PATH1=$(get_real_path $3) +PATH2=$(get_real_path $4) + if [ ! -d $PATH1 ] then @@ -60,6 +79,6 @@ cp *.sh ./eval cp -r ../src ./eval cd ./eval || exit env > env.log -echo "start infering for device $DEVICE_ID" -python eval.py --do_eval=True --dataset_path=$PATH1 --checkpoint_path=$PATH2 &> log & +echo "start evaluation for device $DEVICE_ID" +python eval.py --net=$1 --dataset=$2 --dataset_path=$PATH1 --checkpoint_path=$PATH2 &> log & cd .. diff --git a/model_zoo/resnet101/scripts/run_standalone_train.sh b/model_zoo/resnet/scripts/run_standalone_train.sh similarity index 56% rename from model_zoo/resnet101/scripts/run_standalone_train.sh rename to model_zoo/resnet/scripts/run_standalone_train.sh index 7214d114d5..2272dbd88b 100755 --- a/model_zoo/resnet101/scripts/run_standalone_train.sh +++ b/model_zoo/resnet/scripts/run_standalone_train.sh @@ -14,12 +14,31 @@ # limitations under the License. # ============================================================================ -if [ $# != 1 ] && [ $# != 2 ] +if [ $# != 3 ] && [ $# != 4 ] then - echo "Usage: sh run_standalone_train.sh [DATASET_PATH] [PRETRAINED_PATH](optional)" + echo "Usage: sh run_standalone_train.sh [resnet50|resnet101] [cifar10|imagenet2012] [DATASET_PATH] [PRETRAINED_CKPT_PATH](optional)" exit 1 fi +if [ $1 != "resnet50" ] && [ $1 != "resnet101" ] +then + echo "error: the selected net is neither resnet50 nor resnet101" +exit 1 +fi + +if [ $2 != "cifar10" ] && [ $2 != "imagenet2012" ] +then + echo "error: the selected dataset is neither cifar10 nor imagenet2012" +exit 1 +fi + +if [ $1 == "resnet101" ] && [ $2 == "cifar10" ] +then + echo "error: training resnet101 with cifar10 dataset is unsupported now!" +exit 1 +fi + + get_real_path(){ if [ "${1:0:1}" == "/" ]; then echo "$1" @@ -27,12 +46,12 @@ get_real_path(){ echo "$(realpath -m $PWD/$1)" fi } -PATH1=$(get_real_path $1) -echo $PATH1 -if [ $# == 2 ] + +PATH1=$(get_real_path $3) + +if [ $# == 4 ] then - PATH2=$(get_real_path $2) - echo $PATH2 + PATH2=$(get_real_path $4) fi if [ ! -d $PATH1 ] @@ -41,9 +60,9 @@ then exit 1 fi -if [ $# == 2 ] && [ ! -f $PATH2 ] +if [ $# == 4 ] && [ ! -f $PATH2 ] then - echo "error: PRETRAINED_PATH=$PATH2 is not a file" + echo "error: PRETRAINED_CKPT_PATH=$PATH2 is not a file" exit 1 fi @@ -64,13 +83,13 @@ cp -r ../src ./train cd ./train || exit echo "start training for device $DEVICE_ID" env > env.log -if [ $# == 1 ] +if [ $# == 3 ] then - python train.py --do_train=True --dataset_path=$PATH1 &> log & + python train.py --net=$1 --dataset=$2 --dataset_path=$PATH1 &> log & fi -if [ $# == 2 ] +if [ $# == 4 ] then - python train.py --do_train=True --dataset_path=$PATH1 --pre_trained=$PATH2 &> log & + python train.py --net=$1 --dataset=$2 --dataset_path=$PATH1 --pre_trained=$PATH2 &> log & fi cd .. diff --git a/example/resnet50_imagenet2012/config.py b/model_zoo/resnet/src/config.py similarity index 54% rename from example/resnet50_imagenet2012/config.py rename to model_zoo/resnet/src/config.py index cf5093d245..7b1759fde0 100755 --- a/example/resnet50_imagenet2012/config.py +++ b/model_zoo/resnet/src/config.py @@ -17,17 +17,34 @@ network config setting, will be used in train.py and eval.py """ from easydict import EasyDict as ed -config = ed({ +# config for resent50, cifar10 +config1 = ed({ + "class_num": 10, + "batch_size": 32, + "loss_scale": 1024, + "momentum": 0.9, + "weight_decay": 1e-4, + "epoch_size": 90, + "save_checkpoint": True, + "save_checkpoint_epochs": 5, + "keep_checkpoint_max": 10, + "save_checkpoint_path": "./", + "warmup_epochs": 5, + "lr_decay_mode": "poly", + "lr_init": 0.01, + "lr_end": 0.00001, + "lr_max": 0.1 +}) + +# config for resnet50, imagenet2012 +config2 = ed({ "class_num": 1001, "batch_size": 32, "loss_scale": 1024, "momentum": 0.9, "weight_decay": 1e-4, "epoch_size": 90, - "pretrained_epoch_size": 1, - "buffer_size": 1000, - "image_height": 224, - "image_width": 224, + "pretrain_epoch_size": 1, "save_checkpoint": True, "save_checkpoint_epochs": 5, "keep_checkpoint_max": 10, @@ -40,3 +57,23 @@ config = ed({ "lr_max": 0.1 }) + +# config for resent101, imagenet2012 +config3 = ed({ + "class_num": 1001, + "batch_size": 32, + "loss_scale": 1024, + "momentum": 0.9, + "weight_decay": 1e-4, + "epoch_size": 120, + "pretrain_epoch_size": 0, + "save_checkpoint": True, + "save_checkpoint_epochs": 5, + "keep_checkpoint_max": 10, + "save_checkpoint_path": "./", + "warmup_epochs": 0, + "lr_decay_mode": "cosine", + "use_label_smooth": True, + "label_smooth_factor": 0.1, + "lr": 0.1 +}) diff --git a/model_zoo/resnet101/src/crossentropy.py b/model_zoo/resnet/src/crossentropy.py similarity index 98% rename from model_zoo/resnet101/src/crossentropy.py rename to model_zoo/resnet/src/crossentropy.py index 1145a41804..5118cb5161 100755 --- a/model_zoo/resnet101/src/crossentropy.py +++ b/model_zoo/resnet/src/crossentropy.py @@ -20,15 +20,18 @@ from mindspore import Tensor from mindspore.common import dtype as mstype import mindspore.nn as nn + class CrossEntropy(_Loss): """the redefined loss function with SoftmaxCrossEntropyWithLogits""" + def __init__(self, smooth_factor=0., num_classes=1001): super(CrossEntropy, self).__init__() self.onehot = P.OneHot() self.on_value = Tensor(1.0 - smooth_factor, mstype.float32) - self.off_value = Tensor(1.0 * smooth_factor / (num_classes -1), mstype.float32) + self.off_value = Tensor(1.0 * smooth_factor / (num_classes - 1), mstype.float32) self.ce = nn.SoftmaxCrossEntropyWithLogits() self.mean = P.ReduceMean(False) + def construct(self, logit, label): one_hot_label = self.onehot(label, F.shape(logit)[1], self.on_value, self.off_value) loss = self.ce(logit, one_hot_label) diff --git a/model_zoo/resnet/src/dataset.py b/model_zoo/resnet/src/dataset.py new file mode 100755 index 0000000000..ac0adc4bc9 --- /dev/null +++ b/model_zoo/resnet/src/dataset.py @@ -0,0 +1,205 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +create train or eval dataset. +""" +import os +import mindspore.common.dtype as mstype +import mindspore.dataset.engine as de +import mindspore.dataset.transforms.vision.c_transforms as C +import mindspore.dataset.transforms.c_transforms as C2 +from mindspore.communication.management import init, get_rank, get_group_size + + +def create_dataset1(dataset_path, do_train, repeat_num=1, batch_size=32, target="Ascend"): + """ + create a train or evaluate cifar10 dataset for resnet50 + Args: + dataset_path(string): the path of dataset. + do_train(bool): whether dataset is used for train or eval. + repeat_num(int): the repeat times of dataset. Default: 1 + batch_size(int): the batch size of dataset. Default: 32 + target(str): the device target. Default: Ascend + + Returns: + dataset + """ + if target == "Ascend": + device_num = int(os.getenv("DEVICE_NUM")) + rank_id = int(os.getenv("RANK_ID")) + else: + init("nccl") + rank_id = get_rank() + device_num = get_group_size() + + if device_num == 1: + ds = de.Cifar10Dataset(dataset_path, num_parallel_workers=8, shuffle=True) + else: + ds = de.Cifar10Dataset(dataset_path, num_parallel_workers=8, shuffle=True, + num_shards=device_num, shard_id=rank_id) + + # define map operations + trans = [] + if do_train: + trans += [ + C.RandomCrop((32, 32), (4, 4, 4, 4)), + C.RandomHorizontalFlip(prob=0.5) + ] + + trans += [ + C.Resize((224, 224)), + C.Rescale(1.0 / 255.0, 0.0), + C.Normalize([0.4914, 0.4822, 0.4465], [0.2023, 0.1994, 0.2010]), + C.HWC2CHW() + ] + + type_cast_op = C2.TypeCast(mstype.int32) + + ds = ds.map(input_columns="label", num_parallel_workers=8, operations=type_cast_op) + ds = ds.map(input_columns="image", num_parallel_workers=8, operations=trans) + + # apply batch operations + ds = ds.batch(batch_size, drop_remainder=True) + # apply dataset repeat operation + ds = ds.repeat(repeat_num) + + return ds + + +def create_dataset2(dataset_path, do_train, repeat_num=1, batch_size=32, target="Ascend"): + """ + create a train or eval imagenet2012 dataset for resnet50 + + Args: + dataset_path(string): the path of dataset. + do_train(bool): whether dataset is used for train or eval. + repeat_num(int): the repeat times of dataset. Default: 1 + batch_size(int): the batch size of dataset. Default: 32 + target(str): the device target. Default: Ascend + + Returns: + dataset + """ + if target == "Ascend": + device_num = int(os.getenv("DEVICE_NUM")) + rank_id = int(os.getenv("RANK_ID")) + else: + init("nccl") + rank_id = get_rank() + device_num = get_group_size() + + if device_num == 1: + ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True) + else: + ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True, + num_shards=device_num, shard_id=rank_id) + + image_size = 224 + mean = [0.485 * 255, 0.456 * 255, 0.406 * 255] + std = [0.229 * 255, 0.224 * 255, 0.225 * 255] + + # define map operations + if do_train: + trans = [ + C.RandomCropDecodeResize(image_size, scale=(0.08, 1.0), ratio=(0.75, 1.333)), + C.RandomHorizontalFlip(prob=0.5), + C.Normalize(mean=mean, std=std), + C.HWC2CHW() + ] + else: + trans = [ + C.Decode(), + C.Resize((256, 256)), + C.CenterCrop(image_size), + C.Normalize(mean=mean, std=std), + C.HWC2CHW() + ] + + type_cast_op = C2.TypeCast(mstype.int32) + + ds = ds.map(input_columns="image", num_parallel_workers=8, operations=trans) + ds = ds.map(input_columns="label", num_parallel_workers=8, operations=type_cast_op) + + # apply batch operations + ds = ds.batch(batch_size, drop_remainder=True) + + # apply dataset repeat operation + ds = ds.repeat(repeat_num) + + return ds + + +def create_dataset3(dataset_path, do_train, repeat_num=1, batch_size=32): + """ + create a train or eval imagenet2012 dataset for resnet101 + Args: + dataset_path(string): the path of dataset. + do_train(bool): whether dataset is used for train or eval. + repeat_num(int): the repeat times of dataset. Default: 1 + batch_size(int): the batch size of dataset. Default: 32 + + Returns: + dataset + """ + device_num = int(os.getenv("RANK_SIZE")) + rank_id = int(os.getenv("RANK_ID")) + + if device_num == 1: + ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True) + else: + ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True, + num_shards=device_num, shard_id=rank_id) + resize_height = 224 + rescale = 1.0 / 255.0 + shift = 0.0 + + # define map operations + decode_op = C.Decode() + + random_resize_crop_op = C.RandomResizedCrop(resize_height, (0.08, 1.0), (0.75, 1.33), max_attempts=100) + horizontal_flip_op = C.RandomHorizontalFlip(rank_id / (rank_id + 1)) + resize_op_256 = C.Resize((256, 256)) + center_crop = C.CenterCrop(224) + rescale_op = C.Rescale(rescale, shift) + normalize_op = C.Normalize((0.475, 0.451, 0.392), (0.275, 0.267, 0.278)) + changeswap_op = C.HWC2CHW() + + if do_train: + trans = [decode_op, + random_resize_crop_op, + horizontal_flip_op, + rescale_op, + normalize_op, + changeswap_op] + + else: + trans = [decode_op, + resize_op_256, + center_crop, + rescale_op, + normalize_op, + changeswap_op] + + type_cast_op = C2.TypeCast(mstype.int32) + + ds = ds.map(input_columns="image", operations=trans, num_parallel_workers=8) + ds = ds.map(input_columns="label", operations=type_cast_op, num_parallel_workers=8) + + # apply batch operations + ds = ds.batch(batch_size, drop_remainder=True) + # apply dataset repeat operation + ds = ds.repeat(repeat_num) + + return ds diff --git a/example/resnet50_imagenet2012/lr_generator.py b/model_zoo/resnet/src/lr_generator.py similarity index 64% rename from example/resnet50_imagenet2012/lr_generator.py rename to model_zoo/resnet/src/lr_generator.py index 4a57be2f01..2af8971715 100755 --- a/example/resnet50_imagenet2012/lr_generator.py +++ b/model_zoo/resnet/src/lr_generator.py @@ -28,7 +28,7 @@ def get_lr(lr_init, lr_end, lr_max, warmup_epochs, total_epochs, steps_per_epoch warmup_epochs(int): number of warmup epochs total_epochs(int): total epoch of training steps_per_epoch(int): steps of one epoch - lr_decay_mode(string): learning rate decay mode, including steps, poly, cosine or default + lr_decay_mode(string): learning rate decay mode, including steps, poly or default Returns: np.array, learning rate array @@ -62,18 +62,6 @@ def get_lr(lr_init, lr_end, lr_max, warmup_epochs, total_epochs, steps_per_epoch if lr < 0.0: lr = 0.0 lr_each_step.append(lr) - elif lr_decay_mode == 'cosine': - decay_steps = total_steps - warmup_steps - for i in range(total_steps): - if i < warmup_steps: - lr_inc = (float(lr_max) - float(lr_init)) / float(warmup_steps) - lr = float(lr_init) + lr_inc * (i + 1) - else: - linear_decay = (total_steps - i) / decay_steps - cosine_decay = 0.5 * (1 + math.cos(math.pi * 2 * 0.47 * i / decay_steps)) - decayed = linear_decay * cosine_decay + 0.00001 - lr = lr_max * decayed - lr_each_step.append(lr) else: for i in range(total_steps): if i < warmup_steps: @@ -82,6 +70,47 @@ def get_lr(lr_init, lr_end, lr_max, warmup_epochs, total_epochs, steps_per_epoch lr = lr_max - (lr_max - lr_end) * (i - warmup_steps) / (total_steps - warmup_steps) lr_each_step.append(lr) - learning_rate = np.array(lr_each_step).astype(np.float32) + lr_each_step = np.array(lr_each_step).astype(np.float32) + + return lr_each_step + + +def linear_warmup_lr(current_step, warmup_steps, base_lr, init_lr): + lr_inc = (float(base_lr) - float(init_lr)) / float(warmup_steps) + lr = float(init_lr) + lr_inc * current_step + return lr + + +def warmup_cosine_annealing_lr(lr, steps_per_epoch, warmup_epochs, max_epoch=120, global_step=0): + """ + generate learning rate array with cosine + + Args: + lr(float): base learning rate + steps_per_epoch(int): steps size of one epoch + warmup_epochs(int): number of warmup epochs + max_epoch(int): total epochs of training + global_step(int): the current start index of lr array + Returns: + np.array, learning rate array + """ + base_lr = lr + warmup_init_lr = 0 + total_steps = int(max_epoch * steps_per_epoch) + warmup_steps = int(warmup_epochs * steps_per_epoch) + decay_steps = total_steps - warmup_steps + + lr_each_step = [] + for i in range(total_steps): + if i < warmup_steps: + lr = linear_warmup_lr(i + 1, warmup_steps, base_lr, warmup_init_lr) + else: + linear_decay = (total_steps - i) / decay_steps + cosine_decay = 0.5 * (1 + math.cos(math.pi * 2 * 0.47 * i / decay_steps)) + decayed = linear_decay * cosine_decay + 0.00001 + lr = base_lr * decayed + lr_each_step.append(lr) + lr_each_step = np.array(lr_each_step).astype(np.float32) + learning_rate = lr_each_step[global_step:] return learning_rate diff --git a/model_zoo/resnet101/src/resnet101.py b/model_zoo/resnet/src/resnet.py similarity index 94% rename from model_zoo/resnet101/src/resnet101.py rename to model_zoo/resnet/src/resnet.py index 33f10fd6cb..0e21222d21 100755 --- a/model_zoo/resnet101/src/resnet101.py +++ b/model_zoo/resnet/src/resnet.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ -"""ResNet101.""" +"""ResNet.""" import numpy as np import mindspore.nn as nn from mindspore.ops import operations as P @@ -240,6 +240,28 @@ class ResNet(nn.Cell): return out + +def resnet50(class_num=10): + """ + Get ResNet50 neural network. + + Args: + class_num (int): Class number. + + Returns: + Cell, cell instance of ResNet50 neural network. + + Examples: + >>> net = resnet50(10) + """ + return ResNet(ResidualBlock, + [3, 4, 6, 3], + [64, 256, 512, 1024], + [256, 512, 1024, 2048], + [1, 2, 2, 2], + class_num) + + def resnet101(class_num=1001): """ Get ResNet101 neural network. diff --git a/model_zoo/resnet/train.py b/model_zoo/resnet/train.py new file mode 100755 index 0000000000..89ce62d733 --- /dev/null +++ b/model_zoo/resnet/train.py @@ -0,0 +1,162 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""train resnet.""" +import os +import random +import argparse +import numpy as np +from mindspore import context +from mindspore import Tensor +from mindspore import dataset as de +from mindspore.parallel._auto_parallel_context import auto_parallel_context +from mindspore.nn.optim.momentum import Momentum +from mindspore.train.model import Model, ParallelMode +from mindspore.train.callback import ModelCheckpoint, CheckpointConfig, LossMonitor, TimeMonitor +from mindspore.nn.loss import SoftmaxCrossEntropyWithLogits +from mindspore.train.loss_scale_manager import FixedLossScaleManager +from mindspore.train.serialization import load_checkpoint, load_param_into_net +from mindspore.communication.management import init, get_rank, get_group_size +import mindspore.nn as nn +import mindspore.common.initializer as weight_init +from src.lr_generator import get_lr, warmup_cosine_annealing_lr +from src.crossentropy import CrossEntropy + +parser = argparse.ArgumentParser(description='Image classification') +parser.add_argument('--net', type=str, default=None, help='Resnet Model, either resnet50 or resnet101') +parser.add_argument('--dataset', type=str, default=None, help='Dataset, either cifar10 or imagenet2012') +parser.add_argument('--run_distribute', type=bool, default=False, help='Run distribute') +parser.add_argument('--device_num', type=int, default=1, help='Device num.') + +parser.add_argument('--dataset_path', type=str, default=None, help='Dataset path') +parser.add_argument('--device_target', type=str, default='Ascend', help='Device target') +parser.add_argument('--pre_trained', type=str, default=None, help='Pretrained checkpoint path') +args_opt = parser.parse_args() + +random.seed(1) +np.random.seed(1) +de.config.set_seed(1) + +if args_opt.net == "resnet50": + from src.resnet import resnet50 as resnet + + if args_opt.dataset == "cifar10": + from src.config import config1 as config + from src.dataset import create_dataset1 as create_dataset + else: + from src.config import config2 as config + from src.dataset import create_dataset2 as create_dataset +else: + from src.resnet import resnet101 as resnet + from src.config import config3 as config + from src.dataset import create_dataset3 as create_dataset + +if __name__ == '__main__': + target = args_opt.device_target + ckpt_save_dir = config.save_checkpoint_path + + # init context + context.set_context(mode=context.GRAPH_MODE, device_target=target, save_graphs=False) + if args_opt.run_distribute: + if target == "Ascend": + device_id = int(os.getenv('DEVICE_ID')) + context.set_context(device_id=device_id, enable_auto_mixed_precision=True) + context.set_auto_parallel_context(device_num=args_opt.device_num, parallel_mode=ParallelMode.DATA_PARALLEL, + mirror_mean=True) + if args_opt.net == "resnet50": + auto_parallel_context().set_all_reduce_fusion_split_indices([107, 160]) + else: + auto_parallel_context().set_all_reduce_fusion_split_indices([180, 313]) + init() + # GPU target + else: + init("nccl") + context.set_auto_parallel_context(device_num=get_group_size(), parallel_mode=ParallelMode.DATA_PARALLEL, + mirror_mean=True) + ckpt_save_dir = config.save_checkpoint_path + "ckpt_" + str(get_rank()) + "/" + + # create dataset + if args_opt.net == "resnet50": + dataset = create_dataset(dataset_path=args_opt.dataset_path, do_train=True, repeat_num=config.epoch_size, + batch_size=config.batch_size, target=target) + else: + dataset = create_dataset(dataset_path=args_opt.dataset_path, do_train=True, repeat_num=config.epoch_size, + batch_size=config.batch_size) + step_size = dataset.get_dataset_size() + + # define net + net = resnet(class_num=config.class_num) + + # init weight + if args_opt.pre_trained: + param_dict = load_checkpoint(args_opt.pre_trained) + load_param_into_net(net, param_dict) + else: + for _, cell in net.cells_and_names(): + if isinstance(cell, nn.Conv2d): + cell.weight.default_input = weight_init.initializer(weight_init.XavierUniform(), + cell.weight.default_input.shape, + cell.weight.default_input.dtype).to_tensor() + if isinstance(cell, nn.Dense): + cell.weight.default_input = weight_init.initializer(weight_init.TruncatedNormal(), + cell.weight.default_input.shape, + cell.weight.default_input.dtype).to_tensor() + + # init lr + if args_opt.net == "resnet50": + if args_opt.dataset == "cifar10": + lr = get_lr(lr_init=config.lr_init, lr_end=config.lr_end, lr_max=config.lr_max, + warmup_epochs=config.warmup_epochs, total_epochs=config.epoch_size, steps_per_epoch=step_size, + lr_decay_mode='poly') + else: + lr = get_lr(lr_init=config.lr_init, lr_end=0.0, lr_max=config.lr_max, warmup_epochs=config.warmup_epochs, + total_epochs=config.epoch_size, steps_per_epoch=step_size, lr_decay_mode='cosine') + else: + lr = warmup_cosine_annealing_lr(config.lr, step_size, config.warmup_epochs, 120, + config.pretrain_epoch_size * step_size) + lr = Tensor(lr) + + # define opt + opt = Momentum(filter(lambda x: x.requires_grad, net.get_parameters()), lr, config.momentum, + config.weight_decay, config.loss_scale) + + # define loss, model + if target == "Ascend": + if args_opt.dataset == "imagenet2012": + if not config.use_label_smooth: + config.label_smooth_factor = 0.0 + loss = CrossEntropy(smooth_factor=config.label_smooth_factor, num_classes=config.class_num) + else: + loss = SoftmaxCrossEntropyWithLogits(sparse=True, reduction='mean') + loss_scale = FixedLossScaleManager(config.loss_scale, drop_overflow_update=False) + model = Model(net, loss_fn=loss, optimizer=opt, loss_scale_manager=loss_scale, metrics={'acc'}, + amp_level="O2", keep_batchnorm_fp32=False) + else: + # GPU target + loss = SoftmaxCrossEntropyWithLogits(sparse=True, is_grad=False, reduction='mean') + opt = Momentum(filter(lambda x: x.requires_grad, net.get_parameters()), lr, config.momentum) + model = Model(net, loss_fn=loss, optimizer=opt, metrics={'acc'}) + + # define callbacks + time_cb = TimeMonitor(data_size=step_size) + loss_cb = LossMonitor() + cb = [time_cb, loss_cb] + if config.save_checkpoint: + config_ck = CheckpointConfig(save_checkpoint_steps=config.save_checkpoint_epochs * step_size, + keep_checkpoint_max=config.keep_checkpoint_max) + ckpt_cb = ModelCheckpoint(prefix="resnet", directory=ckpt_save_dir, config=config_ck) + cb += [ckpt_cb] + + # train model + model.train(config.epoch_size, dataset, callbacks=cb) diff --git a/model_zoo/resnet101/README.md b/model_zoo/resnet101/README.md deleted file mode 100644 index 86744be372..0000000000 --- a/model_zoo/resnet101/README.md +++ /dev/null @@ -1,147 +0,0 @@ -# ResNet101 Example - -## Description - -This is an example of training ResNet101 with ImageNet dataset in MindSpore. - -## Requirements - -- Install [MindSpore](https://www.mindspore.cn/install/en). - -- Download the dataset ImageNet2012. - -> Unzip the ImageNet2012 dataset to any path you want, the folder should include train and eval dataset as follows: - -``` -. -└─dataset - ├─ilsvrc - │ - └─validation_preprocess -``` - -## Structure - -```shell -. -└─resnet101 - ├─README.md - ├─scripts - ├─run_standalone_train.sh # launch standalone training(1p) - ├─run_distribute_train.sh # launch distributed training(8p) - └─run_eval.sh # launch evaluating - ├─src - ├─config.py # parameter configuration - ├─crossentropy.py # CrossEntropy loss function - ├─dataset.py # data preprocessin - ├─lr_generator.py # generate learning rate - ├─resnet101.py # resnet101 backbone - ├─eval.py # eval net - └─train.py # train net -``` - -## Parameter configuration - -Parameters for both training and evaluating can be set in config.py. - -``` -"class_num": 1001, # dataset class number -"batch_size": 32, # batch size of input tensor -"loss_scale": 1024, # loss scale -"momentum": 0.9, # momentum optimizer -"weight_decay": 1e-4, # weight decay -"epoch_size": 120, # epoch sizes for training -"pretrain_epoch_size": 0, # epoch size of pretrain checkpoint -"buffer_size": 1000, # number of queue size in data preprocessing -"image_height": 224, # image height -"image_width": 224, # image width -"save_checkpoint": True, # whether save checkpoint or not -"save_checkpoint_epochs": 1, # the epoch interval between two checkpoints. By default, the last checkpoint will be saved after the last epoch -"keep_checkpoint_max": 10, # only keep the last keep_checkpoint_max checkpoint -"save_checkpoint_path": "./", # path to save checkpoint relative to the executed path -"warmup_epochs": 0, # number of warmup epoch -"lr_decay_mode": "cosine" # decay mode for generating learning rate -"label_smooth": 1, # label_smooth -"label_smooth_factor": 0.1, # label_smooth_factor -"lr": 0.1 # base learning rate -``` - -## Running the example - -### Train - -#### Usage - -``` -# distributed training -sh run_distribute_train.sh [MINDSPORE_HCCL_CONFIG_PATH] [DATASET_PATH] [PRETRAINED_PATH](optional) - -# standalone training -sh run_standalone_train.sh [DATASET_PATH] [PRETRAINED_PATH](optional) -``` - -#### Launch - -```bash -# distributed training example(8p) -sh run_distribute_train.sh rank_table_8p.json dataset/ilsvrc - -If you want to load pretrained ckpt file, -sh run_distribute_train.sh rank_table_8p.json dataset/ilsvrc ./ckpt/pretrained.ckpt - -# standalone training example（1p） -sh run_standalone_train.sh dataset/ilsvrc - -If you want to load pretrained ckpt file, -sh run_standalone_train.sh dataset/ilsvrc ./ckpt/pretrained.ckpt -``` - -> About rank_table.json, you can refer to the [distributed training tutorial](https://www.mindspore.cn/tutorial/en/master/advanced_use/distributed_training.html). - -#### Result - -Training result will be stored in the scripts path, whose folder name begins with "train" or "train_parallel". You can find checkpoint file together with result like the followings in log. - - -``` -# distribute training result(8p) -epoch: 1 step: 5004, loss is 4.805483 -epoch: 2 step: 5004, loss is 3.2121816 -epoch: 3 step: 5004, loss is 3.429647 -epoch: 4 step: 5004, loss is 3.3667371 -epoch: 5 step: 5004, loss is 3.1718972 -... -epoch: 67 step: 5004, loss is 2.2768745 -epoch: 68 step: 5004, loss is 1.7223864 -epoch: 69 step: 5004, loss is 2.0665488 -epoch: 70 step: 5004, loss is 1.8717369 -... -``` - -### Infer - -#### Usage - -``` -# infer -sh run_eval.sh [VALIDATION_DATASET_PATH] [CHECKPOINT_PATH] -``` - -#### Launch - -```bash -# infer with checkpoint -sh run_eval.sh dataset/validation_preprocess/ train_parallel0/resnet-120_5004.ckpt - -``` - -> checkpoint can be produced in training process. - - -#### Result - -Inference result will be stored in the scripts path, whose folder name is "eval". Under this, you can find result like the followings in log. - -``` -result: {'top_5_accuracy': 0.9429417413572343, 'top_1_accuracy': 0.7853513124199744} ckpt=train_parallel0/resnet-120_5004.ckpt -``` diff --git a/model_zoo/resnet101/eval.py b/model_zoo/resnet101/eval.py deleted file mode 100755 index 73c0289ebd..0000000000 --- a/model_zoo/resnet101/eval.py +++ /dev/null @@ -1,75 +0,0 @@ -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ -""" -eval. -""" -import os -import argparse -import random -import numpy as np -from mindspore import context -from mindspore.parallel._auto_parallel_context import auto_parallel_context -from mindspore.train.model import Model, ParallelMode -from mindspore.train.serialization import load_checkpoint, load_param_into_net -import mindspore.dataset.engine as de -from mindspore.communication.management import init -from src.resnet101 import resnet101 -from src.dataset import create_dataset -from src.config import config -from src.crossentropy import CrossEntropy - -random.seed(1) -np.random.seed(1) -de.config.set_seed(1) - -parser = argparse.ArgumentParser(description='Image classification') -parser.add_argument('--run_distribute', type=bool, default=False, help='Run distribute') -parser.add_argument('--device_num', type=int, default=1, help='Device num.') -parser.add_argument('--do_train', type=bool, default=False, help='Do train or not.') -parser.add_argument('--do_eval', type=bool, default=True, help='Do eval or not.') -parser.add_argument('--checkpoint_path', type=str, default=None, help='Checkpoint file path') -parser.add_argument('--dataset_path', type=str, default=None, help='Dataset path') -args_opt = parser.parse_args() - -device_id = int(os.getenv('DEVICE_ID')) - -context.set_context(mode=context.GRAPH_MODE, device_target="Ascend", save_graphs=False, device_id=device_id) - -if __name__ == '__main__': - if not args_opt.do_eval and args_opt.run_distribute: - context.set_auto_parallel_context(device_num=args_opt.device_num, parallel_mode=ParallelMode.DATA_PARALLEL, - mirror_mean=True, parameter_broadcast=True) - auto_parallel_context().set_all_reduce_fusion_split_indices([180, 313]) - init() - - epoch_size = config.epoch_size - net = resnet101(class_num=config.class_num) - - if not config.label_smooth: - config.label_smooth_factor = 0.0 - loss = CrossEntropy(smooth_factor=config.label_smooth_factor, num_classes=config.class_num) - - if args_opt.do_eval: - dataset = create_dataset(dataset_path=args_opt.dataset_path, do_train=False, batch_size=config.batch_size) - step_size = dataset.get_dataset_size() - - if args_opt.checkpoint_path: - param_dict = load_checkpoint(args_opt.checkpoint_path) - load_param_into_net(net, param_dict) - net.set_train(False) - - model = Model(net, loss_fn=loss, metrics={'top_1_accuracy', 'top_5_accuracy'}) - res = model.eval(dataset) - print("result:", res, "ckpt=", args_opt.checkpoint_path) diff --git a/model_zoo/resnet101/src/config.py b/model_zoo/resnet101/src/config.py deleted file mode 100755 index 594b28522a..0000000000 --- a/model_zoo/resnet101/src/config.py +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ -""" -network config setting, will be used in train.py and eval.py -""" -from easydict import EasyDict as ed - -config = ed({ - "class_num": 1001, - "batch_size": 32, - "loss_scale": 1024, - "momentum": 0.9, - "weight_decay": 1e-4, - "epoch_size": 120, - "pretrain_epoch_size": 0, - "buffer_size": 1000, - "image_height": 224, - "image_width": 224, - "save_checkpoint": True, - "save_checkpoint_epochs": 5, - "keep_checkpoint_max": 10, - "save_checkpoint_path": "./", - "warmup_epochs": 0, - "lr_decay_mode": "cosine", - "label_smooth": 1, - "label_smooth_factor": 0.1, - "lr": 0.1 -}) diff --git a/model_zoo/resnet101/src/dataset.py b/model_zoo/resnet101/src/dataset.py deleted file mode 100755 index b2a074a535..0000000000 --- a/model_zoo/resnet101/src/dataset.py +++ /dev/null @@ -1,89 +0,0 @@ -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ -""" -create train or eval dataset. -""" -import os -import mindspore.common.dtype as mstype -import mindspore.dataset.engine as de -import mindspore.dataset.transforms.vision.c_transforms as C -import mindspore.dataset.transforms.c_transforms as C2 -from src.config import config - -def create_dataset(dataset_path, do_train, repeat_num=1, batch_size=32): - """ - create a train or evaluate dataset - Args: - dataset_path(string): the path of dataset. - do_train(bool): whether dataset is used for train or eval. - repeat_num(int): the repeat times of dataset. Default: 1 - batch_size(int): the batch size of dataset. Default: 32 - - Returns: - dataset - """ - device_num = int(os.getenv("RANK_SIZE")) - rank_id = int(os.getenv("RANK_ID")) - - if device_num == 1: - ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True) - else: - ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True, - num_shards=device_num, shard_id=rank_id) - resize_height = 224 - rescale = 1.0 / 255.0 - shift = 0.0 - - # define map operations - decode_op = C.Decode() - - random_resize_crop_op = C.RandomResizedCrop(resize_height, (0.08, 1.0), (0.75, 1.33), max_attempts=100) - horizontal_flip_op = C.RandomHorizontalFlip(rank_id / (rank_id + 1)) - resize_op_256 = C.Resize((256, 256)) - center_crop = C.CenterCrop(224) - rescale_op = C.Rescale(rescale, shift) - normalize_op = C.Normalize((0.475, 0.451, 0.392), (0.275, 0.267, 0.278)) - changeswap_op = C.HWC2CHW() - - trans = [] - if do_train: - trans = [decode_op, - random_resize_crop_op, - horizontal_flip_op, - rescale_op, - normalize_op, - changeswap_op] - - else: - trans = [decode_op, - resize_op_256, - center_crop, - rescale_op, - normalize_op, - changeswap_op] - - type_cast_op = C2.TypeCast(mstype.int32) - - ds = ds.map(input_columns="image", operations=trans, num_parallel_workers=8) - ds = ds.map(input_columns="label", operations=type_cast_op, num_parallel_workers=8) - - # apply shuffle operations - ds = ds.shuffle(buffer_size=config.buffer_size) - # apply batch operations - ds = ds.batch(batch_size, drop_remainder=True) - # apply dataset repeat operation - ds = ds.repeat(repeat_num) - - return ds diff --git a/model_zoo/resnet101/src/lr_generator.py b/model_zoo/resnet101/src/lr_generator.py deleted file mode 100755 index 2392e7a7bf..0000000000 --- a/model_zoo/resnet101/src/lr_generator.py +++ /dev/null @@ -1,56 +0,0 @@ -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ -"""learning rate generator""" -import math -import numpy as np - -def linear_warmup_lr(current_step, warmup_steps, base_lr, init_lr): - lr_inc = (float(base_lr) - float(init_lr)) / float(warmup_steps) - lr = float(init_lr) + lr_inc * current_step - return lr - -def warmup_cosine_annealing_lr(lr, steps_per_epoch, warmup_epochs, max_epoch=120, global_step=0): - """ - generate learning rate array with cosine - - Args: - lr(float): base learning rate - steps_per_epoch(int): steps size of one epoch - warmup_epochs(int): number of warmup epochs - max_epoch(int): total epochs of training - global_step(int): the current start index of lr array - Returns: - np.array, learning rate array - """ - base_lr = lr - warmup_init_lr = 0 - total_steps = int(max_epoch * steps_per_epoch) - warmup_steps = int(warmup_epochs * steps_per_epoch) - decay_steps = total_steps - warmup_steps - - lr_each_step = [] - for i in range(total_steps): - if i < warmup_steps: - lr = linear_warmup_lr(i + 1, warmup_steps, base_lr, warmup_init_lr) - else: - linear_decay = (total_steps - i) / decay_steps - cosine_decay = 0.5 * (1 + math.cos(math.pi * 2 * 0.47 * i / decay_steps)) - decayed = linear_decay * cosine_decay + 0.00001 - lr = base_lr * decayed - lr_each_step.append(lr) - - lr_each_step = np.array(lr_each_step).astype(np.float32) - learning_rate = lr_each_step[global_step:] - return learning_rate diff --git a/model_zoo/resnet101/train.py b/model_zoo/resnet101/train.py deleted file mode 100755 index 1cd3627a11..0000000000 --- a/model_zoo/resnet101/train.py +++ /dev/null @@ -1,102 +0,0 @@ -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ -"""train_imagenet.""" -import os -import argparse -import random -import numpy as np -from mindspore import context -from mindspore import Tensor -from mindspore.parallel._auto_parallel_context import auto_parallel_context -from mindspore.nn.optim.momentum import Momentum -from mindspore.train.model import Model, ParallelMode -from mindspore.train.callback import ModelCheckpoint, CheckpointConfig, LossMonitor, TimeMonitor -from mindspore.train.loss_scale_manager import FixedLossScaleManager -from mindspore.train.serialization import load_checkpoint, load_param_into_net -import mindspore.dataset.engine as de -from mindspore.communication.management import init -import mindspore.nn as nn -import mindspore.common.initializer as weight_init -from src.resnet101 import resnet101 -from src.dataset import create_dataset -from src.lr_generator import warmup_cosine_annealing_lr -from src.config import config -from src.crossentropy import CrossEntropy - -random.seed(1) -np.random.seed(1) -de.config.set_seed(1) - -parser = argparse.ArgumentParser(description='Image classification') -parser.add_argument('--run_distribute', type=bool, default=False, help='Run distribute') -parser.add_argument('--device_num', type=int, default=1, help='Device num.') -parser.add_argument('--do_train', type=bool, default=True, help='Do train or not.') -parser.add_argument('--do_eval', type=bool, default=False, help='Do eval or not.') -parser.add_argument('--dataset_path', type=str, default=None, help='Dataset path') -parser.add_argument('--pre_trained', type=str, default=None, help='Pretrained checkpoint path') -args_opt = parser.parse_args() - -device_id = int(os.getenv('DEVICE_ID')) - -context.set_context(mode=context.GRAPH_MODE, device_target="Ascend", save_graphs=False, device_id=device_id, - enable_auto_mixed_precision=True) - -if __name__ == '__main__': - if not args_opt.do_eval and args_opt.run_distribute: - context.set_auto_parallel_context(device_num=args_opt.device_num, parallel_mode=ParallelMode.DATA_PARALLEL, - mirror_mean=True, parameter_broadcast=True) - auto_parallel_context().set_all_reduce_fusion_split_indices([180, 313]) - init() - - epoch_size = config.epoch_size - net = resnet101(class_num=config.class_num) - # weight init - for _, cell in net.cells_and_names(): - if isinstance(cell, nn.Conv2d): - cell.weight.default_input = weight_init.initializer(weight_init.XavierUniform(), - cell.weight.default_input.shape, - cell.weight.default_input.dtype).to_tensor() - if isinstance(cell, nn.Dense): - cell.weight.default_input = weight_init.initializer(weight_init.TruncatedNormal(), - cell.weight.default_input.shape, - cell.weight.default_input.dtype).to_tensor() - if not config.label_smooth: - config.label_smooth_factor = 0.0 - loss = CrossEntropy(smooth_factor=config.label_smooth_factor, num_classes=config.class_num) - if args_opt.do_train: - dataset = create_dataset(dataset_path=args_opt.dataset_path, do_train=True, - repeat_num=epoch_size, batch_size=config.batch_size) - step_size = dataset.get_dataset_size() - loss_scale = FixedLossScaleManager(config.loss_scale, drop_overflow_update=False) - if args_opt.pre_trained: - param_dict = load_checkpoint(args_opt.pre_trained) - load_param_into_net(net, param_dict) - - # learning rate strategy with cosine - lr = Tensor(warmup_cosine_annealing_lr(config.lr, step_size, config.warmup_epochs, 120, - config.pretrain_epoch_size*step_size)) - opt = Momentum(filter(lambda x: x.requires_grad, net.get_parameters()), lr, config.momentum, - config.weight_decay, config.loss_scale) - model = Model(net, loss_fn=loss, optimizer=opt, amp_level='O2', keep_batchnorm_fp32=False, - loss_scale_manager=loss_scale, metrics={'acc'}) - time_cb = TimeMonitor(data_size=step_size) - loss_cb = LossMonitor() - cb = [time_cb, loss_cb] - if config.save_checkpoint: - config_ck = CheckpointConfig(save_checkpoint_steps=config.save_checkpoint_epochs*step_size, - keep_checkpoint_max=config.keep_checkpoint_max) - ckpt_cb = ModelCheckpoint(prefix="resnet", directory=config.save_checkpoint_path, config=config_ck) - cb += [ckpt_cb] - model.train(epoch_size, dataset, callbacks=cb) From 748e07eb9e35cec25d95b9e67a1dc2e9e47b47f9 Mon Sep 17 00:00:00 2001 From: liyong Date: Sun, 28 Jun 2020 17:09:56 +0800 Subject: [PATCH 128/254] adjust model zoo utils --- example/nlp_to_mindrecord/CLUERNER2020/run.sh | 40 ------------------- model_zoo/gat/scripts/run_process_data.sh | 2 +- model_zoo/gcn/scripts/run_process_data.sh | 2 +- .../Caltech-UCSD-Birds-200-2011/README.md | 0 .../create_dataset.py | 0 .../data/README.md | 0 .../gen_mindrecord.py | 0 .../output/README.md | 0 .../Caltech-UCSD-Birds-200-2011/run.sh | 0 .../Caltech-UCSD-Birds-200-2011/run_read.sh | 0 .../ImageNet_Similar_Perf/README.md | 0 .../imagenet/__init__.py | 0 .../ImageNet_Similar_Perf/imagenet/mr_api.py | 0 .../ImageNet_Similar_Perf/run_imagenet.sh | 0 .../ImageNet_Similar_Perf/run_template.sh | 0 .../template/__init__.py | 0 .../ImageNet_Similar_Perf/template/mr_api.py | 0 .../ImageNet_Similar_Perf/writer.py | 0 .../utils}/cv_to_mindrecord/README.md | 0 .../utils}/graph_to_mindrecord/README.md | 0 .../graph_to_mindrecord/citeseer/__init__.py | 0 .../graph_to_mindrecord/citeseer/mr_api.py | 0 .../graph_to_mindrecord/cora/__init__.py | 0 .../utils}/graph_to_mindrecord/cora/mr_api.py | 0 .../graph_to_mindrecord/graph_map_schema.py | 0 .../graph_to_mindrecord/read_citeseer.sh | 0 .../utils}/graph_to_mindrecord/read_cora.sh | 0 .../utils}/graph_to_mindrecord/reader.py | 0 .../graph_to_mindrecord/sns/__init__.py | 0 .../utils}/graph_to_mindrecord/sns/mr_api.py | 0 .../graph_to_mindrecord/write_citeseer.sh | 0 .../utils}/graph_to_mindrecord/write_cora.sh | 0 .../utils}/graph_to_mindrecord/write_sns.sh | 0 .../utils}/graph_to_mindrecord/writer.py | 0 .../nlp_to_mindrecord/CLUERNER2020/README.md | 0 .../CLUERNER2020/create_dataset.py | 0 .../CLUERNER2020/data/.gitignore | 0 .../CLUERNER2020/data/README.md | 0 .../CLUERNER2020/output/README.md | 0 .../nlp_to_mindrecord/CLUERNER2020/run.sh | 40 +++++++++++++++++++ .../CLUERNER2020/run_read.sh | 0 .../utils}/nlp_to_mindrecord/README.md | 0 .../nlp_to_mindrecord/aclImdb/README.md | 0 .../aclImdb/create_dataset.py | 0 .../nlp_to_mindrecord/aclImdb/data/README.md | 0 .../aclImdb/gen_mindrecord.py | 0 .../aclImdb/output/README.md | 0 .../utils}/nlp_to_mindrecord/aclImdb/run.sh | 0 .../nlp_to_mindrecord/aclImdb/run_read.sh | 0 .../aclImdb_preprocess/README.md | 0 .../aclImdb_preprocess/create_dataset.py | 0 .../aclImdb_preprocess/data/README.md | 0 .../aclImdb_preprocess/gen_mindrecord.py | 0 .../aclImdb_preprocess/output/README.md | 0 .../aclImdb_preprocess/run.sh | 0 .../aclImdb_preprocess/run_read.sh | 0 .../utils}/nlp_to_mindrecord/enwiki/README.md | 0 .../enwiki/create_dataset.py | 0 .../utils}/nlp_to_mindrecord/enwiki/run.sh | 14 +++---- .../nlp_to_mindrecord/enwiki/run_read.sh | 0 .../utils}/nlp_to_mindrecord/zhwiki/README.md | 2 +- .../zhwiki/create_dataset.py | 0 .../nlp_to_mindrecord/zhwiki/data/.gitignore | 0 .../nlp_to_mindrecord/zhwiki/data/README.md | 0 .../nlp_to_mindrecord/zhwiki/output/README.md | 0 .../utils}/nlp_to_mindrecord/zhwiki/run.sh | 16 ++++---- .../nlp_to_mindrecord/zhwiki/run_read.sh | 0 .../zhwiki/run_read_simple.sh | 2 +- .../nlp_to_mindrecord/zhwiki/run_simple.sh | 20 +++++----- 69 files changed, 69 insertions(+), 69 deletions(-) delete mode 100644 example/nlp_to_mindrecord/CLUERNER2020/run.sh rename {example => model_zoo/utils}/cv_to_mindrecord/Caltech-UCSD-Birds-200-2011/README.md (100%) rename {example => model_zoo/utils}/cv_to_mindrecord/Caltech-UCSD-Birds-200-2011/create_dataset.py (100%) rename {example => model_zoo/utils}/cv_to_mindrecord/Caltech-UCSD-Birds-200-2011/data/README.md (100%) rename {example => model_zoo/utils}/cv_to_mindrecord/Caltech-UCSD-Birds-200-2011/gen_mindrecord.py (100%) rename {example => model_zoo/utils}/cv_to_mindrecord/Caltech-UCSD-Birds-200-2011/output/README.md (100%) rename {example => model_zoo/utils}/cv_to_mindrecord/Caltech-UCSD-Birds-200-2011/run.sh (100%) rename {example => model_zoo/utils}/cv_to_mindrecord/Caltech-UCSD-Birds-200-2011/run_read.sh (100%) rename {example => model_zoo/utils}/cv_to_mindrecord/ImageNet_Similar_Perf/README.md (100%) rename {example => model_zoo/utils}/cv_to_mindrecord/ImageNet_Similar_Perf/imagenet/__init__.py (100%) rename {example => model_zoo/utils}/cv_to_mindrecord/ImageNet_Similar_Perf/imagenet/mr_api.py (100%) rename {example => model_zoo/utils}/cv_to_mindrecord/ImageNet_Similar_Perf/run_imagenet.sh (100%) rename {example => model_zoo/utils}/cv_to_mindrecord/ImageNet_Similar_Perf/run_template.sh (100%) rename {example => model_zoo/utils}/cv_to_mindrecord/ImageNet_Similar_Perf/template/__init__.py (100%) rename {example => model_zoo/utils}/cv_to_mindrecord/ImageNet_Similar_Perf/template/mr_api.py (100%) rename {example => model_zoo/utils}/cv_to_mindrecord/ImageNet_Similar_Perf/writer.py (100%) rename {example => model_zoo/utils}/cv_to_mindrecord/README.md (100%) rename {example => model_zoo/utils}/graph_to_mindrecord/README.md (100%) rename {example => model_zoo/utils}/graph_to_mindrecord/citeseer/__init__.py (100%) rename {example => model_zoo/utils}/graph_to_mindrecord/citeseer/mr_api.py (100%) rename {example => model_zoo/utils}/graph_to_mindrecord/cora/__init__.py (100%) rename {example => model_zoo/utils}/graph_to_mindrecord/cora/mr_api.py (100%) rename {example => model_zoo/utils}/graph_to_mindrecord/graph_map_schema.py (100%) rename {example => model_zoo/utils}/graph_to_mindrecord/read_citeseer.sh (100%) rename {example => model_zoo/utils}/graph_to_mindrecord/read_cora.sh (100%) rename {example => model_zoo/utils}/graph_to_mindrecord/reader.py (100%) rename {example => model_zoo/utils}/graph_to_mindrecord/sns/__init__.py (100%) rename {example => model_zoo/utils}/graph_to_mindrecord/sns/mr_api.py (100%) rename {example => model_zoo/utils}/graph_to_mindrecord/write_citeseer.sh (100%) rename {example => model_zoo/utils}/graph_to_mindrecord/write_cora.sh (100%) rename {example => model_zoo/utils}/graph_to_mindrecord/write_sns.sh (100%) rename {example => model_zoo/utils}/graph_to_mindrecord/writer.py (100%) rename {example => model_zoo/utils}/nlp_to_mindrecord/CLUERNER2020/README.md (100%) rename {example => model_zoo/utils}/nlp_to_mindrecord/CLUERNER2020/create_dataset.py (100%) rename {example => model_zoo/utils}/nlp_to_mindrecord/CLUERNER2020/data/.gitignore (100%) rename {example => model_zoo/utils}/nlp_to_mindrecord/CLUERNER2020/data/README.md (100%) rename {example => model_zoo/utils}/nlp_to_mindrecord/CLUERNER2020/output/README.md (100%) create mode 100644 model_zoo/utils/nlp_to_mindrecord/CLUERNER2020/run.sh rename {example => model_zoo/utils}/nlp_to_mindrecord/CLUERNER2020/run_read.sh (100%) rename {example => model_zoo/utils}/nlp_to_mindrecord/README.md (100%) rename {example => model_zoo/utils}/nlp_to_mindrecord/aclImdb/README.md (100%) rename {example => model_zoo/utils}/nlp_to_mindrecord/aclImdb/create_dataset.py (100%) rename {example => model_zoo/utils}/nlp_to_mindrecord/aclImdb/data/README.md (100%) rename {example => model_zoo/utils}/nlp_to_mindrecord/aclImdb/gen_mindrecord.py (100%) rename {example => model_zoo/utils}/nlp_to_mindrecord/aclImdb/output/README.md (100%) rename {example => model_zoo/utils}/nlp_to_mindrecord/aclImdb/run.sh (100%) rename {example => model_zoo/utils}/nlp_to_mindrecord/aclImdb/run_read.sh (100%) rename {example => model_zoo/utils}/nlp_to_mindrecord/aclImdb_preprocess/README.md (100%) rename {example => model_zoo/utils}/nlp_to_mindrecord/aclImdb_preprocess/create_dataset.py (100%) rename {example => model_zoo/utils}/nlp_to_mindrecord/aclImdb_preprocess/data/README.md (100%) rename {example => model_zoo/utils}/nlp_to_mindrecord/aclImdb_preprocess/gen_mindrecord.py (100%) rename {example => model_zoo/utils}/nlp_to_mindrecord/aclImdb_preprocess/output/README.md (100%) rename {example => model_zoo/utils}/nlp_to_mindrecord/aclImdb_preprocess/run.sh (100%) rename {example => model_zoo/utils}/nlp_to_mindrecord/aclImdb_preprocess/run_read.sh (100%) rename {example => model_zoo/utils}/nlp_to_mindrecord/enwiki/README.md (100%) rename {example => model_zoo/utils}/nlp_to_mindrecord/enwiki/create_dataset.py (100%) rename {example => model_zoo/utils}/nlp_to_mindrecord/enwiki/run.sh (83%) rename {example => model_zoo/utils}/nlp_to_mindrecord/enwiki/run_read.sh (100%) rename {example => model_zoo/utils}/nlp_to_mindrecord/zhwiki/README.md (98%) rename {example => model_zoo/utils}/nlp_to_mindrecord/zhwiki/create_dataset.py (100%) rename {example => model_zoo/utils}/nlp_to_mindrecord/zhwiki/data/.gitignore (100%) rename {example => model_zoo/utils}/nlp_to_mindrecord/zhwiki/data/README.md (100%) rename {example => model_zoo/utils}/nlp_to_mindrecord/zhwiki/output/README.md (100%) rename {example => model_zoo/utils}/nlp_to_mindrecord/zhwiki/run.sh (80%) rename {example => model_zoo/utils}/nlp_to_mindrecord/zhwiki/run_read.sh (100%) rename {example => model_zoo/utils}/nlp_to_mindrecord/zhwiki/run_read_simple.sh (91%) rename {example => model_zoo/utils}/nlp_to_mindrecord/zhwiki/run_simple.sh (53%) diff --git a/example/nlp_to_mindrecord/CLUERNER2020/run.sh b/example/nlp_to_mindrecord/CLUERNER2020/run.sh deleted file mode 100644 index 15c6aa4362..0000000000 --- a/example/nlp_to_mindrecord/CLUERNER2020/run.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/bash -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ - -rm -f output/train.mindrecord* -rm -f output/dev.mindrecord* - -if [ ! -d "../../../third_party/to_mindrecord/CLUERNER2020" ]; then - echo "The patch base dir ../../../third_party/to_mindrecord/CLUERNER2020 is not exist." - exit 1 -fi - -if [ ! -f "../../../third_party/patch/to_mindrecord/CLUERNER2020/data_processor_seq.patch" ]; then - echo "The patch file ../../../third_party/patch/to_mindrecord/CLUERNER2020/data_processor_seq.patch is not exist." - exit 1 -fi - -# patch for data_processor_seq.py -patch -p0 -d ../../../third_party/to_mindrecord/CLUERNER2020/ -o data_processor_seq_patched.py < ../../../third_party/patch/to_mindrecord/CLUERNER2020/data_processor_seq.patch -if [ $? -ne 0 ]; then - echo "Patch ../../../third_party/to_mindrecord/CLUERNER2020/data_processor_seq.py failed" - exit 1 -fi - -# use patched script -python ../../../third_party/to_mindrecord/CLUERNER2020/data_processor_seq_patched.py \ ---vocab_file=../../../third_party/to_mindrecord/CLUERNER2020/vocab.txt \ ---label2id_file=../../../third_party/to_mindrecord/CLUERNER2020/label2id.json diff --git a/model_zoo/gat/scripts/run_process_data.sh b/model_zoo/gat/scripts/run_process_data.sh index 4501f3c67f..3bf6672301 100755 --- a/model_zoo/gat/scripts/run_process_data.sh +++ b/model_zoo/gat/scripts/run_process_data.sh @@ -42,7 +42,7 @@ MINDRECORD_PATH=`pwd`/data_mr rm -f $MINDRECORD_PATH/* -cd ../../../example/graph_to_mindrecord || exit +cd ../../utils/graph_to_mindrecord || exit python writer.py --mindrecord_script $DATASET_NAME \ --mindrecord_file "$MINDRECORD_PATH/$DATASET_NAME" \ diff --git a/model_zoo/gcn/scripts/run_process_data.sh b/model_zoo/gcn/scripts/run_process_data.sh index d51d915943..013cf2f28c 100755 --- a/model_zoo/gcn/scripts/run_process_data.sh +++ b/model_zoo/gcn/scripts/run_process_data.sh @@ -43,7 +43,7 @@ MINDRECORD_PATH=`pwd`/data_mr rm -f $MINDRECORD_PATH/$DATASET_NAME rm -f $MINDRECORD_PATH/$DATASET_NAME.db -cd ../../../example/graph_to_mindrecord || exit +cd ../../utils/graph_to_mindrecord || exit python writer.py --mindrecord_script $DATASET_NAME \ --mindrecord_file "$MINDRECORD_PATH/$DATASET_NAME" \ diff --git a/example/cv_to_mindrecord/Caltech-UCSD-Birds-200-2011/README.md b/model_zoo/utils/cv_to_mindrecord/Caltech-UCSD-Birds-200-2011/README.md similarity index 100% rename from example/cv_to_mindrecord/Caltech-UCSD-Birds-200-2011/README.md rename to model_zoo/utils/cv_to_mindrecord/Caltech-UCSD-Birds-200-2011/README.md diff --git a/example/cv_to_mindrecord/Caltech-UCSD-Birds-200-2011/create_dataset.py b/model_zoo/utils/cv_to_mindrecord/Caltech-UCSD-Birds-200-2011/create_dataset.py similarity index 100% rename from example/cv_to_mindrecord/Caltech-UCSD-Birds-200-2011/create_dataset.py rename to model_zoo/utils/cv_to_mindrecord/Caltech-UCSD-Birds-200-2011/create_dataset.py diff --git a/example/cv_to_mindrecord/Caltech-UCSD-Birds-200-2011/data/README.md b/model_zoo/utils/cv_to_mindrecord/Caltech-UCSD-Birds-200-2011/data/README.md similarity index 100% rename from example/cv_to_mindrecord/Caltech-UCSD-Birds-200-2011/data/README.md rename to model_zoo/utils/cv_to_mindrecord/Caltech-UCSD-Birds-200-2011/data/README.md diff --git a/example/cv_to_mindrecord/Caltech-UCSD-Birds-200-2011/gen_mindrecord.py b/model_zoo/utils/cv_to_mindrecord/Caltech-UCSD-Birds-200-2011/gen_mindrecord.py similarity index 100% rename from example/cv_to_mindrecord/Caltech-UCSD-Birds-200-2011/gen_mindrecord.py rename to model_zoo/utils/cv_to_mindrecord/Caltech-UCSD-Birds-200-2011/gen_mindrecord.py diff --git a/example/cv_to_mindrecord/Caltech-UCSD-Birds-200-2011/output/README.md b/model_zoo/utils/cv_to_mindrecord/Caltech-UCSD-Birds-200-2011/output/README.md similarity index 100% rename from example/cv_to_mindrecord/Caltech-UCSD-Birds-200-2011/output/README.md rename to model_zoo/utils/cv_to_mindrecord/Caltech-UCSD-Birds-200-2011/output/README.md diff --git a/example/cv_to_mindrecord/Caltech-UCSD-Birds-200-2011/run.sh b/model_zoo/utils/cv_to_mindrecord/Caltech-UCSD-Birds-200-2011/run.sh similarity index 100% rename from example/cv_to_mindrecord/Caltech-UCSD-Birds-200-2011/run.sh rename to model_zoo/utils/cv_to_mindrecord/Caltech-UCSD-Birds-200-2011/run.sh diff --git a/example/cv_to_mindrecord/Caltech-UCSD-Birds-200-2011/run_read.sh b/model_zoo/utils/cv_to_mindrecord/Caltech-UCSD-Birds-200-2011/run_read.sh similarity index 100% rename from example/cv_to_mindrecord/Caltech-UCSD-Birds-200-2011/run_read.sh rename to model_zoo/utils/cv_to_mindrecord/Caltech-UCSD-Birds-200-2011/run_read.sh diff --git a/example/cv_to_mindrecord/ImageNet_Similar_Perf/README.md b/model_zoo/utils/cv_to_mindrecord/ImageNet_Similar_Perf/README.md similarity index 100% rename from example/cv_to_mindrecord/ImageNet_Similar_Perf/README.md rename to model_zoo/utils/cv_to_mindrecord/ImageNet_Similar_Perf/README.md diff --git a/example/cv_to_mindrecord/ImageNet_Similar_Perf/imagenet/__init__.py b/model_zoo/utils/cv_to_mindrecord/ImageNet_Similar_Perf/imagenet/__init__.py similarity index 100% rename from example/cv_to_mindrecord/ImageNet_Similar_Perf/imagenet/__init__.py rename to model_zoo/utils/cv_to_mindrecord/ImageNet_Similar_Perf/imagenet/__init__.py diff --git a/example/cv_to_mindrecord/ImageNet_Similar_Perf/imagenet/mr_api.py b/model_zoo/utils/cv_to_mindrecord/ImageNet_Similar_Perf/imagenet/mr_api.py similarity index 100% rename from example/cv_to_mindrecord/ImageNet_Similar_Perf/imagenet/mr_api.py rename to model_zoo/utils/cv_to_mindrecord/ImageNet_Similar_Perf/imagenet/mr_api.py diff --git a/example/cv_to_mindrecord/ImageNet_Similar_Perf/run_imagenet.sh b/model_zoo/utils/cv_to_mindrecord/ImageNet_Similar_Perf/run_imagenet.sh similarity index 100% rename from example/cv_to_mindrecord/ImageNet_Similar_Perf/run_imagenet.sh rename to model_zoo/utils/cv_to_mindrecord/ImageNet_Similar_Perf/run_imagenet.sh diff --git a/example/cv_to_mindrecord/ImageNet_Similar_Perf/run_template.sh b/model_zoo/utils/cv_to_mindrecord/ImageNet_Similar_Perf/run_template.sh similarity index 100% rename from example/cv_to_mindrecord/ImageNet_Similar_Perf/run_template.sh rename to model_zoo/utils/cv_to_mindrecord/ImageNet_Similar_Perf/run_template.sh diff --git a/example/cv_to_mindrecord/ImageNet_Similar_Perf/template/__init__.py b/model_zoo/utils/cv_to_mindrecord/ImageNet_Similar_Perf/template/__init__.py similarity index 100% rename from example/cv_to_mindrecord/ImageNet_Similar_Perf/template/__init__.py rename to model_zoo/utils/cv_to_mindrecord/ImageNet_Similar_Perf/template/__init__.py diff --git a/example/cv_to_mindrecord/ImageNet_Similar_Perf/template/mr_api.py b/model_zoo/utils/cv_to_mindrecord/ImageNet_Similar_Perf/template/mr_api.py similarity index 100% rename from example/cv_to_mindrecord/ImageNet_Similar_Perf/template/mr_api.py rename to model_zoo/utils/cv_to_mindrecord/ImageNet_Similar_Perf/template/mr_api.py diff --git a/example/cv_to_mindrecord/ImageNet_Similar_Perf/writer.py b/model_zoo/utils/cv_to_mindrecord/ImageNet_Similar_Perf/writer.py similarity index 100% rename from example/cv_to_mindrecord/ImageNet_Similar_Perf/writer.py rename to model_zoo/utils/cv_to_mindrecord/ImageNet_Similar_Perf/writer.py diff --git a/example/cv_to_mindrecord/README.md b/model_zoo/utils/cv_to_mindrecord/README.md similarity index 100% rename from example/cv_to_mindrecord/README.md rename to model_zoo/utils/cv_to_mindrecord/README.md diff --git a/example/graph_to_mindrecord/README.md b/model_zoo/utils/graph_to_mindrecord/README.md similarity index 100% rename from example/graph_to_mindrecord/README.md rename to model_zoo/utils/graph_to_mindrecord/README.md diff --git a/example/graph_to_mindrecord/citeseer/__init__.py b/model_zoo/utils/graph_to_mindrecord/citeseer/__init__.py similarity index 100% rename from example/graph_to_mindrecord/citeseer/__init__.py rename to model_zoo/utils/graph_to_mindrecord/citeseer/__init__.py diff --git a/example/graph_to_mindrecord/citeseer/mr_api.py b/model_zoo/utils/graph_to_mindrecord/citeseer/mr_api.py similarity index 100% rename from example/graph_to_mindrecord/citeseer/mr_api.py rename to model_zoo/utils/graph_to_mindrecord/citeseer/mr_api.py diff --git a/example/graph_to_mindrecord/cora/__init__.py b/model_zoo/utils/graph_to_mindrecord/cora/__init__.py similarity index 100% rename from example/graph_to_mindrecord/cora/__init__.py rename to model_zoo/utils/graph_to_mindrecord/cora/__init__.py diff --git a/example/graph_to_mindrecord/cora/mr_api.py b/model_zoo/utils/graph_to_mindrecord/cora/mr_api.py similarity index 100% rename from example/graph_to_mindrecord/cora/mr_api.py rename to model_zoo/utils/graph_to_mindrecord/cora/mr_api.py diff --git a/example/graph_to_mindrecord/graph_map_schema.py b/model_zoo/utils/graph_to_mindrecord/graph_map_schema.py similarity index 100% rename from example/graph_to_mindrecord/graph_map_schema.py rename to model_zoo/utils/graph_to_mindrecord/graph_map_schema.py diff --git a/example/graph_to_mindrecord/read_citeseer.sh b/model_zoo/utils/graph_to_mindrecord/read_citeseer.sh similarity index 100% rename from example/graph_to_mindrecord/read_citeseer.sh rename to model_zoo/utils/graph_to_mindrecord/read_citeseer.sh diff --git a/example/graph_to_mindrecord/read_cora.sh b/model_zoo/utils/graph_to_mindrecord/read_cora.sh similarity index 100% rename from example/graph_to_mindrecord/read_cora.sh rename to model_zoo/utils/graph_to_mindrecord/read_cora.sh diff --git a/example/graph_to_mindrecord/reader.py b/model_zoo/utils/graph_to_mindrecord/reader.py similarity index 100% rename from example/graph_to_mindrecord/reader.py rename to model_zoo/utils/graph_to_mindrecord/reader.py diff --git a/example/graph_to_mindrecord/sns/__init__.py b/model_zoo/utils/graph_to_mindrecord/sns/__init__.py similarity index 100% rename from example/graph_to_mindrecord/sns/__init__.py rename to model_zoo/utils/graph_to_mindrecord/sns/__init__.py diff --git a/example/graph_to_mindrecord/sns/mr_api.py b/model_zoo/utils/graph_to_mindrecord/sns/mr_api.py similarity index 100% rename from example/graph_to_mindrecord/sns/mr_api.py rename to model_zoo/utils/graph_to_mindrecord/sns/mr_api.py diff --git a/example/graph_to_mindrecord/write_citeseer.sh b/model_zoo/utils/graph_to_mindrecord/write_citeseer.sh similarity index 100% rename from example/graph_to_mindrecord/write_citeseer.sh rename to model_zoo/utils/graph_to_mindrecord/write_citeseer.sh diff --git a/example/graph_to_mindrecord/write_cora.sh b/model_zoo/utils/graph_to_mindrecord/write_cora.sh similarity index 100% rename from example/graph_to_mindrecord/write_cora.sh rename to model_zoo/utils/graph_to_mindrecord/write_cora.sh diff --git a/example/graph_to_mindrecord/write_sns.sh b/model_zoo/utils/graph_to_mindrecord/write_sns.sh similarity index 100% rename from example/graph_to_mindrecord/write_sns.sh rename to model_zoo/utils/graph_to_mindrecord/write_sns.sh diff --git a/example/graph_to_mindrecord/writer.py b/model_zoo/utils/graph_to_mindrecord/writer.py similarity index 100% rename from example/graph_to_mindrecord/writer.py rename to model_zoo/utils/graph_to_mindrecord/writer.py diff --git a/example/nlp_to_mindrecord/CLUERNER2020/README.md b/model_zoo/utils/nlp_to_mindrecord/CLUERNER2020/README.md similarity index 100% rename from example/nlp_to_mindrecord/CLUERNER2020/README.md rename to model_zoo/utils/nlp_to_mindrecord/CLUERNER2020/README.md diff --git a/example/nlp_to_mindrecord/CLUERNER2020/create_dataset.py b/model_zoo/utils/nlp_to_mindrecord/CLUERNER2020/create_dataset.py similarity index 100% rename from example/nlp_to_mindrecord/CLUERNER2020/create_dataset.py rename to model_zoo/utils/nlp_to_mindrecord/CLUERNER2020/create_dataset.py diff --git a/example/nlp_to_mindrecord/CLUERNER2020/data/.gitignore b/model_zoo/utils/nlp_to_mindrecord/CLUERNER2020/data/.gitignore similarity index 100% rename from example/nlp_to_mindrecord/CLUERNER2020/data/.gitignore rename to model_zoo/utils/nlp_to_mindrecord/CLUERNER2020/data/.gitignore diff --git a/example/nlp_to_mindrecord/CLUERNER2020/data/README.md b/model_zoo/utils/nlp_to_mindrecord/CLUERNER2020/data/README.md similarity index 100% rename from example/nlp_to_mindrecord/CLUERNER2020/data/README.md rename to model_zoo/utils/nlp_to_mindrecord/CLUERNER2020/data/README.md diff --git a/example/nlp_to_mindrecord/CLUERNER2020/output/README.md b/model_zoo/utils/nlp_to_mindrecord/CLUERNER2020/output/README.md similarity index 100% rename from example/nlp_to_mindrecord/CLUERNER2020/output/README.md rename to model_zoo/utils/nlp_to_mindrecord/CLUERNER2020/output/README.md diff --git a/model_zoo/utils/nlp_to_mindrecord/CLUERNER2020/run.sh b/model_zoo/utils/nlp_to_mindrecord/CLUERNER2020/run.sh new file mode 100644 index 0000000000..9de6f0e9fc --- /dev/null +++ b/model_zoo/utils/nlp_to_mindrecord/CLUERNER2020/run.sh @@ -0,0 +1,40 @@ +#!/bin/bash +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +rm -f output/train.mindrecord* +rm -f output/dev.mindrecord* + +if [ ! -d "../../../../third_party/to_mindrecord/CLUERNER2020" ]; then + echo "The patch base dir ../../../../third_party/to_mindrecord/CLUERNER2020 is not exist." + exit 1 +fi + +if [ ! -f "../../../../third_party/patch/to_mindrecord/CLUERNER2020/data_processor_seq.patch" ]; then + echo "The patch file ../../../../third_party/patch/to_mindrecord/CLUERNER2020/data_processor_seq.patch is not exist." + exit 1 +fi + +# patch for data_processor_seq.py +patch -p0 -d ../../../../third_party/to_mindrecord/CLUERNER2020/ -o data_processor_seq_patched.py < ../../../../third_party/patch/to_mindrecord/CLUERNER2020/data_processor_seq.patch +if [ $? -ne 0 ]; then + echo "Patch ../../../../third_party/to_mindrecord/CLUERNER2020/data_processor_seq.py failed" + exit 1 +fi + +# use patched script +python ../../../../third_party/to_mindrecord/CLUERNER2020/data_processor_seq_patched.py \ +--vocab_file=../../../../third_party/to_mindrecord/CLUERNER2020/vocab.txt \ +--label2id_file=../../../../third_party/to_mindrecord/CLUERNER2020/label2id.json diff --git a/example/nlp_to_mindrecord/CLUERNER2020/run_read.sh b/model_zoo/utils/nlp_to_mindrecord/CLUERNER2020/run_read.sh similarity index 100% rename from example/nlp_to_mindrecord/CLUERNER2020/run_read.sh rename to model_zoo/utils/nlp_to_mindrecord/CLUERNER2020/run_read.sh diff --git a/example/nlp_to_mindrecord/README.md b/model_zoo/utils/nlp_to_mindrecord/README.md similarity index 100% rename from example/nlp_to_mindrecord/README.md rename to model_zoo/utils/nlp_to_mindrecord/README.md diff --git a/example/nlp_to_mindrecord/aclImdb/README.md b/model_zoo/utils/nlp_to_mindrecord/aclImdb/README.md similarity index 100% rename from example/nlp_to_mindrecord/aclImdb/README.md rename to model_zoo/utils/nlp_to_mindrecord/aclImdb/README.md diff --git a/example/nlp_to_mindrecord/aclImdb/create_dataset.py b/model_zoo/utils/nlp_to_mindrecord/aclImdb/create_dataset.py similarity index 100% rename from example/nlp_to_mindrecord/aclImdb/create_dataset.py rename to model_zoo/utils/nlp_to_mindrecord/aclImdb/create_dataset.py diff --git a/example/nlp_to_mindrecord/aclImdb/data/README.md b/model_zoo/utils/nlp_to_mindrecord/aclImdb/data/README.md similarity index 100% rename from example/nlp_to_mindrecord/aclImdb/data/README.md rename to model_zoo/utils/nlp_to_mindrecord/aclImdb/data/README.md diff --git a/example/nlp_to_mindrecord/aclImdb/gen_mindrecord.py b/model_zoo/utils/nlp_to_mindrecord/aclImdb/gen_mindrecord.py similarity index 100% rename from example/nlp_to_mindrecord/aclImdb/gen_mindrecord.py rename to model_zoo/utils/nlp_to_mindrecord/aclImdb/gen_mindrecord.py diff --git a/example/nlp_to_mindrecord/aclImdb/output/README.md b/model_zoo/utils/nlp_to_mindrecord/aclImdb/output/README.md similarity index 100% rename from example/nlp_to_mindrecord/aclImdb/output/README.md rename to model_zoo/utils/nlp_to_mindrecord/aclImdb/output/README.md diff --git a/example/nlp_to_mindrecord/aclImdb/run.sh b/model_zoo/utils/nlp_to_mindrecord/aclImdb/run.sh similarity index 100% rename from example/nlp_to_mindrecord/aclImdb/run.sh rename to model_zoo/utils/nlp_to_mindrecord/aclImdb/run.sh diff --git a/example/nlp_to_mindrecord/aclImdb/run_read.sh b/model_zoo/utils/nlp_to_mindrecord/aclImdb/run_read.sh similarity index 100% rename from example/nlp_to_mindrecord/aclImdb/run_read.sh rename to model_zoo/utils/nlp_to_mindrecord/aclImdb/run_read.sh diff --git a/example/nlp_to_mindrecord/aclImdb_preprocess/README.md b/model_zoo/utils/nlp_to_mindrecord/aclImdb_preprocess/README.md similarity index 100% rename from example/nlp_to_mindrecord/aclImdb_preprocess/README.md rename to model_zoo/utils/nlp_to_mindrecord/aclImdb_preprocess/README.md diff --git a/example/nlp_to_mindrecord/aclImdb_preprocess/create_dataset.py b/model_zoo/utils/nlp_to_mindrecord/aclImdb_preprocess/create_dataset.py similarity index 100% rename from example/nlp_to_mindrecord/aclImdb_preprocess/create_dataset.py rename to model_zoo/utils/nlp_to_mindrecord/aclImdb_preprocess/create_dataset.py diff --git a/example/nlp_to_mindrecord/aclImdb_preprocess/data/README.md b/model_zoo/utils/nlp_to_mindrecord/aclImdb_preprocess/data/README.md similarity index 100% rename from example/nlp_to_mindrecord/aclImdb_preprocess/data/README.md rename to model_zoo/utils/nlp_to_mindrecord/aclImdb_preprocess/data/README.md diff --git a/example/nlp_to_mindrecord/aclImdb_preprocess/gen_mindrecord.py b/model_zoo/utils/nlp_to_mindrecord/aclImdb_preprocess/gen_mindrecord.py similarity index 100% rename from example/nlp_to_mindrecord/aclImdb_preprocess/gen_mindrecord.py rename to model_zoo/utils/nlp_to_mindrecord/aclImdb_preprocess/gen_mindrecord.py diff --git a/example/nlp_to_mindrecord/aclImdb_preprocess/output/README.md b/model_zoo/utils/nlp_to_mindrecord/aclImdb_preprocess/output/README.md similarity index 100% rename from example/nlp_to_mindrecord/aclImdb_preprocess/output/README.md rename to model_zoo/utils/nlp_to_mindrecord/aclImdb_preprocess/output/README.md diff --git a/example/nlp_to_mindrecord/aclImdb_preprocess/run.sh b/model_zoo/utils/nlp_to_mindrecord/aclImdb_preprocess/run.sh similarity index 100% rename from example/nlp_to_mindrecord/aclImdb_preprocess/run.sh rename to model_zoo/utils/nlp_to_mindrecord/aclImdb_preprocess/run.sh diff --git a/example/nlp_to_mindrecord/aclImdb_preprocess/run_read.sh b/model_zoo/utils/nlp_to_mindrecord/aclImdb_preprocess/run_read.sh similarity index 100% rename from example/nlp_to_mindrecord/aclImdb_preprocess/run_read.sh rename to model_zoo/utils/nlp_to_mindrecord/aclImdb_preprocess/run_read.sh diff --git a/example/nlp_to_mindrecord/enwiki/README.md b/model_zoo/utils/nlp_to_mindrecord/enwiki/README.md similarity index 100% rename from example/nlp_to_mindrecord/enwiki/README.md rename to model_zoo/utils/nlp_to_mindrecord/enwiki/README.md diff --git a/example/nlp_to_mindrecord/enwiki/create_dataset.py b/model_zoo/utils/nlp_to_mindrecord/enwiki/create_dataset.py similarity index 100% rename from example/nlp_to_mindrecord/enwiki/create_dataset.py rename to model_zoo/utils/nlp_to_mindrecord/enwiki/create_dataset.py diff --git a/example/nlp_to_mindrecord/enwiki/run.sh b/model_zoo/utils/nlp_to_mindrecord/enwiki/run.sh similarity index 83% rename from example/nlp_to_mindrecord/enwiki/run.sh rename to model_zoo/utils/nlp_to_mindrecord/enwiki/run.sh index cf66bed0fd..c36b6b9903 100644 --- a/example/nlp_to_mindrecord/enwiki/run.sh +++ b/model_zoo/utils/nlp_to_mindrecord/enwiki/run.sh @@ -66,20 +66,20 @@ getdir "${data_dir}" # echo "The input files: "${file_list[@]} # echo "The output files: "${output_filename[@]} -if [ ! -d "../../../third_party/to_mindrecord/zhwiki" ]; then - echo "The patch base dir ../../../third_party/to_mindrecord/zhwiki is not exist." +if [ ! -d "../../../../third_party/to_mindrecord/zhwiki" ]; then + echo "The patch base dir ../../../../third_party/to_mindrecord/zhwiki is not exist." exit 1 fi -if [ ! -f "../../../third_party/patch/to_mindrecord/zhwiki/create_pretraining_data.patch" ]; then - echo "The patch file ../../../third_party/patch/to_mindrecord/zhwiki/create_pretraining_data.patch is not exist." +if [ ! -f "../../../../third_party/patch/to_mindrecord/zhwiki/create_pretraining_data.patch" ]; then + echo "The patch file ../../../../third_party/patch/to_mindrecord/zhwiki/create_pretraining_data.patch is not exist." exit 1 fi # patch for create_pretraining_data.py -patch -p0 -d ../../../third_party/to_mindrecord/zhwiki/ -o create_pretraining_data_patched.py < ../../../third_party/patch/to_mindrecord/zhwiki/create_pretraining_data.patch +patch -p0 -d ../../../../third_party/to_mindrecord/zhwiki/ -o create_pretraining_data_patched.py < ../../../../third_party/patch/to_mindrecord/zhwiki/create_pretraining_data.patch if [ $? -ne 0 ]; then - echo "Patch ../../../third_party/to_mindrecord/zhwiki/create_pretraining_data.py failed" + echo "Patch ../../../../third_party/to_mindrecord/zhwiki/create_pretraining_data.py failed" exit 1 fi @@ -94,7 +94,7 @@ file_list_len=`expr ${#file_list[*]} - 1` for index in $(seq 0 $file_list_len); do echo "Begin preprocess input file: ${file_list[$index]}" echo "Begin output file: ${output_filename[$index]}" - python ../../../third_party/to_mindrecord/zhwiki/create_pretraining_data_patched.py \ + python ../../../../third_party/to_mindrecord/zhwiki/create_pretraining_data_patched.py \ --input_file=${file_list[$index]} \ --output_file=${output_dir}/${output_filename[$index]} \ --partition_number=1 \ diff --git a/example/nlp_to_mindrecord/enwiki/run_read.sh b/model_zoo/utils/nlp_to_mindrecord/enwiki/run_read.sh similarity index 100% rename from example/nlp_to_mindrecord/enwiki/run_read.sh rename to model_zoo/utils/nlp_to_mindrecord/enwiki/run_read.sh diff --git a/example/nlp_to_mindrecord/zhwiki/README.md b/model_zoo/utils/nlp_to_mindrecord/zhwiki/README.md similarity index 98% rename from example/nlp_to_mindrecord/zhwiki/README.md rename to model_zoo/utils/nlp_to_mindrecord/zhwiki/README.md index 1a9de05114..ad7afdd662 100644 --- a/example/nlp_to_mindrecord/zhwiki/README.md +++ b/model_zoo/utils/nlp_to_mindrecord/zhwiki/README.md @@ -26,7 +26,7 @@ This example is based on [zhwiki](https://dumps.wikimedia.org/zhwiki) training d Follow the step: ```bash -bash run_simple.sh # generate output/simple.mindrecord* by ../../../third_party/to_mindrecord/zhwiki/sample_text.txt +bash run_simple.sh # generate output/simple.mindrecord* by ../../../../third_party/to_mindrecord/zhwiki/sample_text.txt bash run_read_simple.sh # use MindDataset to read output/simple.mindrecord* ``` diff --git a/example/nlp_to_mindrecord/zhwiki/create_dataset.py b/model_zoo/utils/nlp_to_mindrecord/zhwiki/create_dataset.py similarity index 100% rename from example/nlp_to_mindrecord/zhwiki/create_dataset.py rename to model_zoo/utils/nlp_to_mindrecord/zhwiki/create_dataset.py diff --git a/example/nlp_to_mindrecord/zhwiki/data/.gitignore b/model_zoo/utils/nlp_to_mindrecord/zhwiki/data/.gitignore similarity index 100% rename from example/nlp_to_mindrecord/zhwiki/data/.gitignore rename to model_zoo/utils/nlp_to_mindrecord/zhwiki/data/.gitignore diff --git a/example/nlp_to_mindrecord/zhwiki/data/README.md b/model_zoo/utils/nlp_to_mindrecord/zhwiki/data/README.md similarity index 100% rename from example/nlp_to_mindrecord/zhwiki/data/README.md rename to model_zoo/utils/nlp_to_mindrecord/zhwiki/data/README.md diff --git a/example/nlp_to_mindrecord/zhwiki/output/README.md b/model_zoo/utils/nlp_to_mindrecord/zhwiki/output/README.md similarity index 100% rename from example/nlp_to_mindrecord/zhwiki/output/README.md rename to model_zoo/utils/nlp_to_mindrecord/zhwiki/output/README.md diff --git a/example/nlp_to_mindrecord/zhwiki/run.sh b/model_zoo/utils/nlp_to_mindrecord/zhwiki/run.sh similarity index 80% rename from example/nlp_to_mindrecord/zhwiki/run.sh rename to model_zoo/utils/nlp_to_mindrecord/zhwiki/run.sh index a057031e6b..3142056b11 100644 --- a/example/nlp_to_mindrecord/zhwiki/run.sh +++ b/model_zoo/utils/nlp_to_mindrecord/zhwiki/run.sh @@ -45,20 +45,20 @@ getdir "${data_dir}" # echo "The input files: "${file_list[@]} # echo "The output files: "${output_filename[@]} -if [ ! -d "../../../third_party/to_mindrecord/zhwiki" ]; then - echo "The patch base dir ../../../third_party/to_mindrecord/zhwiki is not exist." +if [ ! -d "../../../../third_party/to_mindrecord/zhwiki" ]; then + echo "The patch base dir ../../../../third_party/to_mindrecord/zhwiki is not exist." exit 1 fi -if [ ! -f "../../../third_party/patch/to_mindrecord/zhwiki/create_pretraining_data.patch" ]; then - echo "The patch file ../../../third_party/patch/to_mindrecord/zhwiki/create_pretraining_data.patch is not exist." +if [ ! -f "../../../../third_party/patch/to_mindrecord/zhwiki/create_pretraining_data.patch" ]; then + echo "The patch file ../../../../third_party/patch/to_mindrecord/zhwiki/create_pretraining_data.patch is not exist." exit 1 fi # patch for create_pretraining_data.py -patch -p0 -d ../../../third_party/to_mindrecord/zhwiki/ -o create_pretraining_data_patched.py < ../../../third_party/patch/to_mindrecord/zhwiki/create_pretraining_data.patch +patch -p0 -d ../../../../third_party/to_mindrecord/zhwiki/ -o create_pretraining_data_patched.py < ../../../../third_party/patch/to_mindrecord/zhwiki/create_pretraining_data.patch if [ $? -ne 0 ]; then - echo "Patch ../../../third_party/to_mindrecord/zhwiki/create_pretraining_data.py failed" + echo "Patch ../../../../third_party/to_mindrecord/zhwiki/create_pretraining_data.py failed" exit 1 fi @@ -73,11 +73,11 @@ file_list_len=`expr ${#file_list[*]} - 1` for index in $(seq 0 $file_list_len); do echo "Begin preprocess input file: ${file_list[$index]}" echo "Begin output file: ${output_filename[$index]}" - python ../../../third_party/to_mindrecord/zhwiki/create_pretraining_data_patched.py \ + python ../../../../third_party/to_mindrecord/zhwiki/create_pretraining_data_patched.py \ --input_file=${file_list[$index]} \ --output_file=output/${output_filename[$index]} \ --partition_number=1 \ - --vocab_file=../../../third_party/to_mindrecord/zhwiki/vocab.txt \ + --vocab_file=../../../../third_party/to_mindrecord/zhwiki/vocab.txt \ --do_lower_case=True \ --max_seq_length=128 \ --max_predictions_per_seq=20 \ diff --git a/example/nlp_to_mindrecord/zhwiki/run_read.sh b/model_zoo/utils/nlp_to_mindrecord/zhwiki/run_read.sh similarity index 100% rename from example/nlp_to_mindrecord/zhwiki/run_read.sh rename to model_zoo/utils/nlp_to_mindrecord/zhwiki/run_read.sh diff --git a/example/nlp_to_mindrecord/zhwiki/run_read_simple.sh b/model_zoo/utils/nlp_to_mindrecord/zhwiki/run_read_simple.sh similarity index 91% rename from example/nlp_to_mindrecord/zhwiki/run_read_simple.sh rename to model_zoo/utils/nlp_to_mindrecord/zhwiki/run_read_simple.sh index 1c26dec449..abecc20187 100644 --- a/example/nlp_to_mindrecord/zhwiki/run_read_simple.sh +++ b/model_zoo/utils/nlp_to_mindrecord/zhwiki/run_read_simple.sh @@ -15,4 +15,4 @@ # ============================================================================ # create dataset for train -python create_dataset.py --input_file=output/simple.mindrecord0 +python create_dataset.py --input_file=output/simple.mindrecord diff --git a/example/nlp_to_mindrecord/zhwiki/run_simple.sh b/model_zoo/utils/nlp_to_mindrecord/zhwiki/run_simple.sh similarity index 53% rename from example/nlp_to_mindrecord/zhwiki/run_simple.sh rename to model_zoo/utils/nlp_to_mindrecord/zhwiki/run_simple.sh index 20c1d98d66..a41df700b4 100644 --- a/example/nlp_to_mindrecord/zhwiki/run_simple.sh +++ b/model_zoo/utils/nlp_to_mindrecord/zhwiki/run_simple.sh @@ -16,29 +16,29 @@ rm -f output/simple.mindrecord* -if [ ! -d "../../../third_party/to_mindrecord/zhwiki" ]; then - echo "The patch base dir ../../../third_party/to_mindrecord/zhwiki is not exist." +if [ ! -d "../../../../third_party/to_mindrecord/zhwiki" ]; then + echo "The patch base dir ../../../../third_party/to_mindrecord/zhwiki is not exist." exit 1 fi -if [ ! -f "../../../third_party/patch/to_mindrecord/zhwiki/create_pretraining_data.patch" ]; then - echo "The patch file ../../../third_party/patch/to_mindrecord/zhwiki/create_pretraining_data.patch is not exist." +if [ ! -f "../../../../third_party/patch/to_mindrecord/zhwiki/create_pretraining_data.patch" ]; then + echo "The patch file ../../../../third_party/patch/to_mindrecord/zhwiki/create_pretraining_data.patch is not exist." exit 1 fi # patch for create_pretraining_data.py -patch -p0 -d ../../../third_party/to_mindrecord/zhwiki/ -o create_pretraining_data_patched.py < ../../../third_party/patch/to_mindrecord/zhwiki/create_pretraining_data.patch +patch -p0 -d ../../../../third_party/to_mindrecord/zhwiki/ -o create_pretraining_data_patched.py < ../../../../third_party/patch/to_mindrecord/zhwiki/create_pretraining_data.patch if [ $? -ne 0 ]; then - echo "Patch ../../../third_party/to_mindrecord/zhwiki/create_pretraining_data.py failed" + echo "Patch ../../../../third_party/to_mindrecord/zhwiki/create_pretraining_data.py failed" exit 1 fi # using patched script to generate mindrecord -python ../../../third_party/to_mindrecord/zhwiki/create_pretraining_data_patched.py \ ---input_file=../../../third_party/to_mindrecord/zhwiki/sample_text.txt \ +python ../../../../third_party/to_mindrecord/zhwiki/create_pretraining_data_patched.py \ +--input_file=../../../../third_party/to_mindrecord/zhwiki/sample_text.txt \ --output_file=output/simple.mindrecord \ ---partition_number=4 \ ---vocab_file=../../../third_party/to_mindrecord/zhwiki/vocab.txt \ +--partition_number=1 \ +--vocab_file=../../../../third_party/to_mindrecord/zhwiki/vocab.txt \ --do_lower_case=True \ --max_seq_length=128 \ --max_predictions_per_seq=20 \ From 699dc99bc8c24f3f6df041183b1aa01b8bf3a9ee Mon Sep 17 00:00:00 2001 From: buxue Date: Sun, 28 Jun 2020 19:13:36 +0800 Subject: [PATCH 129/254] add typeid to type conversion scene --- mindspore/ccsrc/debug/anf_ir_utils.cc | 2 +- mindspore/ccsrc/ir/dtype.h | 5 ++++ mindspore/ccsrc/ir/dtype/empty.cc | 2 ++ mindspore/ccsrc/ir/dtype/empty.h | 14 ++++++----- mindspore/ccsrc/ir/dtype_extends.cc | 23 +++++++++++++++++-- mindspore/ccsrc/ir/func_graph.cc | 8 +++---- mindspore/ccsrc/ir/named.cc | 8 +++---- mindspore/ccsrc/ir/named.h | 16 ++++++------- .../pipeline/static_analysis/abstract_value.h | 4 ++-- mindspore/ccsrc/utils/convert_utils.cc | 2 +- mindspore/common/dtype.py | 2 +- 11 files changed, 57 insertions(+), 29 deletions(-) diff --git a/mindspore/ccsrc/debug/anf_ir_utils.cc b/mindspore/ccsrc/debug/anf_ir_utils.cc index e4d21a74a4..c797b8efea 100644 --- a/mindspore/ccsrc/debug/anf_ir_utils.cc +++ b/mindspore/ccsrc/debug/anf_ir_utils.cc @@ -403,7 +403,7 @@ std::string AnfExporter::GetValueText(const FuncGraphPtr &func_graph, const Valu } else if (value->isa()) { auto tensor_ptr = dyn_cast(value); oss << value->DumpText() << "@" << DumpObject(TensorPy::AsNumpy(*tensor_ptr), "T"); - } else if (value->isa() || value->isa() || value->isa()) { + } else if (value->isa() || value->isa() || value->isa()) { oss << value->DumpText(); } else if (value->isa()) { oss << GetSequenceText(func_graph, value); diff --git a/mindspore/ccsrc/ir/dtype.h b/mindspore/ccsrc/ir/dtype.h index 104a2ec40a..9659a27e36 100644 --- a/mindspore/ccsrc/ir/dtype.h +++ b/mindspore/ccsrc/ir/dtype.h @@ -275,6 +275,11 @@ extern const TypePtr kTypeExternal; extern const TypePtr kTypeEnv; extern const TypePtr kTypeType; extern const TypePtr kString; +extern const TypePtr kList; +extern const TypePtr kTuple; +extern const TypePtr kDict; +extern const TypePtr kSlice; +extern const TypePtr kKeyword; extern const TypePtr kTensorType; } // namespace mindspore diff --git a/mindspore/ccsrc/ir/dtype/empty.cc b/mindspore/ccsrc/ir/dtype/empty.cc index 5cb3a91806..c6abe9a408 100644 --- a/mindspore/ccsrc/ir/dtype/empty.cc +++ b/mindspore/ccsrc/ir/dtype/empty.cc @@ -18,5 +18,7 @@ namespace mindspore { const TypePtr kTypeNone = std::make_shared(); +const TypePtr kTypeNull = std::make_shared(); +const TypePtr kTypeEllipsis = std::make_shared(); const TypePtr kAnyType = std::make_shared(); } // namespace mindspore diff --git a/mindspore/ccsrc/ir/dtype/empty.h b/mindspore/ccsrc/ir/dtype/empty.h index 76cf8ea0eb..e3b46ec7d9 100644 --- a/mindspore/ccsrc/ir/dtype/empty.h +++ b/mindspore/ccsrc/ir/dtype/empty.h @@ -71,20 +71,22 @@ class TypeNull : public Type { }; using TypeNullPtr = std::shared_ptr; -class Ellipsis : public Type { +class TypeEllipsis : public Type { public: - Ellipsis() : Type(kMetaTypeEllipsis) {} - ~Ellipsis() override {} - MS_DECLARE_PARENT(Ellipsis, Type) + TypeEllipsis() : Type(kMetaTypeEllipsis) {} + ~TypeEllipsis() override {} + MS_DECLARE_PARENT(TypeEllipsis, Type) TypeId generic_type_id() const override { return kMetaTypeEllipsis; } - TypePtr DeepCopy() const override { return std::make_shared(); } + TypePtr DeepCopy() const override { return std::make_shared(); } std::string ToReprString() const override { return "Ellipsis"; } std::string DumpText() const override { return "Ellipsis"; } }; -using EllipsisPtr = std::shared_ptr; +using TypeEllipsisPtr = std::shared_ptr; extern const TypePtr kTypeNone; +extern const TypePtr kTypeNull; +extern const TypePtr kTypeEllipsis; extern const TypePtr kAnyType; } // namespace mindspore diff --git a/mindspore/ccsrc/ir/dtype_extends.cc b/mindspore/ccsrc/ir/dtype_extends.cc index 90e181949c..e7af812922 100644 --- a/mindspore/ccsrc/ir/dtype_extends.cc +++ b/mindspore/ccsrc/ir/dtype_extends.cc @@ -95,6 +95,10 @@ TypePtr TypeIdToType(TypeId id) { return kAnyType; case kMetaTypeNone: return kTypeNone; + case kMetaTypeNull: + return kTypeNull; + case kMetaTypeEllipsis: + return kTypeEllipsis; case kObjectTypeEnvType: return kTypeEnv; case kObjectTypeRefKey: @@ -105,6 +109,16 @@ TypePtr TypeIdToType(TypeId id) { return kTypeType; case kObjectTypeString: return kString; + case kObjectTypeList: + return kList; + case kObjectTypeTuple: + return kTuple; + case kObjectTypeDictionary: + return kDict; + case kObjectTypeSlice: + return kSlice; + case kObjectTypeKeyword: + return kKeyword; case kTypeUnknown: return kTypeNone; default: @@ -278,7 +292,7 @@ TypePtr StringToType(const std::string &type_name) { if (type_name.compare("None") == 0) { type = std::make_shared(); } else if (type_name.compare("Ellipsis") == 0) { - type = std::make_shared(); + type = std::make_shared(); } else if (type_name.compare("TypeType") == 0) { type = std::make_shared(); } else if (type_name.compare("SymbolicKeyType") == 0) { @@ -480,7 +494,7 @@ REGISTER_PYBIND_DEFINE( (void)py::class_>(m_sub, "RefType").def(py::init()); (void)py::class_>(m_sub, "TypeAnything").def(py::init()); (void)py::class_>(m_sub, "Slice").def(py::init()); - (void)py::class_>(m_sub, "Ellipsis").def(py::init()); + (void)py::class_>(m_sub, "TypeEllipsis").def(py::init()); })); const TypePtr kTypeExternal = std::make_shared(); @@ -488,4 +502,9 @@ const TypePtr kTypeEnv = std::make_shared(); const TypePtr kTypeType = std::make_shared(); const TypePtr kTensorType = std::make_shared(); const TypePtr kString = std::make_shared(); +const TypePtr kList = std::make_shared(); +const TypePtr kTuple = std::make_shared(); +const TypePtr kDict = std::make_shared(); +const TypePtr kSlice = std::make_shared(); +const TypePtr kKeyword = std::make_shared(); } // namespace mindspore diff --git a/mindspore/ccsrc/ir/func_graph.cc b/mindspore/ccsrc/ir/func_graph.cc index cdca98fc61..803c910d1d 100644 --- a/mindspore/ccsrc/ir/func_graph.cc +++ b/mindspore/ccsrc/ir/func_graph.cc @@ -432,7 +432,7 @@ AnfNodePtr FuncGraph::GetDefaultValueByName(const std::string &name) { if (default_value == nullptr) { MS_LOG(EXCEPTION) << "Graph parameter " << name << " not exist"; } - if (IsValueNode(default_value)) { + if (IsValueNode(default_value)) { return nullptr; } return default_value; @@ -440,8 +440,8 @@ AnfNodePtr FuncGraph::GetDefaultValueByName(const std::string &name) { // set the default values void FuncGraph::SetDefaultValues(const std::vector &name_list, const std::vector &value_list) { - auto all_is_null = std::all_of(value_list.begin(), value_list.end(), - [](const AnfNodePtr &node) { return IsValueNode(node); }); + auto all_is_null = + std::all_of(value_list.begin(), value_list.end(), [](const AnfNodePtr &node) { return IsValueNode(node); }); if (value_list.empty()) { all_is_null = true; } @@ -457,7 +457,7 @@ void FuncGraph::ClearDefaultValues() { parameter_default_value_.clear(); } size_t FuncGraph::GetDefaultValueCount() { int null_count = std::count_if(parameter_default_value_.begin(), parameter_default_value_.end(), - [](const std::pair &pair) { return IsValueNode(pair.second); }); + [](const std::pair &pair) { return IsValueNode(pair.second); }); return parameter_default_value_.size() - IntToSize(null_count); } diff --git a/mindspore/ccsrc/ir/named.cc b/mindspore/ccsrc/ir/named.cc index 0a679e6011..9e1a7968b8 100644 --- a/mindspore/ccsrc/ir/named.cc +++ b/mindspore/ccsrc/ir/named.cc @@ -30,9 +30,9 @@ bool Named::operator==(const Value &other) const { abstract::AbstractBasePtr None::ToAbstract() { return std::make_shared(); } const NamedPtr kNone = std::make_shared(); -abstract::AbstractBasePtr NullObj::ToAbstract() { return std::make_shared(); } -const NamedPtr kNull = std::make_shared(); +abstract::AbstractBasePtr Null::ToAbstract() { return std::make_shared(); } +const NamedPtr kNull = std::make_shared(); -abstract::AbstractBasePtr EllipsisObj::ToAbstract() { return std::make_shared(); } -const NamedPtr kEllipsis = std::make_shared(); +abstract::AbstractBasePtr Ellipsis::ToAbstract() { return std::make_shared(); } +const NamedPtr kEllipsis = std::make_shared(); } // namespace mindspore diff --git a/mindspore/ccsrc/ir/named.h b/mindspore/ccsrc/ir/named.h index fbc7969cab..40e544c129 100644 --- a/mindspore/ccsrc/ir/named.h +++ b/mindspore/ccsrc/ir/named.h @@ -71,20 +71,20 @@ class None : public Named { }; extern const NamedPtr kNone; -class NullObj : public Named { +class Null : public Named { public: - NullObj() : Named("Null") {} - ~NullObj() override = default; - MS_DECLARE_PARENT(NullObj, Named); + Null() : Named("Null") {} + ~Null() override = default; + MS_DECLARE_PARENT(Null, Named); abstract::AbstractBasePtr ToAbstract() override; }; extern const NamedPtr kNull; -class EllipsisObj : public Named { +class Ellipsis : public Named { public: - EllipsisObj() : Named("Ellipsis") {} - ~EllipsisObj() override = default; - MS_DECLARE_PARENT(EllipsisObj, Named); + Ellipsis() : Named("Ellipsis") {} + ~Ellipsis() override = default; + MS_DECLARE_PARENT(Ellipsis, Named); abstract::AbstractBasePtr ToAbstract() override; }; extern const NamedPtr kEllipsis; diff --git a/mindspore/ccsrc/pipeline/static_analysis/abstract_value.h b/mindspore/ccsrc/pipeline/static_analysis/abstract_value.h index f3375d22d6..5b54c749b6 100644 --- a/mindspore/ccsrc/pipeline/static_analysis/abstract_value.h +++ b/mindspore/ccsrc/pipeline/static_analysis/abstract_value.h @@ -515,11 +515,11 @@ using AbstractNullPtr = std::shared_ptr; class AbstractEllipsis : public AbstractBase { public: - AbstractEllipsis() : AbstractBase(kEllipsis) { set_type(std::make_shared()); } + AbstractEllipsis() : AbstractBase(kEllipsis) { set_type(std::make_shared()); } ~AbstractEllipsis() override = default; MS_DECLARE_PARENT(AbstractEllipsis, AbstractBase) - TypePtr BuildType() const override { return std::make_shared(); } + TypePtr BuildType() const override { return std::make_shared(); } bool operator==(const AbstractEllipsis &other) const; bool operator==(const AbstractBase &other) const override; AbstractBasePtr Clone() const override { return std::make_shared(); } diff --git a/mindspore/ccsrc/utils/convert_utils.cc b/mindspore/ccsrc/utils/convert_utils.cc index 15e08f7794..8cb071b769 100644 --- a/mindspore/ccsrc/utils/convert_utils.cc +++ b/mindspore/ccsrc/utils/convert_utils.cc @@ -105,7 +105,7 @@ py::object ValuePtrToPyData(const ValuePtr &value) { i++; } ret = rets; - } else if (value->isa()) { + } else if (value->isa()) { ret = py::ellipsis(); } else if (value->isa()) { auto slice = value->cast(); diff --git a/mindspore/common/dtype.py b/mindspore/common/dtype.py index 46b111d2f6..73aa67f67a 100644 --- a/mindspore/common/dtype.py +++ b/mindspore/common/dtype.py @@ -96,7 +96,7 @@ type_refkey = typing.RefKeyType() tensor_type = typing.TensorType anything_type = typing.TypeAnything slice_type = typing.Slice -ellipsis_type = typing.Ellipsis +ellipsis_type = typing.TypeEllipsis number_type = (int8, int16, From 2097a0e90a15fb2b9c15dcda076985f5d4bfa878 Mon Sep 17 00:00:00 2001 From: liuxiao Date: Wed, 24 Jun 2020 16:28:55 +0800 Subject: [PATCH 130/254] Add optimizer operators for VM. --- mindspore/ccsrc/kernel/tbe/tbe_adapter.cc | 2 + mindspore/ops/_op_impl/tbe/__init__.py | 4 + mindspore/ops/_op_impl/tbe/apply_add_sign.py | 65 ++++ .../_op_impl/tbe/apply_gradient_descent.py | 44 +++ .../ops/_op_impl/tbe/apply_power_sign.py | 65 ++++ .../tbe/apply_proximal_gradient_descent.py | 54 +++ mindspore/ops/operations/__init__.py | 5 + mindspore/ops/operations/nn_ops.py | 361 +++++++++++++++++- tests/ut/python/ops/test_ops.py | 74 ++++ 9 files changed, 665 insertions(+), 9 deletions(-) create mode 100644 mindspore/ops/_op_impl/tbe/apply_add_sign.py create mode 100644 mindspore/ops/_op_impl/tbe/apply_gradient_descent.py create mode 100644 mindspore/ops/_op_impl/tbe/apply_power_sign.py create mode 100644 mindspore/ops/_op_impl/tbe/apply_proximal_gradient_descent.py diff --git a/mindspore/ccsrc/kernel/tbe/tbe_adapter.cc b/mindspore/ccsrc/kernel/tbe/tbe_adapter.cc index deb858ff39..cbc31415ec 100644 --- a/mindspore/ccsrc/kernel/tbe/tbe_adapter.cc +++ b/mindspore/ccsrc/kernel/tbe/tbe_adapter.cc @@ -77,6 +77,8 @@ static std::map tbe_func_adapter_map = { {"sparse_apply_adagrad", "sparse_apply_adagrad_d"}, {"apply_proximal_adagrad", "apply_proximal_adagrad_d"}, {"sparse_apply_proximal_adagrad", "sparse_apply_proximal_adagrad_d"}, + {"apply_add_sign", "apply_add_sign_d"}, + {"apply_power_sign", "apply_power_sign_d"}, {"transpose", "transpose_d"}, {"fill", "fill_d"}, {"unsorted_segment_sum", "unsorted_segment_sum_d"}, diff --git a/mindspore/ops/_op_impl/tbe/__init__.py b/mindspore/ops/_op_impl/tbe/__init__.py index 7207e5ee69..c16e16f96c 100644 --- a/mindspore/ops/_op_impl/tbe/__init__.py +++ b/mindspore/ops/_op_impl/tbe/__init__.py @@ -34,6 +34,10 @@ from .apply_ada_max import _apply_ada_max_tbe from .apply_adadelta import _apply_adadelta_tbe from .apply_adagrad import _apply_adagrad_tbe from .apply_adagrad_v2 import _apply_adagrad_v2_tbe +from .apply_add_sign import _apply_add_sign_tbe +from .apply_power_sign import _apply_power_sign_tbe +from .apply_gradient_descent import _apply_gradient_descent_tbe +from .apply_proximal_gradient_descent import _apply_proximal_gradient_descent_tbe from .approximate_equal import _approximate_equal_tbe from .adam_apply_one import _adam_apply_one_tbe from .assign import _assign_tbe diff --git a/mindspore/ops/_op_impl/tbe/apply_add_sign.py b/mindspore/ops/_op_impl/tbe/apply_add_sign.py new file mode 100644 index 0000000000..4a9c3c4be2 --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/apply_add_sign.py @@ -0,0 +1,65 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""ApplyAddSignD op""" +from mindspore.ops.op_info_register import op_info_register, TBERegOp, DataType + +apply_add_sign_d_op_info = TBERegOp("ApplyAddSign") \ + .fusion_type("OPAQUE") \ + .async_flag(False) \ + .binfile_name("apply_add_sign_d.so") \ + .compute_cost(10) \ + .kernel_name("apply_add_sign_d") \ + .partial_flag(True) \ + .input(0, "var", False, "required", "all") \ + .input(1, "m", False, "required", "all") \ + .input(2, "lr", False, "required", "all") \ + .input(3, "alpha", False, "required", "all") \ + .input(4, "sign_decay", False, "required", "all") \ + .input(5, "beta", False, "required", "all") \ + .input(6, "grad", False, "required", "all") \ + .output(0, "var", False, "required", "all") \ + .output(1, "m", False, "required", "all") \ + .dtype_format(DataType.F16_5HD, DataType.F16_5HD, DataType.F16_Default, DataType.F16_Default, + DataType.F16_Default, DataType.F16_Default, DataType.F16_5HD, DataType.F16_5HD, + DataType.F16_5HD) \ + .dtype_format(DataType.F16_C1HWNCoC0, DataType.F16_C1HWNCoC0, DataType.F16_Default, DataType.F16_Default, + DataType.F16_Default, DataType.F16_Default, DataType.F16_C1HWNCoC0, DataType.F16_C1HWNCoC0, + DataType.F16_C1HWNCoC0) \ + .dtype_format(DataType.F16_Default, DataType.F16_Default, DataType.F16_Default, DataType.F16_Default, + DataType.F16_Default, DataType.F16_Default, DataType.F16_Default, DataType.F16_Default, + DataType.F16_Default) \ + .dtype_format(DataType.F16_FracZ, DataType.F16_FracZ, DataType.F16_Default, DataType.F16_Default, + DataType.F16_Default, DataType.F16_Default, DataType.F16_FracZ, DataType.F16_FracZ, + DataType.F16_FracZ) \ + .dtype_format(DataType.F32_5HD, DataType.F32_5HD, DataType.F32_Default, DataType.F32_Default, + DataType.F32_Default, DataType.F32_Default, DataType.F32_5HD, DataType.F32_5HD, + DataType.F32_5HD) \ + .dtype_format(DataType.F32_C1HWNCoC0, DataType.F32_C1HWNCoC0, DataType.F32_Default, DataType.F32_Default, + DataType.F32_Default, DataType.F32_Default, DataType.F32_C1HWNCoC0, DataType.F32_C1HWNCoC0, + DataType.F32_C1HWNCoC0) \ + .dtype_format(DataType.F32_Default, DataType.F32_Default, DataType.F32_Default, DataType.F32_Default, + DataType.F32_Default, DataType.F32_Default, DataType.F32_Default, DataType.F32_Default, + DataType.F32_Default) \ + .dtype_format(DataType.F32_FracZ, DataType.F32_FracZ, DataType.F32_Default, DataType.F32_Default, + DataType.F32_Default, DataType.F32_Default, DataType.F32_FracZ, DataType.F32_FracZ, + DataType.F32_FracZ) \ + .get_op_info() + + +@op_info_register(apply_add_sign_d_op_info) +def _apply_add_sign_tbe(): + """ApplyAddSignD TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/apply_gradient_descent.py b/mindspore/ops/_op_impl/tbe/apply_gradient_descent.py new file mode 100644 index 0000000000..c3276f8595 --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/apply_gradient_descent.py @@ -0,0 +1,44 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""ApplyGradientDescent op""" +from mindspore.ops.op_info_register import op_info_register, TBERegOp, DataType + +apply_gradient_descent_op_info = TBERegOp("ApplyGradientDescent") \ + .fusion_type("OPAQUE") \ + .async_flag(False) \ + .binfile_name("apply_gradient_descent.so") \ + .compute_cost(10) \ + .kernel_name("apply_gradient_descent") \ + .partial_flag(True) \ + .input(0, "var", False, "required", "all") \ + .input(1, "alpha", False, "required", "all") \ + .input(2, "delta", False, "required", "all") \ + .output(0, "var", False, "required", "all") \ + .dtype_format(DataType.F16_5HD, DataType.F16_Default, DataType.F16_5HD, DataType.F16_5HD) \ + .dtype_format(DataType.F16_FracZ, DataType.F16_Default, DataType.F16_FracZ, DataType.F16_FracZ) \ + .dtype_format(DataType.F16_C1HWNCoC0, DataType.F16_Default, DataType.F16_C1HWNCoC0, DataType.F16_C1HWNCoC0) \ + .dtype_format(DataType.F16_Default, DataType.F16_Default, DataType.F16_Default, DataType.F16_Default) \ + .dtype_format(DataType.F32_5HD, DataType.F32_Default, DataType.F32_5HD, DataType.F32_5HD) \ + .dtype_format(DataType.F32_FracZ, DataType.F32_Default, DataType.F32_FracZ, DataType.F32_FracZ) \ + .dtype_format(DataType.F32_C1HWNCoC0, DataType.F32_Default, DataType.F32_C1HWNCoC0, DataType.F32_C1HWNCoC0) \ + .dtype_format(DataType.F32_Default, DataType.F32_Default, DataType.F32_Default, DataType.F32_Default) \ + .get_op_info() + + +@op_info_register(apply_gradient_descent_op_info) +def _apply_gradient_descent_tbe(): + """ApplyGradientDescent TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/apply_power_sign.py b/mindspore/ops/_op_impl/tbe/apply_power_sign.py new file mode 100644 index 0000000000..136391e91e --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/apply_power_sign.py @@ -0,0 +1,65 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""ApplyPowerSignD op""" +from mindspore.ops.op_info_register import op_info_register, TBERegOp, DataType + +apply_power_sign_d_op_info = TBERegOp("ApplyPowerSign") \ + .fusion_type("OPAQUE") \ + .async_flag(False) \ + .binfile_name("apply_power_sign_d.so") \ + .compute_cost(10) \ + .kernel_name("apply_power_sign_d") \ + .partial_flag(True) \ + .input(0, "var", False, "required", "all") \ + .input(1, "m", False, "required", "all") \ + .input(2, "lr", False, "required", "all") \ + .input(3, "logbase", False, "required", "all") \ + .input(4, "sign_decay", False, "required", "all") \ + .input(5, "beta", False, "required", "all") \ + .input(6, "grad", False, "required", "all") \ + .output(0, "var", False, "required", "all") \ + .output(1, "m", False, "required", "all") \ + .dtype_format(DataType.F16_5HD, DataType.F16_5HD, DataType.F16_Default, DataType.F16_Default, + DataType.F16_Default, DataType.F16_Default, DataType.F16_5HD, DataType.F16_5HD, + DataType.F16_5HD) \ + .dtype_format(DataType.F16_C1HWNCoC0, DataType.F16_C1HWNCoC0, DataType.F16_Default, DataType.F16_Default, + DataType.F16_Default, DataType.F16_Default, DataType.F16_C1HWNCoC0, DataType.F16_C1HWNCoC0, + DataType.F16_C1HWNCoC0) \ + .dtype_format(DataType.F16_Default, DataType.F16_Default, DataType.F16_Default, DataType.F16_Default, + DataType.F16_Default, DataType.F16_Default, DataType.F16_Default, DataType.F16_Default, + DataType.F16_Default) \ + .dtype_format(DataType.F16_FracZ, DataType.F16_FracZ, DataType.F16_Default, DataType.F16_Default, + DataType.F16_Default, DataType.F16_Default, DataType.F16_FracZ, DataType.F16_FracZ, + DataType.F16_FracZ) \ + .dtype_format(DataType.F32_5HD, DataType.F32_5HD, DataType.F32_Default, DataType.F32_Default, + DataType.F32_Default, DataType.F32_Default, DataType.F32_5HD, DataType.F32_5HD, + DataType.F32_5HD) \ + .dtype_format(DataType.F32_C1HWNCoC0, DataType.F32_C1HWNCoC0, DataType.F32_Default, DataType.F32_Default, + DataType.F32_Default, DataType.F32_Default, DataType.F32_C1HWNCoC0, DataType.F32_C1HWNCoC0, + DataType.F32_C1HWNCoC0) \ + .dtype_format(DataType.F32_Default, DataType.F32_Default, DataType.F32_Default, DataType.F32_Default, + DataType.F32_Default, DataType.F32_Default, DataType.F32_Default, DataType.F32_Default, + DataType.F32_Default) \ + .dtype_format(DataType.F32_FracZ, DataType.F32_FracZ, DataType.F32_Default, DataType.F32_Default, + DataType.F32_Default, DataType.F32_Default, DataType.F32_FracZ, DataType.F32_FracZ, + DataType.F32_FracZ) \ + .get_op_info() + + +@op_info_register(apply_power_sign_d_op_info) +def _apply_power_sign_tbe(): + """ApplyPowerSignD TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/apply_proximal_gradient_descent.py b/mindspore/ops/_op_impl/tbe/apply_proximal_gradient_descent.py new file mode 100644 index 0000000000..aca521d84b --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/apply_proximal_gradient_descent.py @@ -0,0 +1,54 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""ApplyProximalGradientDescent op""" +from mindspore.ops.op_info_register import op_info_register, TBERegOp, DataType + +apply_proximal_gradient_descent_op_info = TBERegOp("ApplyProximalGradientDescent") \ + .fusion_type("OPAQUE") \ + .async_flag(False) \ + .binfile_name("apply_proximal_gradient_descent.so") \ + .compute_cost(10) \ + .kernel_name("apply_proximal_gradient_descent") \ + .partial_flag(True) \ + .input(0, "var", False, "required", "all") \ + .input(1, "alpha", False, "required", "all") \ + .input(2, "l1", False, "required", "all") \ + .input(3, "l2", False, "required", "all") \ + .input(4, "delta", False, "required", "all") \ + .output(0, "var", False, "required", "all") \ + .dtype_format(DataType.F16_5HD, DataType.F16_Default, DataType.F16_Default, DataType.F16_Default, + DataType.F16_5HD, DataType.F16_5HD) \ + .dtype_format(DataType.F16_FracZ, DataType.F16_Default, DataType.F16_Default, DataType.F16_Default, + DataType.F16_FracZ, DataType.F16_FracZ) \ + .dtype_format(DataType.F16_C1HWNCoC0, DataType.F16_Default, DataType.F16_Default, DataType.F16_Default, + DataType.F16_C1HWNCoC0, DataType.F16_C1HWNCoC0) \ + .dtype_format(DataType.F16_Default, DataType.F16_Default, DataType.F16_Default, DataType.F16_Default, + DataType.F16_Default, DataType.F16_Default) \ + .dtype_format(DataType.F32_5HD, DataType.F32_Default, DataType.F32_Default, DataType.F32_Default, + DataType.F32_5HD, DataType.F32_5HD) \ + .dtype_format(DataType.F32_FracZ, DataType.F32_Default, DataType.F32_Default, DataType.F32_Default, + DataType.F32_FracZ, DataType.F32_FracZ) \ + .dtype_format(DataType.F32_C1HWNCoC0, DataType.F32_Default, DataType.F32_Default, DataType.F32_Default, + DataType.F32_C1HWNCoC0, DataType.F32_C1HWNCoC0) \ + .dtype_format(DataType.F32_Default, DataType.F32_Default, DataType.F32_Default, DataType.F32_Default, + DataType.F32_Default, DataType.F32_Default) \ + .get_op_info() + + +@op_info_register(apply_proximal_gradient_descent_op_info) +def _apply_proximal_gradient_descent_tbe(): + """ApplyProximalGradientDescent TBE register""" + return diff --git a/mindspore/ops/operations/__init__.py b/mindspore/ops/operations/__init__.py index 55db22ecb9..78db290784 100644 --- a/mindspore/ops/operations/__init__.py +++ b/mindspore/ops/operations/__init__.py @@ -74,6 +74,7 @@ from .nn_ops import (LSTM, SGD, Adam, SparseApplyAdam, SparseApplyLazyAdam, Appl TopK, BinaryCrossEntropy, SparseApplyAdagrad, LARSUpdate, ApplyFtrl, SparseApplyFtrl, ApplyProximalAdagrad, SparseApplyProximalAdagrad, ApplyAdaMax, ApplyAdadelta, ApplyAdagrad, ApplyAdagradV2, + ApplyAddSign, ApplyPowerSign, ApplyGradientDescent, ApplyProximalGradientDescent, ApplyRMSProp, ApplyCenteredRMSProp, BasicLSTMCell, InTopK) from .other_ops import (Assign, IOU, BoundingBoxDecode, BoundingBoxEncode, CheckValid, MakeRefKey, Partial, Depend, CheckBprop) @@ -295,6 +296,10 @@ __all__ = [ "ApplyAdadelta", "ApplyAdagrad", "ApplyAdagradV2", + "ApplyAddSign", + "ApplyPowerSign", + "ApplyGradientDescent", + "ApplyProximalGradientDescent", "BatchToSpace", "Atan2", "ApplyRMSProp", diff --git a/mindspore/ops/operations/nn_ops.py b/mindspore/ops/operations/nn_ops.py index 24b081bdad..0c5677871f 100644 --- a/mindspore/ops/operations/nn_ops.py +++ b/mindspore/ops/operations/nn_ops.py @@ -3382,7 +3382,7 @@ class ApplyAdagrad(PrimitiveWithInfer): - **var** (Parameter) - Variable to be updated. With float32 or float16 data type. - **accum** (Parameter) - Accum to be updated. The shape and dtype should be the same as `var`. With float32 or float16 data type. - - **lr** (Union[Number, Tensor]): The learning rate value, should be scalar. With float32 or float16 data type. + - **lr** (Union[Number, Tensor]) - The learning rate value, should be scalar. With float32 or float16 data type. - **grad** (Tensor) - A tensor for gradient. The shape and dtype should be the same as `var`. With float32 or float16 data type. @@ -3458,7 +3458,7 @@ class ApplyAdagradV2(PrimitiveWithInfer): - **var** (Parameter) - Variable to be updated. With float32 or float16 data type. - **accum** (Parameter) - Accum to be updated. The shape and dtype should be the same as `var`. With float32 or float16 data type. - - **lr** (Union[Number, Tensor]): The learning rate value, should be scalar. With float32 or float16 data type. + - **lr** (Union[Number, Tensor]) - The learning rate value, should be scalar. With float32 or float16 data type. - **grad** (Tensor) - A tensor for gradient. The shape and dtype should be the same as `var`. With float32 or float16 data type. @@ -3615,11 +3615,11 @@ class ApplyProximalAdagrad(PrimitiveWithInfer): Inputs: - **var** (Parameter) - Variable to be updated. The data type should be float16 or float32. - **accum** (Parameter) - Accum to be updated. Must has the same shape and dtype as `var`. - - **lr** (Union[Number, Tensor]): The learning rate value, should be scalar. The data type should be + - **lr** (Union[Number, Tensor]) - The learning rate value, should be scalar. The data type should be float16 or float32. - - **l1** (Union[Number, Tensor]): l1 regularization strength, should be scalar. The data type should be + - **l1** (Union[Number, Tensor]) - l1 regularization strength, should be scalar. The data type should be float16 or float32. - - **l2** (Union[Number, Tensor]): l2 regularization strength, should be scalar. The data type should be + - **l2** (Union[Number, Tensor]) - l2 regularization strength, should be scalar. The data type should be float16 or float32. - **grad** (Tensor) - Gradient. Must has the same shape and dtype as `var`. @@ -3710,9 +3710,9 @@ class SparseApplyProximalAdagrad(PrimitiveWithInfer): Inputs: - **var** (Parameter) - Variable tensor to be updated. The data type must be float32. - **accum** (Parameter) - Variable tensor to be updated. Has the same dtype as `var`. - - **lr** (Union[Number, Tensor]): The learning rate value. The data type must be float32. - - **l1** (Union[Number, Tensor]): l1 regularization strength. The data type must be float32. - - **l2** (Union[Number, Tensor]): l2 regularization strength. The data type must be float32. + - **lr** (Union[Number, Tensor]) - The learning rate value. The data type must be float32. + - **l1** (Union[Number, Tensor]) - l1 regularization strength. The data type must be float32. + - **l2** (Union[Number, Tensor]) - l2 regularization strength. The data type must be float32. - **grad** (Tensor) - A tensor of the same type as `var`, for the gradient. The data type must be float32. - **indices** (Tensor) - A vector of indices into the first dimension of `var` and `accum`. @@ -3759,7 +3759,7 @@ class SparseApplyProximalAdagrad(PrimitiveWithInfer): @prim_attr_register def __init__(self, use_locking=False): self.init_prim_io_names(inputs=['var', 'accum', 'lr', 'l1', 'l2', 'grad', 'indices'], - outputs=['output']) + outputs=['var', 'accum']) self.use_locking = validator.check_value_type("use_locking", use_locking, [bool], self.name) def infer_shape(self, var_shape, accum_shape, lr_shape, l1_shape, l2_shape, grad_shape, indices_shape): @@ -3778,6 +3778,349 @@ class SparseApplyProximalAdagrad(PrimitiveWithInfer): return var_dtype, accum_dtype +class ApplyAddSign(PrimitiveWithInfer): + r""" + Update relevant entries according to the AddSign algorithm. + + .. math:: + \begin{array}{ll} \\ + m_{t} = \beta * m_{t-1} + (1 - \beta) * g \\ + \text{update} = (\alpha + \text{sign_decay} * sign(g) * sign(m)) * g \\ + var = var - lr_{t} * \text{update} + \end{array} + + :math:`t` represents updating step while, :math:`m` represents the 1st moment vector, :math:`m_{t-1}` + is the last momentent of :math:`m_{t}`, :math:`lr` represents scaling factor `lr`, :math:`g` represents `grad`. + + Inputs: + - **var** (Parameter) - Variable tensor to be updated. With float32 or float16 data type. + - **m** (Parameter) - Variable tensor to be updated. Has the same dtype as `var`. + - **lr** (Union[Number, Tensor]) - The learning rate value, should be a scalar. + With float32 or float16 data type. + - **alpha** (Union[Number, Tensor]) - Should be a scalar. With float32 or float16 data type. + - **sign_decay** (Union[Number, Tensor]) - Should be a scalar. With float32 or float16 data type. + - **beta** (Union[Number, Tensor]) - The exponential decay rate, should be a scalar. + With float32 or float16 data type. + - **grad** (Tensor) - A tensor of the same type as `var`, for the gradient. + + Outputs: + Tuple of 2 Tensor, the updated parameters. + + - **var** (Tensor) - The same shape and data type as `var`. + - **m** (Tensor) - The same shape and data type as `m`. + + Examples: + >>> import numpy as np + >>> import mindspore.nn as nn + >>> from mindspore import Tensor, Parameter + >>> from mindspore.ops import operations as P + >>> class Net(nn.Cell): + >>> def __init__(self): + >>> super(Net, self).__init__() + >>> self.apply_add_sign = P.ApplyAddSign() + >>> self.var = Parameter(Tensor(np.random.rand(3, 3).astype(np.float32)), name="var") + >>> self.m = Parameter(Tensor(np.random.rand(3, 3).astype(np.float32)), name="m") + >>> self.lr = 0.001 + >>> self.alpha = 1.0 + >>> self.sign_decay = 0.99 + >>> self.beta = 0.9 + >>> def construct(self, grad): + >>> out = self.apply_add_sign(self.var, self.m, self.lr, self.alpha, self.sign_decay, self.beta, grad) + >>> return out + >>> net = Net() + >>> grad = Tensor(np.random.rand(3, 3).astype(np.float32)) + >>> output = net(grad) + """ + + __mindspore_signature__ = ( + ('var', sig_rw.RW_WRITE, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), + ('m', sig_rw.RW_WRITE, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), + ('lr', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T1), + ('alpha', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T2), + ('sign_decay', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, + sig_dtype.T3), + ('beta', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T4), + ('grad', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T) + ) + + @prim_attr_register + def __init__(self): + "init ApplyAddSign" + + def infer_shape(self, var_shape, m_shape, lr_shape, alpha_shape, sign_decay_shape, beta_shape, grad_shape): + validator.check('m_shape', m_shape, 'var_shape', var_shape, Rel.EQ, self.name) + validator.check('grad_shape', grad_shape, 'var_shape', var_shape, Rel.EQ, self.name) + lr_shape_len = len(lr_shape) + validator.check_integer("lr's rank", lr_shape_len, 1, Rel.LE, self.name) + if lr_shape_len == 1: + validator.check_integer("lr_shape[0]", lr_shape[0], 1, Rel.EQ, self.name) + alpha_shape_len = len(alpha_shape) + validator.check_integer("alpha's rank", alpha_shape_len, 1, Rel.LE, self.name) + if alpha_shape_len == 1: + validator.check_integer("alpha_shape[0]", alpha_shape[0], 1, Rel.EQ, self.name) + sign_decay_shape_len = len(sign_decay_shape) + validator.check_integer("sign_decay's rank", sign_decay_shape_len, 1, Rel.LE, self.name) + if sign_decay_shape_len == 1: + validator.check_integer("sign_decay_shape[0]", sign_decay_shape[0], 1, Rel.EQ, self.name) + beta_shape_len = len(beta_shape) + validator.check_integer("beta's rank", beta_shape_len, 1, Rel.LE, self.name) + if beta_shape_len == 1: + validator.check_integer("beta_shape[0]", beta_shape[0], 1, Rel.EQ, self.name) + return var_shape, m_shape + + def infer_dtype(self, var_dtype, m_dtype, lr_dtype, alpha_dtype, sign_decay_dtype, beta_dtype, grad_dtype): + valid_types = [mstype.float16, mstype.float32] + args = {'var': var_dtype, 'm': m_dtype, 'grad': grad_dtype} + validator.check_tensor_type_same(args, valid_types, self.name) + validator.check_scalar_or_tensor_type_same({"lr": lr_dtype}, valid_types, self.name) + validator.check_scalar_or_tensor_type_same({"alpha": alpha_dtype}, valid_types, self.name) + validator.check_scalar_or_tensor_type_same({"sign_decay": sign_decay_dtype}, valid_types, self.name) + validator.check_scalar_or_tensor_type_same({"beta": beta_dtype}, valid_types, self.name) + return var_dtype, m_dtype + + +class ApplyPowerSign(PrimitiveWithInfer): + r""" + Update relevant entries according to the AddSign algorithm. + + .. math:: + \begin{array}{ll} \\ + m_{t} = \beta * m_{t-1} + (1 - \beta) * g \\ + \text{update} = \exp(\text{logbase} * \text{sign_decay} * sign(g) * sign(m)) * g \\ + var = var - lr_{t} * \text{update} + \end{array} + + :math:`t` represents updating step while, :math:`m` represents the 1st moment vector, :math:`m_{t-1}` + is the last momentent of :math:`m_{t}`, :math:`lr` represents scaling factor `lr`, :math:`g` represents `grad`. + + Inputs: + - **var** (Parameter) - Variable tensor to be updated. With float32 or float16 data type. + - **m** (Parameter) - Variable tensor to be updated. Has the same dtype as `var`. + - **lr** (Union[Number, Tensor]) - The learning rate value, should be a scalar. + With float32 or float16 data type. + - **logbase** (Union[Number, Tensor]) - Should be a scalar. With float32 or float16 data type. + - **sign_decay** (Union[Number, Tensor]) - Should be a scalar. With float32 or float16 data type. + - **beta** (Union[Number, Tensor]) - The exponential decay rate, should be a scalar. + With float32 or float16 data type. + - **grad** (Tensor) - A tensor of the same type as `var`, for the gradient. + + Outputs: + Tuple of 2 Tensor, the updated parameters. + + - **var** (Tensor) - The same shape and data type as `var`. + - **m** (Tensor) - The same shape and data type as `m`. + + Examples: + >>> import numpy as np + >>> import mindspore.nn as nn + >>> from mindspore import Tensor, Parameter + >>> from mindspore.ops import operations as P + >>> class Net(nn.Cell): + >>> def __init__(self): + >>> super(Net, self).__init__() + >>> self.apply_power_sign = P.ApplyPowerSign() + >>> self.var = Parameter(Tensor(np.random.rand(3, 3).astype(np.float32)), name="var") + >>> self.m = Parameter(Tensor(np.random.rand(3, 3).astype(np.float32)), name="m") + >>> self.lr = 0.001 + >>> self.logbase = np.e + >>> self.sign_decay = 0.99 + >>> self.beta = 0.9 + >>> def construct(self, grad): + >>> out = self.apply_power_sign(self.var, self.m, self.lr, self.logbase, + self.sign_decay, self.beta, grad) + >>> return out + >>> net = Net() + >>> grad = Tensor(np.random.rand(3, 3).astype(np.float32)) + >>> output = net(grad) + """ + + __mindspore_signature__ = ( + ('var', sig_rw.RW_WRITE, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), + ('m', sig_rw.RW_WRITE, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), + ('lr', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T1), + ('logbase', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T2), + ('sign_decay', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, + sig_dtype.T3), + ('beta', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T4), + ('grad', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T) + ) + + @prim_attr_register + def __init__(self): + "init ApplyPowerSign" + + def infer_shape(self, var_shape, m_shape, lr_shape, logbase_shape, sign_decay_shape, beta_shape, grad_shape): + validator.check('m_shape', m_shape, 'var_shape', var_shape, Rel.EQ, self.name) + validator.check('grad_shape', grad_shape, 'var_shape', var_shape, Rel.EQ, self.name) + lr_shape_len = len(lr_shape) + validator.check_integer("lr's rank", lr_shape_len, 1, Rel.LE, self.name) + if lr_shape_len == 1: + validator.check_integer("lr_shape[0]", lr_shape[0], 1, Rel.EQ, self.name) + logbase_shape_len = len(logbase_shape) + validator.check_integer("logbase's rank", logbase_shape_len, 1, Rel.LE, self.name) + if logbase_shape_len == 1: + validator.check_integer("logbase_shape[0]", logbase_shape[0], 1, Rel.EQ, self.name) + sign_decay_shape_len = len(sign_decay_shape) + validator.check_integer("sign_decay's rank", sign_decay_shape_len, 1, Rel.LE, self.name) + if sign_decay_shape_len == 1: + validator.check_integer("sign_decay_shape[0]", sign_decay_shape[0], 1, Rel.EQ, self.name) + beta_shape_len = len(beta_shape) + validator.check_integer("beta's rank", beta_shape_len, 1, Rel.LE, self.name) + if beta_shape_len == 1: + validator.check_integer("beta_shape[0]", beta_shape[0], 1, Rel.EQ, self.name) + return var_shape, m_shape + + def infer_dtype(self, var_dtype, m_dtype, lr_dtype, logbase_dtype, sign_decay_dtype, beta_dtype, grad_dtype): + valid_types = [mstype.float16, mstype.float32] + args = {'var': var_dtype, 'm': m_dtype, 'grad': grad_dtype} + validator.check_tensor_type_same(args, valid_types, self.name) + validator.check_scalar_or_tensor_type_same({"lr": lr_dtype}, valid_types, self.name) + validator.check_scalar_or_tensor_type_same({"logbase": logbase_dtype}, valid_types, self.name) + validator.check_scalar_or_tensor_type_same({"sign_decay": sign_decay_dtype}, valid_types, self.name) + validator.check_scalar_or_tensor_type_same({"beta": beta_dtype}, valid_types, self.name) + return var_dtype, m_dtype + + +class ApplyGradientDescent(PrimitiveWithInfer): + r""" + Update relevant entries according to the following formula. + + .. math:: + var = var - \alpha * \delta + + Inputs: + - **var** (Parameter) - Variable tensor to be updated. With float32 or float16 data type. + - **alpha** (Union[Number, Tensor]) - Scaling factor, should be a scalar. With float32 or float16 data type. + - **delta** (Tensor) - A tensor for the change. Has the same type as `var`. + + Outputs: + Tensor, representing the updated var. + + Examples: + >>> import numpy as np + >>> import mindspore.nn as nn + >>> from mindspore import Tensor, Parameter + >>> from mindspore.ops import operations as P + >>> class Net(nn.Cell): + >>> def __init__(self): + >>> super(Net, self).__init__() + >>> self.apply_gradient_descent = P.ApplyGradientDescent() + >>> self.var = Parameter(Tensor(np.random.rand(3, 3).astype(np.float32)), name="var") + >>> self.alpha = 0.001 + >>> def construct(self, delta): + >>> out = self.apply_gradient_descent(self.var, self.alpha, delta) + >>> return out + >>> net = Net() + >>> delta = Tensor(np.random.rand(3, 3).astype(np.float32)) + >>> output = net(delta) + """ + + __mindspore_signature__ = ( + ('var', sig_rw.RW_WRITE, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), + ('alpha', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T1), + ('delta', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T) + ) + + @prim_attr_register + def __init__(self): + "init ApplyGradientDescent" + + def infer_shape(self, var_shape, alpha_shape, delta_shape): + validator.check('delta shape', delta_shape, 'var shape', var_shape, Rel.EQ, self.name) + alpha_shape_len = len(alpha_shape) + validator.check_integer("alpha's rank", alpha_shape_len, 1, Rel.LE, self.name) + if alpha_shape_len == 1: + validator.check_integer("alpha_shape[0]", alpha_shape[0], 1, Rel.EQ, self.name) + return var_shape + + def infer_dtype(self, var_dtype, alpha_dtype, delta_dtype): + valid_types = [mstype.float16, mstype.float32] + args = {'var': var_dtype, 'delta': delta_dtype} + validator.check_tensor_type_same(args, valid_types, self.name) + validator.check_scalar_or_tensor_type_same({"alpha": alpha_dtype}, valid_types, self.name) + return var_dtype + + +class ApplyProximalGradientDescent(PrimitiveWithInfer): + r""" + Update relevant entries according to the FOBOS(Forward Backward Splitting) algorithm. + + .. math:: + \text{prox_v} = var - \alpha * \delta + .. math:: + var = \frac{sign(\text{prox_v})}{1 + \alpha * l2} * \max(\left| \text{prox_v} \right| - alpha * l1, 0) + + Inputs: + - **var** (Parameter) - Variable tensor to be updated. With float32 or float16 data type. + - **alpha** (Union[Number, Tensor]) - Saling factor, should be a scalar. With float32 or float16 data type. + - **l1** (Union[Number, Tensor]) - l1 regularization strength, should be scalar. + With float32 or float16 data type. + - **l2** (Union[Number, Tensor]) - l2 regularization strength, should be scalar. + With float32 or float16 data type. + - **delta** (Tensor) - A tensor for the change. Has the same type as `var`. + + Outputs: + Tensor, representing the updated var. + + Examples: + >>> import numpy as np + >>> import mindspore.nn as nn + >>> from mindspore import Tensor, Parameter + >>> from mindspore.ops import operations as P + >>> class Net(nn.Cell): + >>> def __init__(self): + >>> super(Net, self).__init__() + >>> self.apply_proximal_gradient_descent = P.ApplyProximalGradientDescent() + >>> self.var = Parameter(Tensor(np.random.rand(3, 3).astype(np.float32)), name="var") + >>> self.alpha = 0.001 + >>> self.l1 = 0.0 + >>> self.l2 = 0.0 + >>> def construct(self, delta): + >>> out = self.apply_proximal_gradient_descent(self.var, self.alpha, self.l1, self.l2, delta) + >>> return out + >>> net = Net() + >>> delta = Tensor(np.random.rand(3, 3).astype(np.float32)) + >>> output = net(delta) + """ + + __mindspore_signature__ = ( + ('var', sig_rw.RW_WRITE, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), + ('alpha', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T1), + ('l1', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T2), + ('l2', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T3), + ('delta', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T) + ) + + @prim_attr_register + def __init__(self): + "init ApplyGradientDescent" + + def infer_shape(self, var_shape, alpha_shape, l1_shape, l2_shape, delta_shape): + validator.check('delta shape', delta_shape, 'var shape', var_shape, Rel.EQ, self.name) + alpha_shape_len = len(alpha_shape) + validator.check_integer("alpha's rank", alpha_shape_len, 1, Rel.LE, self.name) + if alpha_shape_len == 1: + validator.check_integer("alpha_shape[0]", alpha_shape[0], 1, Rel.EQ, self.name) + l1_shape_len = len(l1_shape) + validator.check_integer("l1's rank", l1_shape_len, 1, Rel.LE, self.name) + if l1_shape_len == 1: + validator.check_integer("l1_shape[0]", l1_shape[0], 1, Rel.EQ, self.name) + l2_shape_len = len(l2_shape) + validator.check_integer("l2's rank", l2_shape_len, 1, Rel.LE, self.name) + if l2_shape_len == 1: + validator.check_integer("l2_shape[0]", l2_shape[0], 1, Rel.EQ, self.name) + return var_shape + + def infer_dtype(self, var_dtype, alpha_dtype, l1_dtype, l2_dtype, delta_dtype): + valid_types = [mstype.float16, mstype.float32] + args = {'var': var_dtype, 'delta': delta_dtype} + validator.check_tensor_type_same(args, valid_types, self.name) + validator.check_scalar_or_tensor_type_same({"alpha": alpha_dtype}, valid_types, self.name) + validator.check_scalar_or_tensor_type_same({"l1": l1_dtype}, valid_types, self.name) + validator.check_scalar_or_tensor_type_same({"l2": l2_dtype}, valid_types, self.name) + return var_dtype + + class LARSUpdate(PrimitiveWithInfer): """ Conduct lars (layer-wise adaptive rate scaling) update on the square sum of gradient. diff --git a/tests/ut/python/ops/test_ops.py b/tests/ut/python/ops/test_ops.py index cf6a6705ab..19d79031eb 100755 --- a/tests/ut/python/ops/test_ops.py +++ b/tests/ut/python/ops/test_ops.py @@ -351,6 +351,64 @@ class ApplyAdagradV2Net(nn.Cell): return out +class ApplyAddSignNet(nn.Cell): + def __init__(self): + super(ApplyAddSignNet, self).__init__() + self.apply_add_sign = P.ApplyAddSign() + self.lr = 0.001 + self.alpha = 1.0 + self.sign_decay = 0.99 + self.beta = 0.99 + self.var = Parameter(Tensor(np.random.rand(3, 3).astype(np.float32)), name="var") + self.m = Parameter(Tensor(np.random.rand(3, 3).astype(np.float32)), name="m") + + def construct(self, grad): + out = self.apply_add_sign(self.var, self.m, self.lr, self.alpha, self.sign_decay, self.beta, grad) + return out + + +class ApplyPowerSignNet(nn.Cell): + def __init__(self): + super(ApplyPowerSignNet, self).__init__() + self.apply_power_sign = P.ApplyPowerSign() + self.lr = 0.001 + self.logbase = np.e + self.sign_decay = 0.99 + self.beta = 0.99 + self.var = Parameter(Tensor(np.random.rand(3, 3).astype(np.float32)), name="var") + self.m = Parameter(Tensor(np.random.rand(3, 3).astype(np.float32)), name="m") + + def construct(self, grad): + out = self.apply_power_sign(self.var, self.m, self.lr, self.logbase, self.sign_decay, self.beta, grad) + return out + + +class ApplyGradientDescentNet(nn.Cell): + def __init__(self): + super(ApplyGradientDescentNet, self).__init__() + self.apply_gradient_descent = P.ApplyGradientDescent() + self.alpha = 0.001 + self.var = Parameter(Tensor(np.random.rand(3, 3).astype(np.float32)), name="var") + + def construct(self, delta): + out = self.apply_gradient_descent(self.var, self.alpha, delta) + return out + + +class ApplyProximalGradientDescentNet(nn.Cell): + def __init__(self): + super(ApplyProximalGradientDescentNet, self).__init__() + self.apply_proximal_gradient_descent = P.ApplyProximalGradientDescent() + self.alpha = 0.001 + self.l1 = 0.0 + self.l2 = 0.0 + self.var = Parameter(Tensor(np.random.rand(3, 3).astype(np.float32)), name="var") + + def construct(self, delta): + out = self.apply_proximal_gradient_descent(self.var, self.alpha, self.l1, self.l2, delta) + return out + + class SparseApplyAdagradNet(nn.Cell): def __init__(self): super(SparseApplyAdagradNet, self).__init__() @@ -1241,6 +1299,22 @@ test_case_nn_ops = [ 'block': ApplyAdagradV2Net(), 'desc_inputs': [[3, 3]], 'skip': ['backward']}), + ('ApplyAddSign', { + 'block': ApplyAddSignNet(), + 'desc_inputs': [[3, 3]], + 'skip': ['backward']}), + ('ApplyPowerSign', { + 'block': ApplyPowerSignNet(), + 'desc_inputs': [[3, 3]], + 'skip': ['backward']}), + ('ApplyGradientDescent', { + 'block': ApplyGradientDescentNet(), + 'desc_inputs': [[3, 3]], + 'skip': ['backward']}), + ('ApplyProximalGradientDescent', { + 'block': ApplyProximalGradientDescentNet(), + 'desc_inputs': [[3, 3]], + 'skip': ['backward']}), ('Flatten_1', { 'block': NetForFlatten(), 'desc_inputs': [Tensor(np.ones([2, 3, 4]).astype(np.int32)), Tensor(np.ones([2, 12]).astype(np.int32))], From 7bbe183e6f915579dfd53ca3ccebd7e95d771da9 Mon Sep 17 00:00:00 2001 From: lizhenyu Date: Mon, 29 Jun 2020 11:38:08 +0800 Subject: [PATCH 131/254] bugfix:Attr of axis of ReduceSum is invalid --- mindspore/ccsrc/kernel/gpu/arrays/array_reduce_gpu_kernel.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mindspore/ccsrc/kernel/gpu/arrays/array_reduce_gpu_kernel.h b/mindspore/ccsrc/kernel/gpu/arrays/array_reduce_gpu_kernel.h index e1f995d648..4a52439305 100644 --- a/mindspore/ccsrc/kernel/gpu/arrays/array_reduce_gpu_kernel.h +++ b/mindspore/ccsrc/kernel/gpu/arrays/array_reduce_gpu_kernel.h @@ -94,7 +94,8 @@ class ArrayReduceGpuKernel : public GpuKernel { } int input_dim_length = SizeToInt(AnfAlgo::GetPrevNodeOutputInferShape(kernel_node, 0).size()); - if (AnfAlgo::GetCNodePrimitive(kernel_node)->GetAttr("axis")->isa()) { + if (AnfAlgo::GetCNodePrimitive(kernel_node)->GetAttr("axis")->isa() || + AnfAlgo::GetCNodePrimitive(kernel_node)->GetAttr("axis")->isa()) { auto attr_axis = GetAttr>(kernel_node, "axis"); if (attr_axis.empty()) { axis_.push_back(-1); From 58982e598e4f705618375a7fe5f84e2d4b7ff88a Mon Sep 17 00:00:00 2001 From: panbingao Date: Mon, 29 Jun 2020 11:10:24 +0800 Subject: [PATCH 132/254] move resnet_thor series from example to model_zoo --- model_zoo/resnet_thor/README.md | 128 ++++ model_zoo/resnet_thor/eval.py | 62 ++ .../scripts/run_distribute_train.sh | 57 ++ model_zoo/resnet_thor/scripts/run_eval.sh | 67 ++ model_zoo/resnet_thor/src/config.py | 37 + model_zoo/resnet_thor/src/crossentropy.py | 41 + model_zoo/resnet_thor/src/dataset_helper.py | 125 +++ model_zoo/resnet_thor/src/dataset_imagenet.py | 80 ++ .../resnet_thor/src/grad_reducer_thor.py | 183 +++++ model_zoo/resnet_thor/src/model_thor.py | 725 ++++++++++++++++++ model_zoo/resnet_thor/src/resnet50.py | 262 +++++++ model_zoo/resnet_thor/src/resnet_thor.py | 359 +++++++++ model_zoo/resnet_thor/src/thor.py | 199 +++++ model_zoo/resnet_thor/src/thor_layer.py | 477 ++++++++++++ model_zoo/resnet_thor/train.py | 132 ++++ 15 files changed, 2934 insertions(+) create mode 100644 model_zoo/resnet_thor/README.md create mode 100644 model_zoo/resnet_thor/eval.py create mode 100644 model_zoo/resnet_thor/scripts/run_distribute_train.sh create mode 100644 model_zoo/resnet_thor/scripts/run_eval.sh create mode 100644 model_zoo/resnet_thor/src/config.py create mode 100644 model_zoo/resnet_thor/src/crossentropy.py create mode 100644 model_zoo/resnet_thor/src/dataset_helper.py create mode 100644 model_zoo/resnet_thor/src/dataset_imagenet.py create mode 100644 model_zoo/resnet_thor/src/grad_reducer_thor.py create mode 100644 model_zoo/resnet_thor/src/model_thor.py create mode 100644 model_zoo/resnet_thor/src/resnet50.py create mode 100644 model_zoo/resnet_thor/src/resnet_thor.py create mode 100644 model_zoo/resnet_thor/src/thor.py create mode 100644 model_zoo/resnet_thor/src/thor_layer.py create mode 100644 model_zoo/resnet_thor/train.py diff --git a/model_zoo/resnet_thor/README.md b/model_zoo/resnet_thor/README.md new file mode 100644 index 0000000000..5fb17007ae --- /dev/null +++ b/model_zoo/resnet_thor/README.md @@ -0,0 +1,128 @@ +# ResNet-50-THOR Example + +## Description + +This is an example of training ResNet-50 V1.5 with ImageNet2012 dataset by second-order optimizer THOR. THOR is a novel approximate seond-order optimization method in MindSpore. With fewer iterations, THOR can finish ResNet-50 V1.5 training in 72 minutes to top-1 accuracy of 75.9% using 8 Ascend 910, which is much faster than SGD with Momentum. + +## Requirements + +- Install [MindSpore](https://www.mindspore.cn/install/en). + +- Download the dataset ImageNet2012 + +> Unzip the ImageNet2012 dataset to any path you want and the folder structure should include train and eval dataset as follows: +> ``` +> . +> ├── ilsvrc # train dataset +> └── ilsvrc_eval # infer dataset +> ``` + + +## Example structure + +```shell +. +├── resnet_thor + ├── README.md + ├── src + ├── crossentropy.py # CrossEntropy loss function + ├── config.py # parameter configuration + ├── resnet50.py # resnet50 backbone + ├── dataset_helper.py # dataset help for minddata dataset + ├── grad_reducer_thor.py # grad reducer for thor + ├── model_thor.py # model + ├── resnet_thor.py # resnet50_thor backone + ├── thor.py # thor + ├── thor_layer.py # thor layer + └── dataset_imagenet.py # data preprocessing + ├── scripts + ├── run_distribute_train.sh # launch distributed training(8 pcs) + └── run_eval.sh # launch infering + ├── eval.py # infer script + └── train.py # train script +``` + + +## Parameter configuration + +Parameters for both training and inference can be set in config.py. + +``` +"class_num": 1000, # dataset class number +"batch_size": 32, # batch size of input tensor +"loss_scale": 128, # loss scale +"momentum": 0.9, # momentum of THOR optimizer +"weight_decay": 5e-4, # weight decay +"epoch_size": 45, # only valid for taining, which is always 1 for inference +"buffer_size": 1000, # number of queue size in data preprocessing +"image_height": 224, # image height +"image_width": 224, # image width +"save_checkpoint": True, # whether save checkpoint or not +"save_checkpoint_steps": 5004, # the step interval between two checkpoints. By default, the checkpoint will be saved every epoch +"keep_checkpoint_max": 20, # only keep the last keep_checkpoint_max checkpoint +"save_checkpoint_path": "./", # path to save checkpoint relative to the executed path +"label_smooth": True, # label smooth +"label_smooth_factor": 0.1, # label smooth factor +"frequency": 834, # the step interval to update second-order information matrix +``` + +## Running the example + +### Train + +#### Usage + +``` +# distributed training +Usage: sh run_distribute_train.sh [MINDSPORE_HCCL_CONFIG_PATH] [DATASET_PATH] [DEVICE_NUM] +``` + + +#### Launch + +```bash +# distributed training example(8 pcs) +sh run_distribute_train.sh rank_table_8p.json dataset/ilsvrc +``` + +> About rank_table.json, you can refer to the [distributed training tutorial](https://www.mindspore.cn/tutorial/en/master/advanced_use/distributed_training.html). + +#### Result + +Training result will be stored in the example path, whose folder name begins with "train_parallel". Under this, you can find checkpoint file together with result like the followings in log. + +``` +# distribute training result(8 pcs) +epoch: 1 step: 5004, loss is 4.4182425 +epoch: 2 step: 5004, loss is 3.740064 +epoch: 3 step: 5004, loss is 4.0546017 +epoch: 4 step: 5004, loss is 3.7598825 +epoch: 5 step: 5004, loss is 3.3744206 +...... +``` + +### Infer + +#### Usage + +``` +# infer +Usage: sh run_eval.sh [DATASET_PATH] [CHECKPOINT_PATH] +``` + +#### Launch + +```bash +# infer with checkpoint +sh run_eval.sh dataset/ilsvrc_eval train_parallel0/resnet-42_5004.ckpt +``` + +> checkpoint can be produced in training process. + +#### Result + +Inference result will be stored in the example path, whose folder name is "infer". Under this, you can find result like the followings in log. + +``` +result: {'acc': 0.759503041} ckpt=train_parallel0/resnet-42_5004.ckpt +``` diff --git a/model_zoo/resnet_thor/eval.py b/model_zoo/resnet_thor/eval.py new file mode 100644 index 0000000000..fb911787ba --- /dev/null +++ b/model_zoo/resnet_thor/eval.py @@ -0,0 +1,62 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +eval. +""" +import os +import argparse + +from mindspore import context +from mindspore.train.model import Model +from mindspore.train.serialization import load_checkpoint, load_param_into_net + +from src.dataset_imagenet import create_dataset +from src.config import config +from src.crossentropy import CrossEntropy +from src.resnet50 import resnet50 + +parser = argparse.ArgumentParser(description='Image classification') +parser.add_argument('--run_distribute', type=bool, default=False, help='Run distribute') +parser.add_argument('--device_num', type=int, default=1, help='Device num.') +parser.add_argument('--do_train', type=bool, default=False, help='Do train or not.') +parser.add_argument('--do_eval', type=bool, default=True, help='Do eval or not.') +parser.add_argument('--checkpoint_path', type=str, default=None, help='Checkpoint file path') +parser.add_argument('--dataset_path', type=str, default=None, help='Dataset path') +args_opt = parser.parse_args() + +device_id = int(os.getenv('DEVICE_ID')) + +context.set_context(mode=context.GRAPH_MODE, device_target="Ascend", save_graphs=False) +context.set_context(device_id=device_id) + +if __name__ == '__main__': + + net = resnet50(class_num=config.class_num) + if not config.label_smooth: + config.label_smooth_factor = 0.0 + loss = CrossEntropy(smooth_factor=config.label_smooth_factor, num_classes=config.class_num) + + if args_opt.do_eval: + dataset = create_dataset(dataset_path=args_opt.dataset_path, do_train=False, batch_size=config.batch_size) + step_size = dataset.get_dataset_size() + + if args_opt.checkpoint_path: + param_dict = load_checkpoint(args_opt.checkpoint_path) + load_param_into_net(net, param_dict) + net.set_train(False) + + model = Model(net, loss_fn=loss, metrics={'acc'}) + res = model.eval(dataset) + print("result:", res, "ckpt=", args_opt.checkpoint_path) diff --git a/model_zoo/resnet_thor/scripts/run_distribute_train.sh b/model_zoo/resnet_thor/scripts/run_distribute_train.sh new file mode 100644 index 0000000000..6fa7457227 --- /dev/null +++ b/model_zoo/resnet_thor/scripts/run_distribute_train.sh @@ -0,0 +1,57 @@ +#!/bin/bash +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +if [ $# != 3 ] +then + echo "Usage: sh run_distribute_train.sh [MINDSPORE_HCCL_CONFIG_PATH] [DATASET_PATH] [DEVICE_NUM]" +exit 1 +fi + +if [ ! -f $1 ] +then + echo "error: DMINDSPORE_HCCL_CONFIG_PATH=$1 is not a file" +exit 1 +fi + +if [ ! -d $2 ] +then + echo "error: DATASET_PATH=$2 is not a directory" +exit 1 +fi + +BASE_PATH=$(cd "`dirname $0`" || exit; pwd) +cd $BASE_PATH/../ || exit + +ulimit -u unlimited +export DEVICE_NUM=$3 +export RANK_SIZE=$3 +export MINDSPORE_HCCL_CONFIG_PATH=$1 + +for((i=0; i<${DEVICE_NUM}; i++)) +do + export DEVICE_ID=$i + export RANK_ID=$i + rm -rf ./train_parallel$i + mkdir ./train_parallel$i + cp *.py ./train_parallel$i + cp -r ./src ./train_parallel$i + cd ./train_parallel$i || exit + echo "start training for rank $RANK_ID, device $DEVICE_ID" + + env > env.log + python train.py --do_train=True --run_distribute=True --device_num=$DEVICE_NUM --dataset_path=$2 > log 2>&1 & + cd .. +done diff --git a/model_zoo/resnet_thor/scripts/run_eval.sh b/model_zoo/resnet_thor/scripts/run_eval.sh new file mode 100644 index 0000000000..eafba5fbea --- /dev/null +++ b/model_zoo/resnet_thor/scripts/run_eval.sh @@ -0,0 +1,67 @@ +#!/bin/bash +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +if [ $# != 2 ] +then + echo "Usage: sh run_eval.sh [DATASET_PATH] [CHECKPOINT_PATH]" +exit 1 +fi + +get_real_path(){ + if [ "${1:0:1}" == "/" ]; then + echo "$1" + else + echo "$(realpath -m $PWD/$1)" + fi +} + +PATH1=$(get_real_path $1) +PATH2=$(get_real_path $2) + + +if [ ! -d $PATH1 ] +then + echo "error: DATASET_PATH=$PATH1 is not a directory" +exit 1 +fi + +if [ ! -f $PATH2 ] +then + echo "error: CHECKPOINT_PATH=$PATH2 is not a file" +exit 1 +fi + +BASE_PATH=$(cd "`dirname $0`" || exit; pwd) +cd $BASE_PATH/../ || exit + +ulimit -u unlimited +export DEVICE_NUM=1 +export DEVICE_ID=0 +export RANK_SIZE=$DEVICE_NUM +export RANK_ID=0 + +if [ -d "eval" ]; +then + rm -rf ./eval +fi +mkdir ./eval +cp *.py ./eval +cp -r ./src ./eval +cd ./eval || exit +env > env.log +echo "start infering for device $DEVICE_ID" +python eval.py --do_eval=True --dataset_path=$PATH1 --checkpoint_path=$PATH2 &> log & +cd .. diff --git a/model_zoo/resnet_thor/src/config.py b/model_zoo/resnet_thor/src/config.py new file mode 100644 index 0000000000..cd0a81d5e6 --- /dev/null +++ b/model_zoo/resnet_thor/src/config.py @@ -0,0 +1,37 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +network config setting, will be used in train.py and eval.py +""" +from easydict import EasyDict as ed + +config = ed({ + "class_num": 1000, + "batch_size": 32, + "loss_scale": 128, + "momentum": 0.9, + "weight_decay": 5e-4, + "epoch_size": 45, + "buffer_size": 1000, + "image_height": 224, + "image_width": 224, + "save_checkpoint": True, + "save_checkpoint_steps": 5004, + "keep_checkpoint_max": 20, + "save_checkpoint_path": "./", + "label_smooth": 1, + "label_smooth_factor": 0.1, + "frequency": 834 +}) diff --git a/model_zoo/resnet_thor/src/crossentropy.py b/model_zoo/resnet_thor/src/crossentropy.py new file mode 100644 index 0000000000..e8681ff497 --- /dev/null +++ b/model_zoo/resnet_thor/src/crossentropy.py @@ -0,0 +1,41 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""CrossEntropy""" +import mindspore.nn as nn +from mindspore import Tensor +from mindspore.common import dtype as mstype +from mindspore.nn.loss.loss import _Loss +from mindspore.ops import functional as F +from mindspore.ops import operations as P + + +class CrossEntropy(_Loss): + """CrossEntropy""" + def __init__(self, smooth_factor=0., num_classes=1000): + super(CrossEntropy, self).__init__() + self.onehot = P.OneHot() + self.on_value = Tensor(1.0 - smooth_factor, mstype.float32) + self.off_value = Tensor(1.0 * smooth_factor / (num_classes - 1), mstype.float32) + # self.cast = P.Cast() + self.ce = nn.SoftmaxCrossEntropyWithLogits() + self.mean = P.ReduceMean(False) + + def construct(self, logit, label): + # one_hot_label = self.onehot(self.cast(label, mstype.int32), + # F.shape(logit)[1], self.on_value, self.off_value)、 + one_hot_label = self.onehot(label, F.shape(logit)[1], self.on_value, self.off_value) + loss = self.ce(logit, one_hot_label) + loss = self.mean(loss, 0) + return loss diff --git a/model_zoo/resnet_thor/src/dataset_helper.py b/model_zoo/resnet_thor/src/dataset_helper.py new file mode 100644 index 0000000000..77f67344c2 --- /dev/null +++ b/model_zoo/resnet_thor/src/dataset_helper.py @@ -0,0 +1,125 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Dataset help for minddata dataset""" +from mindspore._checkparam import check_bool +from mindspore.parallel._utils import _get_device_num, _get_parallel_mode +from mindspore.train.dataset_helper import _send_data +from mindspore.train._utils import _exec_datagraph, _get_types_and_shapes, \ + _to_full_shapes +from mindspore.train.parallel_utils import ParallelMode + + +class DatasetHelper: + """ + Help function to use the Minddata dataset. + + According to different context, change the iter of dataset, to use the same for loop in different context. + + Note: + The iter of DatasetHelper will give one epoch data. + + Args: + dataset (DataSet): The dataset. + dataset_sink_mode (bool): If true use GetNext to fetch the data, or else feed the data from host. + Default: True. + + Examples: + >>> dataset_helper = DatasetHelper(dataset) + >>> for inputs in dataset_helper: + >>> outputs = network(*inputs) + """ + + def __init__(self, dataset, dataset_sink_mode=True, iter_first_order=0): + check_bool(dataset_sink_mode) + self.iter = _DatasetIterMSLoopSink(dataset, iter_first_order) + + def __iter__(self): + return self.iter.__iter__() + + # A temp solution for loop sink. Delete later + def types_shapes(self): + """Get the types and shapes from dataset on current config.""" + return self.iter.types_shapes() + + def loop_size(self): + """Get loop_size for every iteration.""" + return self.iter.loop_size + + +class _DatasetIter: + """Base iter for dataset help""" + + def __init__(self, dataset): + self.loop_size = 1 + if not hasattr(dataset, '__ME_INITED__'): + if not hasattr(dataset, '__loop_size__'): + self.loop_size = dataset.get_dataset_size() + else: + self.loop_size = dataset.__loop_size__ + dataset.__TRANSFER_DATASET__ = _exec_datagraph(dataset, self.loop_size) + dataset.__ME_INITED__ = dataset.__TRANSFER_DATASET__.queue_name + + if not hasattr(dataset, '__no_send__'): + _send_data(dataset) + else: + _send_data(dataset) + + self.ind = 0 + self.dataset = dataset + dataset_types, dataset_shapes = _get_types_and_shapes(dataset) + self.dataset_types, self.dataset_shapes = dataset_types, dataset_shapes + + def __iter__(self): + self.ind = 0 + return self + + def __next__(self): + if self.ind >= self.loop_count: + raise StopIteration() + self.ind += 1 + return self.op() + + def types_shapes(self): + return self.dataset_types, self.dataset_shapes + + def get_loop_count(self, dataset): + loop_count = 1 + if hasattr(dataset, '__loop_size__'): + loop_size = dataset.__loop_size__ + if dataset.get_dataset_size() % loop_size != 0: + raise ValueError(f'Dataset size {dataset.get_dataset_size()} and ' + f'loop_size {loop_size} are not matched.') + loop_count = int(dataset.get_dataset_size() / loop_size) + return loop_count + + +class _DatasetIterMSLoopSink(_DatasetIter): + """Iter for context (device_target=Ascend)""" + + def __init__(self, dataset, iter_first_order): + super(_DatasetIterMSLoopSink, self).__init__(dataset) + loop_size = dataset.__loop_size__ + iter_first_order + self.loop_count = int(dataset.get_dataset_size() / loop_size) * 2 + # for self._parallel_mode equal to semi_auto_parallel or auto_parallel, use a complete tensor to + # compile, and slice tensor to run. The batch dimension of tensors for compile is device_number + # times the batch dimension of tensors for run. Now only support LoopSink. + if _get_parallel_mode() in (ParallelMode.SEMI_AUTO_PARALLEL, ParallelMode.AUTO_PARALLEL): + device_num = _get_device_num() + self.dataset_shapes = _to_full_shapes(self.dataset_shapes, device_num) + + def op(): + return tuple() + + self.op = op diff --git a/model_zoo/resnet_thor/src/dataset_imagenet.py b/model_zoo/resnet_thor/src/dataset_imagenet.py new file mode 100644 index 0000000000..296b675136 --- /dev/null +++ b/model_zoo/resnet_thor/src/dataset_imagenet.py @@ -0,0 +1,80 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +create train or eval dataset. +""" +import os + +import mindspore.common.dtype as mstype +import mindspore.dataset.engine as de +import mindspore.dataset.transforms.c_transforms as C2 +import mindspore.dataset.transforms.vision.c_transforms as V_C + + +def create_dataset(dataset_path, do_train, repeat_num=1, batch_size=32): + """ + create a train or eval dataset + Args: + dataset_path(string): the path of dataset. + do_train(bool): whether dataset is used for train or eval. + repeat_num(int): the repeat times of dataset. Default: 1 + batch_size(int): the batch size of dataset. Default: 32 + Returns: + dataset + """ + + device_num = int(os.getenv("RANK_SIZE")) + rank_id = int(os.getenv("RANK_ID")) + + if device_num == 1: + ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=False) + else: + ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True, + num_shards=device_num, shard_id=rank_id) + + image_size = 224 + mean = [0.485 * 255, 0.456 * 255, 0.406 * 255] + std = [0.229 * 255, 0.224 * 255, 0.225 * 255] + if do_train: + transform_img = [ + V_C.RandomCropDecodeResize(image_size, scale=(0.08, 1.0), ratio=(0.75, 1.333)), + V_C.RandomHorizontalFlip(prob=0.5), + V_C.Normalize(mean=mean, std=std), + V_C.HWC2CHW() + ] + else: + transform_img = [ + V_C.Decode(), + V_C.Resize((256, 256)), + V_C.CenterCrop(image_size), + V_C.Normalize(mean=mean, std=std), + V_C.HWC2CHW() + ] + # type_cast_op = C2.TypeCast(mstype.float16) + type_cast_op = C2.TypeCast(mstype.int32) + + ds = ds.map(input_columns="image", operations=transform_img, num_parallel_workers=8) + ds = ds.map(input_columns="label", operations=type_cast_op, num_parallel_workers=8) + + # apply shuffle operations + # ds = ds.shuffle(buffer_size=config.buffer_size) + + # apply batch operations + ds = ds.batch(batch_size, drop_remainder=True) + + # apply dataset repeat operation + ds = ds.repeat(repeat_num) + + return ds diff --git a/model_zoo/resnet_thor/src/grad_reducer_thor.py b/model_zoo/resnet_thor/src/grad_reducer_thor.py new file mode 100644 index 0000000000..ad8d8dd8e4 --- /dev/null +++ b/model_zoo/resnet_thor/src/grad_reducer_thor.py @@ -0,0 +1,183 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""grad_reducer_thor""" +import mindspore.common.dtype as mstype +from mindspore.communication.management import GlobalComm, get_group_size +from mindspore.nn.cell import Cell +from mindspore.ops import functional as F, composite as C, operations as P +from mindspore.ops.operations.comm_ops import AllReduce, ReduceOp + +reduce_opt = C.MultitypeFuncGraph("reduce_opt") + +_all_reduce_A = AllReduce() + + +def _init_optimizer_allreduce(group): + global _all_reduce_A + _all_reduce_A = AllReduce(ReduceOp.SUM, GlobalComm.WORLD_COMM_GROUP) + _all_reduce_A.add_prim_attr('fusion', group) + + +@reduce_opt.register("Function", "Number", "Tensor") +def _tensors_allreduce_mean(mul, degree, grad): + degree = F.scalar_cast(degree, F.dtype(grad)) + grad = _all_reduce_A(grad) + cast_op = P.Cast() + return mul(grad, cast_op(F.scalar_to_array(1.0 / degree), F.dtype(grad))) + + +@reduce_opt.register("Bool", "Tensor") +def _tensors_allreduce(allreduce_filter, grad): + if allreduce_filter: + return _all_reduce_A(grad) + return grad + + +_get_datatype = C.MultitypeFuncGraph("_get_datatype") + + +@_get_datatype.register("Tensor") +def _tensors_get_datatype(grad): + """ + Acquire gradient datatype. + + Args: + grad (Tensor): The gradient tensor before operation. + + Returns: + mstype, the datatype of gradient. + """ + return F.dtype(grad) + + +_cast_datatype = C.MultitypeFuncGraph("_cast_datatype") + + +@_cast_datatype.register("TypeType", "Tensor") +def _tensors_cast_datatype(datatype, grad): + """ + Cast gradient to datatype. + + Args: + datatype (mstype): the destination datatype of gradient. + grad (Tensor): The gradient tensor before operation. + + Returns: + Tensor, the gradient tensor after operation. + """ + return F.cast(grad, datatype) + + +class DistributedGradReducerThor(Cell): + """ + A distributed optimizer. + + Constructs a gradient reducer Cell, which applies communication and average operations on + single-process gradient values. + + Args: + parameters (list): the parameters to be updated. + mean (bool): When mean is true, the mean coefficient (degree) would apply on gradients. Default: False. + degree (int): The mean coefficient. Usually it equals to device number. Default: None. + + Raises: + ValueError: If degree is not a int or less than 0. + + Examples: + >>> from mindspore.communication import init, get_group_size + >>> from mindspore.ops import composite as C + >>> from mindspore.ops import operations as P + >>> from mindspore.ops import functional as F + >>> from mindspore import context + >>> from mindspore import nn + >>> from mindspore import ParallelMode, ParameterTuple + >>> + >>> device_id = int(os.environ["DEVICE_ID"]) + >>> context.set_context(mode=context.GRAPH_MODE, device_target="Ascend", save_graphs=True, + >>> device_id=int(device_id), enable_hccl=True) + >>> init() + >>> context.reset_auto_parallel_context() + >>> context.set_auto_parallel_context(parallel_mode=ParallelMode.DATA_PARALLEL) + >>> + >>> + >>> class TrainingWrapper(nn.Cell): + >>> def __init__(self, network, optimizer, sens=1.0): + >>> super(TrainingWrapper, self).__init__(auto_prefix=False) + >>> self.network = network + >>> self.network.add_flags(defer_inline=True) + >>> self.weights = ParameterTuple(network.trainable_params()) + >>> self.optimizer = optimizer + >>> self.grad = C.GradOperation('grad', get_by_list=True, sens_param=True) + >>> self.sens = sens + >>> self.reducer_flag = False + >>> self.grad_reducer = None + >>> self.parallel_mode = context.get_auto_parallel_context("parallel_mode") + >>> if self.parallel_mode in [ParallelMode.DATA_PARALLEL, + >>> ParallelMode.HYBRID_PARALLEL]: + >>> self.reducer_flag = True + >>> if self.reducer_flag: + >>> mean = context.get_auto_parallel_context("mirror_mean") + >>> if mean.get_device_num_is_set(): + >>> degree = context.get_auto_parallel_context("device_num") + >>> else: + >>> degree = get_group_size() + >>> self.grad_reducer = nn.DistributedGradReducer(optimizer.parameters, mean, degree) + >>> + >>> def construct(self, *args): + >>> weights = self.weights + >>> loss = self.network(*args) + >>> sens = P.Fill()(P.DType()(loss), P.Shape()(loss), self.sens) + >>> grads = self.grad(self.network, weights)(*args, sens) + >>> if self.reducer_flag: + >>> # apply grad reducer on grads + >>> grads = self.grad_reducer(grads) + >>> return F.depend(loss, self.optimizer(grads)) + >>> + >>> network = Net() + >>> optimizer = nn.Momentum(network.trainable_params(), learning_rate=0.1, momentum=0.9) + >>> train_cell = TrainingWrapper(network, optimizer) + >>> inputs = Tensor(np.ones([16, 16]).astype(np.float32)) + >>> label = Tensor(np.zeros([16, 16]).astype(np.float32)) + >>> grads = train_cell(inputs, label) + """ + + def __init__(self, parameters, group, mean=True, degree=None): + super(DistributedGradReducerThor, self).__init__(auto_prefix=False) + self.hyper_map = C.HyperMap() + self.mul = P.Mul() + if degree is None: + self.degree = get_group_size() + else: + if not isinstance(degree, int) or degree <= 0: + raise ValueError("Parameter 'degree' in DistributedGradReducer should large than 0 and be int") + self.degree = degree + self.mean = mean + self.allreduce_filter = tuple(x.layerwise_parallel is False for x in parameters) + _init_optimizer_allreduce(group) + + def construct(self, grads): + # In some circumstances, the data precision of grads could be mixed with float16 and float32. Thus, the + # result of AllReduce is unreliable. To solve the problem, grads should be cast to float32 before AllReduce, + # and cast back after the operation. + datatypes = self.hyper_map(F.partial(_get_datatype), grads) + grads = self.hyper_map(F.partial(_cast_datatype, mstype.float32), grads) + + if self.mean: + new_grad = self.hyper_map(F.partial(reduce_opt, self.mul, self.degree), grads) + else: + new_grad = self.hyper_map(F.partial(reduce_opt), self.allreduce_filter, grads) + + new_grad = self.hyper_map(F.partial(_cast_datatype), datatypes, new_grad) + return new_grad diff --git a/model_zoo/resnet_thor/src/model_thor.py b/model_zoo/resnet_thor/src/model_thor.py new file mode 100644 index 0000000000..13b8c49996 --- /dev/null +++ b/model_zoo/resnet_thor/src/model_thor.py @@ -0,0 +1,725 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Model.""" + +import numpy as np +from mindspore import context +from mindspore import log as logger +from mindspore import nn +from mindspore._c_expression import init_exec_dataset +from mindspore._checkparam import check_input_data, check_output_data, check_int_positive, check_bool +from mindspore.common import dtype as mstype +from mindspore.common.dtype import pytype_to_dtype +from mindspore.common.tensor import Tensor +from mindspore.nn.metrics import Loss +from mindspore.nn.metrics import get_metrics +from mindspore.nn.wrap.cell_wrapper import _VirtualDatasetCell +from mindspore.parallel._utils import _get_parallel_mode, _get_device_num, _get_global_rank, \ + _get_parameter_broadcast, _device_number_check, _parameter_broadcast_check +from mindspore.train import amp +from mindspore.train.callback import _InternalCallbackParam, RunContext, _CallbackManager +from mindspore.train.parallel_utils import ParallelMode + +from src.dataset_helper import DatasetHelper + + +def _convert_type(types): + """ + Convert from numpy type to tensor type. + + Args: + types (list): Numpy type list of element in dataset. + + Returns: + list, list of element in dataset. + """ + ms_types = [] + for np_type in types: + ms_type = pytype_to_dtype(np_type) + ms_types.append(ms_type) + return ms_types + + +def _get_types_and_shapes(dataset): + """Get dataset types and shapes.""" + dataset_types = _convert_type(dataset.output_types()) + dataset_shapes = dataset.output_shapes() + return dataset_types, dataset_shapes + + +def _exec_datagraph(exec_dataset, dataset_size, phase='dataset'): + """Initialize and execute the dataset graph.""" + batch_size = exec_dataset.get_batch_size() + input_indexs = exec_dataset.input_indexs + + # transform data format + dataset_types, dataset_shapes = _get_types_and_shapes(exec_dataset) + init_exec_dataset(exec_dataset.__ME_INITED__, + dataset_size, + batch_size, + dataset_types, + dataset_shapes, + input_indexs, + phase=phase, + need_run=False) + + +class Model: + """ + High-Level API for Training or Testing. + + `Model` groups layers into an object with training and inference features. + + Args: + network (Cell): The training or testing network. + loss_fn (Cell): Objective function, if loss_fn is None, the + network should contain the logic of loss and grads calculation, and the logic + of parallel if needed. Default: None. + optimizer (Cell): Optimizer for updating the weights. Default: None. + metrics (Union[dict, set]): Dict or set of metrics to be evaluated by the model during + training and testing. eg: {'accuracy', 'recall'}. Default: None. + eval_network (Cell): Network for evaluation. If not defined, `network` and `loss_fn` would be wrapped as + `eval_network`. Default: None. + eval_indexes (list): In case of defining the `eval_network`, if `eval_indexes` is None, all outputs of + `eval_network` would be passed to metrics, otherwise `eval_indexes` must contain three + elements, representing the positions of loss value, predict value and label, the loss + value would be passed to `Loss` metric, predict value and label would be passed to other + metric. Default: None. + amp_level (str): Option for argument `level` in `mindspore.amp.build_train_network`, level for mixed + precision training. Supports [O0, O2]. Default: "O0". + + - O0: Do not change. + - O2: Cast network to float16, keep batchnorm run in float32, using dynamic loss scale. + + loss_scale_manager (Union[None, LossScaleManager]): If None, not scale the loss, or else + scale the loss by LossScaleManager. If it is set, overwrite the level setting. It's a eyword argument. + e.g. Use `loss_scale_manager=None` to set the value. + keep_batchnorm_fp32 (bool): Keep Batchnorm run in `float32`. If set, overwrite the level setting. Default: True. + + Examples: + >>> class Net(nn.Cell): + >>> def __init__(self): + >>> super(Net, self).__init__() + >>> self.conv = nn.Conv2d(3, 64, 3, has_bias=False, weight_init='normal') + >>> self.bn = nn.BatchNorm2d(64) + >>> self.relu = nn.ReLU() + >>> self.flatten = nn.Flatten() + >>> self.fc = nn.Dense(64*224*224, 12) # padding=0 + >>> + >>> def construct(self, x): + >>> x = self.conv(x) + >>> x = self.bn(x) + >>> x = self.relu(x) + >>> x = self.flatten(x) + >>> out = self.fc(x) + >>> return out + >>> + >>> net = Net() + >>> loss = nn.SoftmaxCrossEntropyWithLogits(is_grad=False, sparse=True) + >>> optim = Momentum(params=net.trainable_params(), learning_rate=0.1, momentum=0.9) + >>> model = Model(net, loss_fn=loss, optimizer=optim, metrics=None) + >>> dataset = get_dataset() + >>> model.train(2, dataset) + """ + + def __init__(self, network, loss_fn=None, optimizer=None, metrics=None, eval_network=None, + eval_indexes=None, amp_level="O0", frequency=278, stop_epoch=100, **kwargs): + self._network = network + self._loss_fn = loss_fn + self._optimizer = optimizer + self._loss_scale_manager = None + self._loss_scale_manager_set = False + self._keep_bn_fp32 = True + self._check_kwargs(kwargs) + self._amp_level = amp_level + self._process_amp_args(kwargs) + self._parallel_mode = _get_parallel_mode() + self._device_number = _get_device_num() + self._global_rank = _get_global_rank() + self._parameter_broadcast = _get_parameter_broadcast() + self._frequency = frequency + self._stop_epoch = stop_epoch + + self._train_network = self._build_train_network() + self._build_eval_network(metrics, eval_network, eval_indexes) + self._build_predict_network() + + def _process_amp_args(self, kwargs): + if self._amp_level == "O0": + self._keep_bn_fp32 = False + if 'keep_batchnorm_fp32' in kwargs: + self._keep_bn_fp32 = kwargs['keep_batchnorm_fp32'] + if 'loss_scale_manager' in kwargs: + self._loss_scale_manager = kwargs['loss_scale_manager'] + self._loss_scale_manager_set = True + + def _check_kwargs(self, kwargs): + for arg in kwargs: + if arg not in ['loss_scale_manager', 'keep_batchnorm_fp32']: + raise ValueError(f"Unsupport arg '{arg}'") + + def _build_train_network(self): + """Build train network""" + network = self._network + if self._optimizer: + if self._loss_scale_manager_set: + network = amp.build_train_network(network, + self._optimizer, + self._loss_fn, + level=self._amp_level, + loss_scale_manager=self._loss_scale_manager, + keep_batchnorm_fp32=self._keep_bn_fp32) + else: + network = amp.build_train_network(network, + self._optimizer, + self._loss_fn, + level=self._amp_level, + keep_batchnorm_fp32=self._keep_bn_fp32) + elif self._loss_fn: + network = nn.WithLossCell(network, self._loss_fn) + # If need to check if loss_fn is not None, but optimizer is None + + if self._parallel_mode in (ParallelMode.SEMI_AUTO_PARALLEL, ParallelMode.AUTO_PARALLEL): + network.set_auto_parallel() + return network + + def _build_eval_network(self, metrics, eval_network, eval_indexes): + """Build the network for evaluation.""" + self._metric_fns = get_metrics(metrics) + if not self._metric_fns: + return + + if eval_network is not None: + if eval_indexes is not None and not (isinstance(eval_indexes, list) and len(eval_indexes) == 3): + raise ValueError("Eval_indexes must be a list or None. If eval_indexes is a list, length of it \ + must be three. But got {}".format(eval_indexes)) + + self._eval_network = eval_network + self._eval_indexes = eval_indexes + else: + if self._loss_fn is None: + raise ValueError("loss_fn can not be None.") + self._eval_network = nn.WithEvalCell(self._network, self._loss_fn, self._amp_level == "O2") + self._eval_indexes = [0, 1, 2] + + if self._parallel_mode in (ParallelMode.SEMI_AUTO_PARALLEL, ParallelMode.AUTO_PARALLEL): + self._eval_network.set_auto_parallel() + + def _build_predict_network(self): + """Build the network for prediction.""" + self._predict_network = self._network + if self._parallel_mode in (ParallelMode.SEMI_AUTO_PARALLEL, ParallelMode.AUTO_PARALLEL): + self._predict_network = _VirtualDatasetCell(self._network) + self._predict_network.set_auto_parallel() + + def _clear_metrics(self): + """Clear metrics local values.""" + for metric in self._metric_fns.values(): + metric.clear() + + def _update_metrics(self, outputs): + """Update metrics local values.""" + if not isinstance(outputs, tuple): + raise ValueError("The `outputs` is not tuple.") + + if self._eval_indexes is not None and len(outputs) < 3: + raise ValueError("The length of `outputs` must be greater than or equal to 3, \ + but got {}".format(len(outputs))) + + for metric in self._metric_fns.values(): + if self._eval_indexes is None: + metric.update(*outputs) + else: + if isinstance(metric, Loss): + metric.update(outputs[self._eval_indexes[0]]) + else: + metric.update(outputs[self._eval_indexes[1]], outputs[self._eval_indexes[2]]) + + def _get_metrics(self): + """Get metrics local values.""" + metrics = dict() + for key, value in self._metric_fns.items(): + metrics[key] = value.eval() + return metrics + + def _get_scaling_sens(self): + """get the scaling sens""" + scaling_sens = 1 + if self._loss_scale_manager is not None: + scaling_sens = self._loss_scale_manager.get_loss_scale() + if self._parallel_mode == ParallelMode.DATA_PARALLEL: + scaling_sens /= self._device_number + return scaling_sens + + def _exec_preprocess(self, network, is_train, phase, dataset, dataset_sink_mode, iter_first_order): + """Initializes dataset.""" + need_wrap = False + if dataset_sink_mode: + # remove later to deal with loop sink + if not hasattr(dataset, '__ME_INITED__') and context.get_context("device_target") == "Ascend" \ + and not context.get_context("enable_ge"): + need_wrap = True + + if not is_train: + dataset.__loop_size__ = 1 + + dataset_helper = DatasetHelper(dataset, dataset_sink_mode, iter_first_order) + + # remove later to deal with loop sink + if need_wrap: + network = nn.DataWrapper(network, *(dataset_helper.types_shapes()), dataset.__ME_INITED__) + network.set_train(is_train) + network.phase = phase + + return dataset_helper, network + + def init(self, train_dataset=None, valid_dataset=None): + """ + Initializes compute graphs and data graphs with sink mode. + + Note: + Pre-init process only supports `GRAPH_MODE` and `Ascend` target currently. + + Args: + train_dataset (Dataset): A training dataset iterator. If define `train_dataset`, training graphs will be + initialized. Default: None. + valid_dataset (Dataset): A evaluating dataset iterator. If define `valid_dataset`, evaluation graphs will + be initialized, and `metrics` in `Model` can not be None. Default: None. + + Examples: + >>> train_dataset = get_train_dataset() + >>> valid_dataset = get_valid_dataset() + >>> net = Net() + >>> loss = nn.SoftmaxCrossEntropyWithLogits(is_grad=False, sparse=True) + >>> optim = Momentum(params=net.trainable_params(), learning_rate=0.1, momentum=0.9) + >>> model = Model(net, loss_fn=loss, optimizer=optim, metrics={'acc'}) + >>> model.init(train_dataset, valid_dataset) + >>> model.train(2, train_dataset) + >>> model.eval(valid_dataset) + """ + if context.get_context("mode") != context.GRAPH_MODE or context.get_context("device_target") != "Ascend": + raise RuntimeError('Pre-init process only supports GRAPH MODE and Ascend target currently.') + + if not train_dataset and not valid_dataset: + raise ValueError('Both train_dataset and valid_dataset can not be None or empty.') + + _device_number_check(self._parallel_mode, self._device_number) + + if train_dataset: + _parameter_broadcast_check(self._parallel_mode, self._parameter_broadcast) + self._train_network.set_train() + self._train_network.phase = 'train' + + if self._parameter_broadcast: + self._train_network.set_broadcast_flag() + + train_dataset_helper, train_network = self._exec_preprocess(self._train_network, + is_train=True, + phase='train', + dataset=train_dataset, + dataset_sink_mode=True) + self._train_network = train_network + for inputs in train_dataset_helper: + self._train_network.compile(*inputs) + break + + if valid_dataset: + if not self._metric_fns: + raise RuntimeError('If define `valid_dataset`, metric fn can not be None or empty.') + + self._eval_network.set_train(False) + self._eval_network.phase = 'eval' + valid_dataset_helper, eval_network = self._exec_preprocess(self._eval_network, + is_train=False, + phase='eval', + dataset=valid_dataset, + dataset_sink_mode=True) + self._eval_network = eval_network + for inputs in valid_dataset_helper: + self._eval_network.compile(*inputs) + break + + def _train(self, epoch, train_dataset, callbacks=None, dataset_sink_mode=True): + """ + Training. + + Args: + epoch (int): Total number of iterations on the data. + train_dataset (Dataset): A training dataset iterator. If there is no + loss_fn, a tuple with multiply data (data1, data2, data3, ...) will be + returned and passed to the network. Otherwise, a tuple (data, label) will + be returned, and the data and label are passed to the network and loss + function respectively. + callbacks (list): List of callback object. Callbacks which should be executed while training. Default: None. + dataset_sink_mode (bool): Determines whether to pass the data through dataset channel. Default: True. + Configure pynative mode, the training process will be performed with + dataset not sink. + """ + epoch = check_int_positive(epoch) + self._train_network.set_train() + + if self._parameter_broadcast: + self._train_network.set_broadcast_flag() + + # build callback list + cb_params = _InternalCallbackParam() + cb_params.train_network = self._train_network + cb_params.epoch_num = epoch + cb_params.batch_num = train_dataset.get_dataset_size() + cb_params.mode = "train" + cb_params.loss_fn = self._loss_fn + cb_params.optimizer = self._optimizer + cb_params.parallel_mode = self._parallel_mode + cb_params.device_number = self._device_number + cb_params.train_dataset = train_dataset + cb_params.list_callback = callbacks + + with _CallbackManager(callbacks) as list_callback: + if not dataset_sink_mode: + self._train_process(epoch, train_dataset, list_callback, cb_params) + elif context.get_context("mode") == context.PYNATIVE_MODE: + logger.warning("The pynative mode cannot support dataset sink mode currently." + "So the training process will be performed with dataset not sink.") + self._train_process(epoch, train_dataset, list_callback, cb_params) + else: + self._train_dataset_sink_process(epoch, train_dataset, list_callback, cb_params) + + def _train_dataset_sink_process(self, epoch, train_dataset, list_callback=None, cb_params=None): + """ + Training process. The data would be passed to network through dataset channel. + + Args: + epoch (int): Total number of iterations on the data. + train_dataset (Dataset): A training dataset iterator. If there is no + loss_fn, a tuple with multiply data (data1, data2, data3, ...) should be + returned and passed to the network. Otherwise, a tuple (data, label) should + be returned, and the data and label are passed to the network and loss + function respectively. + list_callback (Callback): Executor of callback list. Default: None. + cb_params (_InternalCallbackParam): Callback parameters. Default: None. + """ + iter_first_order = self._frequency - 1 + iter_second_order = 1 + train_dataset.__loop_size__ = iter_second_order + dataset_helper, train_network = self._exec_preprocess(self._train_network, + is_train=True, + phase='train', + dataset=train_dataset, + dataset_sink_mode=True, + iter_first_order=iter_first_order) + self._train_network = train_network + cb_params.train_network = self._train_network + cb_params.cur_step_num = 0 + + loop_size = dataset_helper.loop_size() + run_context = RunContext(cb_params) + list_callback.begin(run_context) + + # used to stop training for early stop, such as stopAtTIme or stopATStep + should_stop = False + has_do_dataset_init = False + switch_branch_one = True + for i in range(epoch): + cb_params.cur_epoch_num = i + 1 + list_callback.epoch_begin(run_context) + + # for data sink dataset_helper only iter once, other wise iter epoch_size times. + for inputs in dataset_helper: + list_callback.step_begin(run_context) + if switch_branch_one: + cb_params.cur_step_num += loop_size + self._train_network.add_flags_recursive(thor=True) + self._train_network.phase = 'train0' + else: + cb_params.cur_step_num += iter_first_order + self._train_network.add_flags_recursive(thor=False) + self._train_network.phase = 'train1' + if not has_do_dataset_init: + _exec_datagraph(train_dataset, iter_first_order, phase='train1_dataset') + has_do_dataset_init = True + switch_branch_one = not switch_branch_one + outputs = self._train_network(*inputs) + cb_params.net_outputs = outputs + list_callback.step_end(run_context) + + list_callback.epoch_end(run_context) + should_stop = should_stop or run_context.get_stop_requested() + if should_stop: + break + + list_callback.end(run_context) + + def _train_process(self, epoch, train_dataset, list_callback=None, cb_params=None): + """ + Training process. The data would be passed to network directly. + + Args: + epoch (int): Total number of iterations on the data. + train_dataset (Dataset): A training dataset iterator. If there is no + loss_fn, a tuple with multiply data (data1, data2, data3, ...) should be + returned and passed to the network. Otherwise, a tuple (data, label) should + be returned, and the data and label are passed to the network and loss + function respectively. + list_callback (Callback): Executor of callback list. Default: None. + cb_params (_InternalCallbackParam): Callback parameters. Default: None. + """ + dataset_helper, _ = self._exec_preprocess(self._train_network, + is_train=True, + phase='train', + dataset=train_dataset, + dataset_sink_mode=False) + cb_params.cur_step_num = 0 + run_context = RunContext(cb_params) + list_callback.begin(run_context) + # used to stop training for early stop, such as stopAtTIme or stopATStep + should_stop = False + + for i in range(epoch): + cb_params.cur_epoch_num = i + 1 + + list_callback.epoch_begin(run_context) + + for next_element in dataset_helper: + len_element = len(next_element) + if self._loss_fn and len_element != 2: + raise ValueError("when loss_fn is not None, train_dataset should" + "return two elements, but got {}".format(len_element)) + cb_params.cur_step_num += 1 + list_callback.step_begin(run_context) + + overflow = False + if self._loss_scale_manager and self._loss_scale_manager.get_drop_overflow_update(): + scaling_sens = self._get_scaling_sens() + next_element = tuple(next_element) + (Tensor(scaling_sens, mstype.float32),) + + outputs = self._train_network(*next_element) + cb_params.net_outputs = outputs + if self._loss_scale_manager and self._loss_scale_manager.get_drop_overflow_update(): + _, overflow, _ = outputs + overflow = np.all(overflow.asnumpy()) + self._loss_scale_manager.update_loss_scale(overflow) + + list_callback.step_end(run_context) + should_stop = should_stop or run_context.get_stop_requested() + if should_stop: + break + + train_dataset.reset() + + list_callback.epoch_end(run_context) + should_stop = should_stop or run_context.get_stop_requested() + if should_stop: + break + + list_callback.end(run_context) + + def train(self, epoch, train_dataset, callbacks=None, dataset_sink_mode=True): + """ + Training API where the iteration is controlled by python front-end. + + When setting pynative mode, the training process will be performed with dataset not sink. + + Note: + CPU is not supported when dataset_sink_mode is true. + If dataset_sink_mode is True, epoch of training should be equal to the count of repeat + operation in dataset processing. Otherwise, errors could occur since the amount of data + is not the amount training requires. + If dataset_sink_mode is True, data will be sent to device. If device is Ascend, features + of data will be transferred one by one. The limitation of data transmission per time is 256M. + + Args: + epoch (int): Total number of iterations on the data. + train_dataset (Dataset): A training dataset iterator. If there is no + loss_fn, a tuple with multiply data (data1, data2, data3, ...) should be + returned and passed to the network. Otherwise, a tuple (data, label) should + be returned, and the data and label are passed to the network and loss + function respectively. + callbacks (list): List of callback object. Callbacks which should be excuted while training. Default: None. + dataset_sink_mode (bool): Determines whether to pass the data through dataset channel. Default: True. + Configure pynative mode, the training process will be performed with + dataset not sink. + + + Examples: + >>> dataset = get_dataset() + >>> net = Net() + >>> loss = nn.SoftmaxCrossEntropyWithLogits(is_grad=False, sparse=True) + >>> loss_scale_manager = FixedLossScaleManager() + >>> optim = Momentum(params=net.trainable_params(), learning_rate=0.1, momentum=0.9) + >>> model = Model(net, loss_fn=loss, optimizer=optim, metrics=None, loss_scale_manager=loss_scale_manager) + >>> model.train(2, dataset) + """ + repeat_count = train_dataset.get_repeat_count() + if epoch != repeat_count and dataset_sink_mode is True: + logger.warning(f"The epoch_size {epoch} is not the same with dataset repeat_count {repeat_count}") + check_bool(dataset_sink_mode) + _device_number_check(self._parallel_mode, self._device_number) + _parameter_broadcast_check(self._parallel_mode, self._parameter_broadcast) + + self._train(epoch, + train_dataset, + callbacks=callbacks, + dataset_sink_mode=dataset_sink_mode) + + def _eval_dataset_sink_process(self, valid_dataset, list_callback=None, cb_params=None): + """ + Evaluation. The data would be passed to network through dataset channel. + + Args: + valid_dataset (Dataset): Dataset to evaluate the model. + list_callback (Callback): Executor of callback list. Default: None. + cb_params (_InternalCallbackParam): Callback parameters. Default: None. + + Returns: + Dict, returns the loss value & metrics values for the model in test mode. + """ + run_context = RunContext(cb_params) + + dataset_helper, eval_network = self._exec_preprocess(self._eval_network, + is_train=False, + phase='eval', + dataset=valid_dataset, + dataset_sink_mode=True) + self._eval_network = eval_network + cb_params.eval_network = self._eval_network + list_callback.begin(run_context) + + for inputs in dataset_helper: + cb_params.cur_step_num += 1 + list_callback.step_begin(run_context) + + outputs = self._eval_network(*inputs) + + cb_params.net_outputs = outputs + list_callback.step_end(run_context) + self._update_metrics(outputs) + + metrics = self._get_metrics() + cb_params.metrics = metrics + list_callback.end(run_context) + + return metrics + + def _eval_process(self, valid_dataset, list_callback=None, cb_params=None): + """ + Evaluation. The data would be passed to network directly. + + Args: + valid_dataset (Dataset): Dataset to evaluate the model. + list_callback (Callback): Executor of callback list. Default: None. + cb_params (_InternalCallbackParam): Callback parameters. Default: None. + + Returns: + Dict, returns the loss value & metrics values for the model in test mode. + """ + run_context = RunContext(cb_params) + list_callback.begin(run_context) + + dataset_helper, _ = self._exec_preprocess(self._eval_network, + is_train=False, + phase='eval', + dataset=valid_dataset, + dataset_sink_mode=False) + for next_element in dataset_helper: + cb_params.cur_step_num += 1 + list_callback.step_begin(run_context) + outputs = self._eval_network(*next_element) + cb_params.net_outputs = outputs + list_callback.step_end(run_context) + self._update_metrics(outputs) + + metrics = self._get_metrics() + cb_params.metrics = metrics + list_callback.end(run_context) + return metrics + + def eval(self, valid_dataset, callbacks=None, dataset_sink_mode=True): + """ + Evaluation API where the iteration is controlled by python front-end. + + Configure to pynative mode, the evaluation will be performed with dataset non-sink mode. + + Note: + CPU is not supported when dataset_sink_mode is true. + If dataset_sink_mode is True, data will be sent to device. If device is Ascend, features + of data will be transferred one by one. The limitation of data transmission per time is 256M. + + Args: + valid_dataset (Dataset): Dataset to evaluate the model. + callbacks (list): List of callback object. Callbacks which should be excuted + while training. Default: None. + dataset_sink_mode (bool): Determines whether to pass the data through dataset channel. Default: True. + + Returns: + Dict, returns the loss value & metrics values for the model in test mode. + + Examples: + >>> dataset = get_dataset() + >>> net = Net() + >>> loss = nn.SoftmaxCrossEntropyWithLogits(is_grad=False, sparse=True) + >>> model = Model(net, loss_fn=loss, optimizer=None, metrics={'acc'}) + >>> model.eval(dataset) + """ + check_bool(dataset_sink_mode) + _device_number_check(self._parallel_mode, self._device_number) + if not self._metric_fns: + raise ValueError("metric fn can not be None or empty.") + + cb_params = _InternalCallbackParam() + cb_params.eval_network = self._eval_network + cb_params.valid_dataset = valid_dataset + cb_params.batch_num = valid_dataset.get_dataset_size() + cb_params.mode = "eval" + cb_params.cur_step_num = 0 + + self._eval_network.set_train(mode=False) + self._eval_network.phase = 'eval' + + self._clear_metrics() + + with _CallbackManager(callbacks) as list_callback: + if dataset_sink_mode: + return self._eval_dataset_sink_process(valid_dataset, list_callback, cb_params) + return self._eval_process(valid_dataset, list_callback, cb_params) + + def predict(self, *predict_data): + """ + Generates output predictions for the input samples. + + Data could be single tensor, or list of tensor, tuple of tensor. + + Note: + Batch data should be put together in one tensor. + + Args: + predict_data (Tensor): Tensor of predict data. can be array, list or tuple. + + Returns: + Tensor, array(s) of predictions. + + Examples: + >>> input_data = Tensor(np.random.randint(0, 255, [1, 3, 224, 224]), mindspore.float32) + >>> model = Model(Net()) + >>> model.predict(input_data) + """ + self._predict_network.set_train(False) + check_input_data(*predict_data, data_class=Tensor) + result = self._predict_network(*predict_data) + + check_output_data(result) + return result + + +__all__ = ["Model"] diff --git a/model_zoo/resnet_thor/src/resnet50.py b/model_zoo/resnet_thor/src/resnet50.py new file mode 100644 index 0000000000..00beac2fdf --- /dev/null +++ b/model_zoo/resnet_thor/src/resnet50.py @@ -0,0 +1,262 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""ResNet.""" +import numpy as np +import mindspore.nn as nn +from mindspore.ops import operations as P +from mindspore.common.tensor import Tensor + + +def _weight_variable(shape, factor=0.01): + init_value = np.random.randn(*shape).astype(np.float32) * factor + return Tensor(init_value) + + +def _conv3x3(in_channel, out_channel, stride=1): + weight_shape = (out_channel, in_channel, 3, 3) + weight = _weight_variable(weight_shape) + return nn.Conv2d(in_channel, out_channel, + kernel_size=3, stride=stride, padding=0, pad_mode='same', weight_init=weight) + + +def _conv1x1(in_channel, out_channel, stride=1): + weight_shape = (out_channel, in_channel, 1, 1) + weight = _weight_variable(weight_shape) + return nn.Conv2d(in_channel, out_channel, + kernel_size=1, stride=stride, padding=0, pad_mode='same', weight_init=weight) + + +def _conv7x7(in_channel, out_channel, stride=1): + weight_shape = (out_channel, in_channel, 7, 7) + weight = _weight_variable(weight_shape) + return nn.Conv2d(in_channel, out_channel, + kernel_size=7, stride=stride, padding=0, pad_mode='same', weight_init=weight) + + +def _bn(channel): + return nn.BatchNorm2d(channel, eps=1e-4, momentum=0.9, + gamma_init=1, beta_init=0, moving_mean_init=0, moving_var_init=1) + + +def _bn_last(channel): + return nn.BatchNorm2d(channel, eps=1e-4, momentum=0.9, + gamma_init=0, beta_init=0, moving_mean_init=0, moving_var_init=1) + + +def _fc(in_channel, out_channel): + weight_shape = (out_channel, in_channel) + weight = _weight_variable(weight_shape) + return nn.Dense(in_channel, out_channel, has_bias=True, weight_init=weight, bias_init=0) + + +class ResidualBlock(nn.Cell): + """ + ResNet V1 residual block definition. + + Args: + in_channel (int): Input channel. + out_channel (int): Output channel. + stride (int): Stride size for the first convolutional layer. Default: 1. + + Returns: + Tensor, output tensor. + + Examples: + >>> ResidualBlock(3, 256, stride=2) + """ + expansion = 4 + + def __init__(self, + in_channel, + out_channel, + stride=1): + super(ResidualBlock, self).__init__() + + channel = out_channel // self.expansion + self.conv1 = _conv1x1(in_channel, channel, stride=1) + self.bn1 = _bn(channel) + + self.conv2 = _conv3x3(channel, channel, stride=stride) + self.bn2 = _bn(channel) + + self.conv3 = _conv1x1(channel, out_channel, stride=1) + self.bn3 = _bn_last(out_channel) + + self.relu = nn.ReLU() + + self.down_sample = False + + if stride != 1 or in_channel != out_channel: + self.down_sample = True + self.down_sample_layer = None + + if self.down_sample: + self.down_sample_layer = nn.SequentialCell([_conv1x1(in_channel, out_channel, stride), + _bn(out_channel)]) + self.add = P.TensorAdd() + + def construct(self, x): + identity = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + out = self.relu(out) + + out = self.conv3(out) + out = self.bn3(out) + + if self.down_sample: + identity = self.down_sample_layer(identity) + + out = self.add(out, identity) + out = self.relu(out) + + return out + + +class ResNet(nn.Cell): + """ + ResNet architecture. + + Args: + block (Cell): Block for network. + layer_nums (list): Numbers of block in different layers. + in_channels (list): Input channel in each layer. + out_channels (list): Output channel in each layer. + strides (list): Stride size in each layer. + num_classes (int): The number of classes that the training images are belonging to. + Returns: + Tensor, output tensor. + + Examples: + >>> ResNet(ResidualBlock, + >>> [3, 4, 6, 3], + >>> [64, 256, 512, 1024], + >>> [256, 512, 1024, 2048], + >>> [1, 2, 2, 2], + >>> 10) + """ + + def __init__(self, + block, + layer_nums, + in_channels, + out_channels, + strides, + num_classes): + super(ResNet, self).__init__() + + if not len(layer_nums) == len(in_channels) == len(out_channels) == 4: + raise ValueError("the length of layer_num, in_channels, out_channels list must be 4!") + + self.conv1 = _conv7x7(3, 64, stride=2) + self.bn1 = _bn(64) + self.relu = P.ReLU() + self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, pad_mode="same") + + self.layer1 = self._make_layer(block, + layer_nums[0], + in_channel=in_channels[0], + out_channel=out_channels[0], + stride=strides[0]) + self.layer2 = self._make_layer(block, + layer_nums[1], + in_channel=in_channels[1], + out_channel=out_channels[1], + stride=strides[1]) + self.layer3 = self._make_layer(block, + layer_nums[2], + in_channel=in_channels[2], + out_channel=out_channels[2], + stride=strides[2]) + self.layer4 = self._make_layer(block, + layer_nums[3], + in_channel=in_channels[3], + out_channel=out_channels[3], + stride=strides[3]) + + self.mean = P.ReduceMean(keep_dims=True) + self.flatten = nn.Flatten() + self.end_point = _fc(out_channels[3], num_classes) + + def _make_layer(self, block, layer_num, in_channel, out_channel, stride): + """ + Make stage network of ResNet. + + Args: + block (Cell): Resnet block. + layer_num (int): Layer number. + in_channel (int): Input channel. + out_channel (int): Output channel. + stride (int): Stride size for the first convolutional layer. + + Returns: + SequentialCell, the output layer. + + Examples: + >>> _make_layer(ResidualBlock, 3, 128, 256, 2) + """ + layers = [] + + resnet_block = block(in_channel, out_channel, stride=stride) + layers.append(resnet_block) + + for _ in range(1, layer_num): + resnet_block = block(out_channel, out_channel, stride=1) + layers.append(resnet_block) + + return nn.SequentialCell(layers) + + def construct(self, x): + x = self.conv1(x) + x = self.bn1(x) + x = self.relu(x) + c1 = self.maxpool(x) + + c2 = self.layer1(c1) + c3 = self.layer2(c2) + c4 = self.layer3(c3) + c5 = self.layer4(c4) + + out = self.mean(c5, (2, 3)) + out = self.flatten(out) + out = self.end_point(out) + + return out + + +def resnet50(class_num=10): + """ + Get ResNet50 neural network. + + Args: + class_num (int): Class number. + + Returns: + Cell, cell instance of ResNet50 neural network. + + Examples: + >>> net = resnet50(10) + """ + return ResNet(ResidualBlock, + [3, 4, 6, 3], + [64, 256, 512, 1024], + [256, 512, 1024, 2048], + [1, 2, 2, 2], + class_num) diff --git a/model_zoo/resnet_thor/src/resnet_thor.py b/model_zoo/resnet_thor/src/resnet_thor.py new file mode 100644 index 0000000000..50e694854f --- /dev/null +++ b/model_zoo/resnet_thor/src/resnet_thor.py @@ -0,0 +1,359 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""ResNet.""" +import math +import numpy as np +import mindspore.nn as nn +from mindspore.common.tensor import Tensor +from mindspore.ops import operations as P + +from src.thor_layer import Conv2d_Thor, Dense_Thor + + +def calculate_gain(nonlinearity, param=None): + """calculate_gain""" + linear_fns = ['linear', 'conv1d', 'conv2d', 'conv3d', 'conv_transpose1d', 'conv_transpose2d', 'conv_transpose3d'] + res = 0 + if nonlinearity in linear_fns or nonlinearity == 'sigmoid': + res = 1 + elif nonlinearity == 'tanh': + res = 5.0 / 3 + elif nonlinearity == 'relu': + res = math.sqrt(2.0) + elif nonlinearity == 'leaky_relu': + if param is None: + negative_slope = 0.01 + elif not isinstance(param, bool) and isinstance(param, int) or isinstance(param, float): + # True/False are instances of int, hence check above + negative_slope = param + else: + raise ValueError("negative_slope {} not a valid number".format(param)) + res = math.sqrt(2.0 / (1 + negative_slope ** 2)) + else: + raise ValueError("Unsupported nonlinearity {}".format(nonlinearity)) + return res + + +def _calculate_fan_in_and_fan_out(tensor): + """_calculate_fan_in_and_fan_out""" + dimensions = len(tensor) + if dimensions < 2: + raise ValueError("Fan in and fan out can not be computed for tensor with fewer than 2 dimensions") + if dimensions == 2: # Linear + fan_in = tensor[1] + fan_out = tensor[0] + else: + num_input_fmaps = tensor[1] + num_output_fmaps = tensor[0] + receptive_field_size = 1 + if dimensions > 2: + receptive_field_size = tensor[2] * tensor[3] + fan_in = num_input_fmaps * receptive_field_size + fan_out = num_output_fmaps * receptive_field_size + return fan_in, fan_out + + +def _calculate_correct_fan(tensor, mode): + mode = mode.lower() + valid_modes = ['fan_in', 'fan_out'] + if mode not in valid_modes: + raise ValueError("Mode {} not supported, please use one of {}".format(mode, valid_modes)) + fan_in, fan_out = _calculate_fan_in_and_fan_out(tensor) + return fan_in if mode == 'fan_in' else fan_out + + +def kaiming_normal(inputs_shape, a=0, mode='fan_in', nonlinearity='leaky_relu'): + fan = _calculate_correct_fan(inputs_shape, mode) + gain = calculate_gain(nonlinearity, a) + std = gain / math.sqrt(fan) + return np.random.normal(0, std, size=inputs_shape).astype(np.float32) + + +def kaiming_uniform(inputs_shape, a=0, mode='fan_in', nonlinearity='leaky_relu'): + fan = _calculate_correct_fan(inputs_shape, mode) + gain = calculate_gain(nonlinearity, a) + std = gain / math.sqrt(fan) + bound = math.sqrt(3.0) * std # Calculate uniform bounds from standard deviation + return np.random.uniform(-bound, bound, size=inputs_shape).astype(np.float32) + + +def _conv3x3(in_channel, out_channel, stride=1, damping=0.03, loss_scale=1, frequency=278): + weight_shape = (out_channel, in_channel, 3, 3) + weight = Tensor(kaiming_normal(weight_shape, mode="fan_out", nonlinearity='relu')) + return Conv2d_Thor(in_channel, out_channel, + kernel_size=3, stride=stride, padding=0, pad_mode='same', weight_init=weight, + damping=damping, loss_scale=loss_scale, frequency=frequency) + + +def _conv1x1(in_channel, out_channel, stride=1, damping=0.03, loss_scale=1, frequency=278): + weight_shape = (out_channel, in_channel, 1, 1) + weight = Tensor(kaiming_normal(weight_shape, mode="fan_out", nonlinearity='relu')) + return Conv2d_Thor(in_channel, out_channel, + kernel_size=1, stride=stride, padding=0, pad_mode='same', weight_init=weight, + damping=damping, loss_scale=loss_scale, frequency=frequency) + + +def _conv7x7(in_channel, out_channel, stride=1, damping=0.03, loss_scale=1, frequency=278): + weight_shape = (out_channel, in_channel, 7, 7) + weight = Tensor(kaiming_normal(weight_shape, mode="fan_out", nonlinearity='relu')) + return Conv2d_Thor(in_channel, out_channel, + kernel_size=7, stride=stride, padding=0, pad_mode='same', weight_init=weight, + damping=damping, loss_scale=loss_scale, frequency=frequency) + + +def _bn(channel): + return nn.BatchNorm2d(channel, eps=1e-4, momentum=0.9, + gamma_init=1, beta_init=0, moving_mean_init=0, moving_var_init=1) + + +def _bn_last(channel): + return nn.BatchNorm2d(channel, eps=1e-4, momentum=0.9, + gamma_init=1, beta_init=0, moving_mean_init=0, moving_var_init=1) + + +def _fc(in_channel, out_channel, damping, loss_scale, frequency): + weight_shape = (out_channel, in_channel) + weight = Tensor(kaiming_uniform(weight_shape, a=math.sqrt(5))) + return Dense_Thor(in_channel, out_channel, has_bias=False, weight_init=weight, + bias_init=0, damping=damping, loss_scale=loss_scale, frequency=frequency) + + +class ResidualBlock(nn.Cell): + """ + ResNet V1 residual block definition. + + Args: + in_channel (int): Input channel. + out_channel (int): Output channel. + stride (int): Stride size for the first convolutional layer. Default: 1. + + Returns: + Tensor, output tensor. + + Examples: + >>> ResidualBlock(3, 256, stride=2) + """ + expansion = 4 + + def __init__(self, + in_channel, + out_channel, + stride=1, + damping=0.03, + loss_scale=1, + frequency=278): + super(ResidualBlock, self).__init__() + + channel = out_channel // self.expansion + self.conv1 = _conv1x1(in_channel, channel, stride=1, damping=damping, loss_scale=loss_scale, + frequency=frequency) + self.bn1 = _bn(channel) + + self.conv2 = _conv3x3(channel, channel, stride=stride, damping=damping, loss_scale=loss_scale, + frequency=frequency) + self.bn2 = _bn(channel) + + self.conv3 = _conv1x1(channel, out_channel, stride=1, damping=damping, loss_scale=loss_scale, + frequency=frequency) + self.bn3 = _bn_last(out_channel) + + self.relu = nn.ReLU() + + self.down_sample = False + + if stride != 1 or in_channel != out_channel: + self.down_sample = True + self.down_sample_layer = None + + if self.down_sample: + self.down_sample_layer = nn.SequentialCell([_conv1x1(in_channel, out_channel, stride, + damping=damping, loss_scale=loss_scale, + frequency=frequency), + _bn(out_channel)]) + self.add = P.TensorAdd() + + def construct(self, x): + identity = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + out = self.relu(out) + + out = self.conv3(out) + out = self.bn3(out) + + if self.down_sample: + identity = self.down_sample_layer(identity) + + out = self.add(out, identity) + out = self.relu(out) + + return out + + +class ResNet(nn.Cell): + """ + ResNet architecture. + + Args: + block (Cell): Block for network. + layer_nums (list): Numbers of block in different layers. + in_channels (list): Input channel in each layer. + out_channels (list): Output channel in each layer. + strides (list): Stride size in each layer. + num_classes (int): The number of classes that the training images are belonging to. + Returns: + Tensor, output tensor. + + Examples: + >>> ResNet(ResidualBlock, + >>> [3, 4, 6, 3], + >>> [64, 256, 512, 1024], + >>> [256, 512, 1024, 2048], + >>> [1, 2, 2, 2], + >>> 10) + """ + + def __init__(self, + block, + layer_nums, + in_channels, + out_channels, + strides, + num_classes, + damping, + loss_scale, + frequency): + super(ResNet, self).__init__() + + if not len(layer_nums) == len(in_channels) == len(out_channels) == 4: + raise ValueError("the length of layer_num, in_channels, out_channels list must be 4!") + + self.conv1 = _conv7x7(3, 64, stride=2, damping=damping, loss_scale=loss_scale, frequency=frequency) + self.bn1 = _bn(64) + self.relu = P.ReLU() + self.maxpool = P.MaxPoolWithArgmax(padding="same", ksize=3, strides=2) + + self.layer1 = self._make_layer(block, + layer_nums[0], + in_channel=in_channels[0], + out_channel=out_channels[0], + stride=strides[0], + damping=damping, + loss_scale=loss_scale, + frequency=frequency) + self.layer2 = self._make_layer(block, + layer_nums[1], + in_channel=in_channels[1], + out_channel=out_channels[1], + stride=strides[1], + damping=damping, + loss_scale=loss_scale, + frequency=frequency) + self.layer3 = self._make_layer(block, + layer_nums[2], + in_channel=in_channels[2], + out_channel=out_channels[2], + stride=strides[2], damping=damping, + loss_scale=loss_scale, + frequency=frequency) + self.layer4 = self._make_layer(block, + layer_nums[3], + in_channel=in_channels[3], + out_channel=out_channels[3], + stride=strides[3], + damping=damping, + loss_scale=loss_scale, + frequency=frequency) + + self.mean = P.ReduceMean(keep_dims=True) + self.flatten = nn.Flatten() + self.end_point = _fc(out_channels[3], num_classes, damping=damping, loss_scale=loss_scale, frequency=frequency) + + def _make_layer(self, block, layer_num, in_channel, out_channel, stride, + damping, loss_scale, frequency): + """ + Make stage network of ResNet. + + Args: + block (Cell): Resnet block. + layer_num (int): Layer number. + in_channel (int): Input channel. + out_channel (int): Output channel. + stride (int): Stride size for the first convolutional layer. + + Returns: + SequentialCell, the output layer. + + Examples: + >>> _make_layer(ResidualBlock, 3, 128, 256, 2) + """ + layers = [] + + resnet_block = block(in_channel, out_channel, stride=stride, + damping=damping, loss_scale=loss_scale, frequency=frequency) + layers.append(resnet_block) + + for _ in range(1, layer_num): + resnet_block = block(out_channel, out_channel, stride=1, + damping=damping, loss_scale=loss_scale, frequency=frequency) + layers.append(resnet_block) + + return nn.SequentialCell(layers) + + def construct(self, x): + x = self.conv1(x) + x = self.bn1(x) + x = self.relu(x) + c1, _ = self.maxpool(x) + + c2 = self.layer1(c1) + c3 = self.layer2(c2) + c4 = self.layer3(c3) + c5 = self.layer4(c4) + + out = self.mean(c5, (2, 3)) + out = self.flatten(out) + out = self.end_point(out) + + return out + + +def resnet50(class_num=10, damping=0.03, loss_scale=1, frequency=278): + """ + Get ResNet50 neural network. + + Args: + class_num (int): Class number. + + Returns: + Cell, cell instance of ResNet50 neural network. + + Examples: + >>> net = resnet50(10) + """ + return ResNet(ResidualBlock, + [3, 4, 6, 3], + [64, 256, 512, 1024], + [256, 512, 1024, 2048], + [1, 2, 2, 2], + class_num, + damping, + loss_scale, + frequency) diff --git a/model_zoo/resnet_thor/src/thor.py b/model_zoo/resnet_thor/src/thor.py new file mode 100644 index 0000000000..cc82e6e369 --- /dev/null +++ b/model_zoo/resnet_thor/src/thor.py @@ -0,0 +1,199 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""momentum""" +import mindspore.common.dtype as mstype +from mindspore.common.initializer import initializer +from mindspore.common.parameter import Parameter +from mindspore.common.parameter import ParameterTuple +from mindspore.common.tensor import Tensor +from mindspore.nn.optim.optimizer import Optimizer +from mindspore.ops import functional as F, composite as C, operations as P +from mindspore.parallel._utils import _get_device_num, _get_mirror_mean +from src.grad_reducer_thor import DistributedGradReducerThor + +momentum_opt = C.MultitypeFuncGraph("momentum_opt") + + +@momentum_opt.register("Function", "Tensor", "Tensor", "Tensor", "Tensor", "Tensor") +def _tensor_run_opt_ext(opt, learning_rate, momentum, gradient, weight, moment): + """Apply momentum optimizer to the weight parameter using Tensor.""" + success = True + success = F.depend(success, opt(weight, moment, learning_rate, gradient, momentum)) + return success + + +op_add = P.AddN() +apply_decay = C.MultitypeFuncGraph("apply_decay") + + +@apply_decay.register("Number", "Bool", "Tensor", "Tensor") +def _tensor_apply_decay(weight_decay, if_apply, weight, gradient): + """Get grad with weight_decay.""" + if if_apply: + return op_add((weight * weight_decay, gradient)) + return gradient + + +class THOR(Optimizer): + """THOR""" + def __init__(self, params, learning_rate, momentum, matrix_A, matrix_G, A_inv_max, G_inv_max, weight_decay=0.0, + loss_scale=1.0, + decay_filter=lambda x: x.name not in []): + super(THOR, self).__init__(learning_rate, params, weight_decay, loss_scale) + if isinstance(momentum, float) and momentum < 0.0: + raise ValueError("momentum should be at least 0.0, but got momentum {}".format(momentum)) + self.momentum = Parameter(Tensor(momentum, mstype.float32), name="momentum") + self.params = self.parameters + self.moments = self.params.clone(prefix="moments", init='zeros') + self.hyper_map = C.HyperMap() + self.opt = P.ApplyMomentum() + self.matrix_A = ParameterTuple(matrix_A) + self.matrix_G = ParameterTuple(matrix_G) + self.A_inv_max = ParameterTuple(A_inv_max) + self.G_inv_max = ParameterTuple(G_inv_max) + self.cube_matmul_left = P.CusMatMulCubeFraczLeftCast() + self.cube_matmul_left_fc = P.CusMatMulCubeDenseLeft() + self.cube_matmul_right_fc = P.CusMatMulCubeDenseRight() + self.cube_matmul_right_mul = P.CusMatMulCubeFraczRightMul() + self.transpose = P.Transpose() + self.shape = P.Shape() + self.reshape = P.Reshape() + self.mul = P.Mul() + self.weight_idx = [] + for i in range(len(self.params)): + if "conv" in self.params[i].name or "end_point" in self.params[i].name: + self.weight_idx.append(i) + self.weight_idx.append(len(self.params)) + self.feature_map = [1.0 / 12544, 1.0 / 3136, 1.0 / 3136, 1.0 / 3136, 1.0 / 3136, 1.0 / 3136, 1.0 / 3136, + 1.0 / 3136, 1.0 / 3136, 1.0 / 3136, 1.0 / 3136, 1.0 / 3136, + 1.0 / 784, 1.0 / 784, 1.0 / 784, 1.0 / 784, 1.0 / 784, 1.0 / 784, 1.0 / 784, 1.0 / 784, + 1.0 / 784, 1.0 / 784, 1.0 / 784, 1.0 / 784, 1.0 / 784, + 1.0 / 196, 1.0 / 196, 1.0 / 196, 1.0 / 196, 1.0 / 196, 1.0 / 196, 1.0 / 196, 1.0 / 196, + 1.0 / 196, 1.0 / 196, 1.0 / 196, 1.0 / 196, 1.0 / 196, 1.0 / 196, 1.0 / 196, 1.0 / 196, + 1.0 / 196, 1.0 / 196, 1.0 / 196, + 1.0 / 49, 1.0 / 49, 1.0 / 49, 1.0 / 49, 1.0 / 49, 1.0 / 49, 1.0 / 49, 1.0 / 49, 1.0 / 49, + 1.0] + mean = _get_mirror_mean() + degree = _get_device_num() + self.grad_reducer_Amax = DistributedGradReducerThor(self.parameters, 2, mean, degree) + self.grad_reducer_Gmax = DistributedGradReducerThor(self.parameters, 5, mean, degree) + self.grad_reducer_A = DistributedGradReducerThor(self.parameters, 3, mean, degree) + self.grad_reducer_G = DistributedGradReducerThor(self.parameters, 4, mean, degree) + self.matrix_A_inv = () + self.matrix_G_inv = () + self.matrix_max_inv = () + + for i in range(54): + self.matrix_max_inv = self.matrix_max_inv + ( + Parameter(initializer(1, [1], mstype.float32), name="matrix_max" + str(i), requires_grad=False),) + self.log = P.Log() + self.exp = P.Exp() + self.sqrt = P.Sqrt() + self.matrix_max_inv = ParameterTuple(self.matrix_max_inv) + self.assign = P.Assign() + self.cast = P.Cast() + self.thor = True + self.weight_decay = weight_decay * loss_scale + self.decay_flags = tuple(decay_filter(x) for x in self.parameters) + + def construct(self, gradients): + params = self.params + moments = self.moments + if self.thor: + matrix_A_allreduce = () + matrix_G_allreduce = () + matrix_A_max_allreduce = () + matrix_G_max_allreduce = () + for i in range(54): + g = gradients[i * 3] + matrix_A = self.matrix_A[i] + matrix_G = self.matrix_G[i] + A_max = self.A_inv_max[i] + G_max = self.G_inv_max[i] + matrix_A = F.depend(matrix_A, g) + matrix_G = F.depend(matrix_G, g) + A_max = F.depend(A_max, g) + G_max = F.depend(G_max, g) + matrix_A_allreduce = matrix_A_allreduce + (matrix_A,) + matrix_G_allreduce = matrix_G_allreduce + (matrix_G,) + matrix_A_max_allreduce = matrix_A_max_allreduce + (A_max,) + matrix_G_max_allreduce = matrix_G_max_allreduce + (G_max,) + matrix_A_allreduce = self.grad_reducer_A(matrix_A_allreduce) + matrix_G_allreduce = self.grad_reducer_G(matrix_G_allreduce) + matrix_A_max_allreduce = self.grad_reducer_Amax(matrix_A_max_allreduce) + matrix_G_max_allreduce = self.grad_reducer_Gmax(matrix_G_max_allreduce) + new_grads = () + for i in range(54): + g = gradients[i * 3] + temp_a = matrix_A_allreduce[i] + temp_g = matrix_G_allreduce[i] + temp_a = self.cast(temp_a, mstype.float32) + temp_g = self.cast(temp_g, mstype.float32) + matrix_A_inv_max = self.log(matrix_A_max_allreduce[i]) + matrix_A_inv_max = self.mul(matrix_A_inv_max, -1) + matrix_A_inv_max = self.exp(matrix_A_inv_max) + temp_a = self.mul(temp_a, matrix_A_inv_max) + matrix_G_inv_max = self.log(matrix_G_max_allreduce[i]) + matrix_G_inv_max = self.mul(matrix_G_inv_max, -1) + matrix_G_inv_max = self.exp(matrix_G_inv_max) + temp_g = self.mul(temp_g, matrix_G_inv_max) + temp_max = self.mul(matrix_A_max_allreduce[i], matrix_G_max_allreduce[i]) + temp_max = self.mul(temp_max, self.feature_map[i]) + temp_a = self.cast(temp_a, mstype.float16) + temp_g = self.cast(temp_g, mstype.float16) + if i == 53: + g = self.cube_matmul_left_fc(temp_g, g) + g = self.cube_matmul_right_fc(g, temp_a, temp_max) + else: + g = self.cube_matmul_left(temp_g, g) + g = self.cube_matmul_right_mul(g, temp_a, temp_max) + fake_A = self.assign(self.matrix_A[i], temp_a) + fake_G = self.assign(self.matrix_G[i], temp_g) + fake_max = self.assign(self.matrix_max_inv[i], temp_max) + g = F.depend(g, fake_A) + g = F.depend(g, fake_G) + g = F.depend(g, fake_max) + if i == 53: + new_grads = new_grads + (g,) + else: + new_grads = new_grads + (g, gradients[i * 3 + 1], gradients[i * 3 + 2]) + gradients = new_grads + else: + new_grads = () + for i in range(54): + g = gradients[i * 3] + matrix_A = self.matrix_A[i] + matrix_G = self.matrix_G[i] + matrix_max = self.matrix_max_inv[i] + matrix_A = F.depend(matrix_A, g) + matrix_G = F.depend(matrix_G, g) + matrix_max = F.depend(matrix_max, g) + if i == 53: + g = self.cube_matmul_left_fc(matrix_G, g) + g = self.cube_matmul_right_fc(g, matrix_A, matrix_max) + new_grads = new_grads + (g,) + else: + g = self.cube_matmul_left(matrix_G, g) + g = self.cube_matmul_right_mul(g, matrix_A, matrix_max) + new_grads = new_grads + (g, gradients[i * 3 + 1], gradients[i * 3 + 2]) + gradients = new_grads + + if self.weight_decay > 0: + gradients = self.hyper_map(F.partial(apply_decay, self.weight_decay), self.decay_flags, + params, gradients) + gradients = self.scale_grad(gradients) + lr = self.get_lr() + success = self.hyper_map(F.partial(momentum_opt, self.opt, lr, self.momentum), gradients, params, moments) + return success diff --git a/model_zoo/resnet_thor/src/thor_layer.py b/model_zoo/resnet_thor/src/thor_layer.py new file mode 100644 index 0000000000..d84cbf7a93 --- /dev/null +++ b/model_zoo/resnet_thor/src/thor_layer.py @@ -0,0 +1,477 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""thor_layer""" +import numpy as np + +import mindspore as ms +import mindspore.common.dtype as mstype +from mindspore._checkparam import check_bool, twice, check_int_positive +from mindspore._extends import cell_attr_register +from mindspore.common.initializer import initializer +from mindspore.common.parameter import Parameter +from mindspore.common.tensor import Tensor +from mindspore.nn.cell import Cell +from mindspore.nn.layer.activation import get_activation +from mindspore.ops import operations as P +C0 = 16 + +def caculate_device_shape(matrix_dim, channel, is_A): + ll = (0) + if is_A: + if channel // C0 == 0: + matrix_dim = (matrix_dim / channel) * C0 + ll = (int(matrix_dim // C0), int(matrix_dim // C0), C0, C0), int(matrix_dim) + else: + ll = (int(matrix_dim // C0), int(matrix_dim // C0), C0, C0), int(matrix_dim) + return ll + +class _Conv(Cell): + r"""Applies a N-D convolution over an input signal composed of several input + planes. + """ + + def __init__(self, + in_channels, + out_channels, + kernel_size, + stride, + pad_mode, + padding, + dilation, + group, + data_format, + has_bias, + weight_init, + bias_init, + ): + super(_Conv, self).__init__() + self.in_channels = in_channels + self.out_channels = out_channels + self.kernel_size = kernel_size + self.stride = stride + self.pad_mode = pad_mode + self.padding = padding + self.dilation = dilation + self.group = group + self.data_format = data_format + self.has_bias = has_bias + if not (isinstance(in_channels, int) and in_channels > 0): + raise ValueError('Attr \'in_channels\' of \'Conv2D\' Op passed ' + + str(in_channels) + ', should be a int and greater than 0.') + if (not isinstance(kernel_size, tuple)) or len(kernel_size) != 2 or \ + (not isinstance(kernel_size[0], int)) or (not isinstance(kernel_size[1], int)) or \ + kernel_size[0] < 1 or kernel_size[1] < 1: + raise ValueError('Attr \'kernel_size\' of \'Conv2D\' Op passed ' + + str(self.kernel_size) + ', should be a int or tuple and equal to or greater than 1.') + if in_channels % group != 0: + raise ValueError('Attr \'in_channels\' of \'Conv2D\' Op must be divisible by ' + 'attr \'group\' of \'Conv2D\' Op.') + if out_channels % group != 0: + raise ValueError('Attr \'out_channels\' of \'Conv2D\' Op must be divisible by ' + 'attr \'group\' of \'Conv2D\' Op.') + + self.weight = Parameter(initializer( + weight_init, [out_channels, in_channels // group, *kernel_size]), name='weight') + + if check_bool(has_bias): + self.bias = Parameter(_initializer( + bias_init, [out_channels]), name='bias') + else: + if bias_init != 'zeros': + logger.warning("Value of 'has_bias' is False, value of 'bias_init' will be ignored.") + self.bias = None + + def construct(self, *inputs): + raise NotImplementedError + + +class Conv2d_Thor(_Conv): + """Conv2d_Thor""" + def __init__(self, + in_channels, + out_channels, + kernel_size, + stride=1, + pad_mode='same', + padding=0, + dilation=1, + group=1, + data_format='NCHW', + has_bias=False, + weight_init='normal', + damping=0.03, + loss_scale=1, + frequency=278, + bias_init='zeros'): + self.thor = True + ksizes = (1, kernel_size, kernel_size, 1) + self.hw = kernel_size * kernel_size + strides = (1, stride, stride, 1) + kernel_size = twice(kernel_size) + super(Conv2d_Thor, self).__init__( + in_channels, + out_channels, + kernel_size, + stride, + pad_mode, + padding, + dilation, + group, + data_format, + has_bias, + weight_init, + bias_init, + ) + self.conv2d = P.Conv2D(out_channel=self.out_channels, + kernel_size=self.kernel_size, + mode=1, + pad_mode=self.pad_mode, + pad=self.padding, + stride=self.stride, + dilation=self.dilation, + group=self.group + ) + + self.img2col = P.CusImg2Col(ksizes=ksizes, strides=strides) + self.cube_matmul = P.CusMatMulCube(transpose_a=True) + self.matrix_combine = P.CusMatrixCombine() + self.cholesky = P.CusCholeskyTrsm() + self.transpose02314 = P.CusTranspose02314() + self.matrix_A_dim = self.in_channels * self.kernel_size[0] * self.kernel_size[1] + self.matrix_G_dim = self.out_channels + self.matrix_A_device_shape, self.matrix_A_device_dim = caculate_device_shape(self.matrix_A_dim, + self.in_channels, True) + self.matrix_G_device_shape, self.matrix_G_device_dim = caculate_device_shape(self.matrix_G_dim, + self.in_channels, False) + self.matrix_A_device_temp_shape = ( + self.matrix_A_device_shape[0], self.matrix_A_device_shape[2], self.matrix_A_device_shape[1], + self.matrix_A_device_shape[3]) + self.matrix_G_device_temp_shape = ( + self.matrix_G_device_shape[0], self.matrix_G_device_shape[2], self.matrix_G_device_shape[1], + self.matrix_G_device_shape[3]) + self.matrix_A_inv = Parameter( + Tensor(np.reshape(np.identity(self.matrix_A_device_dim).astype(np.float16), self.matrix_A_device_shape)), + name='matrix_A_inv', requires_grad=False) + self.A_inv_max = Parameter(initializer(0, [1], mstype.float32), name="A_inv_max", requires_grad=False) + self.matrix_G_inv = Parameter( + Tensor(np.reshape(np.identity(self.matrix_G_device_dim).astype(np.float16), self.matrix_G_device_shape)), + name="matrix_G_inv", requires_grad=False) + + self.G_inv_max = Parameter(initializer(0, [1], mstype.float32), name="G_inv_max", requires_grad=False) + self.fake_G = Tensor( + np.reshape(np.identity(self.matrix_G_device_dim).astype(np.float16), self.matrix_G_device_shape)) + + self.shape = P.Shape() + self.reshape = P.Reshape() + self.transpose = P.Transpose() + self.cov_step = Parameter(initializer(0, [1], mstype.int32), name="cov_step", requires_grad=False) + self.mul = P.Mul() + self.cast = P.Cast() + self.damping = Tensor(damping) + self.vector_matmul = P.CusBatchMatMul() + self.diag_block_dim = 128 + self.channels_slice_flag = False + if self.in_channels % C0 != 0: + self.channels_slice_flag = True + + self.padA_flag = False + if (self.matrix_A_dim // self.diag_block_dim) * self.diag_block_dim != self.matrix_A_dim \ + and self.matrix_A_dim > self.diag_block_dim: + self.padA_flag = True + pad_dim = self.diag_block_dim - self.matrix_A_dim % self.diag_block_dim + self.padA = P.Pad(((0, pad_dim), (0, pad_dim))) + self.device_shape_pad_flag = False + if self.matrix_A_dim != self.matrix_A_device_dim: + self.device_shape_pad_flag = True + self.device_shape_pad = P.Pad(((0, 0), (0, C0 - self.in_channels), (0, 0), (0, C0 - self.in_channels))) + self.slice = P.Slice() + self.gather = P.GatherV2() + self.freq = Tensor(frequency, mstype.int32) + self.loss_scale = Tensor(1 / loss_scale, mstype.float16) + self.axis = 0 + + dampingA_dim = self.matrix_A_dim + if (self.matrix_A_dim % self.diag_block_dim) != 0 and self.matrix_A_dim > self.diag_block_dim: + dampingA_dim = (self.matrix_A_dim // self.diag_block_dim + 1) * self.diag_block_dim + dampingG_dim = self.matrix_G_dim + if (self.matrix_G_dim % self.diag_block_dim) != 0 and self.matrix_G_dim > self.diag_block_dim: + dampingG_dim = (self.matrix_G_dim // self.diag_block_dim + 1) * self.diag_block_dim + + self.dampingA = Tensor(np.identity(dampingA_dim), mstype.float32) + self.dampingG = Tensor(np.identity(dampingG_dim), mstype.float32) + self.fused_abs_max1 = P.CusFusedAbsMax1([self.matrix_A_dim, self.matrix_A_dim]) + self.fused_abs_max2 = P.CusFusedAbsMax1() + self.log = P.Log() + self.exp = P.Exp() + self.sqrt = P.Sqrt() + self.getG = P.InsertGradientOf(self.save_gradient) + + def save_gradient(self, dout): + """save_gradient""" + out = dout + dout = self.mul(dout, self.loss_scale) + dout = self.mul(dout, 32.0) + dout = self.transpose02314(dout) + dout_shape = self.shape(dout) + normalizer = dout_shape[0] + + matrix_G = self.cube_matmul(dout, dout) + normalizer = self.cast(normalizer, ms.float32) + matrix_G = self.mul(matrix_G, 1.0 / normalizer) + damping_step = self.gather(self.damping, self.cov_step, 0) + self.cov_step = self.cov_step + self.freq + damping_step = self.cast(damping_step, mstype.float32) + damping = self.mul(damping_step, 32.0 / normalizer) + damping = self.sqrt(damping) + dampingG = self.cast(self.dampingG, mstype.float32) + matrix_G = matrix_G + damping * dampingG + + matrix_G_inv = self.cholesky(matrix_G) + matrix_G_inv = self.vector_matmul(matrix_G_inv, matrix_G_inv) + matrix_G_inv_max = self.fused_abs_max2(matrix_G_inv) + matrix_G_inv_max = self.fused_abs_max2(matrix_G_inv_max) + self.G_inv_max = matrix_G_inv_max + matrix_G_inv = self.matrix_combine(matrix_G_inv) + matrix_G_inv = self.reshape(matrix_G_inv, self.matrix_G_device_temp_shape) + matrix_G_inv = self.transpose(matrix_G_inv, (2, 0, 1, 3)) + matrix_G = self.cast(matrix_G_inv, mstype.float16) + self.matrix_G_inv = matrix_G + return out + + def construct(self, x): + if self.thor: + matrix_A = self.img2col(x) + matrix_A_shape = self.shape(matrix_A) + normalizer = matrix_A_shape[0] + matrix_A = self.cube_matmul(matrix_A, matrix_A) + + if self.channels_slice_flag: + matrix_A = self.reshape(matrix_A, (self.hw, C0, self.hw, C0)) + matrix_A = self.slice(matrix_A, (0, 0, 0, 0), (self.hw, self.in_channels, self.hw, self.in_channels)) + matrix_A = self.reshape(matrix_A, (self.matrix_A_dim, self.matrix_A_dim)) + normalizer = self.cast(normalizer, ms.float32) + matrix_A = self.mul(matrix_A, 1.0 / normalizer) + if self.padA_flag: + matrix_A = self.padA(matrix_A) + damping_step = self.gather(self.damping, self.cov_step, self.axis) + damping_step = self.cast(damping_step, mstype.float32) + damping = self.mul(damping_step, 32.0 / normalizer) + damping = self.sqrt(damping) + damping_A = self.cast(self.dampingA, mstype.float32) + matrix_A = matrix_A + damping * damping_A + matrix_A_inv = self.cholesky(matrix_A) + matrix_A_inv = self.vector_matmul(matrix_A_inv, matrix_A_inv) + matrix_A_inv_max = self.fused_abs_max1(matrix_A_inv) + matrix_A_inv_max = self.fused_abs_max2(matrix_A_inv_max) + self.A_inv_max = matrix_A_inv_max + matrix_A_inv = self.matrix_combine(matrix_A_inv) + matrix_A_inv = self.cast(matrix_A_inv, mstype.float16) + if self.padA_flag: + matrix_A_inv = self.slice(matrix_A_inv, (0, 0), (self.matrix_A_dim, self.matrix_A_dim)) + + if self.device_shape_pad_flag: + matrix_A_inv = self.reshape(matrix_A_inv, (self.hw, self.in_channels, self.hw, self.in_channels)) + matrix_A_inv = self.device_shape_pad(matrix_A_inv) + matrix_A_inv = self.reshape(matrix_A_inv, self.matrix_A_device_temp_shape) + matrix_A_inv = self.transpose(matrix_A_inv, (2, 0, 1, 3)) + self.matrix_A_inv = matrix_A_inv + self.matrix_G_inv = self.fake_G + out = self.conv2d(x, self.weight) + out = self.getG(out) + else: + out = self.conv2d(x, self.weight) + + return out + + def extra_repr(self): + """extra_repr""" + s = 'input_channels={}, output_channels={}, kernel_size={},' \ + 'stride={}, pad_mode={}, padding={}, dilation={}, ' \ + 'group={}, data_format={}, has_bias={},' \ + 'weight_init={}, bias_init={}'.format( + self.in_channels, + self.out_channels, + self.kernel_size, + self.stride, + self.pad_mode, + self.padding, + self.dilation, + self.group, + self.data_format, + self.has_bias, + self.weight, + self.bias) + + if self.has_bias: + s += ', bias={}'.format(self.bias) + return s + + +class Dense_Thor(Cell): + """Dense_Thor""" + @cell_attr_register(attrs=['has_bias', 'activation']) + def __init__(self, + in_channels, + out_channels, + weight_init='normal', + bias_init='zeros', + damping=0.03, + loss_scale=1, + frequency=278, + has_bias=True, + activation=None): + super(Dense_Thor, self).__init__() + self.in_channels = check_int_positive(in_channels) + self.out_channels = check_int_positive(out_channels) + self.has_bias = check_bool(has_bias) + self.thor = True + if isinstance(weight_init, Tensor): + if weight_init.dim() != 2 or weight_init.shape[0] != out_channels or \ + weight_init.shape[1] != in_channels: + raise ValueError("weight_init shape error") + + self.weight = Parameter(initializer(weight_init, [out_channels, in_channels]), name="weight") + + if self.has_bias: + if isinstance(bias_init, Tensor): + if bias_init.dim() != 1 or bias_init.shape[0] != out_channels: + raise ValueError("bias_init shape error") + + self.bias = Parameter(initializer(bias_init, [out_channels]), name="bias") + + self.matmul = P.MatMul(transpose_b=True) + self.bias_add = P.BiasAdd() + + self.activation = get_activation(activation) + self.activation_flag = self.activation is not None + + self.matrix_A_inv = Parameter(Tensor(np.zeros([128, 128, 16, 16]).astype(np.float16)), name='matrix_A_inv', + requires_grad=False) + self.matrix_G_inv = Parameter(Tensor(np.zeros([63, 63, 16, 16]).astype(np.float16)), name="matrix_G_inv", + requires_grad=False) + self.fake_G = Tensor(np.zeros([63, 63, 16, 16]).astype(np.float16)) + + self.matmul = P.MatMul(transpose_b=True) + self.cube_matmul = P.CusMatMulCube(transpose_a=True) + self.matrix_combine = P.CusMatrixCombine() + self.cholesky = P.CusCholeskyTrsm() + self.shape = P.Shape() + self.reshape = P.Reshape() + self.transpose = P.Transpose() + self.cov_step = Parameter(initializer(0, [1], mstype.int32), name="cov_step", requires_grad=False) + self.mul = P.Mul() + self.cast = P.Cast() + self.damping = Tensor(damping) + self.loss_scale = Tensor(1 / loss_scale, mstype.float16) + self.vector_matmul = P.CusBatchMatMul() + self.pad = P.Pad(((0, 24), (0, 24))) + self.pad1 = P.Pad(((0, 8), (0, 8))) + self.slice = P.Slice() + self.gather = P.GatherV2() + self.assignadd = P.AssignAdd() + self.freq = Tensor(frequency, mstype.int32) + self.axis = 0 + self.A_inv_max = Parameter(initializer(0, [1], mstype.float32), name="A_inv_max", requires_grad=False) + self.G_inv_max = Parameter(initializer(0, [1], mstype.float32), name="G_inv_max", requires_grad=False) + self.fused_abs_max1 = P.CusFusedAbsMax1([1000, 1000]) + self.fused_abs_max2 = P.CusFusedAbsMax1() + self.log = P.Log() + self.exp = P.Exp() + self.dampingA = Tensor(np.identity(2048), mstype.float32) + self.dampingG = Tensor(np.identity(1024), mstype.float32) + self.add = P.TensorAdd() + self.sqrt = P.Sqrt() + self.getG = P.InsertGradientOf(self.save_gradient) + + def save_gradient(self, dout): + """save_gradient""" + out = dout + dout = self.mul(dout, self.loss_scale) + dout = self.mul(dout, 32.0) + normalizer = 32 + matrix_G = self.cube_matmul(dout, dout) + normalizer = self.cast(normalizer, ms.float32) + matrix_G = self.mul(matrix_G, 1.0 / normalizer) + matrix_G = self.pad(matrix_G) + damping_step = self.gather(self.damping, self.cov_step, 0) + damping_step = self.cast(damping_step, mstype.float32) + self.cov_step = self.cov_step + self.freq + damping = self.sqrt(damping_step) + dampingG = self.cast(self.dampingG, mstype.float32) + matrix_G = matrix_G + damping * dampingG + matrix_G_inv = self.cholesky(matrix_G) + matrix_G_inv = self.vector_matmul(matrix_G_inv, matrix_G_inv) + matrix_G_inv_max = self.fused_abs_max1(matrix_G_inv) + matrix_G_inv_max = self.fused_abs_max2(matrix_G_inv_max) + self.G_inv_max = matrix_G_inv_max + matrix_G_inv = self.matrix_combine(matrix_G_inv) + matrix_G_inv = self.slice(matrix_G_inv, (0, 0), (1000, 1000)) + matrix_G_inv = self.pad1(matrix_G_inv) + matrix_G_inv_shape = self.shape(matrix_G_inv) + matrix_G_inv = self.reshape(matrix_G_inv, (matrix_G_inv_shape[0] / 16, 16, matrix_G_inv_shape[0] / 16, 16)) + matrix_G_inv = self.transpose(matrix_G_inv, (2, 0, 1, 3)) + matrix_G_inv = self.cast(matrix_G_inv, mstype.float16) + self.matrix_G_inv = matrix_G_inv + return out + + def construct(self, x): + """construct""" + if self.thor: + inputs = self.cube_matmul(x, x) + normalizer = 32 + normalizer = self.cast(normalizer, ms.float32) + matrix_A = self.mul(inputs, 1.0 / normalizer) + + damping_step = self.gather(self.damping, self.cov_step, self.axis) + damping_step = self.cast(damping_step, mstype.float32) + damping = self.sqrt(damping_step) + dampingA = self.cast(self.dampingA, mstype.float32) + matrix_A = matrix_A + damping * dampingA + matrix_A_inv = self.cholesky(matrix_A) + matrix_A_inv = self.vector_matmul(matrix_A_inv, matrix_A_inv) + + matrix_A_inv_max = self.fused_abs_max2(matrix_A_inv) + matrix_A_inv_max = self.fused_abs_max2(matrix_A_inv_max) + self.A_inv_max = matrix_A_inv_max + + matrix_A_inv = self.matrix_combine(matrix_A_inv) + matrix_A_inv_shape = self.shape(matrix_A_inv) + matrix_A_inv = self.reshape(matrix_A_inv, (matrix_A_inv_shape[0] / 16, 16, matrix_A_inv_shape[0] / 16, 16)) + matrix_A_inv = self.transpose(matrix_A_inv, (2, 0, 1, 3)) + matrix_A_inv = self.cast(matrix_A_inv, mstype.float16) + self.matrix_A_inv = matrix_A_inv + self.matrix_G_inv = self.fake_G + output = self.matmul(x, self.weight) + output = self.getG(output) + else: + output = self.matmul(x, self.weight) + + if self.has_bias: + output = self.bias_add(output, self.bias) + if self.activation_flag: + return self.activation(output) + return output + + def extend_repr(self): + """extend_repr""" + str_info = 'in_channels={}, out_channels={}, weight={}, has_bias={}' \ + .format(self.in_channels, self.out_channels, self.weight, self.has_bias) + if self.has_bias: + str_info = str_info + ', bias={}'.format(self.bias) + + if self.activation_flag: + str_info = str_info + ', activation={}'.format(self.activation) + + return str_info diff --git a/model_zoo/resnet_thor/train.py b/model_zoo/resnet_thor/train.py new file mode 100644 index 0000000000..47f56a0676 --- /dev/null +++ b/model_zoo/resnet_thor/train.py @@ -0,0 +1,132 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""train_imagenet.""" +import argparse +import os +import random + +import numpy as np + +from mindspore import Tensor +from mindspore import context +from mindspore.communication.management import init +from mindspore.parallel._auto_parallel_context import auto_parallel_context +from mindspore.train.callback import ModelCheckpoint, CheckpointConfig, LossMonitor, TimeMonitor +from mindspore.train.loss_scale_manager import FixedLossScaleManager +from mindspore.train.model import ParallelMode +from src.model_thor import Model +from src.resnet_thor import resnet50 +from src.thor import THOR +from src.config import config +from src.crossentropy import CrossEntropy +from src.dataset_imagenet import create_dataset + +random.seed(1) +np.random.seed(1) + +parser = argparse.ArgumentParser(description='Image classification') +parser.add_argument('--run_distribute', type=bool, default=False, help='Run distribute') +parser.add_argument('--device_num', type=int, default=1, help='Device num.') +parser.add_argument('--do_train', type=bool, default=True, help='Do train or not.') +parser.add_argument('--do_eval', type=bool, default=False, help='Do eval or not.') +parser.add_argument('--dataset_path', type=str, default=None, help='Dataset path') + +args_opt = parser.parse_args() +device_id = int(os.getenv('DEVICE_ID')) + +context.set_context(mode=context.GRAPH_MODE, device_target="Ascend", save_graphs=False, device_id=device_id) + + +def get_model_lr(global_step, lr_init, decay, total_epochs, steps_per_epoch): + """get_model_lr""" + lr_each_step = [] + total_steps = steps_per_epoch * total_epochs + for i in range(total_steps): + epoch = (i + 1) / steps_per_epoch + base = (1.0 - float(epoch) / total_epochs) ** decay + lr_local = lr_init * base + if epoch >= 39: + lr_local = lr_local * 0.5 + if epoch >= 40: + lr_local = lr_local * 0.5 + lr_each_step.append(lr_local) + current_step = global_step + lr_each_step = np.array(lr_each_step).astype(np.float32) + learning_rate = lr_each_step[current_step:] + return learning_rate + + +def get_model_damping(global_step, damping_init, decay_rate, total_epochs, steps_per_epoch): + """get_model_damping""" + damping_each_step = [] + total_steps = steps_per_epoch * total_epochs + for step in range(total_steps): + epoch = (step + 1) / steps_per_epoch + damping_here = damping_init * (decay_rate ** (epoch / 10)) + damping_each_step.append(damping_here) + + current_step = global_step + damping_each_step = np.array(damping_each_step).astype(np.float32) + damping_now = damping_each_step[current_step:] + return damping_now + + +if __name__ == '__main__': + if not args_opt.do_eval and args_opt.run_distribute: + context.set_auto_parallel_context(device_num=args_opt.device_num, parallel_mode=ParallelMode.DATA_PARALLEL, + mirror_mean=True, parameter_broadcast=True) + auto_parallel_context().set_all_reduce_fusion_split_indices([107], "hccl_world_groupsum1") + auto_parallel_context().set_all_reduce_fusion_split_indices([27], "hccl_world_groupsum2") + auto_parallel_context().set_all_reduce_fusion_split_indices([27], "hccl_world_groupsum3") + auto_parallel_context().set_all_reduce_fusion_split_indices([27], "hccl_world_groupsum4") + auto_parallel_context().set_all_reduce_fusion_split_indices([27], "hccl_world_groupsum5") + + init() + + epoch_size = config.epoch_size + damping = get_model_damping(0, 0.03, 0.87, 50, 5004) + net = resnet50(class_num=config.class_num, damping=damping, loss_scale=config.loss_scale, + frequency=config.frequency) + + if not config.label_smooth: + config.label_smooth_factor = 0.0 + loss = CrossEntropy(smooth_factor=config.label_smooth_factor, num_classes=config.class_num) + if args_opt.do_train: + dataset = create_dataset(dataset_path=args_opt.dataset_path, do_train=True, + repeat_num=epoch_size, batch_size=config.batch_size) + step_size = dataset.get_dataset_size() + + loss_scale = FixedLossScaleManager(config.loss_scale, drop_overflow_update=False) + lr = Tensor(get_model_lr(0, 0.045, 6, 70, 5004)) + opt = THOR(filter(lambda x: x.requires_grad, net.get_parameters()), lr, config.momentum, + filter(lambda x: 'matrix_A' in x.name, net.get_parameters()), + filter(lambda x: 'matrix_G' in x.name, net.get_parameters()), + filter(lambda x: 'A_inv_max' in x.name, net.get_parameters()), + filter(lambda x: 'G_inv_max' in x.name, net.get_parameters()), + config.weight_decay, config.loss_scale) + + model = Model(net, loss_fn=loss, optimizer=opt, amp_level='O2', loss_scale_manager=loss_scale, + keep_batchnorm_fp32=False, metrics={'acc'}, frequency=config.frequency) + + time_cb = TimeMonitor(data_size=step_size) + loss_cb = LossMonitor() + cb = [time_cb, loss_cb] + if config.save_checkpoint: + config_ck = CheckpointConfig(save_checkpoint_steps=config.save_checkpoint_steps, + keep_checkpoint_max=config.keep_checkpoint_max) + ckpt_cb = ModelCheckpoint(prefix="resnet", directory=config.save_checkpoint_path, config=config_ck) + cb += [ckpt_cb] + + model.train(epoch_size, dataset, callbacks=cb) From 89c9f46a9742badda0c1bf8df4f82accaea9d4c4 Mon Sep 17 00:00:00 2001 From: kswang Date: Mon, 29 Jun 2020 11:37:19 +0800 Subject: [PATCH 133/254] format device ascend code --- .../device/ascend/ascend_kernel_runtime.cc | 68 +++++++-------- .../device/ascend/ascend_stream_assign.cc | 86 +++++++++---------- .../ccsrc/device/cpu/cpu_kernel_runtime.cc | 56 +++++++----- .../ccsrc/device/cpu/cpu_kernel_runtime.h | 4 + .../ccsrc/device/cpu/cpu_simple_mem_plan.cc | 8 +- .../ccsrc/device/cpu/cpu_simple_mem_plan.h | 2 +- 6 files changed, 122 insertions(+), 102 deletions(-) diff --git a/mindspore/ccsrc/device/ascend/ascend_kernel_runtime.cc b/mindspore/ccsrc/device/ascend/ascend_kernel_runtime.cc index dd1be39e55..efdcb98755 100644 --- a/mindspore/ccsrc/device/ascend/ascend_kernel_runtime.cc +++ b/mindspore/ccsrc/device/ascend/ascend_kernel_runtime.cc @@ -68,9 +68,9 @@ std::string GetRankId() { int rank_offset = std::stoi(offset); rank_id += rank_offset; } catch (std::invalid_argument) { - MS_LOG(EXCEPTION) << "stoi invalid argument:" << offset; + MS_LOG(EXCEPTION) << "Call stoi invalid argument:" << offset; } catch (std::out_of_range) { - MS_LOG(EXCEPTION) << "stoi out_of_range:" << offset; + MS_LOG(EXCEPTION) << "Call stoi out_of_range:" << offset; } } rank_id_str = std::to_string(rank_id); @@ -81,7 +81,7 @@ std::string GetRankId() { rank_id_str = std::getenv("RANK_ID"); #endif if (rank_id_str.empty()) { - MS_LOG(ERROR) << "get hccl rankid failed, please set env RANK_ID"; + MS_LOG(ERROR) << "Get hccl rankid failed, please set env RANK_ID"; } return rank_id_str; } @@ -100,7 +100,7 @@ void AscendKernelRuntime::ClearGraphModelMap() { } void AscendKernelRuntime::ClearGraphRuntimeResource(uint32_t graph_id) { - MS_LOG(DEBUG) << "clear graph:" << graph_id << " runtime resource"; + MS_LOG(DEBUG) << "Clear graph:" << graph_id << " runtime resource"; auto iter = graph_model_map_.find(graph_id); if (iter == graph_model_map_.end()) { MS_LOG(DEBUG) << "GraphId:" << graph_id << " not found"; @@ -118,7 +118,7 @@ bool AscendKernelRuntime::NeedDestroyHccl() { auto context_ptr = MsContext::GetInstance(); MS_EXCEPTION_IF_NULL(context_ptr); if (!context_ptr->enable_hccl()) { - MS_LOG(INFO) << "hccl is not enabled"; + MS_LOG(INFO) << "Hccl is not enabled"; return false; } // Note: make sure hcom_connectivity_detection api never be used. @@ -126,7 +126,7 @@ bool AscendKernelRuntime::NeedDestroyHccl() { } void AscendKernelRuntime::ReleaseDeviceRes() { - MS_LOG(INFO) << "ascend finalize start"; + MS_LOG(INFO) << "Ascend finalize start"; // release ge runtime ClearGraphModelMap(); @@ -134,7 +134,7 @@ void AscendKernelRuntime::ReleaseDeviceRes() { MS_EXCEPTION_IF_NULL(context_ptr); auto ret = rtSetDevice(context_ptr->device_id()); if (ret != RT_ERROR_NONE) { - MS_EXCEPTION(DeviceProcessError) << "rtSetDevice, ret[" << static_cast(ret) << "]"; + MS_EXCEPTION(DeviceProcessError) << "Call rtSetDevice, ret[" << static_cast(ret) << "]"; } if (mem_manager_ != nullptr) { @@ -144,7 +144,7 @@ void AscendKernelRuntime::ReleaseDeviceRes() { (void)DestroyHccl(); (void)ResetDevice(); (void)ProfilingManager::GetInstance().StopProfiling(); - MS_LOG(INFO) << "ascend finalize end"; + MS_LOG(INFO) << "Ascend finalize end"; } bool AscendKernelRuntime::Init() { @@ -155,7 +155,7 @@ bool AscendKernelRuntime::Init() { #ifdef ENABLE_DUMP_E2E ret = SetDumpConf(); if (!ret) { - MS_LOG(INFO) << "no dump conf to set!"; + MS_LOG(INFO) << "No dump conf to set!"; } #endif @@ -263,13 +263,13 @@ void DumpParameters(mindspore::session::KernelGraph *graph, const string &dump_p bool AscendKernelRuntime::DumpData(mindspore::session::KernelGraph *graph) { MS_EXCEPTION_IF_NULL(graph); #ifdef ENABLE_DUMP_E2E - MS_LOG(INFO) << "start dump step"; + MS_LOG(INFO) << "Start dump step"; DumpConfPtr dump_conf = GetDumpConf(); MS_EXCEPTION_IF_NULL(dump_conf); dump_conf->UpdataCurIter(); bool dump_flag = dump_conf->dump_enable(); if (!dump_flag) { - MS_LOG(INFO) << "dump flag is disable, pass dump step"; + MS_LOG(INFO) << "Dump flag is disable, pass dump step"; return true; } uint32_t cur_iter = dump_conf->cur_iter(); @@ -278,7 +278,7 @@ bool AscendKernelRuntime::DumpData(mindspore::session::KernelGraph *graph) { return true; } } - MS_LOG(INFO) << "cur iter is " << cur_iter; + MS_LOG(INFO) << "Cur iter is " << cur_iter; std::string net_name = dump_conf->dump_net_name(); std::string iterator = to_string(cur_iter); std::string dump_path = dump_conf->dump_path(); @@ -369,9 +369,9 @@ void LoadParameters(mindspore::session::KernelGraph *graph, Debugger *debugger) bool AscendKernelRuntime::LoadData(mindspore::session::KernelGraph *graph, Debugger *debugger) { MS_EXCEPTION_IF_NULL(graph); #ifdef ENABLE_DEBUGGER - MS_LOG(INFO) << "start load step"; + MS_LOG(INFO) << "Start load step"; uint32_t cur_iter = 0; - MS_LOG(INFO) << "cur iter is " << cur_iter; + MS_LOG(INFO) << "Cur iter is " << cur_iter; // load output LoadOutput(graph, debugger); // load parameters @@ -421,7 +421,7 @@ bool AscendKernelRuntime::GenTask(const session::KernelGraph *graph) { } // Graph may have no compute node, such TensorAddGrad. if (task_info_list.empty()) { - MS_LOG(WARNING) << "graph " << graph->graph_id() << " have no compute node"; + MS_LOG(WARNING) << "Graph " << graph->graph_id() << " have no compute node"; return true; } AscendStreamAssign &assign_instance = AscendStreamAssign::GetInstance(); @@ -432,7 +432,7 @@ bool AscendKernelRuntime::GenTask(const session::KernelGraph *graph) { assign_instance.GetWaitStreams(&wait_active_stream_list); std::vector force_copy_stream_list; assign_instance.GetHcomStreams(&force_copy_stream_list); - MS_LOG(INFO) << "call DavinciModel total stream num:" << resource_manager.get_cur_stream_num() + MS_LOG(INFO) << "Call DavinciModel total stream num:" << resource_manager.get_cur_stream_num() << ", total event num:" << resource_manager.get_cur_event_num() << ", total label num:" << label_assign_instance.GetLabelNum(NOT_NULL(graph)) << ", wait_active_stream_list size:" << wait_active_stream_list.size() @@ -524,7 +524,7 @@ bool AscendKernelRuntime::RunTask(const session::KernelGraph *graph) { bool status = ge::model_runner::ModelRunner::Instance().RunModel(graph->graph_id(), input_tensors, output_tensors); if (!status) { - MS_LOG(ERROR) << "run task failed"; + MS_LOG(ERROR) << "Run task failed"; DebugTaskIdName(graph->graph_id()); return false; } @@ -543,18 +543,18 @@ bool AscendKernelRuntime::InitDevice() { int device_count = 0; auto ret = rtGetDeviceCount(&device_count); if (ret != RT_ERROR_NONE) { - MS_EXCEPTION(DeviceProcessError) << "rtGetDeviceCount, ret[" << static_cast(ret) << "]"; + MS_EXCEPTION(DeviceProcessError) << "Call rtGetDeviceCount, ret[" << static_cast(ret) << "]"; } ret = rtSetDevice(device_id_); if (ret != RT_ERROR_NONE) { - MS_EXCEPTION(DeviceProcessError) << "rtSetDevice, ret[" << static_cast(ret) << "]"; + MS_EXCEPTION(DeviceProcessError) << "Call rtSetDevice, ret[" << static_cast(ret) << "]"; } auto context_ptr = MsContext::GetInstance(); MS_EXCEPTION_IF_NULL(context_ptr); if (context_ptr == nullptr) { - MS_LOG(ERROR) << "get MsContext instance failed"; + MS_LOG(ERROR) << "Get MsContext instance failed"; return false; } if (context_ptr->enable_hccl()) { @@ -566,17 +566,17 @@ bool AscendKernelRuntime::InitDevice() { ret = rtCtxCreate(&rt_context_, 0, device_id_); if (ret != RT_ERROR_NONE) { - MS_EXCEPTION(DeviceProcessError) << "rtCtxCreate, ret[" << static_cast(ret) << "]"; + MS_EXCEPTION(DeviceProcessError) << "Call rtCtxCreate, ret[" << static_cast(ret) << "]"; } ret = rtCtxSetCurrent(rt_context_); if (ret != RT_ERROR_NONE) { - MS_EXCEPTION(DeviceProcessError) << "rtCtxSetCurrent, ret[" << ret << "]"; + MS_EXCEPTION(DeviceProcessError) << "Call rtCtxSetCurrent, ret[" << ret << "]"; } ret = rtStreamCreate(&stream_, 0); if (ret != RT_ERROR_NONE) { - MS_LOG(EXCEPTION) << "rtStreamCreate, ret[" << ret << "]"; + MS_LOG(EXCEPTION) << "Call rtStreamCreate, ret[" << ret << "]"; } return true; @@ -585,14 +585,14 @@ bool AscendKernelRuntime::InitDevice() { bool AscendKernelRuntime::ResetDevice() { auto ret = rtCtxSetCurrent(rt_context_); if (ret != RT_ERROR_NONE) { - MS_LOG(ERROR) << "call rtCtxSetCurrent failed"; + MS_LOG(ERROR) << "Call rtCtxSetCurrent failed"; return false; } if (stream_ != nullptr) { ret = rtStreamDestroy(stream_); if (ret != RT_ERROR_NONE) { - MS_LOG(EXCEPTION) << "rtStreamDestroy, ret[" << ret << "]"; + MS_LOG(EXCEPTION) << "Call rtStreamDestroy, ret[" << ret << "]"; } stream_ = nullptr; } @@ -600,7 +600,7 @@ bool AscendKernelRuntime::ResetDevice() { if (rt_context_ != nullptr) { ret = rtCtxDestroy(rt_context_); if (ret != RT_ERROR_NONE) { - MS_EXCEPTION(DeviceProcessError) << "rtCtxDestroy, ret[" << ret << "]"; + MS_EXCEPTION(DeviceProcessError) << "Call rtCtxDestroy, ret[" << ret << "]"; } rt_context_ = nullptr; } @@ -613,30 +613,30 @@ bool AscendKernelRuntime::HcclInit() { if (!context_ptr->IsTsdOpened()) { MS_LOG(EXCEPTION) << "Hccl dependent tsd is not open"; } - MS_LOG(INFO) << "do hcom init"; + MS_LOG(INFO) << "Do hcom init"; auto config_path_str = std::getenv("MINDSPORE_HCCL_CONFIG_PATH"); if (config_path_str == nullptr) { config_path_str = std::getenv("RANK_TABLE_FILE"); if (config_path_str == nullptr) { - MS_LOG(ERROR) << "get hccl json config failed, please set env MINDSPORE_HCCL_CONFIG_PATH or RANK_TABLE_FILE"; + MS_LOG(ERROR) << "Get hccl json config failed, please set env MINDSPORE_HCCL_CONFIG_PATH or RANK_TABLE_FILE"; return false; } } if (strlen(config_path_str) > PATH_MAX) { - MS_LOG(ERROR) << "file path oversize"; + MS_LOG(ERROR) << "File path oversize"; return false; } std::string rank_id_str = GetRankId(); auto full_path = realpath(config_path_str, nullptr); if (full_path == nullptr) { - MS_LOG(ERROR) << "file path " << config_path_str << " does not exist"; + MS_LOG(ERROR) << "File path " << config_path_str << " does not exist"; return false; } MS_LOG(INFO) << "MINDSPORE_HCCL_CONFIG_PATH : " << full_path << ", RANK_ID: " << rank_id_str; hcclResult_t res = hcom_init(full_path, rank_id_str.c_str()); free(full_path); if (res != HCCL_SUCCESS) { - MS_LOG(ERROR) << "hcom init failed, res is " << static_cast(res); + MS_LOG(ERROR) << "Hcom init failed, res is " << static_cast(res); return false; } return true; @@ -646,15 +646,15 @@ bool AscendKernelRuntime::DestroyHccl() { auto context_ptr = MsContext::GetInstance(); MS_EXCEPTION_IF_NULL(context_ptr); if (!NeedDestroyHccl()) { - MS_LOG(INFO) << "hccl is not enable, no need to close."; + MS_LOG(INFO) << "Hccl is not enable, no need to close."; return true; } hcclResult_t res = hcom_destroy(); if (res != HCCL_SUCCESS) { - MS_LOG(ERROR) << "hccl destroy failed"; + MS_LOG(ERROR) << "Hccl destroy failed"; return false; } - MS_LOG(INFO) << "hccl destroy successful, status = " << res << "."; + MS_LOG(INFO) << "Hccl destroy successful, status = " << res << "."; context_ptr->set_enable_hccl(false); return true; } diff --git a/mindspore/ccsrc/device/ascend/ascend_stream_assign.cc b/mindspore/ccsrc/device/ascend/ascend_stream_assign.cc index 8f8f022bdb..e3491536ee 100644 --- a/mindspore/ccsrc/device/ascend/ascend_stream_assign.cc +++ b/mindspore/ccsrc/device/ascend/ascend_stream_assign.cc @@ -46,7 +46,7 @@ void AscendStreamAssign::AssignStream(const NotNull &graph_ptr) GetNeedActiveStreams(graph_ptr); graph_ptr->PrintGraphExecuteOrder(); CheckResourceAssign(graph_ptr); - MS_LOG(INFO) << "after finish stream assign"; + MS_LOG(INFO) << "After finish stream assign"; // Get info for D Model AscendResourceMng &resource_manager = AscendResourceMng::GetInstance(); @@ -64,7 +64,7 @@ void AscendStreamAssign::ReorderIndependentOrders(const NotNull std::vector others; auto cnode_ptr_list = graph_ptr->execution_order(); - MS_LOG(INFO) << "before reorder, graph orders size:" << cnode_ptr_list.size(); + MS_LOG(INFO) << "Before reorder, graph orders size:" << cnode_ptr_list.size(); for (size_t i = 0; i < cnode_ptr_list.size(); ++i) { auto cur_cnode_ptr = cnode_ptr_list[i]; MS_EXCEPTION_IF_NULL(cur_cnode_ptr); @@ -76,7 +76,7 @@ void AscendStreamAssign::ReorderIndependentOrders(const NotNull } if (others.empty() || independents.empty()) { - MS_LOG(INFO) << "independent or others is empty, no need reorder"; + MS_LOG(INFO) << "Independent or others is empty, no need reorder"; return; } @@ -107,9 +107,9 @@ void AscendStreamAssign::ReorderIndependentOrders(const NotNull } } - MS_LOG(INFO) << "after reorder, graph orders size:" << exe_orders.size(); + MS_LOG(INFO) << "After reorder, graph orders size:" << exe_orders.size(); if (processed.size() != independents.size()) { - MS_LOG(WARNING) << "processed independent nodes size is not equal to exiting independent nodes size"; + MS_LOG(WARNING) << "Processed independent nodes size is not equal to exiting independent nodes size"; return; } @@ -142,7 +142,7 @@ void AscendStreamAssign::AssignAllNodesStream(const NotNull &gra AssignCommonStreamId(cur_cnode_ptr); } - MS_LOG(INFO) << "common start from 0, common stream nums:" << resource_manager.get_cur_stream_num(); + MS_LOG(INFO) << "Common start from 0, common stream nums:" << resource_manager.get_cur_stream_num(); if (exit_hcom) { uint32_t first_hcom_stream_id = resource_manager.ApplyNewStream(); @@ -157,7 +157,7 @@ void AscendStreamAssign::AssignAllNodesStream(const NotNull &gra AssignHcomStreamId(cur_cnode_ptr); } } - MS_LOG(INFO) << "hcom start from :" << first_hcom_stream_id << ", hcom stream nums:" << hcom_stream_map_.size(); + MS_LOG(INFO) << "Hcom start from :" << first_hcom_stream_id << ", hcom stream nums:" << hcom_stream_map_.size(); } if (exit_independent) { @@ -171,10 +171,10 @@ void AscendStreamAssign::AssignAllNodesStream(const NotNull &gra AssignIndependentStreamId(cur_cnode_ptr); } } - MS_LOG(INFO) << "independ start from:" << first_independ << ", stream nums:" << independent_stream_map_.size(); + MS_LOG(INFO) << "Independ start from:" << first_independ << ", stream nums:" << independent_stream_map_.size(); } - MS_LOG(INFO) << "after stream assign, total stream nums:" << resource_manager.get_cur_stream_num(); + MS_LOG(INFO) << "After stream assign, total stream nums:" << resource_manager.get_cur_stream_num(); } void AscendStreamAssign::AssignCommonStreamId(const CNodePtr &cur_cnode_ptr) { @@ -257,7 +257,7 @@ bool AscendStreamAssign::IsIndependentNode(const CNodePtr &node_ptr) { uint32_t input_nums = AnfAlgo::GetInputTensorNum(node_ptr); if (input_nums == 0) { - MS_LOG(INFO) << "node " << node_ptr->fullname_with_scope() << " is independent, as inputs nums is zero"; + MS_LOG(INFO) << "Node " << node_ptr->fullname_with_scope() << " is independent, as inputs nums is zero"; return true; } @@ -267,13 +267,13 @@ bool AscendStreamAssign::IsIndependentNode(const CNodePtr &node_ptr) { return false; } } - MS_LOG(INFO) << "node " << node_ptr->fullname_with_scope() << " is independent, as inputs is all value node"; + MS_LOG(INFO) << "Node " << node_ptr->fullname_with_scope() << " is independent, as inputs is all value node"; return true; } // section 3: void AscendStreamAssign::UpdateAtomicAddrCleanStreamId(const NotNull &graph_ptr) { - MS_LOG(INFO) << "start"; + MS_LOG(INFO) << "Start"; auto cnode_ptr_list = graph_ptr->execution_order(); for (size_t i = 0; i < cnode_ptr_list.size(); ++i) { CNodePtr cur_cnode_ptr = cnode_ptr_list[i]; @@ -283,12 +283,12 @@ void AscendStreamAssign::UpdateAtomicAddrCleanStreamId(const NotNull &graph_ptr) { - MS_LOG(INFO) << "start"; + MS_LOG(INFO) << "Start"; GetProcessedStream(graph_ptr); std::vector update_cnode_list; CNodePtr cur_cnode_ptr = nullptr; @@ -314,7 +314,7 @@ void AscendStreamAssign::InsertStreamActive(const NotNull &graph bool processed = IsProcessedStream(cur_stream_id); // 1)inner stream assign, need insert active op if (!processed) { - MS_LOG(INFO) << "common stream active info:" << pre_stream_id << "->active" << cur_stream_id; + MS_LOG(INFO) << "Common stream active info:" << pre_stream_id << "->active" << cur_stream_id; CNodePtr active_ptr = KernelAdjust::GetInstance().CreateStreamActiveOp(graph_ptr); // 1.set stream id AnfAlgo::SetStreamId(pre_stream_id, active_ptr.get()); @@ -336,7 +336,7 @@ void AscendStreamAssign::InsertStreamActive(const NotNull &graph pre_cnode_ptr = cur_cnode_ptr; } graph_ptr->set_execution_order(update_cnode_list); - MS_LOG(INFO) << "end"; + MS_LOG(INFO) << "End"; } void AscendStreamAssign::GetProcessedStream(const NotNull &graph_ptr) { @@ -364,7 +364,7 @@ void AscendStreamAssign::GetProcessedStream(const NotNull &graph } } for (const auto &item : processed_streams_) { - MS_LOG(INFO) << "before active:" << item << " is been processed"; + MS_LOG(INFO) << "Before active:" << item << " is been processed"; } } @@ -385,7 +385,7 @@ void AscendStreamAssign::UpdateStreamSwitch(const NotNull &graph MS_EXCEPTION_IF_NULL(switch_ptr); auto true_stream_id = GetValue(primitive->GetAttr(kAttrTrueBranchStream)); - MS_LOG(INFO) << "streamswtich stream id:" << AnfAlgo::GetStreamId(switch_ptr) + MS_LOG(INFO) << "Streamswtich stream id:" << AnfAlgo::GetStreamId(switch_ptr) << "; active stream id:" << true_stream_id; CNodePtr active_ptr = KernelAdjust::GetInstance().CreateStreamActiveOp(graph_ptr); @@ -425,11 +425,11 @@ bool AscendStreamAssign::IsProcessedStream(uint32_t stream_id) { // section5 void AscendStreamAssign::InsertEventForHcomParallel(const NotNull &graph_ptr) { - MS_LOG(INFO) << "start"; + MS_LOG(INFO) << "Start"; InsertEventCommonDependHcom(graph_ptr); InsertEventHcomDependCommon(graph_ptr); InsertEventHcomDependHcom(graph_ptr); - MS_LOG(INFO) << "end"; + MS_LOG(INFO) << "End"; } void AscendStreamAssign::InsertEventCommonDependHcom(const NotNull &graph_ptr) { @@ -447,7 +447,7 @@ void AscendStreamAssign::InsertEventCommonDependHcom(const NotNullfullname_with_scope() + MS_LOG(WARNING) << "Hcom node:" << (*(it - 1))->fullname_with_scope() << ", can't find target for insert recv op, no insert send/recv"; it = cnodes.erase(it); continue; @@ -469,7 +469,7 @@ void AscendStreamAssign::InsertEventCommonDependHcom(const NotNullset_execution_order(cnodes); - MS_LOG(INFO) << "after common depend hcom, total event nums:" << resource_manager.get_cur_event_num(); + MS_LOG(INFO) << "After common depend hcom, total event nums:" << resource_manager.get_cur_event_num(); } void AscendStreamAssign::InsertEventHcomDependCommon(const NotNull &graph_ptr) { @@ -512,7 +512,7 @@ void AscendStreamAssign::InsertEventHcomDependCommon(const NotNullset_execution_order(cnodes); - MS_LOG(INFO) << "after hcom depend common, total event nums:" << resource_manager.get_cur_event_num(); + MS_LOG(INFO) << "After hcom depend common, total event nums:" << resource_manager.get_cur_event_num(); } void AscendStreamAssign::InsertEventHcomDependHcom(const NotNull &graph_ptr) { @@ -547,11 +547,11 @@ void AscendStreamAssign::InsertEventHcomDependHcom(const NotNull } if (hcom_index.size() < 2) { - MS_LOG(INFO) << "different stream hcom size is less than 2, no need insert event between them"; + MS_LOG(INFO) << "Different stream hcom size is less than 2, no need insert event between them"; return; } InsertEventBetweenHcom(graph_ptr, hcom_index, first_hcom_stream, last_hcom_stream); - MS_LOG(INFO) << "after hcom depend hcom, total event nums:" << resource_manager.get_cur_event_num(); + MS_LOG(INFO) << "After hcom depend hcom, total event nums:" << resource_manager.get_cur_event_num(); } void AscendStreamAssign::InsertEventBetweenHcom(const NotNull &graph_ptr, @@ -630,7 +630,7 @@ bool AscendStreamAssign::IsSatisfiedHcom(const std::map // section6 void AscendStreamAssign::InsertEventForIndependentParallel(const NotNull &graph_ptr) { - MS_LOG(INFO) << "start"; + MS_LOG(INFO) << "Start"; AscendResourceMng &resource_manager = AscendResourceMng::GetInstance(); auto cnode_ptr_list = graph_ptr->execution_order(); vector cnodes = cnode_ptr_list; @@ -639,13 +639,13 @@ void AscendStreamAssign::InsertEventForIndependentParallel(const NotNullDebugString() << "]"; + MS_LOG(INFO) << "Deal independent op[" << (*it)->DebugString() << "]"; CNodePtr send_cnode_ptr = CreateSendApplyKernel(graph_ptr, cur_event_id, AnfAlgo::GetStreamId(*it)); it = cnodes.insert(it + 1, send_cnode_ptr); auto target = FindTargetOp(it, cnodes.end(), *(it - 1)); if (target == cnodes.end()) { - MS_LOG(DEBUG) << "independ node[" << (*(it - 1))->fullname_with_scope() + MS_LOG(DEBUG) << "Independ node[" << (*(it - 1))->fullname_with_scope() << "] can't find target for insert recv op, no insert send/recv"; it = cnodes.erase(it); continue; @@ -662,8 +662,8 @@ void AscendStreamAssign::InsertEventForIndependentParallel(const NotNullset_execution_order(cnodes); - MS_LOG(INFO) << "after independent parallel, total event nums:" << resource_manager.get_cur_event_num(); - MS_LOG(INFO) << "end"; + MS_LOG(INFO) << "After independent parallel, total event nums:" << resource_manager.get_cur_event_num(); + MS_LOG(INFO) << "End"; } // section7 @@ -687,7 +687,7 @@ void AscendStreamAssign::GetNeedActiveStreams(const NotNull &gra auto need_active = GetValue(value_ptr); if (need_active) { auto stream_id = AnfAlgo::GetStreamId(cur_cnode_ptr); - MS_LOG(INFO) << "stream id:" << stream_id << " is need actived at first"; + MS_LOG(INFO) << "Stream id:" << stream_id << " is need actived at first"; need_first_active_streams_.push_back(stream_id); } } @@ -724,7 +724,7 @@ void AscendStreamAssign::CheckStreamAssign(const NotNull &graph_ MS_EXCEPTION_IF_NULL(cur_cnode_ptr); uint32_t stream_id = AnfAlgo::GetStreamId(cur_cnode_ptr); if (stream_id == kInvalidStreamId) { - MS_LOG(EXCEPTION) << "node:" << AnfAlgo::GetCNodeName(cur_cnode_ptr) << "had not been assigned stream"; + MS_LOG(EXCEPTION) << "Node:" << AnfAlgo::GetCNodeName(cur_cnode_ptr) << "had not been assigned stream"; } (void)streams.emplace(stream_id); @@ -739,11 +739,11 @@ void AscendStreamAssign::CheckStreamAssign(const NotNull &graph_ // check stream assign if (!streams.empty()) { if (min_stream != 0) { - MS_LOG(EXCEPTION) << "stream should start from 0, now is from " << min_stream; + MS_LOG(EXCEPTION) << "Stream should start from 0, now is from " << min_stream; } uint32_t assigned_stream_num = resource_manager.get_cur_stream_num(); if ((max_stream != assigned_stream_num - 1) || (streams.size() != assigned_stream_num)) { - MS_LOG(EXCEPTION) << "stream should be consecutive, max stream id:" << max_stream + MS_LOG(EXCEPTION) << "Stream should be consecutive, max stream id:" << max_stream << "; alloc stream nums:" << assigned_stream_num << "; streams size:" << streams.size(); } } @@ -779,20 +779,20 @@ void AscendStreamAssign::CheckEventAssign(const NotNull &graph_p // check event assign if (!event_map.empty()) { if (min_event_id != 0) { - MS_LOG(EXCEPTION) << "event should start from 0, now is from " << min_event_id; + MS_LOG(EXCEPTION) << "Event should start from 0, now is from " << min_event_id; } uint32_t assigned_event_num = resource_manager.get_cur_event_num(); if ((max_event_id != assigned_event_num - 1) || (event_map.size() != assigned_event_num)) { - MS_LOG(EXCEPTION) << "event should be consecutive"; + MS_LOG(EXCEPTION) << "Event should be consecutive"; } for (const auto &item : event_map) { if (item.second.size() != 2) { - MS_LOG(EXCEPTION) << "send/recv should be in pair and share one event id"; + MS_LOG(EXCEPTION) << "Send/recv should be in pair and share one event id"; } auto first_name = AnfAlgo::GetCNodeName(item.second[0]); auto second_name = AnfAlgo::GetCNodeName(item.second[1]); if (!(first_name == kSendOpName && second_name == kRecvOpName)) { - MS_LOG(EXCEPTION) << "send should be before recv"; + MS_LOG(EXCEPTION) << "Send should be before recv"; } } } @@ -858,7 +858,7 @@ vector::iterator AscendStreamAssign::FindTargetOp(vector::it } else { auto real_input = AnfAlgo::VisitKernel(input, 0); if (node == real_input.first) { - MS_LOG(INFO) << "find target op[" << (*begin)->DebugString() << "]"; + MS_LOG(INFO) << "Find target op[" << (*begin)->DebugString() << "]"; return begin; } } @@ -872,10 +872,10 @@ bool AscendStreamAssign::IsTaskSink() { auto ms_context = MsContext::GetInstance(); MS_EXCEPTION_IF_NULL(ms_context); if (!ms_context->enable_task_sink()) { - MS_LOG(INFO) << "task sink mode is not enable"; + MS_LOG(INFO) << "Task sink mode is not enable"; return false; } else { - MS_LOG(INFO) << "task sink mode is enable"; + MS_LOG(INFO) << "Task sink mode is enable"; return true; } } @@ -885,7 +885,7 @@ void AscendStreamAssign::GetWaitStreams(vector *wait_active_stream_lis AscendResourceMng &resource_manager = AscendResourceMng::GetInstance(); uint32_t total_stream_num = resource_manager.get_cur_stream_num(); if (total_stream_num == 0) { - MS_LOG(INFO) << "total_common_stream_num is zero"; + MS_LOG(INFO) << "The total_common_stream_num is zero"; return; } @@ -893,7 +893,7 @@ void AscendStreamAssign::GetWaitStreams(vector *wait_active_stream_lis for (uint32_t i = 0; i < total_stream_num; i++) { auto it = std::find(need_first_active_streams_.begin(), need_first_active_streams_.end(), i); if (it == need_first_active_streams_.end()) { - MS_LOG(INFO) << "wait common stream id = " << i; + MS_LOG(INFO) << "Wait common stream id = " << i; wait_active_stream_list->push_back(i); } } diff --git a/mindspore/ccsrc/device/cpu/cpu_kernel_runtime.cc b/mindspore/ccsrc/device/cpu/cpu_kernel_runtime.cc index 463797daac..f46d10ed82 100644 --- a/mindspore/ccsrc/device/cpu/cpu_kernel_runtime.cc +++ b/mindspore/ccsrc/device/cpu/cpu_kernel_runtime.cc @@ -142,6 +142,37 @@ DeviceAddressPtr CPUKernelRuntime::CreateDeviceAddress(void *device_ptr, size_t return std::make_shared(device_ptr, device_size, format, type_id); } +tensor::TensorPtr CPUKernelRuntime::CreatTensorForOutput(const CNodePtr &node, size_t index, + std::set *bound_addresses, + std::vector *need_sync_outputs) { + MS_EXCEPTION_IF_NULL(node); + MS_EXCEPTION_IF_NULL(bound_addresses); + MS_EXCEPTION_IF_NULL(need_sync_outputs); + size_t output_size = AnfAlgo::GetOutputTensorNum(node); + if (index >= output_size) { + MS_LOG(EXCEPTION) << "Invalid input index " << index; + } + auto address = AnfAlgo::GetMutableOutputAddr(node, index); + MS_EXCEPTION_IF_NULL(address); + auto shape = AnfAlgo::GetOutputInferShape(node, index); + std::vector temp_shape; + (void)temp_shape.insert(temp_shape.end(), shape.begin(), shape.end()); + TypeId type_id = AnfAlgo::GetOutputInferDataType(node, index); + type_id = GetCPUSupportOutputTypeId(type_id); + tensor::TensorPtr tensor = std::make_shared(type_id, temp_shape); + MS_EXCEPTION_IF_NULL(tensor); + if (bound_addresses->find(address) != bound_addresses->end()) { + tensor->set_device_address(address); + need_sync_outputs->emplace_back(tensor); + } else { + address->ptr_ = tensor->data_c(); + address->ref_count_ = INIT_NODE_REF; + (void)bound_addresses->insert(address); + } + tensor->set_dirty(false); + return tensor; +} + BaseRef CPUKernelRuntime::CreatTensorForOutput(const session::KernelWithIndex &kernel_with_index, const std::unordered_map &input_map, std::set *bound_addresses, @@ -161,29 +192,7 @@ BaseRef CPUKernelRuntime::CreatTensorForOutput(const session::KernelWithIndex &k } return ret; } - size_t output_size = AnfAlgo::GetOutputTensorNum(node); - if (index >= output_size) { - MS_LOG(EXCEPTION) << "Invalid input index " << index; - } - auto address = AnfAlgo::GetMutableOutputAddr(node, index); - MS_EXCEPTION_IF_NULL(address); - auto shape = AnfAlgo::GetOutputInferShape(node, index); - std::vector temp_shape; - (void)temp_shape.insert(temp_shape.end(), shape.begin(), shape.end()); - TypeId type_id = AnfAlgo::GetOutputInferDataType(node, index); - type_id = GetCPUSupportOutputTypeId(type_id); - tensor::TensorPtr tensor = std::make_shared(type_id, temp_shape); - MS_EXCEPTION_IF_NULL(tensor); - if (bound_addresses->find(address) != bound_addresses->end()) { - tensor->set_device_address(address); - need_sync_outputs->emplace_back(tensor); - } else { - address->ptr_ = tensor->data_c(); - address->ref_count_ = INIT_NODE_REF; - (void)bound_addresses->insert(address); - } - tensor->set_dirty(false); - return tensor; + return CreatTensorForOutput(node, index, bound_addresses, need_sync_outputs); } else if (input_node->isa() || input_node->isa()) { auto iter = input_map.find(input_node.get()); if (iter != input_map.end()) { @@ -247,6 +256,7 @@ void CPUKernelRuntime::BindInputOutput(const session::KernelGraph *kernel_graph, void CPUKernelRuntime::AddRuntimeAddress(DeviceAddress *address, std::vector *input_list) { MS_EXCEPTION_IF_NULL(address); + MS_EXCEPTION_IF_NULL(input_list); kernel::AddressPtr input = std::make_shared(); MS_EXCEPTION_IF_NULL(input); if (address->ptr_ == nullptr) { diff --git a/mindspore/ccsrc/device/cpu/cpu_kernel_runtime.h b/mindspore/ccsrc/device/cpu/cpu_kernel_runtime.h index 27dcefdba9..354d2922c2 100644 --- a/mindspore/ccsrc/device/cpu/cpu_kernel_runtime.h +++ b/mindspore/ccsrc/device/cpu/cpu_kernel_runtime.h @@ -49,6 +49,10 @@ class CPUKernelRuntime : public KernelRuntime { TypeId type_id) override; private: + tensor::TensorPtr CreatTensorForOutput(const CNodePtr &node, size_t index, + std::set *bound_addresses, + std::vector *need_sync_outputs); + BaseRef CreatTensorForOutput(const session::KernelWithIndex &kernel_with_index, const std::unordered_map &input_map, std::set *bound_addresses, diff --git a/mindspore/ccsrc/device/cpu/cpu_simple_mem_plan.cc b/mindspore/ccsrc/device/cpu/cpu_simple_mem_plan.cc index 3fb76d38db..95f0d25f5b 100644 --- a/mindspore/ccsrc/device/cpu/cpu_simple_mem_plan.cc +++ b/mindspore/ccsrc/device/cpu/cpu_simple_mem_plan.cc @@ -56,7 +56,13 @@ void CPUSimpleMemPlan::MemPlan(const session::KernelGraph *graph) { graph_mem_size_[graph] = total_mem_size; } -size_t CPUSimpleMemPlan::GetGraphMemSize(const session::KernelGraph *graph) { return graph_mem_size_[graph]; } +size_t CPUSimpleMemPlan::GetGraphMemSize(const session::KernelGraph *graph) const { + auto iter = graph_mem_size_.find(graph); + if (iter != graph_mem_size_.end()) { + return iter->second; + } + return 0; +} void CPUSimpleMemPlan::MemAssign(const session::KernelGraph *graph, uint8_t *base_ptr) { MS_EXCEPTION_IF_NULL(graph); diff --git a/mindspore/ccsrc/device/cpu/cpu_simple_mem_plan.h b/mindspore/ccsrc/device/cpu/cpu_simple_mem_plan.h index 5f88966103..7633ef3f45 100644 --- a/mindspore/ccsrc/device/cpu/cpu_simple_mem_plan.h +++ b/mindspore/ccsrc/device/cpu/cpu_simple_mem_plan.h @@ -31,7 +31,7 @@ class CPUSimpleMemPlan { void MemPlan(const session::KernelGraph *graph); void MemAssign(const session::KernelGraph *graph, uint8_t *base_ptr); - size_t GetGraphMemSize(const session::KernelGraph *graph); + size_t GetGraphMemSize(const session::KernelGraph *graph) const; private: std::unordered_map graph_mem_size_; From 1ecaf912f990f40993b52b5311e27993ab2cd7d2 Mon Sep 17 00:00:00 2001 From: chenfei Date: Mon, 29 Jun 2020 15:29:02 +0800 Subject: [PATCH 134/254] if parameter is the second input of control depend and depend mode is 0,this control relation is invalid --- mindspore/ccsrc/session/kernel_graph.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mindspore/ccsrc/session/kernel_graph.cc b/mindspore/ccsrc/session/kernel_graph.cc index 7d40667ec9..306e3351e3 100644 --- a/mindspore/ccsrc/session/kernel_graph.cc +++ b/mindspore/ccsrc/session/kernel_graph.cc @@ -592,8 +592,8 @@ void KernelGraph::UpdateControlDependRelations(const std::vector &de if (prior_node->isa() && depend_mode == 1) { prior_nodes = GetOutputNodes(prior_node); } - if (depend_node->isa() && depend_mode == 1) { - depend_nodes = GetOutputNodes(depend_node); + if (depend_node->isa()) { + depend_nodes = depend_mode == 1 ? GetOutputNodes(depend_node) : std::vector{}; } std::vector real_prior_nodes; From 490f949617414abb25bdc72e1a96d49f1061c252 Mon Sep 17 00:00:00 2001 From: liuchongming74 Date: Mon, 29 Jun 2020 16:19:23 +0800 Subject: [PATCH 135/254] Solve MASS codex warning. --- model_zoo/mass/src/utils/byte_pair_encoding.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/model_zoo/mass/src/utils/byte_pair_encoding.py b/model_zoo/mass/src/utils/byte_pair_encoding.py index fb0e34a30d..d18b09e37f 100644 --- a/model_zoo/mass/src/utils/byte_pair_encoding.py +++ b/model_zoo/mass/src/utils/byte_pair_encoding.py @@ -16,8 +16,8 @@ import os import subprocess -ENCODER = "subword-nmt apply-bpe -c {codes} -i {input} -o {output}" -LEARN_DICT = "subword-nmt get-vocab -i {input} -o {dict_path}" +ENCODER = "subword-nmt apply-bpe -c" +LEARN_DICT = "subword-nmt get-vocab -i" def bpe_encode(codes_path, src_path, output_path, dict_path): @@ -43,10 +43,10 @@ def bpe_encode(codes_path, src_path, output_path, dict_path): raise FileNotFoundError("Dir not found.") # Encoding. - print(f" | Applying BPE encoding.") - subprocess.call(ENCODER.format(codes=codes_path, input=src_path, output=output_path), - shell=True) - print(f" | Fetching vocabulary from single file.") + print(" | Applying BPE encoding.") + commands = ENCODER.split() + [codes_path] + ["-i"] + [src_path] + ["-o"] + [output_path] + subprocess.call(commands) + print(" | Fetching vocabulary from single file.") # Learn vocab. - subprocess.call(LEARN_DICT.format(input=output_path, dict_path=dict_path), - shell=True) + commands = LEARN_DICT.split() + [output_path] + ["-o"] + [dict_path] + subprocess.call(commands) From 40251d9578ccec48c4eac60ca2473e0b08794824 Mon Sep 17 00:00:00 2001 From: lirongzhen1 Date: Mon, 29 Jun 2020 15:15:42 +0800 Subject: [PATCH 136/254] configure auto parallel tensors shape --- mindspore/ccsrc/operator/prim_others.cc | 18 +++++++++--------- mindspore/ccsrc/parallel/step_parallel.cc | 6 ++++++ .../ccsrc/pipeline/static_analysis/prim.h | 1 + 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/mindspore/ccsrc/operator/prim_others.cc b/mindspore/ccsrc/operator/prim_others.cc index 432b12f83b..9350e9aa3b 100644 --- a/mindspore/ccsrc/operator/prim_others.cc +++ b/mindspore/ccsrc/operator/prim_others.cc @@ -118,11 +118,9 @@ const size_t UndeterminedShapeType::fields_num = 6; std::unordered_map g_undetermined_configs; void InitUndeterminedFromEnv(const std::string &sparse_shape_types) { - if (!g_undetermined_configs.empty()) { - return; - } std::string tmp; std::stringstream input(sparse_shape_types); + g_undetermined_configs.clear(); while (std::getline(input, tmp, ';')) { auto config = UndeterminedShapeType(tmp); g_undetermined_configs.insert(std::make_pair(config.param_name(), config)); @@ -145,17 +143,19 @@ AbstractBasePtr InferImplEnvGetItem(const AnalysisEnginePtr &, const PrimitivePt if (!key->sparse_grad().empty()) { // Will be fixed once undetermined type ready - auto sparse_shape_types = common::GetEnv("UNDETERMINED_SPARSE_SHAPE_TYPES"); - if (sparse_shape_types.empty()) { - sparse_shape_types = "sparse_key_w1:2:Int32:2 1 2:Float32:3 1 2;sparse_key_w2:2:Int32:2 1 2:Float32:3 1 2"; + if (g_undetermined_configs.empty()) { + auto sparse_shape_types = common::GetEnv("UNDETERMINED_SPARSE_SHAPE_TYPES"); + MS_LOG(INFO) << "Undetermind sparse shape:" << sparse_shape_types; + if (sparse_shape_types.empty()) { + sparse_shape_types = "sparse_key_w1:2:Int32:2 1 2:Float32:3 1 2;sparse_key_w2:2:Int32:2 1 2:Float32:3 1 2"; + } + InitUndeterminedFromEnv(sparse_shape_types); } - InitUndeterminedFromEnv(sparse_shape_types); auto shape_types = g_undetermined_configs.find(key->sparse_grad()); if (shape_types == g_undetermined_configs.end()) { MS_LOG(EXCEPTION) << "Param " << key->ToString() - << " has sparse_grad, but shape/type is not configured in env UNDETERMINED_SPARSE_SHAPE_TYPES: " - << sparse_shape_types; + << " has sparse_grad, but shape/type is not configured in env UNDETERMINED_SPARSE_SHAPE_TYPES"; } MS_LOG(DEBUG) << "EnvGetItem is sparse_grad " << key->ToString(); AbstractBasePtrList sparse_list; diff --git a/mindspore/ccsrc/parallel/step_parallel.cc b/mindspore/ccsrc/parallel/step_parallel.cc index fc7b48d267..7d1200b190 100644 --- a/mindspore/ccsrc/parallel/step_parallel.cc +++ b/mindspore/ccsrc/parallel/step_parallel.cc @@ -43,6 +43,7 @@ #include "parallel/strategy_checkpoint/parallel_strategy_checkpoint.h" #include "utils/comm_manager.h" #include "utils/symbolic.h" +#include "pipeline/static_analysis/prim.h" using mindspore::tensor::Tensor; @@ -1371,6 +1372,11 @@ void SetClonedTensorShapeForOptimizer(const FuncGraphPtr &root) { << cloned_index << ", but not found the be cloned parameter"; } } + std::string env = common::GetEnv("SLICE_ENV"); + if (!env.empty()) { + MS_LOG(INFO) << "Slice tensors shape will be configured from env:" << env; + abstract::InitUndeterminedFromEnv(env); + } } void SetVirtualDatasetStrategy(const CNodePtr &node) { diff --git a/mindspore/ccsrc/pipeline/static_analysis/prim.h b/mindspore/ccsrc/pipeline/static_analysis/prim.h index 5b3972088a..5954179aa5 100644 --- a/mindspore/ccsrc/pipeline/static_analysis/prim.h +++ b/mindspore/ccsrc/pipeline/static_analysis/prim.h @@ -349,6 +349,7 @@ AbstractBasePtr InferImplControlDepend(const AnalysisEnginePtr &, const Primitiv AbstractBasePtr InferImplDebug(const AnalysisEnginePtr &, const PrimitivePtr &primitive, const AbstractBasePtrList &args_spec_list); +void InitUndeterminedFromEnv(const std::string &sparse_shape_types); } // namespace abstract } // namespace mindspore From 41d16a3c770ab52b9af9b7905d431c57e05729e1 Mon Sep 17 00:00:00 2001 From: jonyguo Date: Mon, 29 Jun 2020 15:47:06 +0800 Subject: [PATCH 137/254] enhance: 1. fix api comment, 2. del dead code --- mindspore/ccsrc/dataset/core/data_type.cc | 1 - mindspore/ccsrc/dataset/core/tensor.cc | 6 +- mindspore/ccsrc/dataset/core/tensor.h | 1 - mindspore/ccsrc/dataset/core/tensor_shape.cc | 1 - mindspore/ccsrc/dataset/engine/connector.h | 10 +- mindspore/ccsrc/dataset/engine/data_schema.cc | 3 +- .../dataset/engine/datasetops/dataset_op.cc | 2 +- .../datasetops/source/image_folder_op.cc | 2 +- .../ccsrc/dataset/engine/jagged_connector.h | 2 +- mindspore/ccsrc/dataset/util/arena.cc | 13 +- mindspore/ccsrc/dataset/util/btree_impl.tpp | 8 +- .../ccsrc/dataset/util/btree_iterator.tpp | 1 - mindspore/ccsrc/dataset/util/buddy.cc | 18 +-- mindspore/ccsrc/dataset/util/circular_pool.cc | 6 +- mindspore/ccsrc/dataset/util/de_error.h | 39 ------ mindspore/ccsrc/dataset/util/list.h | 7 +- mindspore/ccsrc/dataset/util/lock.cc | 22 ++-- mindspore/ccsrc/dataset/util/path.cc | 1 - mindspore/ccsrc/dataset/util/queue.h | 1 - .../ccsrc/dataset/util/storage_container.cc | 5 +- .../ccsrc/dataset/util/storage_manager.cc | 1 - mindspore/ccsrc/dataset/util/task.cc | 1 - mindspore/ccsrc/dataset/util/task.h | 1 - mindspore/ccsrc/utils/error_code.h | 114 ------------------ mindspore/mindrecord/filewriter.py | 8 +- tests/ut/cpp/dataset/arena_test.cc | 1 - tests/ut/cpp/dataset/batch_op_test.cc | 1 - tests/ut/cpp/dataset/btree_test.cc | 1 - tests/ut/cpp/dataset/celeba_op_test.cc | 1 - tests/ut/cpp/dataset/cifar_op_test.cc | 1 - tests/ut/cpp/dataset/coco_op_test.cc | 1 - tests/ut/cpp/dataset/common/common.h | 1 - tests/ut/cpp/dataset/connector_test.cc | 6 +- tests/ut/cpp/dataset/duplicate_op_test.cc | 1 - tests/ut/cpp/dataset/execution_tree_test.cc | 1 - tests/ut/cpp/dataset/global_context_test.cc | 1 - tests/ut/cpp/dataset/image_folder_op_test.cc | 1 - tests/ut/cpp/dataset/interrupt_test.cc | 1 - tests/ut/cpp/dataset/manifest_op_test.cc | 1 - tests/ut/cpp/dataset/mask_test.cc | 1 - tests/ut/cpp/dataset/mnist_op_test.cc | 1 - tests/ut/cpp/dataset/path_test.cc | 1 - .../cpp/dataset/stand_alone_samplers_test.cc | 1 - tests/ut/cpp/dataset/status_test.cc | 1 - tests/ut/cpp/dataset/tensor_string_test.cc | 1 - tests/ut/cpp/dataset/tensor_test.cc | 1 - tests/ut/cpp/dataset/treap_test.cc | 1 - tests/ut/cpp/dataset/type_cast_op_test.cc | 1 - tests/ut/cpp/dataset/voc_op_test.cc | 1 - tests/ut/cpp/dataset/zip_op_test.cc | 1 - 50 files changed, 60 insertions(+), 244 deletions(-) delete mode 100644 mindspore/ccsrc/dataset/util/de_error.h delete mode 100644 mindspore/ccsrc/utils/error_code.h diff --git a/mindspore/ccsrc/dataset/core/data_type.cc b/mindspore/ccsrc/dataset/core/data_type.cc index 71a510d88f..bb10fae52f 100644 --- a/mindspore/ccsrc/dataset/core/data_type.cc +++ b/mindspore/ccsrc/dataset/core/data_type.cc @@ -18,7 +18,6 @@ #include "utils/log_adapter.h" #include "dataset/core/pybind_support.h" -#include "dataset/util/de_error.h" namespace mindspore { namespace dataset { diff --git a/mindspore/ccsrc/dataset/core/tensor.cc b/mindspore/ccsrc/dataset/core/tensor.cc index abab8cf3f4..8de3425c5b 100644 --- a/mindspore/ccsrc/dataset/core/tensor.cc +++ b/mindspore/ccsrc/dataset/core/tensor.cc @@ -152,7 +152,7 @@ Tensor::Tensor(const std::vector &strings, const TensorShape &shape this->data_end_ = data_ + offset_arr[i]; - DS_ASSERT(num_bytes == 0); + MS_ASSERT(num_bytes == 0); if (shape.known()) Tensor::Reshape(shape); } Tensor::Tensor(const dataengine::BytesList &bytes_list, const TensorShape &shape) @@ -191,7 +191,7 @@ Tensor::Tensor(const dataengine::BytesList &bytes_list, const TensorShape &shape data_end_ = data_ + offset_arr[i]; - DS_ASSERT(num_bytes == 0); + MS_ASSERT(num_bytes == 0); if (shape.known()) Tensor::Reshape(shape); } Status Tensor::CreateTensor(std::shared_ptr *ptr, TensorImpl tensor_impl, const TensorShape &shape, @@ -420,7 +420,7 @@ bool Tensor::operator==(const Tensor &rhs) const { // Description: A function that print the value as specified by its index void Tensor::PrintItemAt(const std::vector &index, std::ostream &out) const { Status rc; - DS_ASSERT(data_); + MS_ASSERT(data_); switch (type_.value()) { CASE_PRINT_HEX(DataType::DE_BOOL, bool); diff --git a/mindspore/ccsrc/dataset/core/tensor.h b/mindspore/ccsrc/dataset/core/tensor.h index a3dbb391e5..9fed0bbc97 100644 --- a/mindspore/ccsrc/dataset/core/tensor.h +++ b/mindspore/ccsrc/dataset/core/tensor.h @@ -33,7 +33,6 @@ #include "dataset/core/data_type.h" #include "dataset/core/tensor_shape.h" #include "dataset/util/allocator.h" -#include "dataset/util/de_error.h" #include "dataset/util/status.h" #include "proto/example.pb.h" diff --git a/mindspore/ccsrc/dataset/core/tensor_shape.cc b/mindspore/ccsrc/dataset/core/tensor_shape.cc index 30afdf38bc..a0d6b9cd8d 100644 --- a/mindspore/ccsrc/dataset/core/tensor_shape.cc +++ b/mindspore/ccsrc/dataset/core/tensor_shape.cc @@ -22,7 +22,6 @@ #include "common/utils.h" #include "utils/log_adapter.h" #include "dataset/core/constants.h" -#include "dataset/util/de_error.h" namespace mindspore { namespace dataset { diff --git a/mindspore/ccsrc/dataset/engine/connector.h b/mindspore/ccsrc/dataset/engine/connector.h index cdce592c1b..5701c902da 100644 --- a/mindspore/ccsrc/dataset/engine/connector.h +++ b/mindspore/ccsrc/dataset/engine/connector.h @@ -97,7 +97,7 @@ class Connector { virtual Status Pop(int32_t worker_id, // The worker-id of the caller. See the requirement at the top of this file. T *result) noexcept { { - DS_ASSERT(worker_id < num_consumers_); + MS_ASSERT(worker_id < num_consumers_); std::unique_lock lk(m_); RETURN_IF_NOT_OK(cv_.Wait(&lk, [this, worker_id]() { return expect_consumer_ == worker_id; })); RETURN_IF_NOT_OK(queues_[pop_from_]->PopFront(result)); @@ -114,8 +114,8 @@ class Connector { // @param worker_id The id of a worker thread calling this method. // @param el A const lvalue element to be passed/added/pushed. Status Push(int32_t worker_id, const T &el) noexcept { - DS_ASSERT(worker_id < static_cast(queues_.size())); - DS_ASSERT(queues_[worker_id] != nullptr); + MS_ASSERT(worker_id < static_cast(queues_.size())); + MS_ASSERT(queues_[worker_id] != nullptr); return (queues_[worker_id]->Add(el)); } @@ -125,8 +125,8 @@ class Connector { // @param worker_id The id of a worker thread calling this method. // @param el An element to be passed/added/pushed. virtual Status Push(int32_t worker_id, T &&el) noexcept { - DS_ASSERT(worker_id < static_cast(queues_.size())); - DS_ASSERT(queues_[worker_id] != nullptr); + MS_ASSERT(worker_id < static_cast(queues_.size())); + MS_ASSERT(queues_[worker_id] != nullptr); return (queues_[worker_id]->Add(std::forward(el))); } diff --git a/mindspore/ccsrc/dataset/engine/data_schema.cc b/mindspore/ccsrc/dataset/engine/data_schema.cc index 9b86d18912..419816f30a 100644 --- a/mindspore/ccsrc/dataset/engine/data_schema.cc +++ b/mindspore/ccsrc/dataset/engine/data_schema.cc @@ -27,7 +27,6 @@ #include "dataset/util/status.h" #include "dataset/core/tensor_shape.h" #include "utils/log_adapter.h" -#include "dataset/util/de_error.h" namespace mindspore { namespace dataset { @@ -425,7 +424,7 @@ DataSchema::~DataSchema() = default; // Getter for the ColDescriptor by index const ColDescriptor &DataSchema::column(int32_t idx) const { - DS_ASSERT(idx < static_cast(col_descs_.size())); + MS_ASSERT(idx < static_cast(col_descs_.size())); return col_descs_[idx]; } diff --git a/mindspore/ccsrc/dataset/engine/datasetops/dataset_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/dataset_op.cc index d1129848e3..727c543958 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/dataset_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/dataset_op.cc @@ -110,7 +110,7 @@ void DatasetOp::RemoveParent(const DatasetOp *parent) { // Getter function to get a shared pointer to our childAdds a operator to become our child. std::shared_ptr DatasetOp::child(int32_t child_index) const { - DS_ASSERT(child_index < static_cast(child_.size())); + MS_ASSERT(child_index < static_cast(child_.size())); // Return a shared pointer return child_[child_index]; } diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/image_folder_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/image_folder_op.cc index b2611a67fc..c28ed2d3ab 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/image_folder_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/image_folder_op.cc @@ -104,7 +104,7 @@ Status ImageFolderOp::PrescanMasterEntry(const std::string &filedir) { // following loop puts the 2 level of shuffles together into 1 vector for (size_t ind = 0; ind < v.size(); ++ind) { while (v[ind]->second.empty() == false) { - DS_ASSERT(!(v[ind]->first.empty())); // make sure that v[ind]->first.substr(1) is not out of bound + MS_ASSERT(!(v[ind]->first.empty())); // make sure that v[ind]->first.substr(1) is not out of bound v[ind]->second.front()->second = class_index_.empty() ? ind : class_index_[v[ind]->first.substr(1)]; image_label_pairs_.push_back(v[ind]->second.front()); v[ind]->second.pop(); diff --git a/mindspore/ccsrc/dataset/engine/jagged_connector.h b/mindspore/ccsrc/dataset/engine/jagged_connector.h index 5f9f255f2b..2058c542a8 100644 --- a/mindspore/ccsrc/dataset/engine/jagged_connector.h +++ b/mindspore/ccsrc/dataset/engine/jagged_connector.h @@ -44,7 +44,7 @@ class JaggedConnector : public Connector> { Status Pop(int32_t worker_id, std::unique_ptr *result) noexcept override { { - DS_ASSERT(worker_id < num_consumers_); + MS_ASSERT(worker_id < num_consumers_); std::unique_lock lock(m_); RETURN_IF_NOT_OK(cv_.Wait(&lock, [this, worker_id]() { return expect_consumer_ == worker_id; })); if (is_queue_finished_[pop_from_]) { diff --git a/mindspore/ccsrc/dataset/util/arena.cc b/mindspore/ccsrc/dataset/util/arena.cc index e9dec9d86e..af4f522678 100644 --- a/mindspore/ccsrc/dataset/util/arena.cc +++ b/mindspore/ccsrc/dataset/util/arena.cc @@ -17,7 +17,6 @@ #include #include #include "dataset/util/system_pool.h" -#include "dataset/util/de_error.h" #include "./securec.h" #include "utils/log_adapter.h" @@ -79,7 +78,7 @@ void Arena::Deallocate(void *p) { auto *q = get_base_addr(p); MemHdr hdr(0, 0); MemHdr::getHdr(q, &hdr); - DS_ASSERT(hdr.sig == 0xDEADBEEF); + MS_ASSERT(hdr.sig == 0xDEADBEEF); // We are going to insert a free block back to the treap. But first, check if we can combine // with the free blocks before and after to form a bigger block. std::unique_lock lck(mux_); @@ -103,8 +102,8 @@ void Arena::Deallocate(void *p) { } Status Arena::Reallocate(void **pp, size_t old_sz, size_t new_sz) { - DS_ASSERT(pp); - DS_ASSERT(*pp); + MS_ASSERT(pp); + MS_ASSERT(*pp); uint64_t actual_size = static_cast(new_sz) + ARENA_WALL_OVERHEAD_SZ; if (actual_size > this->get_max_size()) { RETURN_STATUS_UNEXPECTED("Request size too big : " + std::to_string(new_sz)); @@ -114,7 +113,7 @@ Status Arena::Reallocate(void **pp, size_t old_sz, size_t new_sz) { auto *oldHdr = get_base_addr(oldAddr); MemHdr hdr(0, 0); MemHdr::getHdr(oldHdr, &hdr); - DS_ASSERT(hdr.sig == 0xDEADBEEF); + MS_ASSERT(hdr.sig == 0xDEADBEEF); std::unique_lock lck(mux_); if (hdr.blk_size > req_blk) { // Refresh the header with the new smaller size. @@ -239,8 +238,8 @@ bool Arena::BlockEnlarge(uint64_t *addr, uint64_t old_sz, uint64_t new_sz) { } Status Arena::FreeAndAlloc(void **pp, size_t old_sz, size_t new_sz) { - DS_ASSERT(pp); - DS_ASSERT(*pp); + MS_ASSERT(pp); + MS_ASSERT(*pp); void *p = nullptr; void *q = *pp; RETURN_IF_NOT_OK(Allocate(new_sz, &p)); diff --git a/mindspore/ccsrc/dataset/util/btree_impl.tpp b/mindspore/ccsrc/dataset/util/btree_impl.tpp index 8148a8d12c..fc3b05e3a1 100644 --- a/mindspore/ccsrc/dataset/util/btree_impl.tpp +++ b/mindspore/ccsrc/dataset/util/btree_impl.tpp @@ -61,8 +61,8 @@ typename BPlusTree::IndexRc BPlusTree::InnerNode:: template typename BPlusTree::IndexRc BPlusTree::InnerNode::Split( BPlusTree::InnerNode *to, key_type *split_key) { - DS_ASSERT(to); - DS_ASSERT(to->slotuse_ == 0); + MS_ASSERT(to); + MS_ASSERT(to->slotuse_ == 0); // It is simpler to sort first, then split. Other alternative is to move key by key to the // new node. Also we need to deal with the 'holes' after a key is moved. RETURN_IF_BAD_RC(this->Sort()); @@ -153,8 +153,8 @@ typename BPlusTree::IndexRc BPlusTree::LeafNode::S template typename BPlusTree::IndexRc BPlusTree::LeafNode::Split( BPlusTree::LeafNode *to) { - DS_ASSERT(to); - DS_ASSERT(to->slotuse_ == 0); + MS_ASSERT(to); + MS_ASSERT(to->slotuse_ == 0); // It is simpler to sort first, then split. Other alternative is to move key by key to the // new node. Also we need to deal with the 'holes' after a key is moved. RETURN_IF_BAD_RC(this->Sort()); diff --git a/mindspore/ccsrc/dataset/util/btree_iterator.tpp b/mindspore/ccsrc/dataset/util/btree_iterator.tpp index 91ba2acd7a..cffef3fa7a 100644 --- a/mindspore/ccsrc/dataset/util/btree_iterator.tpp +++ b/mindspore/ccsrc/dataset/util/btree_iterator.tpp @@ -15,7 +15,6 @@ #ifndef DATASET_UTIL_BTREE_ITERATOR_H_ #define DATASET_UTIL_BTREE_ITERATOR_H_ -#include "dataset/util/de_error.h" #include "utils/log_adapter.h" #include "btree.h" diff --git a/mindspore/ccsrc/dataset/util/buddy.cc b/mindspore/ccsrc/dataset/util/buddy.cc index 3a14258419..540fa993d6 100644 --- a/mindspore/ccsrc/dataset/util/buddy.cc +++ b/mindspore/ccsrc/dataset/util/buddy.cc @@ -16,9 +16,9 @@ #include "dataset/util/buddy.h" #include #include -#include "dataset/util/de_error.h" #include "dataset/util/memory_pool.h" #include "dataset/util/system_pool.h" +#include "utils/log_adapter.h" #include "./securec.h" inline uint64_t BitLeftShift(uint64_t v, uint64_t n) { return (v << n); } @@ -68,7 +68,7 @@ Status BuddySpace::Alloc(const uint64_t sz, BSpaceDescriptor *desc, addr_t *p) n } addr_t BuddySpace::AllocNoLock(const uint64_t sz, BSpaceDescriptor *desc) noexcept { - DS_ASSERT(sz <= max_); + MS_ASSERT(sz <= max_); uint32_t reqSize = SizeToBlock(sz); rel_addr_t rel_addr = AllocBuddySeg(reqSize); if (rel_addr != static_cast(NOSPACE)) { @@ -84,7 +84,7 @@ addr_t BuddySpace::AllocNoLock(const uint64_t sz, BSpaceDescriptor *desc) noexce } void BuddySpace::FreeNoLock(const BSpaceDescriptor *desc) { - DS_ASSERT(desc->sig == 0XDEADBEEF); + MS_ASSERT(desc->sig == 0XDEADBEEF); rel_addr_t rel_addr = desc->addr; size_t blk_size = desc->blk_size; size_t req_size = desc->req_size; @@ -217,7 +217,7 @@ void BuddySpace::JoinBuddySeg(rel_addr_t addr, size_t blk_sz) { auto log_sz = static_cast(Log2(blk_sz)); rel_addr_t left = (buddy < addr) ? buddy : addr; rel_addr_t right = left + blk_sz; - DS_ASSERT(count_[log_sz] >= 2); + MS_ASSERT(count_[log_sz] >= 2); count_[log_sz] -= 2; SetBuddySegState(right, blk_sz, STATE::kEmpty); SetBuddySegState(left, BitLeftShift(blk_sz, 1), STATE::kFree); @@ -235,7 +235,7 @@ void BuddySpace::JoinBuddySeg(rel_addr_t addr, size_t blk_sz) { } void BuddySpace::TrimBuddySeg(rel_addr_t addr, size_t blk_sz, size_t ask_sz) { - DS_ASSERT(ask_sz < blk_sz); + MS_ASSERT(ask_sz < blk_sz); uint32_t inx = Log2(blk_sz); size_t remaining_sz = ask_sz; for (int i = inx; i > 0; i--) { @@ -256,7 +256,7 @@ void BuddySpace::TrimBuddySeg(rel_addr_t addr, size_t blk_sz, size_t ask_sz) { } void BuddySpace::UnTrimBuddySeg(rel_addr_t addr, size_t blk_sz, size_t ask_sz) { - DS_ASSERT(ask_sz < blk_sz); + MS_ASSERT(ask_sz < blk_sz); uint32_t inx = Log2(blk_sz); size_t remaining_sz = ask_sz; for (int i = inx; i > 0; i--) { @@ -268,7 +268,7 @@ void BuddySpace::UnTrimBuddySeg(rel_addr_t addr, size_t blk_sz, size_t ask_sz) { size_t sz = 0; STATE st; GetBuddySegState(addr, &sz, &st); - DS_ASSERT(sz == half_sz && st == STATE::kAlloc); + MS_ASSERT(sz == half_sz && st == STATE::kAlloc); } #endif SetBuddySegState(addr, half_sz, STATE::kFree); @@ -291,7 +291,7 @@ rel_addr_t BuddySpace::AllocBuddySeg(uint32_t req_size) noexcept { STATE st; size_t sz = 0; for (int i = start_inx; !found && i < num_lvl_; i++) { - DS_ASSERT(count_[i] >= 0); + MS_ASSERT(count_[i] >= 0); if (count_[i] == 0) { continue; } @@ -302,7 +302,7 @@ rel_addr_t BuddySpace::AllocBuddySeg(uint32_t req_size) noexcept { if (st == STATE::kFree && sz == blk_sz) { found = true; } else { - DS_ASSERT(st != STATE::kEmpty); + MS_ASSERT(st != STATE::kEmpty); ask_addr += ((sz > blk_sz) ? sz : blk_sz); } } diff --git a/mindspore/ccsrc/dataset/util/circular_pool.cc b/mindspore/ccsrc/dataset/util/circular_pool.cc index 92b169c94a..0c68dab81b 100644 --- a/mindspore/ccsrc/dataset/util/circular_pool.cc +++ b/mindspore/ccsrc/dataset/util/circular_pool.cc @@ -19,8 +19,8 @@ #include #include #include "./securec.h" -#include "dataset/util/de_error.h" #include "dataset/util/system_pool.h" +#include "utils/log_adapter.h" namespace mindspore { namespace dataset { @@ -65,7 +65,7 @@ void CircularPool::CircularIterator::Reset() { auto list_end = dp_->mem_segments_.end(); auto it = std::find_if(dp_->mem_segments_.begin(), list_end, [this](const std::shared_ptr &b) { return b.get() == cur_tail_; }); - DS_ASSERT(it != list_end); + MS_ASSERT(it != list_end); start_ = std::distance(dp_->mem_segments_.begin(), it); cur_ = start_; has_next_ = true; @@ -153,7 +153,7 @@ Status CircularPool::Reallocate(void **pp, size_t old_sz, size_t new_sz) { return (q > base && q < base + b->get_max_size()); }); lock.Unlock(); - DS_ASSERT(it != mem_segments_.end()); + MS_ASSERT(it != mem_segments_.end()); Arena *ba = it->get(); Status rc = ba->Reallocate(pp, old_sz, new_sz); if (rc.IsOutofMemory()) { diff --git a/mindspore/ccsrc/dataset/util/de_error.h b/mindspore/ccsrc/dataset/util/de_error.h deleted file mode 100644 index d4988c58db..0000000000 --- a/mindspore/ccsrc/dataset/util/de_error.h +++ /dev/null @@ -1,39 +0,0 @@ -/** - * Copyright 2019 Huawei Technologies Co., Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef DATASET_UTIL_DE_ERROR_H_ -#define DATASET_UTIL_DE_ERROR_H_ - -#ifdef DEBUG -#include -#define DS_ASSERT(f) assert(f) -#else -#define DS_ASSERT(f) ((void)0) -#endif - -#include -#include "utils/error_code.h" - -namespace mindspore { -namespace dataset { -DE_ERRORNO_DATASET(CATCH_EXCEPTION, 0, "try catch exception error"); -DE_ERRORNO_DATASET(FILE_NOT_FOUND, 1, "file is not found"); -DE_ERRORNO_DATASET(PARSE_FAILED, 2, "parse failed"); -DE_ERRORNO_DATASET(COPY_ERROR, 3, "copy data error"); -DE_ERRORNO_DATASET(BOUND_ERROR, 4, "variable overflow or lost of precision"); -DE_ERRORNO_DATASET(ALLOC_FAILED, 5, "dynamic memory allocation failed"); -} // namespace dataset -} // namespace mindspore -#endif // DATASET_UTIL_DE_ERROR_H_ diff --git a/mindspore/ccsrc/dataset/util/list.h b/mindspore/ccsrc/dataset/util/list.h index a4c15daa0e..06f26ab57c 100644 --- a/mindspore/ccsrc/dataset/util/list.h +++ b/mindspore/ccsrc/dataset/util/list.h @@ -18,7 +18,8 @@ #include #include -#include "dataset/util/de_error.h" + +#include "utils/log_adapter.h" namespace mindspore { namespace dataset { @@ -90,7 +91,7 @@ struct List { // Insert elem2 after elem1 in the list. virtual void InsertAfter(pointer elem1, pointer elem2) { - DS_ASSERT(elem1 != elem2); + MS_ASSERT(elem1 != elem2); Node &elem1_node = elem1->*node; Node &elem2_node = elem2->*node; elem2_node.prev = elem1; @@ -108,7 +109,7 @@ struct List { // Insert elem2 before elem1 in the list. virtual void InsertBefore(pointer elem1, pointer elem2) { - DS_ASSERT(elem1 != elem2); + MS_ASSERT(elem1 != elem2); Node &elem1_node = elem1->*node; Node &elem2_node = elem2->*node; elem2_node.next = elem1; diff --git a/mindspore/ccsrc/dataset/util/lock.cc b/mindspore/ccsrc/dataset/util/lock.cc index 13a43e3e84..bde9d84005 100644 --- a/mindspore/ccsrc/dataset/util/lock.cc +++ b/mindspore/ccsrc/dataset/util/lock.cc @@ -14,7 +14,7 @@ * limitations under the License. */ #include "dataset/util/lock.h" -#include "dataset/util/de_error.h" +#include "utils/log_adapter.h" namespace mindspore { namespace dataset { @@ -63,7 +63,7 @@ void RWLock::Unlock() noexcept { void RWLock::Upgrade() { std::unique_lock lck(mtx_); - DS_ASSERT(status_); + MS_ASSERT(status_); if (status_ == -1) { // I am a writer already. return; @@ -81,7 +81,7 @@ void RWLock::Upgrade() { void RWLock::Downgrade() { std::unique_lock lck(mtx_); - DS_ASSERT(status_); + MS_ASSERT(status_); if (status_ == -1) { // If there are no other writers waiting, just change the status if (waiting_writers_ == 0) { @@ -111,24 +111,24 @@ SharedLock::~SharedLock() { } void SharedLock::Unlock() { - DS_ASSERT(ownlock_ == true); + MS_ASSERT(ownlock_ == true); rw_->Unlock(); ownlock_ = false; } void SharedLock::Lock() { - DS_ASSERT(ownlock_ == false); + MS_ASSERT(ownlock_ == false); rw_->LockShared(); ownlock_ = true; } void SharedLock::Upgrade() { - DS_ASSERT(ownlock_ == true); + MS_ASSERT(ownlock_ == true); rw_->Upgrade(); } void SharedLock::Downgrade() { - DS_ASSERT(ownlock_ == true); + MS_ASSERT(ownlock_ == true); rw_->Downgrade(); } @@ -146,13 +146,13 @@ UniqueLock::~UniqueLock() { } void UniqueLock::Unlock() { - DS_ASSERT(ownlock_ == true); + MS_ASSERT(ownlock_ == true); rw_->Unlock(); ownlock_ = false; } void UniqueLock::Lock() { - DS_ASSERT(ownlock_ == false); + MS_ASSERT(ownlock_ == false); rw_->LockExclusive(); ownlock_ = true; } @@ -171,13 +171,13 @@ LockGuard::~LockGuard() { } void LockGuard::Unlock() { - DS_ASSERT(own_lock_); + MS_ASSERT(own_lock_); lck_->Unlock(); own_lock_ = false; } void LockGuard::Lock() { - DS_ASSERT(own_lock_ == false); + MS_ASSERT(own_lock_ == false); lck_->Lock(); own_lock_ = true; } diff --git a/mindspore/ccsrc/dataset/util/path.cc b/mindspore/ccsrc/dataset/util/path.cc index 59e5e5232c..cdd2343799 100644 --- a/mindspore/ccsrc/dataset/util/path.cc +++ b/mindspore/ccsrc/dataset/util/path.cc @@ -23,7 +23,6 @@ #include #include "common/utils.h" -#include "dataset/util/de_error.h" #include "utils/log_adapter.h" namespace mindspore { diff --git a/mindspore/ccsrc/dataset/util/queue.h b/mindspore/ccsrc/dataset/util/queue.h index 9a51565861..2d55ded736 100644 --- a/mindspore/ccsrc/dataset/util/queue.h +++ b/mindspore/ccsrc/dataset/util/queue.h @@ -25,7 +25,6 @@ #include #include "common/utils.h" -#include "dataset/util/de_error.h" #include "utils/log_adapter.h" #include "dataset/util/allocator.h" #include "dataset/util/services.h" diff --git a/mindspore/ccsrc/dataset/util/storage_container.cc b/mindspore/ccsrc/dataset/util/storage_container.cc index 96f5b45d0c..3a4c13e2d9 100644 --- a/mindspore/ccsrc/dataset/util/storage_container.cc +++ b/mindspore/ccsrc/dataset/util/storage_container.cc @@ -20,7 +20,6 @@ #include #include #include "common/utils.h" -#include "dataset/util/de_error.h" #include "dataset/util/path.h" #include "dataset/util/status.h" #include "utils/log_adapter.h" @@ -59,7 +58,7 @@ Status StorageContainer::Close() noexcept { } Status StorageContainer::Read(WritableSlice *dest, off64_t offset) const noexcept { - DS_ASSERT(is_open_); + MS_ASSERT(is_open_); RETURN_UNEXPECTED_IF_NULL(dest); auto sz = dest->GetSize(); #if defined(_WIN32) || defined(_WIN64) @@ -83,7 +82,7 @@ Status StorageContainer::Read(WritableSlice *dest, off64_t offset) const noexcep } Status StorageContainer::Write(const ReadableSlice &dest, off64_t offset) const noexcept { - DS_ASSERT(is_open_); + MS_ASSERT(is_open_); auto sz = dest.GetSize(); #if defined(_WIN32) || defined(_WIN64) // Doesn't seem there is any pwrite64 on mingw. diff --git a/mindspore/ccsrc/dataset/util/storage_manager.cc b/mindspore/ccsrc/dataset/util/storage_manager.cc index 8b7a6044e9..1d958576ba 100644 --- a/mindspore/ccsrc/dataset/util/storage_manager.cc +++ b/mindspore/ccsrc/dataset/util/storage_manager.cc @@ -22,7 +22,6 @@ #include "common/utils.h" #include "dataset/util/path.h" #include "dataset/util/services.h" -#include "dataset/util//de_error.h" #include "utils/log_adapter.h" namespace mindspore { diff --git a/mindspore/ccsrc/dataset/util/task.cc b/mindspore/ccsrc/dataset/util/task.cc index f00f26f5ce..93db55d5f9 100644 --- a/mindspore/ccsrc/dataset/util/task.cc +++ b/mindspore/ccsrc/dataset/util/task.cc @@ -16,7 +16,6 @@ #include "dataset/util/task.h" #include "common/utils.h" #include "dataset/util/task_manager.h" -#include "dataset/util/de_error.h" #include "utils/log_adapter.h" namespace mindspore { diff --git a/mindspore/ccsrc/dataset/util/task.h b/mindspore/ccsrc/dataset/util/task.h index b120ed3d7c..49eb16b182 100644 --- a/mindspore/ccsrc/dataset/util/task.h +++ b/mindspore/ccsrc/dataset/util/task.h @@ -27,7 +27,6 @@ #include #include #include -#include "dataset/util/de_error.h" #include "dataset/util/intrp_resource.h" #include "dataset/util/list.h" #include "dataset/util/memory_pool.h" diff --git a/mindspore/ccsrc/utils/error_code.h b/mindspore/ccsrc/utils/error_code.h deleted file mode 100644 index f6d70666c4..0000000000 --- a/mindspore/ccsrc/utils/error_code.h +++ /dev/null @@ -1,114 +0,0 @@ -/** - * Copyright 2019 Huawei Technologies Co., Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef MINDSPORE_CCSRC_UTILS_LOG_DE_ERROR_CODE_H_ -#define MINDSPORE_CCSRC_UTILS_LOG_DE_ERROR_CODE_H_ - -#include -#include - -namespace DataEngineBase { -// system ID -const int SYSID_MD = 20; - -// Runtime location -enum LogRuntime { - RT_HOST = 0b01, - RT_DEVICE = 0b10, -}; - -// sub model -enum SubModuleId { - COMMON_MODULE = 0, - DATAGET_MODULE, - MINDRECORD_MODULE, -}; - -// error code type -enum ErrorCodeType { - ERROR_CODE = 0b01, - EXCEPTION_CODE = 0b10, -}; - -// error level -enum ErrorLevel { - COMMON_LEVEL = 0b000, - SUGGESTION_LEVEL = 0b001, - MINOR_LEVEL = 0b010, - MAJOR_LEVEL = 0b011, - CRITICAL_LEVEL = 0b100, -}; - -// code compose(4 byte), runtime: 2 bit, type: 2 bit, level: 3 bit, sysid: 8 bit, modid: 5 bit, value: 12 bit -#define DE_ERRORNO(runtime, type, level, sysid, submodid, name, value, desc) \ - constexpr DataEngineBase::Status name = ((0xFF & ((uint8_t)runtime)) << 30) | ((0xFF & ((uint8_t)type)) << 28) | \ - ((0xFF & ((uint8_t)level)) << 25) | ((0xFF & ((uint8_t)sysid)) << 17) | \ - ((0xFF & ((uint8_t)submodid)) << 12) | (0x0FFF & ((uint16_t)value)); \ - const DataEngineBase::ErrorNoRegisterar g_##name##_errorno(name, desc); - -// each module defines error codes using the following macros -#define DE_ERRORNO_COMMON(name, value, desc) \ - DE_ERRORNO(DataEngineBase::RT_HOST, DataEngineBase::ERROR_CODE, DataEngineBase::COMMON_LEVEL, \ - DataEngineBase::SYSID_MD, DataEngineBase::COMMON_MODULE, name, value, desc) - -#define DE_ERRORNO_DATASET(name, value, desc) \ - DE_ERRORNO(DataEngineBase::RT_HOST, DataEngineBase::ERROR_CODE, DataEngineBase::COMMON_LEVEL, \ - DataEngineBase::SYSID_MD, DataEngineBase::DATAGET_MODULE, name, value, desc) - -#define DE_ERRORNO_MINDRECORD(name, value, desc) \ - DE_ERRORNO(DataEngineBase::RT_HOST, DataEngineBase::ERROR_CODE, DataEngineBase::COMMON_LEVEL, \ - DataEngineBase::SYSID_MD, DataEngineBase::MINDRECORD_MODULE, name, value, desc) - -// get error code description -#define DE_GET_ERRORNO_STR(value) DataEngineBase::StatusFactory::Instance()->GetErrDesc(value) - -class StatusFactory { - public: - static StatusFactory *Instance() { - static StatusFactory instance; - return &instance; - } - - void RegisterErrorNo(uint32_t err, const std::string &desc) { - if (err_desc_.find(err) != err_desc_.end()) return; - err_desc_[err] = desc; - } - - std::string GetErrDesc(uint32_t err) { - auto iter_find = err_desc_.find(err); - if (iter_find == err_desc_.end()) return ""; - return iter_find->second; - } - - protected: - StatusFactory() = default; - - ~StatusFactory() = default; - - private: - std::map err_desc_; -}; - -class ErrorNoRegisterar { - public: - ErrorNoRegisterar(uint32_t err, const std::string &desc) { StatusFactory::Instance()->RegisterErrorNo(err, desc); } - - ~ErrorNoRegisterar() = default; -}; - -using Status = uint32_t; -} // namespace DataEngineBase -#endif // MINDSPORE_CCSRC_UTILS_LOG_DE_ERROR_CODE_H_ diff --git a/mindspore/mindrecord/filewriter.py b/mindspore/mindrecord/filewriter.py index 62bcc2df79..f8b5448c23 100644 --- a/mindspore/mindrecord/filewriter.py +++ b/mindspore/mindrecord/filewriter.py @@ -240,7 +240,9 @@ class FileWriter: def set_header_size(self, header_size): """ - Set the size of header. + Set the size of header which contains shard information, schema information, \ + page meta information, etc. The larger the header, the more training data \ + a single mindrecord file can store. Args: header_size (int): Size of header, between 16KB and 128MB. @@ -256,7 +258,9 @@ class FileWriter: def set_page_size(self, page_size): """ - Set the size of Page. + Set the size of page which mainly refers to the block to store training data, \ + and the training data will be split into raw page and blob page in mindrecord. \ + The larger the page, the more training data a single page can store. Args: page_size (int): Size of page, between 32KB and 256MB. diff --git a/tests/ut/cpp/dataset/arena_test.cc b/tests/ut/cpp/dataset/arena_test.cc index 0809c40ee3..e8698ad979 100644 --- a/tests/ut/cpp/dataset/arena_test.cc +++ b/tests/ut/cpp/dataset/arena_test.cc @@ -17,7 +17,6 @@ #include #include "dataset/util/arena.h" #include "common/common.h" -#include "dataset/util/de_error.h" #include "utils/log_adapter.h" using namespace mindspore::dataset; diff --git a/tests/ut/cpp/dataset/batch_op_test.cc b/tests/ut/cpp/dataset/batch_op_test.cc index 54972af378..a04da06e4e 100644 --- a/tests/ut/cpp/dataset/batch_op_test.cc +++ b/tests/ut/cpp/dataset/batch_op_test.cc @@ -21,7 +21,6 @@ #include "common/utils.h" #include "gtest/gtest.h" #include "dataset/core/global_context.h" -#include "dataset/util/de_error.h" #include "utils/log_adapter.h" #include "securec.h" #include "dataset/util/status.h" diff --git a/tests/ut/cpp/dataset/btree_test.cc b/tests/ut/cpp/dataset/btree_test.cc index 75d5133e58..67b6c4e6c7 100644 --- a/tests/ut/cpp/dataset/btree_test.cc +++ b/tests/ut/cpp/dataset/btree_test.cc @@ -21,7 +21,6 @@ #include "dataset/util/task_manager.h" #include "common/common.h" #include "gtest/gtest.h" -#include "dataset/util/de_error.h" #include "utils/log_adapter.h" using namespace mindspore::dataset; diff --git a/tests/ut/cpp/dataset/celeba_op_test.cc b/tests/ut/cpp/dataset/celeba_op_test.cc index 5fa50a85ff..a109739fda 100644 --- a/tests/ut/cpp/dataset/celeba_op_test.cc +++ b/tests/ut/cpp/dataset/celeba_op_test.cc @@ -23,7 +23,6 @@ #include "dataset/core/global_context.h" #include "dataset/engine/datasetops/source/celeba_op.h" #include "dataset/engine/datasetops/source/sampler/subset_random_sampler.h" -#include "dataset/util/de_error.h" #include "dataset/util/status.h" #include "gtest/gtest.h" #include "utils/log_adapter.h" diff --git a/tests/ut/cpp/dataset/cifar_op_test.cc b/tests/ut/cpp/dataset/cifar_op_test.cc index 2992bc91a8..b37b9acaee 100644 --- a/tests/ut/cpp/dataset/cifar_op_test.cc +++ b/tests/ut/cpp/dataset/cifar_op_test.cc @@ -26,7 +26,6 @@ #include "dataset/engine/datasetops/source/sampler/sampler.h" #include "dataset/engine/datasetops/source/sampler/random_sampler.h" #include "dataset/engine/datasetops/source/sampler/subset_random_sampler.h" -#include "dataset/util/de_error.h" #include "dataset/util/path.h" #include "dataset/util/status.h" #include "gtest/gtest.h" diff --git a/tests/ut/cpp/dataset/coco_op_test.cc b/tests/ut/cpp/dataset/coco_op_test.cc index b412b66887..bcb82f8ec1 100644 --- a/tests/ut/cpp/dataset/coco_op_test.cc +++ b/tests/ut/cpp/dataset/coco_op_test.cc @@ -30,7 +30,6 @@ #include "dataset/engine/datasetops/source/sampler/sequential_sampler.h" #include "dataset/engine/datasetops/source/sampler/subset_random_sampler.h" #include "dataset/engine/datasetops/source/sampler/weighted_random_sampler.h" -#include "dataset/util/de_error.h" #include "dataset/util/path.h" #include "dataset/util/status.h" #include "gtest/gtest.h" diff --git a/tests/ut/cpp/dataset/common/common.h b/tests/ut/cpp/dataset/common/common.h index 2bd5887dca..539978177a 100644 --- a/tests/ut/cpp/dataset/common/common.h +++ b/tests/ut/cpp/dataset/common/common.h @@ -16,7 +16,6 @@ #ifndef TESTS_DATASET_UT_CORE_COMMON_DE_UT_COMMON_H_ #define TESTS_DATASET_UT_CORE_COMMON_DE_UT_COMMON_H_ -#include "dataset/util/de_error.h" #include "gtest/gtest.h" #include "utils/log_adapter.h" diff --git a/tests/ut/cpp/dataset/connector_test.cc b/tests/ut/cpp/dataset/connector_test.cc index 6d9e538ab4..7ee36cc2c0 100644 --- a/tests/ut/cpp/dataset/connector_test.cc +++ b/tests/ut/cpp/dataset/connector_test.cc @@ -140,7 +140,7 @@ Status MindDataTestConnector::Run_test_0() { auto my_conn = std::make_shared>(1, // num of producers 1, // num of consumers 10); // capacity of each queue - DS_ASSERT(my_conn != nullptr); + MS_ASSERT(my_conn != nullptr); rc = my_conn->Register(tg_.get()); RETURN_IF_NOT_OK(rc); @@ -277,7 +277,7 @@ Status MindDataTestConnector::FirstWorkerPush( int start_in, int offset) { TaskManager::FindMe()->Post(); - DS_ASSERT(my_conn != nullptr); + MS_ASSERT(my_conn != nullptr); Status rc; for (int i = start_in; i < input_.size(); i += offset) { rc = my_conn->Push(tid, input_[i]); @@ -294,7 +294,7 @@ Status MindDataTestConnector::MidWorkerJob( int tid, std::shared_ptr > from_conn, std::shared_ptr > to_conn) { - DS_ASSERT((from_conn != nullptr) && (to_conn != nullptr)); + MS_ASSERT((from_conn != nullptr) && (to_conn != nullptr)); Status rc; TaskManager::FindMe()->Post(); while (1) { diff --git a/tests/ut/cpp/dataset/duplicate_op_test.cc b/tests/ut/cpp/dataset/duplicate_op_test.cc index 6c9c00a30e..b7ce32f655 100644 --- a/tests/ut/cpp/dataset/duplicate_op_test.cc +++ b/tests/ut/cpp/dataset/duplicate_op_test.cc @@ -17,7 +17,6 @@ #include "common/common.h" #include "gtest/gtest.h" #include "dataset/core/tensor.h" -#include "dataset/util/de_error.h" #include "dataset/kernels/data/duplicate_op.h" using namespace mindspore::dataset; diff --git a/tests/ut/cpp/dataset/execution_tree_test.cc b/tests/ut/cpp/dataset/execution_tree_test.cc index 03072c0477..529644331a 100644 --- a/tests/ut/cpp/dataset/execution_tree_test.cc +++ b/tests/ut/cpp/dataset/execution_tree_test.cc @@ -21,7 +21,6 @@ #include "dataset/engine/datasetops/source/tf_reader_op.h" #include "common/common.h" #include "gtest/gtest.h" -#include "dataset/util/de_error.h" #include "utils/log_adapter.h" using namespace mindspore::dataset; diff --git a/tests/ut/cpp/dataset/global_context_test.cc b/tests/ut/cpp/dataset/global_context_test.cc index 1bcd7d938e..bb75d941aa 100644 --- a/tests/ut/cpp/dataset/global_context_test.cc +++ b/tests/ut/cpp/dataset/global_context_test.cc @@ -15,7 +15,6 @@ */ #include "dataset/core/global_context.h" #include "common/common.h" -#include "dataset/util/de_error.h" #include "utils/log_adapter.h" using namespace mindspore::dataset; diff --git a/tests/ut/cpp/dataset/image_folder_op_test.cc b/tests/ut/cpp/dataset/image_folder_op_test.cc index d143a9e06f..576c5abbfc 100644 --- a/tests/ut/cpp/dataset/image_folder_op_test.cc +++ b/tests/ut/cpp/dataset/image_folder_op_test.cc @@ -29,7 +29,6 @@ #include "dataset/engine/datasetops/source/sampler/sequential_sampler.h" #include "dataset/engine/datasetops/source/sampler/subset_random_sampler.h" #include "dataset/engine/datasetops/source/sampler/weighted_random_sampler.h" -#include "dataset/util/de_error.h" #include "dataset/util/path.h" #include "dataset/util/status.h" #include "gtest/gtest.h" diff --git a/tests/ut/cpp/dataset/interrupt_test.cc b/tests/ut/cpp/dataset/interrupt_test.cc index bde9351ca6..7ab608b9ae 100644 --- a/tests/ut/cpp/dataset/interrupt_test.cc +++ b/tests/ut/cpp/dataset/interrupt_test.cc @@ -14,7 +14,6 @@ * limitations under the License. */ #include "common/common.h" -#include "dataset/util/de_error.h" #include "utils/log_adapter.h" #include "dataset/util/services.h" #include "dataset/util/intrp_service.h" diff --git a/tests/ut/cpp/dataset/manifest_op_test.cc b/tests/ut/cpp/dataset/manifest_op_test.cc index 35773f6bbb..6317a6a345 100644 --- a/tests/ut/cpp/dataset/manifest_op_test.cc +++ b/tests/ut/cpp/dataset/manifest_op_test.cc @@ -25,7 +25,6 @@ #include "dataset/engine/datasetops/source/manifest_op.h" #include "dataset/engine/datasetops/source/sampler/sequential_sampler.h" #include "dataset/engine/datasetops/source/sampler/subset_random_sampler.h" -#include "dataset/util/de_error.h" #include "dataset/util/status.h" #include "gtest/gtest.h" #include "utils/log_adapter.h" diff --git a/tests/ut/cpp/dataset/mask_test.cc b/tests/ut/cpp/dataset/mask_test.cc index eb8a49aa36..9ff5f51fce 100644 --- a/tests/ut/cpp/dataset/mask_test.cc +++ b/tests/ut/cpp/dataset/mask_test.cc @@ -22,7 +22,6 @@ #include "dataset/core/tensor.h" #include "dataset/core/cv_tensor.h" #include "dataset/core/data_type.h" -#include "dataset/util/de_error.h" #include "dataset/kernels/data/mask_op.h" #include "dataset/kernels/data/data_utils.h" diff --git a/tests/ut/cpp/dataset/mnist_op_test.cc b/tests/ut/cpp/dataset/mnist_op_test.cc index 26b7335ad3..da78cb6f7f 100644 --- a/tests/ut/cpp/dataset/mnist_op_test.cc +++ b/tests/ut/cpp/dataset/mnist_op_test.cc @@ -30,7 +30,6 @@ #include "dataset/engine/datasetops/source/sampler/sequential_sampler.h" #include "dataset/engine/datasetops/source/sampler/subset_random_sampler.h" #include "dataset/engine/datasetops/source/sampler/weighted_random_sampler.h" -#include "dataset/util/de_error.h" #include "dataset/util/path.h" #include "dataset/util/status.h" #include "gtest/gtest.h" diff --git a/tests/ut/cpp/dataset/path_test.cc b/tests/ut/cpp/dataset/path_test.cc index 3839d659aa..4cf3b17968 100644 --- a/tests/ut/cpp/dataset/path_test.cc +++ b/tests/ut/cpp/dataset/path_test.cc @@ -16,7 +16,6 @@ #include "dataset/util/path.h" #include "common/common.h" #include "gtest/gtest.h" -#include "dataset/util/de_error.h" #include "utils/log_adapter.h" #include diff --git a/tests/ut/cpp/dataset/stand_alone_samplers_test.cc b/tests/ut/cpp/dataset/stand_alone_samplers_test.cc index ebfba3dde6..dfe15a8f15 100644 --- a/tests/ut/cpp/dataset/stand_alone_samplers_test.cc +++ b/tests/ut/cpp/dataset/stand_alone_samplers_test.cc @@ -21,7 +21,6 @@ #include "dataset/engine/datasetops/source/sampler/random_sampler.h" #include "dataset/engine/datasetops/source/sampler/sampler.h" #include "dataset/engine/datasetops/source/sampler/sequential_sampler.h" -#include "dataset/util/de_error.h" #include "dataset/util/status.h" #include "gtest/gtest.h" #include "utils/log_adapter.h" diff --git a/tests/ut/cpp/dataset/status_test.cc b/tests/ut/cpp/dataset/status_test.cc index 30bbd1e909..c64a86b8ba 100644 --- a/tests/ut/cpp/dataset/status_test.cc +++ b/tests/ut/cpp/dataset/status_test.cc @@ -16,7 +16,6 @@ #include "dataset/util/status.h" #include "common/common.h" #include "gtest/gtest.h" -#include "dataset/util/de_error.h" #include "utils/log_adapter.h" using namespace mindspore::dataset; diff --git a/tests/ut/cpp/dataset/tensor_string_test.cc b/tests/ut/cpp/dataset/tensor_string_test.cc index 2480809681..43b235304d 100644 --- a/tests/ut/cpp/dataset/tensor_string_test.cc +++ b/tests/ut/cpp/dataset/tensor_string_test.cc @@ -22,7 +22,6 @@ #include "dataset/core/tensor.h" #include "dataset/core/cv_tensor.h" #include "dataset/core/data_type.h" -#include "dataset/util/de_error.h" using namespace mindspore::dataset; diff --git a/tests/ut/cpp/dataset/tensor_test.cc b/tests/ut/cpp/dataset/tensor_test.cc index 1e75880a35..1aa3cad2fa 100644 --- a/tests/ut/cpp/dataset/tensor_test.cc +++ b/tests/ut/cpp/dataset/tensor_test.cc @@ -22,7 +22,6 @@ #include "dataset/core/tensor.h" #include "dataset/core/cv_tensor.h" #include "dataset/core/data_type.h" -#include "dataset/util/de_error.h" using namespace mindspore::dataset; diff --git a/tests/ut/cpp/dataset/treap_test.cc b/tests/ut/cpp/dataset/treap_test.cc index c92bea6bf8..b454ab108e 100644 --- a/tests/ut/cpp/dataset/treap_test.cc +++ b/tests/ut/cpp/dataset/treap_test.cc @@ -16,7 +16,6 @@ #include "dataset/util/treap.h" #include "common/common.h" #include "gtest/gtest.h" -#include "dataset/util/de_error.h" #include "utils/log_adapter.h" using namespace mindspore::dataset; diff --git a/tests/ut/cpp/dataset/type_cast_op_test.cc b/tests/ut/cpp/dataset/type_cast_op_test.cc index f5fa035579..543eb71637 100644 --- a/tests/ut/cpp/dataset/type_cast_op_test.cc +++ b/tests/ut/cpp/dataset/type_cast_op_test.cc @@ -25,7 +25,6 @@ #include "dataset/core/pybind_support.h" #include "gtest/gtest.h" #include "securec.h" -#include "dataset/util/de_error.h" #define MAX_INT_PRECISION 16777216 // float int precision is 16777216 using namespace mindspore::dataset; diff --git a/tests/ut/cpp/dataset/voc_op_test.cc b/tests/ut/cpp/dataset/voc_op_test.cc index be00e17b0e..05dc28b487 100644 --- a/tests/ut/cpp/dataset/voc_op_test.cc +++ b/tests/ut/cpp/dataset/voc_op_test.cc @@ -30,7 +30,6 @@ #include "dataset/engine/datasetops/source/sampler/sequential_sampler.h" #include "dataset/engine/datasetops/source/sampler/subset_random_sampler.h" #include "dataset/engine/datasetops/source/sampler/weighted_random_sampler.h" -#include "dataset/util/de_error.h" #include "dataset/util/path.h" #include "dataset/util/status.h" #include "gtest/gtest.h" diff --git a/tests/ut/cpp/dataset/zip_op_test.cc b/tests/ut/cpp/dataset/zip_op_test.cc index f8f8fe89db..b387341398 100644 --- a/tests/ut/cpp/dataset/zip_op_test.cc +++ b/tests/ut/cpp/dataset/zip_op_test.cc @@ -32,7 +32,6 @@ #include "dataset/engine/data_buffer.h" #include "gtest/gtest.h" #include "dataset/core/global_context.h" -#include "dataset/util/de_error.h" #include "utils/log_adapter.h" namespace common = mindspore::common; From a8654ac3b5e6a0b1d0fbc420e705011d734e81e2 Mon Sep 17 00:00:00 2001 From: zhaozhenlong Date: Sun, 28 Jun 2020 17:38:20 +0800 Subject: [PATCH 138/254] add ScatterMax ScatterMin ScatterSub for vm --- mindspore/ops/_op_impl/tbe/__init__.py | 3 + mindspore/ops/_op_impl/tbe/scatter_max.py | 40 +++++++ mindspore/ops/_op_impl/tbe/scatter_min.py | 40 +++++++ mindspore/ops/_op_impl/tbe/scatter_sub.py | 42 ++++++++ mindspore/ops/operations/__init__.py | 4 +- mindspore/ops/operations/array_ops.py | 89 ++++++++++++++++ tests/ut/python/ops/test_ops.py | 124 ++++++++++++++++++++-- 7 files changed, 335 insertions(+), 7 deletions(-) create mode 100644 mindspore/ops/_op_impl/tbe/scatter_max.py create mode 100644 mindspore/ops/_op_impl/tbe/scatter_min.py create mode 100644 mindspore/ops/_op_impl/tbe/scatter_sub.py diff --git a/mindspore/ops/_op_impl/tbe/__init__.py b/mindspore/ops/_op_impl/tbe/__init__.py index 7207e5ee69..d835d7a4d9 100644 --- a/mindspore/ops/_op_impl/tbe/__init__.py +++ b/mindspore/ops/_op_impl/tbe/__init__.py @@ -269,3 +269,6 @@ from .matrix_diag_part import _matrix_diag_part_tbe from .matrix_set_diag import _matrix_set_diag_tbe from .lrn import _lrn_tbe from .lrn_grad import _lrn_grad_tbe +from .scatter_max import _scatter_max_tbe +from .scatter_min import _scatter_min_tbe +from .scatter_sub import _scatter_sub_tbe diff --git a/mindspore/ops/_op_impl/tbe/scatter_max.py b/mindspore/ops/_op_impl/tbe/scatter_max.py new file mode 100644 index 0000000000..ba85d63d35 --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/scatter_max.py @@ -0,0 +1,40 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""ScatterMax op""" +from mindspore.ops.op_info_register import op_info_register, TBERegOp, DataType + +scatter_max_op_info = TBERegOp("ScatterMax") \ + .fusion_type("ELEMWISE") \ + .async_flag(False) \ + .binfile_name("scatter_max.so") \ + .compute_cost(10) \ + .kernel_name("scatter_max") \ + .partial_flag(True) \ + .attr("use_locking", "optional", "bool", "all") \ + .input(0, "var", False, "required", "all") \ + .input(1, "indices", False, "required", "all") \ + .input(2, "updates", False, "required", "all") \ + .output(0, "var", False, "required", "all") \ + .dtype_format(DataType.F16_Default, DataType.I32_Default, DataType.F16_Default, DataType.F16_Default) \ + .dtype_format(DataType.F32_Default, DataType.I32_Default, DataType.F32_Default, DataType.F32_Default) \ + .dtype_format(DataType.I32_Default, DataType.I32_Default, DataType.I32_Default, DataType.I32_Default) \ + .get_op_info() + + +@op_info_register(scatter_max_op_info) +def _scatter_max_tbe(): + """ScatterMax TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/scatter_min.py b/mindspore/ops/_op_impl/tbe/scatter_min.py new file mode 100644 index 0000000000..a4ab87a0d7 --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/scatter_min.py @@ -0,0 +1,40 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""ScatterMin op""" +from mindspore.ops.op_info_register import op_info_register, TBERegOp, DataType + +scatter_min_op_info = TBERegOp("ScatterMin") \ + .fusion_type("ELEMWISE") \ + .async_flag(False) \ + .binfile_name("scatter_min.so") \ + .compute_cost(10) \ + .kernel_name("scatter_min") \ + .partial_flag(True) \ + .attr("use_locking", "optional", "bool", "all") \ + .input(0, "var", False, "required", "all") \ + .input(1, "indices", False, "required", "all") \ + .input(2, "updates", False, "required", "all") \ + .output(0, "var", False, "required", "all") \ + .dtype_format(DataType.F16_Default, DataType.I32_Default, DataType.F16_Default, DataType.F16_Default) \ + .dtype_format(DataType.F32_Default, DataType.I32_Default, DataType.F32_Default, DataType.F32_Default) \ + .dtype_format(DataType.I32_Default, DataType.I32_Default, DataType.I32_Default, DataType.I32_Default) \ + .get_op_info() + + +@op_info_register(scatter_min_op_info) +def _scatter_min_tbe(): + """ScatterMin TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/scatter_sub.py b/mindspore/ops/_op_impl/tbe/scatter_sub.py new file mode 100644 index 0000000000..2e5e01389c --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/scatter_sub.py @@ -0,0 +1,42 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""ScatterSub op""" +from mindspore.ops.op_info_register import op_info_register, TBERegOp, DataType + +scatter_sub_op_info = TBERegOp("ScatterSub") \ + .fusion_type("ELEMWISE") \ + .async_flag(False) \ + .binfile_name("scatter_sub.so") \ + .compute_cost(10) \ + .kernel_name("scatter_sub") \ + .partial_flag(True) \ + .attr("use_locking", "optional", "bool", "all") \ + .input(0, "var", False, "required", "all") \ + .input(1, "indices", False, "required", "all") \ + .input(2, "updates", False, "required", "all") \ + .output(0, "var", False, "required", "all") \ + .dtype_format(DataType.F16_Default, DataType.I32_Default, DataType.F16_Default, DataType.F16_Default) \ + .dtype_format(DataType.F32_Default, DataType.I32_Default, DataType.F32_Default, DataType.F32_Default) \ + .dtype_format(DataType.I32_Default, DataType.I32_Default, DataType.I32_Default, DataType.I32_Default) \ + .dtype_format(DataType.I8_Default, DataType.I32_Default, DataType.I8_Default, DataType.I8_Default) \ + .dtype_format(DataType.U8_Default, DataType.I32_Default, DataType.U8_Default, DataType.U8_Default) \ + .get_op_info() + + +@op_info_register(scatter_sub_op_info) +def _scatter_sub_tbe(): + """ScatterSub TBE register""" + return diff --git a/mindspore/ops/operations/__init__.py b/mindspore/ops/operations/__init__.py index 55db22ecb9..aa718b963a 100644 --- a/mindspore/ops/operations/__init__.py +++ b/mindspore/ops/operations/__init__.py @@ -25,7 +25,7 @@ from .array_ops import (Argmax, Argmin, Cast, Concat, Pack, Unpack, Fill, GatherNd, GatherV2, SparseGatherV2, InvertPermutation, IsInstance, IsSubClass, ArgMaxWithValue, OnesLike, ZerosLike, Rank, Reshape, ResizeNearestNeighbor, ArgMinWithValue, - SameTypeShape, ScatterAdd, ScatterMax, ScatterUpdate, + SameTypeShape, ScatterAdd, ScatterSub, ScatterMax, ScatterMin, ScatterUpdate, ScalarToArray, ScalarToTensor, ScatterNd, ScatterNdUpdate, Select, Shape, Size, Slice, Split, Squeeze, StridedSlice, Tile, TensorScatterUpdate, @@ -213,8 +213,10 @@ __all__ = [ 'BoundingBoxDecode', 'L2Normalize', 'ScatterAdd', + 'ScatterSub', 'ScatterNd', 'ScatterMax', + 'ScatterMin', 'ResizeNearestNeighbor', 'HistogramFixedWidth', 'Pad', diff --git a/mindspore/ops/operations/array_ops.py b/mindspore/ops/operations/array_ops.py index f59a3a37e5..59b3a470a6 100644 --- a/mindspore/ops/operations/array_ops.py +++ b/mindspore/ops/operations/array_ops.py @@ -2271,6 +2271,51 @@ class ScatterMax(PrimitiveWithInfer): return x_dtype +class ScatterMin(PrimitiveWithInfer): + """ + Update the value of the input tensor through the min operation. + + Using given values to update tensor value through the min operation, along with the input indices. + This operation outputs the `input_x` after the update is done, which makes it convenient to use the updated value. + + Args: + use_locking (bool): Whether protect the assignment by a lock. Default: False. + + Inputs: + - **input_x** (Parameter) - The target parameter. + - **indices** (Tensor) - The index to do min operation whose data type should be mindspore.int32. + - **updates** (Tensor) - The tensor doing the min operation with `input_x`, + the data type is same as `input_x`, the shape is `indices_shape + x_shape[1:]`. + + Outputs: + Parameter, the updated `input_x`. + + Examples: + >>> input_x = Parameter(Tensor(np.array([[0.0, 1.0, 2.0], [0.0, 0.0, 0.0]]), mindspore.float32), name="input_x") + >>> indices = Tensor(np.array([[0, 0], [1, 1]]), mindspore.int32) + >>> update = Tensor(np.ones([2, 2, 3]), mindspore.float32) + >>> scatter_min = P.ScatterMin() + >>> output = scatter_min(input_x, indices, update) + [[0.0, 1.0, 1.0], [0.0, 0.0, 0.0]] + """ + + @prim_attr_register + def __init__(self, use_locking=False): + """Init ScatterMin""" + self.init_prim_io_names(inputs=['x', 'indices', 'updates'], outputs=['y']) + validator.check_value_type('use_locking', use_locking, (bool,), self.name) + + def infer_shape(self, x_shape, indices_shape, updates_shape): + _check_scatter_shape(x_shape, indices_shape, updates_shape, self.name) + return x_shape + + def infer_dtype(self, x_dtype, indices_dtype, updates_dtype): + validator.check_tensor_type_same({'indices': indices_dtype}, (mstype.int32,), self.name) + args = {"x": x_dtype, "updates": updates_dtype} + validator.check_tensor_type_same(args, mstype.number_type, self.name) + return x_dtype + + class ScatterAdd(PrimitiveWithInfer): """ Update the value of the input tensor through the add operation. @@ -2315,6 +2360,50 @@ class ScatterAdd(PrimitiveWithInfer): return x_dtype +class ScatterSub(PrimitiveWithInfer): + """ + Update the value of the input tensor through the sub operation. + + Using given values to update tensor value through the sub operation, along with the input indices. + This operation outputs the `input_x` after the update is done, which makes it convenient to use the updated value. + + Args: + use_locking (bool): Whether protect the assignment by a lock. Default: False. + + Inputs: + - **input_x** (Parameter) - The target parameter. + - **indices** (Tensor) - The index to do sub operation whose data type should be mindspore.int32. + - **updates** (Tensor) - The tensor doing the sub operation with `input_x`, + the data type is same as `input_x`, the shape is `indices_shape + x_shape[1:]`. + + Outputs: + Parameter, the updated `input_x`. + + Examples: + >>> input_x = Parameter(Tensor(np.array([[0.0, 0.0, 0.0], [1.0, 1.0, 1.0]]), mindspore.float32), name="x") + >>> indices = Tensor(np.array([[0, 1]]), mindspore.int32) + >>> updates = Tensor(np.array([[1.0, 1.0, 1.0], [2.0, 2.0, 2.0]]), mindspore.float32) + >>> scatter_sub = P.ScatterSub() + >>> output = scatter_sub(input_x, indices, updates) + [[-1.0, -1.0, -1.0], [-1.0, -1.0, -1.0]] + """ + + @prim_attr_register + def __init__(self, use_locking=False): + """Init ScatterSub""" + validator.check_value_type('use_locking', use_locking, (bool,), self.name) + + def infer_shape(self, x_shape, indices_shape, updates_shape): + _check_scatter_shape(x_shape, indices_shape, updates_shape, self.name) + return x_shape + + def infer_dtype(self, x_dtype, indices_dtype, updates_dtype): + validator.check_tensor_type_same({'indices': indices_dtype}, (mstype.int32,), self.name) + args = {'x': x_dtype, 'updates': updates_dtype} + validator.check_tensor_type_same(args, mstype.number_type, self.name) + return x_dtype + + class SpaceToDepth(PrimitiveWithInfer): r""" Rearrange blocks of spatial data into depth. diff --git a/tests/ut/python/ops/test_ops.py b/tests/ut/python/ops/test_ops.py index def84be4b3..3806e75fda 100755 --- a/tests/ut/python/ops/test_ops.py +++ b/tests/ut/python/ops/test_ops.py @@ -207,22 +207,35 @@ class HistogramSummaryNet(nn.Cell): class ScatterMax(nn.Cell): """ScatterMax net definition""" - def __init__(self): + def __init__(self, dtype=np.float32, use_locking=False): super(ScatterMax, self).__init__() - self.scatter_max = P.ScatterMax() - self.ref = Parameter(Tensor(np.array([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]], np.float32)), name="ref") + self.scatter_max = P.ScatterMax(use_locking) + self.ref = Parameter(Tensor(np.array([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]], dtype)), name="ref") def construct(self, indices, updates): out = self.scatter_max(self.ref, indices, updates) return out +class ScatterMin(nn.Cell): + """ScatterMin net definition""" + + def __init__(self, dtype=np.float32, use_locking=False): + super(ScatterMin, self).__init__() + self.scatter_min = P.ScatterMin(use_locking) + self.ref = Parameter(Tensor(np.array([[-1.0, 2.0, 3.0], [-4.0, 1.0, 6.0]], dtype)), name="ref") + + def construct(self, indices, updates): + out = self.scatter_min(self.ref, indices, updates) + return out + + class ScatterAdd(nn.Cell): """ScatterAdd net definition""" - def __init__(self, ref_shape, dtype=np.float32): + def __init__(self, ref_shape, dtype=np.float32, use_locking=False): super(ScatterAdd, self).__init__() - self.scatter_add = P.ScatterAdd() + self.scatter_add = P.ScatterAdd(use_locking) self.ref = Parameter(Tensor(np.ones(ref_shape, dtype)), name="ref") def construct(self, indices, updates): @@ -230,6 +243,19 @@ class ScatterAdd(nn.Cell): return out +class ScatterSub(nn.Cell): + """ScatterSub net definition""" + + def __init__(self, ref_shape, dtype=np.float32, use_locking=False): + super(ScatterSub, self).__init__() + self.scatter_sub = P.ScatterSub(use_locking) + self.ref = Parameter(Tensor(np.ones(ref_shape, dtype)), name="ref") + + def construct(self, indices, updates): + out = self.scatter_sub(self.ref, indices, updates) + return out + + class ApplyFtrlNet(nn.Cell): def __init__(self): super(ApplyFtrlNet, self).__init__() @@ -1667,11 +1693,61 @@ test_case_other_ops = [ Tensor(np.array([[0, 1], [1, 2]], np.int32)), Tensor(np.ones([2, 5], np.float32) * 99)), 'desc_bprop': [([3, 4, 5], {'dtype': np.float32})]}), - ('ScatterMax', { + ('ScatterMaxUseLocking', { + 'block': ScatterMax(use_locking=True), + 'desc_inputs': (Tensor(np.array([1, 0], np.int32)), + Tensor(np.array([[5.0, 5.0, 5.0], [4.0, 4.0, 4.0]], np.float32))), + 'skip': ['backward']}), + ('ScatterMax1d', { + 'block': ScatterMax(), + 'desc_inputs': (Tensor(np.array([1, 0], np.int32)), + Tensor(np.array([[5.0, 5.0, 5.0], [4.0, 4.0, 4.0]], np.float32))), + 'skip': ['backward']}), + ('ScatterMaxF32', { 'block': ScatterMax(), 'desc_inputs': (Tensor(np.array([[0, 0], [1, 1]], np.int32)), Tensor(np.ones([2, 2, 3], np.float32) * 99)), 'skip': ['backward']}), + ('ScatterMaxF16', { + 'block': ScatterMax(np.float16), + 'desc_inputs': (Tensor(np.array([[0, 0], [1, 1]], np.int32)), + Tensor(np.ones([2, 2, 3], np.float16) * 99)), + 'skip': ['backward']}), + ('ScatterMaxI32', { + 'block': ScatterMax(np.int32), + 'desc_inputs': (Tensor(np.array([[0, 0], [1, 1]], np.int32)), + Tensor(np.ones([2, 2, 3], np.int32) * 99)), + 'skip': ['backward']}), + ('ScatterMinUseLocking', { + 'block': ScatterMin(use_locking=True), + 'desc_inputs': (Tensor(np.array([1, 0], np.int32)), + Tensor(np.ones([2, 3], np.float32))), + 'skip': ['backward']}), + ('ScatterMin1d', { + 'block': ScatterMin(), + 'desc_inputs': (Tensor(np.array([1, 0], np.int32)), + Tensor(np.ones([2, 3], np.float32))), + 'skip': ['backward']}), + ('ScatterMinF32', { + 'block': ScatterMin(), + 'desc_inputs': (Tensor(np.array([[0, 0], [1, 1]], np.int32)), + Tensor(np.ones([2, 2, 3], np.float32))), + 'skip': ['backward']}), + ('ScatterMinF16', { + 'block': ScatterMin(np.float16), + 'desc_inputs': (Tensor(np.array([[0, 0], [1, 1]], np.int32)), + Tensor(np.ones([2, 2, 3], np.float16))), + 'skip': ['backward']}), + ('ScatterMinI32', { + 'block': ScatterMin(np.int32), + 'desc_inputs': (Tensor(np.array([[0, 0], [1, 1]], np.int32)), + Tensor(np.ones([2, 2, 3], np.int32))), + 'skip': ['backward']}), + ('ScatterAddUseLocking', { + 'block': ScatterAdd((6,), use_locking=True), + 'desc_inputs': (Tensor(np.array([2, 0, 5], np.int32)), + Tensor(np.array([2.0, 3.0, 4.0], np.float32))), + 'skip': ['backward']}), ('ScatterAdd', { 'block': ScatterAdd((6,)), 'desc_inputs': (Tensor(np.array([2, 0, 5], np.int32)), @@ -1708,6 +1784,42 @@ test_case_other_ops = [ 'desc_inputs': (Tensor(np.array([2, 0, 5], np.int32)), Tensor(np.array([2, 3, 4], np.uint8))), 'skip': ['backward']}), + ('ScatterSubUseLocking', { + 'block': ScatterSub((6,), use_locking=True), + 'desc_inputs': (Tensor(np.array([2], np.int32)), + Tensor(np.array([2.0], np.float32))), + 'skip': ['backward']}), + ('ScatterSubScalar', { + 'block': ScatterSub((6,)), + 'desc_inputs': (Tensor(np.array([2], np.int32)), + Tensor(np.array([2.0], np.float32))), + 'skip': ['backward']}), + ('ScatterSub2d', { + 'block': ScatterSub((3, 4)), + 'desc_inputs': (Tensor(np.array([[0, 1], [1, 2]], np.int32)), + Tensor(np.array([[[1, 1, 1, 1], [2, 2, 2, 2]], + [[3, 3, 3, 3], [4, 4, 4, 4]]], np.float32))), + 'skip': ['backward']}), + ('ScatterSubF16', { + 'block': ScatterSub((6,), np.float16), + 'desc_inputs': (Tensor(np.array([2, 0, 5], np.int32)), + Tensor(np.array([2.0, 3.0, 4.0], np.float16))), + 'skip': ['backward']}), + ('ScatterSubI32', { + 'block': ScatterSub((6,), np.int32), + 'desc_inputs': (Tensor(np.array([2, 0, 5], np.int32)), + Tensor(np.array([2, 3, 4], np.int32))), + 'skip': ['backward']}), + ('ScatterSubI8', { + 'block': ScatterSub((6,), np.int8), + 'desc_inputs': (Tensor(np.array([2, 0, 5], np.int32)), + Tensor(np.array([2, 3, 4], np.int8))), + 'skip': ['backward']}), + ('ScatterSubU8', { + 'block': ScatterSub((6,), np.uint8), + 'desc_inputs': (Tensor(np.array([2, 0, 5], np.int32)), + Tensor(np.array([1, 1, 0], np.uint8))), + 'skip': ['backward']}), ('SmoothL1Loss', { 'block': P.SmoothL1Loss(), 'desc_inputs': [[256, 4], [256, 4]], From 71f8ecbcdb8a55adb6313cd5633cd0f351d7c009 Mon Sep 17 00:00:00 2001 From: huangdongrun Date: Sat, 20 Jun 2020 15:52:33 +0800 Subject: [PATCH 139/254] add testcase pytest mark --- tests/st/pynative/test_tensor_index.py | 91 +++++++++++++++++++++++++- 1 file changed, 90 insertions(+), 1 deletion(-) diff --git a/tests/st/pynative/test_tensor_index.py b/tests/st/pynative/test_tensor_index.py index 3d1c7acdbf..774e892d2b 100644 --- a/tests/st/pynative/test_tensor_index.py +++ b/tests/st/pynative/test_tensor_index.py @@ -44,6 +44,10 @@ class NetWorkSlicePositive(Cell): return ret0, ret1, ret2, ret3 +@pytest.mark.level0 +@pytest.mark.platform_arm_ascend_training +@pytest.mark.platform_x86_ascend_training +@pytest.mark.env_onecard def test_slice_positive(): net = NetWorkSlicePositive() input_np = np.arange(6*8*10).reshape(6, 8, 10).astype(np.int32) @@ -143,7 +147,12 @@ class TensorGetItemByThreeTensors(Cell): return ret0, ret1, ret2 -def test_getitem_by_tensors(): +@pytest.mark.level0 +@pytest.mark.platform_arm_ascend_training +@pytest.mark.platform_x86_ascend_training +@pytest.mark.env_onecard +def Xtest_getitem_by_tensors(): + """This testcase may encounter a sync stream error occassionally""" net = TensorGetItemByThreeTensors() input_x = np.arange(6*8*10).reshape(6, 8, 10).astype(np.int32) index_0 = np.random.randint(6, size=(3, 4, 5)).astype(np.int32) @@ -179,6 +188,10 @@ class TensorGetItemByMixedTensorsBasicCase(Cell): return ret0, ret1, ret2, ret3, ret4, ret5 +@pytest.mark.level0 +@pytest.mark.platform_arm_ascend_training +@pytest.mark.platform_x86_ascend_training +@pytest.mark.env_onecard def test_getitem_by_mixed_tensors(): const0 = np.ones((3, 4, 5, 3), np.float32) const1 = np.ones((3, 3, 4, 5, 5), np.float32) @@ -217,6 +230,10 @@ class TensorSetItemByMixedTensors_0(Cell): return ret +@pytest.mark.level0 +@pytest.mark.platform_arm_ascend_training +@pytest.mark.platform_x86_ascend_training +@pytest.mark.env_onecard def test_setitem_by_mixed_tensors_0(): value = 88.0 net = TensorSetItemByMixedTensors_0(value) @@ -247,6 +264,10 @@ class TensorSetItemByMixedTensors_1(Cell): return ret +@pytest.mark.level0 +@pytest.mark.platform_arm_ascend_training +@pytest.mark.platform_x86_ascend_training +@pytest.mark.env_onecard def test_setitem_by_mixed_tensors_1(): value = 88.0 net = TensorSetItemByMixedTensors_1(value) @@ -277,6 +298,10 @@ class TensorSetItemByMixedTensors_2(Cell): return ret +@pytest.mark.level0 +@pytest.mark.platform_arm_ascend_training +@pytest.mark.platform_x86_ascend_training +@pytest.mark.env_onecard def test_setitem_by_mixed_tensors_2(): value = 88.0 net = TensorSetItemByMixedTensors_2(value) @@ -324,6 +349,10 @@ class TensorSetItemByOneTensorWithNumber(Cell): return ret +@pytest.mark.level0 +@pytest.mark.platform_arm_ascend_training +@pytest.mark.platform_x86_ascend_training +@pytest.mark.env_onecard def test_setitem_one_tensor_with_number(): value = 0.0 net = TensorSetItemByOneTensorWithNumber(value) @@ -348,6 +377,10 @@ class TensorSetItemByOneTensorWithTensor(Cell): return ret +@pytest.mark.level0 +@pytest.mark.platform_arm_ascend_training +@pytest.mark.platform_x86_ascend_training +@pytest.mark.env_onecard def test_setitem_by_one_tensor_with_tensor(): net = TensorSetItemByOneTensorWithTensor() index_np = np.random.randint(4, size=(5, 4)) @@ -374,6 +407,10 @@ class TensorSetItemByOneTensorWithTupleOfNumber(Cell): return ret +@pytest.mark.level0 +@pytest.mark.platform_arm_ascend_training +@pytest.mark.platform_x86_ascend_training +@pytest.mark.env_onecard def test_setitem_by_one_tensor_with_tuple_number(): value = (0.0, 1.1, 2.2, 3.3, 4.4, 5.5, 6.6, 7.7) net = TensorSetItemByOneTensorWithTupleOfNumber(value) @@ -398,6 +435,10 @@ class TensorSetItemByOneTensorWithTupleOfTensor(Cell): return ret +@pytest.mark.level0 +@pytest.mark.platform_arm_ascend_training +@pytest.mark.platform_x86_ascend_training +@pytest.mark.env_onecard def test_setitem_by_one_tensor_with_tuple_tensors(): net = TensorSetItemByOneTensorWithTupleOfTensor() input_np = np.random.randint(6, size=(5, 4)).astype(np.int32) @@ -428,6 +469,10 @@ class TensorSetItemByTensorsWithNumber(Cell): return ret +@pytest.mark.level0 +@pytest.mark.platform_arm_ascend_training +@pytest.mark.platform_x86_ascend_training +@pytest.mark.env_onecard def test_setitem_by_tensors_with_number(): value = 0.0 net = TensorSetItemByTensorsWithNumber(value) @@ -456,6 +501,10 @@ class TensorSetItemByTensorsWithTensor(Cell): return ret +@pytest.mark.level0 +@pytest.mark.platform_arm_ascend_training +@pytest.mark.platform_x86_ascend_training +@pytest.mark.env_onecard def test_setitem_by_tensors_with_tensor(): net = TensorSetItemByTensorsWithTensor() index_0 = np.random.randint(6, size=(3, 4, 5)) @@ -485,6 +534,10 @@ class TensorSetItemByTensorsWithTensorNumberError(Cell): return ret +@pytest.mark.level0 +@pytest.mark.platform_arm_ascend_training +@pytest.mark.platform_x86_ascend_training +@pytest.mark.env_onecard def test_setitem_by_tensors_with_tensor_error(): index_0 = Tensor(np.random.randint(6, size=(3, 4, 5)), mstype.int32) index_1 = Tensor(np.random.randint(7, size=(4, 5)), mstype.int32) @@ -509,6 +562,10 @@ class TensorSetItemByTensorsWithTupleOfNumber(Cell): return ret +@pytest.mark.level0 +@pytest.mark.platform_arm_ascend_training +@pytest.mark.platform_x86_ascend_training +@pytest.mark.env_onecard def test_setitem_by_tensors_with_tuple_of_number(): value = (0.0, 1.1, 2.2, 3.3, 4.4) net = TensorSetItemByTensorsWithTupleOfNumber(value) @@ -537,6 +594,10 @@ class TensorSetItemByTensorsWithTupleOfTensor(Cell): return ret +@pytest.mark.level0 +@pytest.mark.platform_arm_ascend_training +@pytest.mark.platform_x86_ascend_training +@pytest.mark.env_onecard def test_setitem_by_tensors_with_tuple_of_tensor(): value_0 = np.zeros((4, 5)) value_1 = np.ones((4, 5)) @@ -570,6 +631,10 @@ class TensorSetItemByTensorsWithTupleOfTensorNumberError(Cell): return ret +@pytest.mark.level0 +@pytest.mark.platform_arm_ascend_training +@pytest.mark.platform_x86_ascend_training +@pytest.mark.env_onecard def test_setitem_by_tensor_with_tuple_of_tensor_error(): net = TensorSetItemByTensorsWithTupleOfTensorNumberError() index_0_ms = Tensor(np.random.randint(6, size=(3, 4, 5)), mstype.int32) @@ -661,6 +726,10 @@ class TensorAssignWithSlice(Cell): return z +@pytest.mark.level0 +@pytest.mark.platform_arm_ascend_training +@pytest.mark.platform_x86_ascend_training +@pytest.mark.env_onecard def test_tensor_assign_slice_value_1(): net = TensorAssignWithSlice() a = np.arange(60).reshape(3, 4, 5) @@ -682,6 +751,10 @@ def test_tensor_assign_slice_value_1(): assert np.all(z == out.asnumpy()) +@pytest.mark.level0 +@pytest.mark.platform_arm_ascend_training +@pytest.mark.platform_x86_ascend_training +@pytest.mark.env_onecard def test_tensor_assign_slice_value_2(): net2 = TensorAssignWithSlice2() a = np.array([1, 2, 3, 4, 5, 6, 7, 8]) @@ -701,6 +774,10 @@ def test_tensor_assign_slice_value_2(): assert np.all(z == out.asnumpy()) +@pytest.mark.level0 +@pytest.mark.platform_arm_ascend_training +@pytest.mark.platform_x86_ascend_training +@pytest.mark.env_onecard def test_tensor_assign_exception(): net = TensorAssignWithSlice() net2 = TensorAssignWithSlice2() @@ -886,6 +963,10 @@ class TensorAssignWithBoolTensorIndex2Error(Cell): return a +@pytest.mark.level0 +@pytest.mark.platform_arm_ascend_training +@pytest.mark.platform_x86_ascend_training +@pytest.mark.env_onecard def test_tensor_assign_bool_index_0(): a = np.arange(60).reshape(3, 4, 5) b = a > 5 @@ -903,6 +984,10 @@ def test_tensor_assign_bool_index_0(): assert np.all(out.asnumpy() == res) +@pytest.mark.level0 +@pytest.mark.platform_arm_ascend_training +@pytest.mark.platform_x86_ascend_training +@pytest.mark.env_onecard def test_tensor_assign_bool_index_1(): a = np.arange(60).reshape(3, 4, 5) Ta = Tensor(a, dtype=mstype.float32) @@ -992,6 +1077,10 @@ def Xtest_tensor_slice_reduce_out_of_bounds_positive(): assert "For 'StridedSlice' the `begin[0]` should be an int and must less than 6, but got `6`" in str(ex.value) +@pytest.mark.level0 +@pytest.mark.platform_arm_ascend_training +@pytest.mark.platform_x86_ascend_training +@pytest.mark.env_onecard def test_tensor_range(): a = np.arange(4*5*6).reshape(4, 5, 6).astype(np.float32) ta = Tensor(a, mstype.float32) From a3e8c7438ff9d752a148a5ad3fe8557e20c2703e Mon Sep 17 00:00:00 2001 From: lizhenyu Date: Mon, 29 Jun 2020 17:25:23 +0800 Subject: [PATCH 140/254] remove useless code --- mindspore/ccsrc/device/gpu/gpu_device_address.cc | 3 --- 1 file changed, 3 deletions(-) diff --git a/mindspore/ccsrc/device/gpu/gpu_device_address.cc b/mindspore/ccsrc/device/gpu/gpu_device_address.cc index 24097f3637..401eb9f34e 100644 --- a/mindspore/ccsrc/device/gpu/gpu_device_address.cc +++ b/mindspore/ccsrc/device/gpu/gpu_device_address.cc @@ -18,7 +18,6 @@ #include #include "device/gpu/gpu_device_manager.h" #include "utils/log_adapter.h" -#include "utils/context/ms_context.h" #include "device/gpu/gpu_memory_allocator.h" namespace mindspore { @@ -55,8 +54,6 @@ GPUDeviceAddress::~GPUDeviceAddress() { if (ptr_ == nullptr) { return; } - auto ms_context = MsContext::GetInstance(); - MS_EXCEPTION_IF_NULL(ms_context); if (from_mem_pool_) { GPUMemoryAllocator::GetInstance().FreeTensorMem(ptr_); ptr_ = nullptr; From 136b456967dcf7765c3c21bbffd82f8d5ad13afb Mon Sep 17 00:00:00 2001 From: VectorSL Date: Mon, 29 Jun 2020 11:33:17 +0800 Subject: [PATCH 141/254] gpu dropout rewrite --- .../kernel/gpu/cuda_impl/dropout_impl.cu | 52 +++++++++--- .../kernel/gpu/cuda_impl/dropout_impl.cuh | 7 +- .../ccsrc/kernel/gpu/nn/dropout_gpu_kernel.cc | 84 ++----------------- .../ccsrc/kernel/gpu/nn/dropout_gpu_kernel.h | 79 +++++++++++++---- .../kernel/gpu/nn/dropout_grad_kernel.cc | 76 ++--------------- .../ccsrc/kernel/gpu/nn/dropout_grad_kernel.h | 70 ++++++++++++---- mindspore/ops/operations/nn_ops.py | 3 + 7 files changed, 184 insertions(+), 187 deletions(-) diff --git a/mindspore/ccsrc/kernel/gpu/cuda_impl/dropout_impl.cu b/mindspore/ccsrc/kernel/gpu/cuda_impl/dropout_impl.cu index 019d71d740..d65f319ab7 100644 --- a/mindspore/ccsrc/kernel/gpu/cuda_impl/dropout_impl.cu +++ b/mindspore/ccsrc/kernel/gpu/cuda_impl/dropout_impl.cu @@ -17,31 +17,59 @@ #include #include "dropout_impl.cuh" #include "include/cuda_runtime.h" - -__global__ void DropoutForwardKernel(const float *input, float *mask, float *output, size_t num_count, +template +__global__ void DropoutForwardKernel(const T *input, T *mask, T *output, float *mask_f, size_t num_count, float keep_prob) { float scale = 1.f / keep_prob; for (size_t i = blockIdx.x * blockDim.x + threadIdx.x; i < num_count; i += blockDim.x * gridDim.x) { - mask[i] = mask[i] <= keep_prob; - output[i] = scale * input[i] * mask[i]; + mask_f[i] = mask_f[i] <= keep_prob; + output[i] = scale * input[i] * mask_f[i]; + mask[i] = mask_f[i]; } } - -void DropoutForward(const float *input, float *mask, float *output, size_t num_count, float drop_prob, +template <> +__global__ void DropoutForwardKernel(const half *input, half *mask, half *output, float *mask_f, + size_t num_count, float keep_prob) { + half scale = __float2half(1.f / keep_prob); + for (size_t i = blockIdx.x * blockDim.x + threadIdx.x; i < num_count; i += blockDim.x * gridDim.x) { + mask_f[i] = mask_f[i] <= keep_prob; + output[i] = scale * input[i] * __float2half(mask_f[i]); + mask[i] = __float2half(mask_f[i]); + } +} +template +void DropoutForward(const T *input, T *mask, T *output, float *mask_f, size_t num_count, float drop_prob, cudaStream_t cuda_stream) { - DropoutForwardKernel<<>>(input, mask, output, num_count, - drop_prob); + DropoutForwardKernel<<>>(input, mask, output, mask_f, + num_count, drop_prob); } - -__global__ void DropoutBackwardKernel(const float *dy, const float *mask, float *dx, size_t num_count, +template +__global__ void DropoutBackwardKernel(const T *dy, const T *mask, T *dx, size_t num_count, float keep_prob) { float scale = 1.f / keep_prob; for (size_t i = blockIdx.x * blockDim.x + threadIdx.x; i < num_count; i += blockDim.x * gridDim.x) { dx[i] = scale * dy[i] * mask[i]; } } - -void DropoutBackward(const float *dy, const float *mask, float *dx, size_t num_count, float drop_prob, +template <> +__global__ void DropoutBackwardKernel(const half *dy, const half *mask, half *dx, size_t num_count, + float keep_prob) { + half scale = __float2half(1.f / keep_prob); + for (size_t i = blockIdx.x * blockDim.x + threadIdx.x; i < num_count; i += blockDim.x * gridDim.x) { + dx[i] = scale * dy[i] * mask[i]; + } +} +template +void DropoutBackward(const T *dy, const T *mask, T *dx, size_t num_count, float drop_prob, cudaStream_t cuda_stream) { DropoutBackwardKernel<<>>(dy, mask, dx, num_count, drop_prob); } + +template void DropoutForward(const float *input, float *mask, float *output, float *mask_f, + size_t num_count, float drop_prob, cudaStream_t cuda_stream); +template void DropoutForward(const half *input, half *mask, half *output, float *mask_f, + size_t num_count, float drop_prob, cudaStream_t cuda_stream); +template void DropoutBackward(const float *dy, const float *mask, float *dx, size_t num_count, + float drop_prob, cudaStream_t cuda_stream); +template void DropoutBackward(const half *dy, const half *mask, half *dx, size_t num_count, + float drop_prob, cudaStream_t cuda_stream); diff --git a/mindspore/ccsrc/kernel/gpu/cuda_impl/dropout_impl.cuh b/mindspore/ccsrc/kernel/gpu/cuda_impl/dropout_impl.cuh index bd3de6524d..f89d42ce49 100644 --- a/mindspore/ccsrc/kernel/gpu/cuda_impl/dropout_impl.cuh +++ b/mindspore/ccsrc/kernel/gpu/cuda_impl/dropout_impl.cuh @@ -18,9 +18,10 @@ #define MINDSPORE_CCSRC_KERNEL_GPU_CUDA_IMPL_DROPOUT_H_ #include "device/gpu/cuda_common.h" -void DropoutForward(const float *input, float *mask, float *output, size_t num_count, float keep_prob, +template +void DropoutForward(const T *input, T *mask, T *output, float *mask_f, size_t num_count, float keep_prob, cudaStream_t cuda_stream); -void DropoutBackward(const float *dy, const float *mask, float *dx, size_t num_count, float keep_prob, - cudaStream_t cuda_stream); +template +void DropoutBackward(const T *dy, const T *mask, T *dx, size_t num_count, float keep_prob, cudaStream_t cuda_stream); #endif // MINDSPORE_CCSRC_KERNEL_GPU_CUDA_IMPL_DROPOUT_H_ diff --git a/mindspore/ccsrc/kernel/gpu/nn/dropout_gpu_kernel.cc b/mindspore/ccsrc/kernel/gpu/nn/dropout_gpu_kernel.cc index b84dc628e0..459010e9e9 100644 --- a/mindspore/ccsrc/kernel/gpu/nn/dropout_gpu_kernel.cc +++ b/mindspore/ccsrc/kernel/gpu/nn/dropout_gpu_kernel.cc @@ -15,84 +15,16 @@ */ #include "kernel/gpu/nn/dropout_gpu_kernel.h" -#include "kernel/gpu/cuda_impl/dropout_impl.cuh" namespace mindspore { namespace kernel { -DropoutGpuFwdKernel::DropoutGpuFwdKernel() - : cudnn_handle_(nullptr), - is_null_input_(false), - num_count_(0), - keep_prob_(0.0), - states_init_(false), - mask_generator_(nullptr) {} - -DropoutGpuFwdKernel::~DropoutGpuFwdKernel() { DestroyResource(); } - -const std::vector &DropoutGpuFwdKernel::GetInputSizeList() const { return input_size_list_; } - -const std::vector &DropoutGpuFwdKernel::GetOutputSizeList() const { return output_size_list_; } - -const std::vector &DropoutGpuFwdKernel::GetWorkspaceSizeList() const { return workspace_size_list_; } - -bool DropoutGpuFwdKernel::Init(const CNodePtr &kernel_node) { - InitResource(); - - size_t input_num = AnfAlgo::GetInputTensorNum(kernel_node); - if (input_num != 1) { - MS_LOG(EXCEPTION) << "Argument number is " << input_num << ", but DropoutGpuFwdKernel needs 1."; - } - - auto input_shape = AnfAlgo::GetPrevNodeOutputInferShape(kernel_node, 0); - is_null_input_ = CHECK_NULL_INPUT(input_shape); - if (is_null_input_) { - InitSizeLists(); - return true; - } - - num_count_ = 1; - for (size_t x : input_shape) { - num_count_ *= x; - } - keep_prob_ = GetValue(AnfAlgo::GetCNodePrimitive(kernel_node)->GetAttr("keep_prob")); - - InitSizeLists(); - return true; -} - -void DropoutGpuFwdKernel::InitResource() { - cudnn_handle_ = device::gpu::GPUDeviceManager::GetInstance().GetCudnnHandle(); -} - -void DropoutGpuFwdKernel::DestroyResource() noexcept {} - -void DropoutGpuFwdKernel::InitSizeLists() { - size_t input_size = num_count_ * sizeof(float); - input_size_list_.push_back(input_size); - output_size_list_.push_back(input_size); // output size: the same with input size - output_size_list_.push_back(input_size); // mask size: the same with input size -} - -bool DropoutGpuFwdKernel::Launch(const std::vector &inputs, const std::vector &, - const std::vector &outputs, void *stream_ptr) { - if (is_null_input_) { - return true; - } - - auto *input = reinterpret_cast(inputs[0]->addr); - auto *output = reinterpret_cast(outputs[0]->addr); - auto *mask = reinterpret_cast(outputs[1]->addr); - - if (!states_init_) { - curandCreateGenerator(&mask_generator_, CURAND_RNG_PSEUDO_DEFAULT); - curandSetPseudoRandomGeneratorSeed(mask_generator_, time(NULL)); - states_init_ = true; - } - - curandGenerateUniform(mask_generator_, mask, num_count_); - DropoutForward(input, mask, output, num_count_, keep_prob_, reinterpret_cast(stream_ptr)); - - return true; -} +MS_REG_GPU_KERNEL_ONE( + Dropout, + KernelAttr().AddInputAttr(kNumberTypeFloat32).AddOutputAttr(kNumberTypeFloat32).AddOutputAttr(kNumberTypeFloat32), + DropoutGpuFwdKernel, float) +MS_REG_GPU_KERNEL_ONE( + Dropout, + KernelAttr().AddInputAttr(kNumberTypeFloat16).AddOutputAttr(kNumberTypeFloat16).AddOutputAttr(kNumberTypeFloat16), + DropoutGpuFwdKernel, half) } // namespace kernel } // namespace mindspore diff --git a/mindspore/ccsrc/kernel/gpu/nn/dropout_gpu_kernel.h b/mindspore/ccsrc/kernel/gpu/nn/dropout_gpu_kernel.h index 81eb78c880..4dfacb7ca1 100644 --- a/mindspore/ccsrc/kernel/gpu/nn/dropout_gpu_kernel.h +++ b/mindspore/ccsrc/kernel/gpu/nn/dropout_gpu_kernel.h @@ -20,35 +20,88 @@ #include #include "kernel/gpu/gpu_kernel.h" #include "kernel/gpu/gpu_kernel_factory.h" +#include "kernel/gpu/cuda_impl/dropout_impl.cuh" #include "include/curand.h" namespace mindspore { namespace kernel { +template class DropoutGpuFwdKernel : public GpuKernel { public: - DropoutGpuFwdKernel(); + DropoutGpuFwdKernel() + : cudnn_handle_(nullptr), + is_null_input_(false), + num_count_(0), + keep_prob_(0.0), + states_init_(false), + mask_generator_(nullptr) {} - ~DropoutGpuFwdKernel() override; + ~DropoutGpuFwdKernel() override = default; - const std::vector &GetInputSizeList() const override; + const std::vector &GetInputSizeList() const override { return input_size_list_; } + const std::vector &GetOutputSizeList() const override { return output_size_list_; } + const std::vector &GetWorkspaceSizeList() const override { return workspace_size_list_; } - const std::vector &GetOutputSizeList() const override; + bool Launch(const std::vector &inputs, const std::vector &workspace, + const std::vector &outputs, void *stream_ptr) override { + if (is_null_input_) { + return true; + } - const std::vector &GetWorkspaceSizeList() const override; + T *input = GetDeviceAddress(inputs, 0); + T *output = GetDeviceAddress(outputs, 0); + T *mask = GetDeviceAddress(outputs, 1); + float *mask_f = GetDeviceAddress(workspace, 0); - bool Launch(const std::vector &inputs, const std::vector &workspace, - const std::vector &outputs, void *stream_ptr) override; + if (!states_init_) { + curandCreateGenerator(&mask_generator_, CURAND_RNG_PSEUDO_DEFAULT); + curandSetPseudoRandomGeneratorSeed(mask_generator_, time(NULL)); + states_init_ = true; + } + // curandGen only support float or double for mask. + curandGenerateUniform(mask_generator_, mask_f, num_count_); + DropoutForward(input, mask, output, mask_f, num_count_, keep_prob_, reinterpret_cast(stream_ptr)); + + return true; + } + + bool Init(const CNodePtr &kernel_node) override { + InitResource(); + + size_t input_num = AnfAlgo::GetInputTensorNum(kernel_node); + if (input_num != 1) { + MS_LOG(EXCEPTION) << "Argument number is " << input_num << ", but DropoutGpuFwdKernel needs 1."; + } - bool Init(const CNodePtr &kernel_node) override; + auto input_shape = AnfAlgo::GetPrevNodeOutputInferShape(kernel_node, 0); + is_null_input_ = CHECK_NULL_INPUT(input_shape); + if (is_null_input_) { + InitSizeLists(); + return true; + } + + num_count_ = 1; + for (size_t x : input_shape) { + num_count_ *= x; + } + keep_prob_ = GetAttr(kernel_node, "keep_prob"); + + InitSizeLists(); + return true; + } protected: - void InitResource() override; + void InitResource() override { cudnn_handle_ = device::gpu::GPUDeviceManager::GetInstance().GetCudnnHandle(); } - void InitSizeLists() override; + void InitSizeLists() override { + size_t input_size = num_count_ * sizeof(T); + input_size_list_.push_back(input_size); + output_size_list_.push_back(input_size); // output size: the same with input size + output_size_list_.push_back(input_size); // mask size: the same with input size + workspace_size_list_.push_back(num_count_ * sizeof(float)); // temp mask_f for curandGen + } private: - void DestroyResource() noexcept; - cudnnHandle_t cudnn_handle_; bool is_null_input_; size_t num_count_; @@ -59,8 +112,6 @@ class DropoutGpuFwdKernel : public GpuKernel { std::vector output_size_list_; std::vector workspace_size_list_; }; - -MS_REG_GPU_KERNEL(Dropout, DropoutGpuFwdKernel) } // namespace kernel } // namespace mindspore diff --git a/mindspore/ccsrc/kernel/gpu/nn/dropout_grad_kernel.cc b/mindspore/ccsrc/kernel/gpu/nn/dropout_grad_kernel.cc index 2194805e92..2fd21c96ee 100644 --- a/mindspore/ccsrc/kernel/gpu/nn/dropout_grad_kernel.cc +++ b/mindspore/ccsrc/kernel/gpu/nn/dropout_grad_kernel.cc @@ -15,76 +15,16 @@ */ #include "kernel/gpu/nn/dropout_grad_kernel.h" -#include "kernel/gpu/cuda_impl/dropout_impl.cuh" namespace mindspore { namespace kernel { -DropoutGradGpuFwdKernel::DropoutGradGpuFwdKernel() - : cudnn_handle_(nullptr), is_null_input_(false), num_count_(0), keep_prob_(0.0) {} - -DropoutGradGpuFwdKernel::~DropoutGradGpuFwdKernel() { DestroyResource(); } - -const std::vector &DropoutGradGpuFwdKernel::GetInputSizeList() const { return input_size_list_; } - -const std::vector &DropoutGradGpuFwdKernel::GetOutputSizeList() const { return output_size_list_; } - -const std::vector &DropoutGradGpuFwdKernel::GetWorkspaceSizeList() const { return workspace_size_list_; } - -bool DropoutGradGpuFwdKernel::Init(const CNodePtr &kernel_node) { - InitResource(); - - size_t input_num = AnfAlgo::GetInputTensorNum(kernel_node); - if (input_num != 2) { - MS_LOG(ERROR) << "Argument number is " << input_num << ", but DropoutGradGpuFwdKernel needs 2."; - return false; - } - - auto input_shape = AnfAlgo::GetOutputInferShape(kernel_node, 0); - is_null_input_ = CHECK_NULL_INPUT(input_shape); - if (is_null_input_) { - InitSizeLists(); - return true; - } - - num_count_ = 1; - for (size_t x : input_shape) { - num_count_ *= x; - } - keep_prob_ = GetValue(AnfAlgo::GetCNodePrimitive(kernel_node)->GetAttr("keep_prob")); - - InitSizeLists(); - return true; -} - -void DropoutGradGpuFwdKernel::InitResource() { - cudnn_handle_ = device::gpu::GPUDeviceManager::GetInstance().GetCudnnHandle(); -} - -void DropoutGradGpuFwdKernel::DestroyResource() noexcept {} - -void DropoutGradGpuFwdKernel::InitSizeLists() { - size_t dy_size = num_count_ * sizeof(float); - size_t mask_size = dy_size; - size_t dx_size = dy_size; - - input_size_list_.push_back(dy_size); - input_size_list_.push_back(mask_size); - output_size_list_.push_back(dx_size); -} - -bool DropoutGradGpuFwdKernel::Launch(const std::vector &inputs, const std::vector &, - const std::vector &outputs, void *stream_ptr) { - if (is_null_input_) { - return true; - } - - auto *dy = reinterpret_cast(inputs[0]->addr); - auto *mask = reinterpret_cast(inputs[1]->addr); - auto *dx = reinterpret_cast(outputs[0]->addr); - - DropoutBackward(dy, mask, dx, num_count_, keep_prob_, reinterpret_cast(stream_ptr)); - - return true; -} +MS_REG_GPU_KERNEL_ONE( + DropoutGrad, + KernelAttr().AddInputAttr(kNumberTypeFloat32).AddInputAttr(kNumberTypeFloat32).AddOutputAttr(kNumberTypeFloat32), + DropoutGradGpuBwdKernel, float) +MS_REG_GPU_KERNEL_ONE( + DropoutGrad, + KernelAttr().AddInputAttr(kNumberTypeFloat16).AddInputAttr(kNumberTypeFloat16).AddOutputAttr(kNumberTypeFloat16), + DropoutGradGpuBwdKernel, half) } // namespace kernel } // namespace mindspore diff --git a/mindspore/ccsrc/kernel/gpu/nn/dropout_grad_kernel.h b/mindspore/ccsrc/kernel/gpu/nn/dropout_grad_kernel.h index 4991b9dad5..e6683e15dd 100644 --- a/mindspore/ccsrc/kernel/gpu/nn/dropout_grad_kernel.h +++ b/mindspore/ccsrc/kernel/gpu/nn/dropout_grad_kernel.h @@ -20,28 +20,72 @@ #include #include "kernel/gpu/gpu_kernel.h" #include "kernel/gpu/gpu_kernel_factory.h" +#include "kernel/gpu/cuda_impl/dropout_impl.cuh" namespace mindspore { namespace kernel { -class DropoutGradGpuFwdKernel : public GpuKernel { +template +class DropoutGradGpuBwdKernel : public GpuKernel { public: - DropoutGradGpuFwdKernel(); - ~DropoutGradGpuFwdKernel() override; + DropoutGradGpuBwdKernel() : cudnn_handle_(nullptr), is_null_input_(false), num_count_(0), keep_prob_(0.0) {} + ~DropoutGradGpuBwdKernel() override = default; - const std::vector &GetInputSizeList() const override; - const std::vector &GetOutputSizeList() const override; - const std::vector &GetWorkspaceSizeList() const override; + const std::vector &GetInputSizeList() const override { return input_size_list_; } + const std::vector &GetOutputSizeList() const override { return output_size_list_; } + const std::vector &GetWorkspaceSizeList() const override { return workspace_size_list_; } bool Launch(const std::vector &inputs, const std::vector &workspace, - const std::vector &outputs, void *stream_ptr) override; - bool Init(const CNodePtr &kernel_node) override; + const std::vector &outputs, void *stream_ptr) override { + if (is_null_input_) { + return true; + } + + T *dy = GetDeviceAddress(inputs, 0); + T *mask = GetDeviceAddress(inputs, 1); + T *dx = GetDeviceAddress(outputs, 0); + + DropoutBackward(dy, mask, dx, num_count_, keep_prob_, reinterpret_cast(stream_ptr)); + + return true; + } + bool Init(const CNodePtr &kernel_node) override { + InitResource(); + + size_t input_num = AnfAlgo::GetInputTensorNum(kernel_node); + if (input_num != 2) { + MS_LOG(ERROR) << "Argument number is " << input_num << ", but DropoutGradGpuBwdKernel needs 2."; + return false; + } + + auto input_shape = AnfAlgo::GetOutputInferShape(kernel_node, 0); + is_null_input_ = CHECK_NULL_INPUT(input_shape); + if (is_null_input_) { + InitSizeLists(); + return true; + } + + num_count_ = 1; + for (size_t x : input_shape) { + num_count_ *= x; + } + keep_prob_ = GetAttr(kernel_node, "keep_prob"); + + InitSizeLists(); + return true; + } protected: - void InitResource() override; - void InitSizeLists() override; + void InitResource() override { cudnn_handle_ = device::gpu::GPUDeviceManager::GetInstance().GetCudnnHandle(); } + void InitSizeLists() override { + size_t dy_size = num_count_ * sizeof(T); + size_t mask_size = dy_size; + size_t dx_size = dy_size; - private: - void DestroyResource() noexcept; + input_size_list_.push_back(dy_size); + input_size_list_.push_back(mask_size); + output_size_list_.push_back(dx_size); + } + private: cudnnHandle_t cudnn_handle_; bool is_null_input_; size_t num_count_; @@ -50,8 +94,6 @@ class DropoutGradGpuFwdKernel : public GpuKernel { std::vector output_size_list_; std::vector workspace_size_list_; }; - -MS_REG_GPU_KERNEL(DropoutGrad, DropoutGradGpuFwdKernel) } // namespace kernel } // namespace mindspore diff --git a/mindspore/ops/operations/nn_ops.py b/mindspore/ops/operations/nn_ops.py index 24b081bdad..10c61285d1 100644 --- a/mindspore/ops/operations/nn_ops.py +++ b/mindspore/ops/operations/nn_ops.py @@ -4117,6 +4117,7 @@ class Dropout(PrimitiveWithInfer): def infer_dtype(self, x_dtype): valid_types = (mstype.float16, mstype.float32) + validator.check_subclass("x", x_dtype, mstype.tensor, self.name) validator.check_tensor_type_same({"x_dtype": x_dtype}, valid_types, self.name) return x_dtype, x_dtype @@ -4151,6 +4152,8 @@ class DropoutGrad(PrimitiveWithInfer): def infer_dtype(self, dy_dtype, mask_dtype): valid_types = (mstype.float16, mstype.float32) + validator.check_subclass("dy", dy_dtype, mstype.tensor, self.name) + validator.check_subclass("mask", mask_dtype, mstype.tensor, self.name) validator.check_tensor_type_same({"dy_dtype": dy_dtype}, valid_types, self.name) return dy_dtype From dd5fba1db93b1348da78184466cedddc0ff6be7f Mon Sep 17 00:00:00 2001 From: jinyaohui Date: Mon, 29 Jun 2020 17:39:25 +0800 Subject: [PATCH 142/254] add notice --- .../python/optimizer/test_while_ScatterNdUpdate.py | 14 ++++++++++++++ .../test_auto_parallel_double_subgraphs.py | 14 ++++++++++++++ .../parallel/test_auto_parallel_inference.py | 14 ++++++++++++++ tests/ut/python/pynative_mode/test_hook.py | 14 ++++++++++++++ tests/ut/python/train/quant/mobilenetv2.py | 14 ++++++++++++++ .../ut/python/train/quant/mobilenetv2_combined.py | 14 ++++++++++++++ 6 files changed, 84 insertions(+) diff --git a/tests/ut/python/optimizer/test_while_ScatterNdUpdate.py b/tests/ut/python/optimizer/test_while_ScatterNdUpdate.py index a21955b2b6..4bd21197ac 100644 --- a/tests/ut/python/optimizer/test_while_ScatterNdUpdate.py +++ b/tests/ut/python/optimizer/test_while_ScatterNdUpdate.py @@ -1,3 +1,17 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ import numpy as np from mindspore import context, nn, Tensor, Parameter from mindspore.common import dtype as mstype diff --git a/tests/ut/python/parallel/test_auto_parallel_double_subgraphs.py b/tests/ut/python/parallel/test_auto_parallel_double_subgraphs.py index e121cecf8e..5dd259ec84 100644 --- a/tests/ut/python/parallel/test_auto_parallel_double_subgraphs.py +++ b/tests/ut/python/parallel/test_auto_parallel_double_subgraphs.py @@ -1,3 +1,17 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ import numpy as np import mindspore as ms diff --git a/tests/ut/python/parallel/test_auto_parallel_inference.py b/tests/ut/python/parallel/test_auto_parallel_inference.py index 5181be717d..30c72d6130 100644 --- a/tests/ut/python/parallel/test_auto_parallel_inference.py +++ b/tests/ut/python/parallel/test_auto_parallel_inference.py @@ -1,3 +1,17 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ import numpy as np import mindspore.nn as nn diff --git a/tests/ut/python/pynative_mode/test_hook.py b/tests/ut/python/pynative_mode/test_hook.py index 023f039a97..07a7a7ad8b 100644 --- a/tests/ut/python/pynative_mode/test_hook.py +++ b/tests/ut/python/pynative_mode/test_hook.py @@ -1,3 +1,17 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ import numpy as np import mindspore.nn as nn diff --git a/tests/ut/python/train/quant/mobilenetv2.py b/tests/ut/python/train/quant/mobilenetv2.py index e214953b97..163b230e1e 100644 --- a/tests/ut/python/train/quant/mobilenetv2.py +++ b/tests/ut/python/train/quant/mobilenetv2.py @@ -1,3 +1,17 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ """MobileNetV2""" from mindspore import nn from mindspore.ops import operations as P diff --git a/tests/ut/python/train/quant/mobilenetv2_combined.py b/tests/ut/python/train/quant/mobilenetv2_combined.py index 7ed1498fb6..b0cbafb29a 100644 --- a/tests/ut/python/train/quant/mobilenetv2_combined.py +++ b/tests/ut/python/train/quant/mobilenetv2_combined.py @@ -1,3 +1,17 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ """mobile net v2""" from mindspore import nn from mindspore.ops import operations as P From 94581f1c4376efc5b203a26529ff74bf84fc5378 Mon Sep 17 00:00:00 2001 From: qianlong Date: Mon, 29 Jun 2020 18:00:01 +0800 Subject: [PATCH 143/254] del JiebaMode and NormalizeForm from python api doc --- mindspore/dataset/text/__init__.py | 4 ++-- mindspore/dataset/text/transforms.py | 24 ++++++++++++++++-------- mindspore/dataset/text/utils.py | 3 +++ 3 files changed, 21 insertions(+), 10 deletions(-) diff --git a/mindspore/dataset/text/__init__.py b/mindspore/dataset/text/__init__.py index 7c43a2888c..04eb90a0b6 100644 --- a/mindspore/dataset/text/__init__.py +++ b/mindspore/dataset/text/__init__.py @@ -24,7 +24,7 @@ from .utils import to_str, to_bytes, JiebaMode, Vocab, NormalizeForm __all__ = [ "Lookup", "JiebaTokenizer", "UnicodeCharTokenizer", "Ngram", - "to_str", "to_bytes", "JiebaMode", "Vocab", "WordpieceTokenizer", "TruncateSequencePair", "ToNumber", + "to_str", "to_bytes", "Vocab", "WordpieceTokenizer", "TruncateSequencePair", "ToNumber", "PythonTokenizer" ] @@ -33,4 +33,4 @@ if platform.system().lower() != 'windows': RegexReplace, RegexTokenizer, BasicTokenizer, BertTokenizer, PythonTokenizer __all__.append(["UnicodeScriptTokenizer", "WhitespaceTokenizer", "CaseFold", "NormalizeUTF8", - "RegexReplace", "RegexTokenizer", "BasicTokenizer", "BertTokenizer", "NormalizeForm"]) + "RegexReplace", "RegexTokenizer", "BasicTokenizer", "BertTokenizer"]) diff --git a/mindspore/dataset/text/transforms.py b/mindspore/dataset/text/transforms.py index c50b8e4f75..8b0d47df25 100644 --- a/mindspore/dataset/text/transforms.py +++ b/mindspore/dataset/text/transforms.py @@ -119,10 +119,12 @@ class JiebaTokenizer(cde.JiebaTokenizerOp): the dictionary can be obtained on the official website of cppjieba. mp_path (str): the dictionary file is used by MPSegment algorithm, the dictionary can be obtained on the official website of cppjieba. - mode (JiebaMode, optional): "MP" model will tokenize with MPSegment algorithm, - "HMM" mode will tokenize with Hiddel Markov Model Segment algorithm, - "MIX" model will tokenize with a mix of MPSegment and HMMSegment algorithm - (default="MIX"). + mode (JiebaMode, optional): Valid values can be any of [JiebaMode.MP, JiebaMode.HMM, + JiebaMode.MIX](default=JiebaMode.MIX). + + - JiebaMode.MP, tokenize with MPSegment algorithm. + - JiebaMode.HMM, tokenize with Hiddel Markov Model Segment algorithm. + - JiebaMode.MIX, tokenize with a mix of MPSegment and HMMSegment algorithm. """ @check_jieba_init @@ -287,10 +289,16 @@ if platform.system().lower() != 'windows': Apply normalize operation on utf-8 string tensor. Args: - normalize_form (NormalizeForm, optional): Valid values are "NONE", "NFC", "NFKC", "NFD", "NFKD". - If set "NONE", will do nothing for input string tensor. - If set to any of "NFC", "NFKC", "NFD", "NFKD", will apply normalize operation(default="NFKC"). - See http://unicode.org/reports/tr15/ for details. + normalize_form (NormalizeForm, optional): Valid values can be any of [NormalizeForm.NONE, + NormalizeForm.NFC, NormalizeForm.NFKC, NormalizeForm.NFD, + NormalizeForm.NFKD](default=NormalizeForm.NFKC). + And you can see http://unicode.org/reports/tr15/ for details. + + - NormalizeForm.NONE, do nothing for input string tensor. + - NormalizeForm.NFC, normalize with Normalization Form C. + - NormalizeForm.NFKC, normalize with Normalization Form KC. + - NormalizeForm.NFD, normalize with Normalization Form D. + - NormalizeForm.NFKD, normalize with Normalization Form KD. """ def __init__(self, normalize_form=NormalizeForm.NFKC): diff --git a/mindspore/dataset/text/utils.py b/mindspore/dataset/text/utils.py index 766de76e01..7347a4b854 100644 --- a/mindspore/dataset/text/utils.py +++ b/mindspore/dataset/text/utils.py @@ -24,6 +24,9 @@ import mindspore._c_dataengine as cde from .validators import check_from_file, check_from_list, check_from_dict, check_from_dataset +__all__ = [ + "Vocab", "to_str", "to_bytes" +] class Vocab(cde.Vocab): """ From 7fa0d9e7e443aa342e42d4c8d44692d3df025e93 Mon Sep 17 00:00:00 2001 From: ms_yan <6576637+ms_yan@user.noreply.gitee.com> Date: Mon, 29 Jun 2020 17:58:51 +0800 Subject: [PATCH 144/254] add paramter check for numpyslices and num_shards --- mindspore/dataset/engine/datasets.py | 9 +++++++-- mindspore/dataset/engine/validators.py | 2 ++ tests/ut/python/dataset/test_minddataset_exception.py | 2 +- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/mindspore/dataset/engine/datasets.py b/mindspore/dataset/engine/datasets.py index 49cab9002f..b66fb70c1f 100644 --- a/mindspore/dataset/engine/datasets.py +++ b/mindspore/dataset/engine/datasets.py @@ -3069,7 +3069,7 @@ class GeneratorDataset(MappableDataset): sampler (Sampler/Iterable, optional): Object used to choose samples from the dataset. Random accessible input is required (default=None, expected order behavior shown in the table). num_shards (int, optional): Number of shards that the dataset should be divided into (default=None). - This argument should be specified only when 'num_samples' is "None". Random accessible input is required. + When this argument is specified, 'num_samples' will not effect. Random accessible input is required. shard_id (int, optional): The shard ID within num_shards (default=None). This argument should be specified only when num_shards is also specified. Random accessible input is required. @@ -4878,6 +4878,11 @@ class _NumpySlicesDataset: else: self.data = (np.array(data),) + # check whether the data length in each column is equal + data_len = [len(data_item) for data_item in self.data] + if data_len[1:] != data_len[:-1]: + raise ValueError("Data length in each column is not equal.") + # Init column_name if column_list is not None: self.column_list = column_list @@ -4966,7 +4971,7 @@ class NumpySlicesDataset(GeneratorDataset): sampler (Sampler/Iterable, optional): Object used to choose samples from the dataset. Random accessible input is required (default=None, expected order behavior shown in the table). num_shards (int, optional): Number of shards that the dataset should be divided into (default=None). - This argument should be specified only when 'num_samples' is "None". Random accessible input is required. + When this argument is specified, 'num_samples' will not effect. Random accessible input is required. shard_id (int, optional): The shard ID within num_shards (default=None). This argument should be specified only when num_shards is also specified. Random accessible input is required. diff --git a/mindspore/dataset/engine/validators.py b/mindspore/dataset/engine/validators.py index 76a5f1b819..9857608c19 100644 --- a/mindspore/dataset/engine/validators.py +++ b/mindspore/dataset/engine/validators.py @@ -153,6 +153,7 @@ def check_sampler_shuffle_shard_options(param_dict): raise RuntimeError("sampler and sharding cannot be specified at the same time.") if num_shards is not None: + check_positive_int32(num_shards, "num_shards") if shard_id is None: raise RuntimeError("num_shards is specified and currently requires shard_id as well.") if shard_id < 0 or shard_id >= num_shards: @@ -529,6 +530,7 @@ def check_generatordataset(method): # These two parameters appear together. raise ValueError("num_shards and shard_id need to be passed in together") if num_shards is not None: + check_positive_int32(num_shards, "num_shards") if shard_id >= num_shards: raise ValueError("shard_id should be less than num_shards") diff --git a/tests/ut/python/dataset/test_minddataset_exception.py b/tests/ut/python/dataset/test_minddataset_exception.py index 14cffc8fb2..b15944d76b 100644 --- a/tests/ut/python/dataset/test_minddataset_exception.py +++ b/tests/ut/python/dataset/test_minddataset_exception.py @@ -185,7 +185,7 @@ def test_minddataset_invalidate_num_shards(): columns_list = ["data", "label"] num_readers = 4 with pytest.raises(Exception, match="shard_id is invalid, "): - data_set = ds.MindDataset(CV_FILE_NAME, columns_list, num_readers, True, 0, 1) + data_set = ds.MindDataset(CV_FILE_NAME, columns_list, num_readers, True, 1, 2) num_iter = 0 for _ in data_set.create_dict_iterator(): num_iter += 1 From b1a1e24e0790d37319ebc6f4ca1243409dab938d Mon Sep 17 00:00:00 2001 From: zhouyaqiang Date: Sun, 28 Jun 2020 15:34:56 +0800 Subject: [PATCH 145/254] add resnext50 --- model_zoo/resnext50/README.md | 128 ++++++++ model_zoo/resnext50/eval.py | 243 +++++++++++++++ .../resnext50/scripts/run_distribute_train.sh | 55 ++++ model_zoo/resnext50/scripts/run_eval.sh | 24 ++ .../resnext50/scripts/run_standalone_train.sh | 30 ++ model_zoo/resnext50/src/__init__.py | 0 model_zoo/resnext50/src/backbone/__init__.py | 16 + model_zoo/resnext50/src/backbone/resnet.py | 273 +++++++++++++++++ model_zoo/resnext50/src/config.py | 45 +++ model_zoo/resnext50/src/crossentropy.py | 41 +++ model_zoo/resnext50/src/dataset.py | 155 ++++++++++ model_zoo/resnext50/src/head.py | 42 +++ .../resnext50/src/image_classification.py | 85 ++++++ model_zoo/resnext50/src/linear_warmup.py | 21 ++ model_zoo/resnext50/src/utils/__init__.py | 0 model_zoo/resnext50/src/utils/cunstom_op.py | 108 +++++++ model_zoo/resnext50/src/utils/logging.py | 82 +++++ .../resnext50/src/utils/optimizers__init__.py | 39 +++ model_zoo/resnext50/src/utils/sampler.py | 53 ++++ model_zoo/resnext50/src/utils/var_init.py | 213 +++++++++++++ .../src/warmup_cosine_annealing_lr.py | 40 +++ model_zoo/resnext50/src/warmup_step_lr.py | 56 ++++ model_zoo/resnext50/train.py | 289 ++++++++++++++++++ 23 files changed, 2038 insertions(+) create mode 100644 model_zoo/resnext50/README.md create mode 100644 model_zoo/resnext50/eval.py create mode 100644 model_zoo/resnext50/scripts/run_distribute_train.sh create mode 100644 model_zoo/resnext50/scripts/run_eval.sh create mode 100644 model_zoo/resnext50/scripts/run_standalone_train.sh create mode 100644 model_zoo/resnext50/src/__init__.py create mode 100644 model_zoo/resnext50/src/backbone/__init__.py create mode 100644 model_zoo/resnext50/src/backbone/resnet.py create mode 100644 model_zoo/resnext50/src/config.py create mode 100644 model_zoo/resnext50/src/crossentropy.py create mode 100644 model_zoo/resnext50/src/dataset.py create mode 100644 model_zoo/resnext50/src/head.py create mode 100644 model_zoo/resnext50/src/image_classification.py create mode 100644 model_zoo/resnext50/src/linear_warmup.py create mode 100644 model_zoo/resnext50/src/utils/__init__.py create mode 100644 model_zoo/resnext50/src/utils/cunstom_op.py create mode 100644 model_zoo/resnext50/src/utils/logging.py create mode 100644 model_zoo/resnext50/src/utils/optimizers__init__.py create mode 100644 model_zoo/resnext50/src/utils/sampler.py create mode 100644 model_zoo/resnext50/src/utils/var_init.py create mode 100644 model_zoo/resnext50/src/warmup_cosine_annealing_lr.py create mode 100644 model_zoo/resnext50/src/warmup_step_lr.py create mode 100644 model_zoo/resnext50/train.py diff --git a/model_zoo/resnext50/README.md b/model_zoo/resnext50/README.md new file mode 100644 index 0000000000..c44844eecc --- /dev/null +++ b/model_zoo/resnext50/README.md @@ -0,0 +1,128 @@ +# ResNext50 Example + +## Description + +This is an example of training ResNext50 with ImageNet dataset in Mindspore. + +## Requirements + +- Install [Mindspore](http://www.mindspore.cn/install/en). +- Downlaod the dataset ImageNet2012. + +## Structure + +```shell +. +└─resnext50 + ├─README.md + ├─scripts + ├─run_standalone_train.sh # launch standalone training(1p) + ├─run_distribute_train.sh # launch distributed training(8p) + └─run_eval.sh # launch evaluating + ├─src + ├─backbone + ├─_init_.py # initalize + ├─resnet.py # resnext50 backbone + ├─utils + ├─_init_.py # initalize + ├─cunstom_op.py # network operation + ├─logging.py # print log + ├─optimizers_init_.py # get parameters + ├─sampler.py # distributed sampler + ├─var_init_.py # calculate gain value + ├─_init_.py # initalize + ├─config.py # parameter configuration + ├─crossentropy.py # CrossEntropy loss function + ├─dataset.py # data preprocessing + ├─head.py # commom head + ├─image_classification.py # get resnet + ├─linear_warmup.py # linear warmup learning rate + ├─warmup_cosine_annealing.py # learning rate each step + ├─warmup_step_lr.py # warmup step learning rate + ├─eval.py # eval net + └─train.py # train net + +``` + +## Parameter Configuration + +Parameters for both training and evaluating can be set in config.py + +``` +"image_height": '224,224' # image size +"num_classes": 1000, # dataset class number +"per_batch_size": 128, # batch size of input tensor +"lr": 0.05, # base learning rate +"lr_scheduler": 'cosine_annealing', # learning rate mode +"lr_epochs": '30,60,90,120', # epoch of lr changing +"lr_gamma": 0.1, # decrease lr by a factor of exponential lr_scheduler +"eta_min": 0, # eta_min in cosine_annealing scheduler +"T_max": 150, # T-max in cosine_annealing scheduler +"max_epoch": 150, # max epoch num to train the model +"backbone": 'resnext50', # backbone metwork +"warmup_epochs" : 1, # warmup epoch +"weight_decay": 0.0001, # weight decay +"momentum": 0.9, # momentum +"is_dynamic_loss_scale": 0, # dynamic loss scale +"loss_scale": 1024, # loss scale +"label_smooth": 1, # label_smooth +"label_smooth_factor": 0.1, # label_smooth_factor +"ckpt_interval": 2000, # ckpt_interval +"ckpt_path": 'outputs/', # checkpoint save location +"is_save_on_master": 1, +"rank": 0, # local rank of distributed +"group_size": 1 # world size of distributed +``` + +## Running the example + +### Train + +#### Usage + +``` +# distribute training example(8p) +sh run_distribute_train.sh MINDSPORE_HCCL_CONFIG_PATH DATA_PATH +# standalone training +sh run_standalone_train.sh DEVICE_ID DATA_PATH +``` + +#### Launch + +```bash +# distributed training example(8p) +sh scripts/run_distribute_train.sh MINDSPORE_HCCL_CONFIG_PATH /ImageNet/train +# standalone training example +sh scripts/run_standalone_train.sh 0 /ImageNet_Original/train +``` + +#### Result + +You can find checkpoint file together with result in log. + +### Evaluation + +#### Usage + +``` +# Evaluation +sh run_eval.sh DEVICE_ID DATA_PATH PRETRAINED_CKPT_PATH +``` + +#### Launch + +```bash +# Evaluation with checkpoint +sh scripts/run_eval.sh 0 /opt/npu/datasets/classification/val /resnext50_100.ckpt +``` + +> checkpoint can be produced in training process. + +#### Result + +Evaluation result will be stored in the scripts path. Under this, you can find result like the followings in log. + +``` +acc=78,16%(TOP1) +acc=93.88%(TOP5) +``` \ No newline at end of file diff --git a/model_zoo/resnext50/eval.py b/model_zoo/resnext50/eval.py new file mode 100644 index 0000000000..ff5c83843e --- /dev/null +++ b/model_zoo/resnext50/eval.py @@ -0,0 +1,243 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Eval""" +import os +import time +import argparse +import datetime +import glob +import numpy as np +import mindspore.nn as nn + +from mindspore import Tensor, context +from mindspore.communication.management import init, get_rank, get_group_size, release +from mindspore.train.serialization import load_checkpoint, load_param_into_net +from mindspore.ops import operations as P +from mindspore.ops import functional as F +from mindspore.common import dtype as mstype + +from src.utils.logging import get_logger +from src.image_classification import get_network +from src.dataset import classification_dataset +from src.config import config + +devid = int(os.getenv('DEVICE_ID')) +context.set_context(mode=context.GRAPH_MODE, enable_auto_mixed_precision=True, + device_target="Ascend", save_graphs=False, device_id=devid) + + + +class ParameterReduce(nn.Cell): + """ParameterReduce""" + def __init__(self): + super(ParameterReduce, self).__init__() + self.cast = P.Cast() + self.reduce = P.AllReduce() + + def construct(self, x): + one = self.cast(F.scalar_to_array(1.0), mstype.float32) + out = x * one + ret = self.reduce(out) + return ret + + +def parse_args(cloud_args=None): + """parse_args""" + parser = argparse.ArgumentParser('mindspore classification test') + + # dataset related + parser.add_argument('--data_dir', type=str, default='/opt/npu/datasets/classification/val', help='eval data dir') + parser.add_argument('--per_batch_size', default=32, type=int, help='batch size for per npu') + # network related + parser.add_argument('--graph_ckpt', type=int, default=1, help='graph ckpt or feed ckpt') + parser.add_argument('--pretrained', default='', type=str, help='fully path of pretrained model to load. ' + 'If it is a direction, it will test all ckpt') + + # logging related + parser.add_argument('--log_path', type=str, default='outputs/', help='path to save log') + parser.add_argument('--is_distributed', type=int, default=0, help='if multi device') + + # roma obs + parser.add_argument('--train_url', type=str, default="", help='train url') + + args, _ = parser.parse_known_args() + args = merge_args(args, cloud_args) + args.image_size = config.image_size + args.num_classes = config.num_classes + args.backbone = config.backbone + args.rank = config.rank + args.group_size = config.group_size + + args.image_size = list(map(int, args.image_size.split(','))) + + return args + + +def get_top5_acc(top5_arg, gt_class): + sub_count = 0 + for top5, gt in zip(top5_arg, gt_class): + if gt in top5: + sub_count += 1 + return sub_count + +def merge_args(args, cloud_args): + """merge_args""" + args_dict = vars(args) + if isinstance(cloud_args, dict): + for key in cloud_args.keys(): + val = cloud_args[key] + if key in args_dict and val: + arg_type = type(args_dict[key]) + if arg_type is not type(None): + val = arg_type(val) + args_dict[key] = val + return args + +def test(cloud_args=None): + """test""" + args = parse_args(cloud_args) + + # init distributed + if args.is_distributed: + init() + args.rank = get_rank() + args.group_size = get_group_size() + + args.outputs_dir = os.path.join(args.log_path, + datetime.datetime.now().strftime('%Y-%m-%d_time_%H_%M_%S')) + + args.logger = get_logger(args.outputs_dir, args.rank) + args.logger.save_args(args) + + # network + args.logger.important_info('start create network') + if os.path.isdir(args.pretrained): + models = list(glob.glob(os.path.join(args.pretrained, '*.ckpt'))) + print(models) + if args.graph_ckpt: + f = lambda x: -1 * int(os.path.splitext(os.path.split(x)[-1])[0].split('-')[-1].split('_')[0]) + else: + f = lambda x: -1 * int(os.path.splitext(os.path.split(x)[-1])[0].split('_')[-1]) + args.models = sorted(models, key=f) + else: + args.models = [args.pretrained,] + + for model in args.models: + de_dataset = classification_dataset(args.data_dir, image_size=args.image_size, + per_batch_size=args.per_batch_size, + max_epoch=1, rank=args.rank, group_size=args.group_size, + mode='eval') + eval_dataloader = de_dataset.create_tuple_iterator() + network = get_network(args.backbone, args.num_classes) + if network is None: + raise NotImplementedError('not implement {}'.format(args.backbone)) + + param_dict = load_checkpoint(model) + param_dict_new = {} + for key, values in param_dict.items(): + if key.startswith('moments.'): + continue + elif key.startswith('network.'): + param_dict_new[key[8:]] = values + else: + param_dict_new[key] = values + + load_param_into_net(network, param_dict_new) + args.logger.info('load model {} success'.format(model)) + + # must add + network.add_flags_recursive(fp16=True) + + img_tot = 0 + top1_correct = 0 + top5_correct = 0 + network.set_train(False) + t_end = time.time() + it = 0 + for data, gt_classes in eval_dataloader: + output = network(Tensor(data, mstype.float32)) + output = output.asnumpy() + + top1_output = np.argmax(output, (-1)) + top5_output = np.argsort(output)[:, -5:] + + t1_correct = np.equal(top1_output, gt_classes).sum() + top1_correct += t1_correct + top5_correct += get_top5_acc(top5_output, gt_classes) + img_tot += args.per_batch_size + + if args.rank == 0 and it == 0: + t_end = time.time() + it = 1 + if args.rank == 0: + time_used = time.time() - t_end + fps = (img_tot - args.per_batch_size) * args.group_size / time_used + args.logger.info('Inference Performance: {:.2f} img/sec'.format(fps)) + results = [[top1_correct], [top5_correct], [img_tot]] + args.logger.info('before results={}'.format(results)) + if args.is_distributed: + model_md5 = model.replace('/', '') + tmp_dir = '/cache' + if not os.path.exists(tmp_dir): + os.mkdir(tmp_dir) + top1_correct_npy = '/cache/top1_rank_{}_{}.npy'.format(args.rank, model_md5) + top5_correct_npy = '/cache/top5_rank_{}_{}.npy'.format(args.rank, model_md5) + img_tot_npy = '/cache/img_tot_rank_{}_{}.npy'.format(args.rank, model_md5) + np.save(top1_correct_npy, top1_correct) + np.save(top5_correct_npy, top5_correct) + np.save(img_tot_npy, img_tot) + while True: + rank_ok = True + for other_rank in range(args.group_size): + top1_correct_npy = '/cache/top1_rank_{}_{}.npy'.format(other_rank, model_md5) + top5_correct_npy = '/cache/top5_rank_{}_{}.npy'.format(other_rank, model_md5) + img_tot_npy = '/cache/img_tot_rank_{}_{}.npy'.format(other_rank, model_md5) + if not os.path.exists(top1_correct_npy) or not os.path.exists(top5_correct_npy) or \ + not os.path.exists(img_tot_npy): + rank_ok = False + if rank_ok: + break + + top1_correct_all = 0 + top5_correct_all = 0 + img_tot_all = 0 + for other_rank in range(args.group_size): + top1_correct_npy = '/cache/top1_rank_{}_{}.npy'.format(other_rank, model_md5) + top5_correct_npy = '/cache/top5_rank_{}_{}.npy'.format(other_rank, model_md5) + img_tot_npy = '/cache/img_tot_rank_{}_{}.npy'.format(other_rank, model_md5) + top1_correct_all += np.load(top1_correct_npy) + top5_correct_all += np.load(top5_correct_npy) + img_tot_all += np.load(img_tot_npy) + results = [[top1_correct_all], [top5_correct_all], [img_tot_all]] + results = np.array(results) + else: + results = np.array(results) + + args.logger.info('after results={}'.format(results)) + top1_correct = results[0, 0] + top5_correct = results[1, 0] + img_tot = results[2, 0] + acc1 = 100.0 * top1_correct / img_tot + acc5 = 100.0 * top5_correct / img_tot + args.logger.info('after allreduce eval: top1_correct={}, tot={},' + 'acc={:.2f}%(TOP1)'.format(top1_correct, img_tot, acc1)) + args.logger.info('after allreduce eval: top5_correct={}, tot={},' + 'acc={:.2f}%(TOP5)'.format(top5_correct, img_tot, acc5)) + if args.is_distributed: + release() + + +if __name__ == "__main__": + test() diff --git a/model_zoo/resnext50/scripts/run_distribute_train.sh b/model_zoo/resnext50/scripts/run_distribute_train.sh new file mode 100644 index 0000000000..226cfe3cb6 --- /dev/null +++ b/model_zoo/resnext50/scripts/run_distribute_train.sh @@ -0,0 +1,55 @@ +#!/bin/bash +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +DATA_DIR=$2 +export RANK_TABLE_FILE=$1 +export RANK_SIZE=8 +PATH_CHECKPOINT="" +if [ $# == 3 ] +then + PATH_CHECKPOINT=$3 +fi + +cores=`cat /proc/cpuinfo|grep "processor" |wc -l` +echo "the number of logical core" $cores +avg_core_per_rank=`expr $cores \/ $RANK_SIZE` +core_gap=`expr $avg_core_per_rank \- 1` +echo "avg_core_per_rank" $avg_core_per_rank +echo "core_gap" $core_gap +for((i=0;i env.log + taskset -c $cmdopt python ../train.py \ + --is_distribute=1 \ + --device_id=$DEVICE_ID \ + --pretrained=$PATH_CHECKPOINT \ + --data_dir=$DATA_DIR > log.txt 2>&1 & + cd ../ +done diff --git a/model_zoo/resnext50/scripts/run_eval.sh b/model_zoo/resnext50/scripts/run_eval.sh new file mode 100644 index 0000000000..610faa874e --- /dev/null +++ b/model_zoo/resnext50/scripts/run_eval.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +DEVICE_ID=$1 +DATA_DIR=$2 +PATH_CHECKPOINT=$3 + +python eval.py \ + --device_id=$DEVICE_ID \ + --pretrained=$PATH_CHECKPOINT \ + --data_dir=$DATA_DIR > log.txt 2>&1 & diff --git a/model_zoo/resnext50/scripts/run_standalone_train.sh b/model_zoo/resnext50/scripts/run_standalone_train.sh new file mode 100644 index 0000000000..ca5d8206f3 --- /dev/null +++ b/model_zoo/resnext50/scripts/run_standalone_train.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +DEVICE_ID=$1 +DATA_DIR=$2 +PATH_CHECKPOINT="" +if [ $# == 3 ] +then + PATH_CHECKPOINT=$3 +fi + +python train.py \ + --is_distribute=0 \ + --device_id=$DEVICE_ID \ + --pretrained=$PATH_CHECKPOINT \ + --data_dir=$DATA_DIR > log.txt 2>&1 & + diff --git a/model_zoo/resnext50/src/__init__.py b/model_zoo/resnext50/src/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/model_zoo/resnext50/src/backbone/__init__.py b/model_zoo/resnext50/src/backbone/__init__.py new file mode 100644 index 0000000000..b757d07410 --- /dev/null +++ b/model_zoo/resnext50/src/backbone/__init__.py @@ -0,0 +1,16 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""resnet""" +from .resnet import * diff --git a/model_zoo/resnext50/src/backbone/resnet.py b/model_zoo/resnext50/src/backbone/resnet.py new file mode 100644 index 0000000000..5b69f9e1f5 --- /dev/null +++ b/model_zoo/resnext50/src/backbone/resnet.py @@ -0,0 +1,273 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +ResNet based ResNext +""" +import mindspore.nn as nn +from mindspore.ops.operations import TensorAdd, Split, Concat +from mindspore.ops import operations as P +from mindspore.common.initializer import TruncatedNormal + +from src.utils.cunstom_op import SEBlock, GroupConv + + +__all__ = ['ResNet', 'resnext50'] + + +def weight_variable(shape, factor=0.1): + return TruncatedNormal(0.02) + + +def conv7x7(in_channels, out_channels, stride=1, padding=3, has_bias=False, groups=1): + return nn.Conv2d(in_channels, out_channels, kernel_size=7, stride=stride, has_bias=has_bias, + padding=padding, pad_mode="pad", group=groups) + + +def conv3x3(in_channels, out_channels, stride=1, padding=1, has_bias=False, groups=1): + return nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=stride, has_bias=has_bias, + padding=padding, pad_mode="pad", group=groups) + + +def conv1x1(in_channels, out_channels, stride=1, padding=0, has_bias=False, groups=1): + return nn.Conv2d(in_channels, out_channels, kernel_size=1, stride=stride, has_bias=has_bias, + padding=padding, pad_mode="pad", group=groups) + + +class _DownSample(nn.Cell): + """ + Downsample for ResNext-ResNet. + + Args: + in_channels (int): Input channels. + out_channels (int): Output channels. + stride (int): Stride size for the 1*1 convolutional layer. + + Returns: + Tensor, output tensor. + + Examples: + >>>DownSample(32, 64, 2) + """ + def __init__(self, in_channels, out_channels, stride): + super(_DownSample, self).__init__() + self.conv = conv1x1(in_channels, out_channels, stride=stride, padding=0) + self.bn = nn.BatchNorm2d(out_channels) + + def construct(self, x): + out = self.conv(x) + out = self.bn(out) + return out + +class BasicBlock(nn.Cell): + """ + ResNet basic block definition. + + Args: + in_channels (int): Input channels. + out_channels (int): Output channels. + stride (int): Stride size for the first convolutional layer. Default: 1. + + Returns: + Tensor, output tensor. + + Examples: + >>>BasicBlock(32, 256, stride=2) + """ + expansion = 1 + + def __init__(self, in_channels, out_channels, stride=1, down_sample=None, use_se=False, **kwargs): + super(BasicBlock, self).__init__() + self.conv1 = conv3x3(in_channels, out_channels, stride=stride) + self.bn1 = nn.BatchNorm2d(out_channels) + self.relu = P.ReLU() + self.conv2 = conv3x3(out_channels, out_channels, stride=1) + self.bn2 = nn.BatchNorm2d(out_channels) + + self.use_se = use_se + if self.use_se: + self.se = SEBlock(out_channels) + + self.down_sample_flag = False + if down_sample is not None: + self.down_sample = down_sample + self.down_sample_flag = True + + self.add = TensorAdd() + + def construct(self, x): + identity = x + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + out = self.conv2(out) + out = self.bn2(out) + + if self.use_se: + out = self.se(out) + + if self.down_sample_flag: + identity = self.down_sample(x) + + out = self.add(out, identity) + out = self.relu(out) + return out + +class Bottleneck(nn.Cell): + """ + ResNet Bottleneck block definition. + + Args: + in_channels (int): Input channels. + out_channels (int): Output channels. + stride (int): Stride size for the initial convolutional layer. Default: 1. + + Returns: + Tensor, the ResNet unit's output. + + Examples: + >>>Bottleneck(3, 256, stride=2) + """ + expansion = 4 + + def __init__(self, in_channels, out_channels, stride=1, down_sample=None, + base_width=64, groups=1, use_se=False, **kwargs): + super(Bottleneck, self).__init__() + + width = int(out_channels * (base_width / 64.0)) * groups + self.groups = groups + self.conv1 = conv1x1(in_channels, width, stride=1) + self.bn1 = nn.BatchNorm2d(width) + self.relu = P.ReLU() + + self.conv3x3s = nn.CellList() + + self.conv2 = GroupConv(width, width, 3, stride, pad=1, groups=groups) + self.op_split = Split(axis=1, output_num=self.groups) + self.op_concat = Concat(axis=1) + + self.bn2 = nn.BatchNorm2d(width) + self.conv3 = conv1x1(width, out_channels * self.expansion, stride=1) + self.bn3 = nn.BatchNorm2d(out_channels * self.expansion) + + self.use_se = use_se + if self.use_se: + self.se = SEBlock(out_channels * self.expansion) + + self.down_sample_flag = False + if down_sample is not None: + self.down_sample = down_sample + self.down_sample_flag = True + + self.cast = P.Cast() + self.add = TensorAdd() + + def construct(self, x): + identity = x + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + out = self.relu(out) + out = self.conv3(out) + out = self.bn3(out) + + if self.use_se: + out = self.se(out) + + if self.down_sample_flag: + identity = self.down_sample(x) + + out = self.add(out, identity) + out = self.relu(out) + return out + +class ResNet(nn.Cell): + """ + ResNet architecture. + + Args: + block (cell): Block for network. + layers (list): Numbers of block in different layers. + width_per_group (int): Width of every group. + groups (int): Groups number. + + Returns: + Tuple, output tensor tuple. + + Examples: + >>>ResNet() + """ + def __init__(self, block, layers, width_per_group=64, groups=1, use_se=False): + super(ResNet, self).__init__() + self.in_channels = 64 + self.groups = groups + self.base_width = width_per_group + + self.conv = conv7x7(3, self.in_channels, stride=2, padding=3) + self.bn = nn.BatchNorm2d(self.in_channels) + self.relu = P.ReLU() + self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, pad_mode='same') + + self.layer1 = self._make_layer(block, 64, layers[0], use_se=use_se) + self.layer2 = self._make_layer(block, 128, layers[1], stride=2, use_se=use_se) + self.layer3 = self._make_layer(block, 256, layers[2], stride=2, use_se=use_se) + self.layer4 = self._make_layer(block, 512, layers[3], stride=2, use_se=use_se) + + self.out_channels = 512 * block.expansion + self.cast = P.Cast() + + def construct(self, x): + x = self.conv(x) + x = self.bn(x) + x = self.relu(x) + x = self.maxpool(x) + x = self.layer1(x) + x = self.layer2(x) + x = self.layer3(x) + x = self.layer4(x) + + return x + + def _make_layer(self, block, out_channels, blocks_num, stride=1, use_se=False): + """_make_layer""" + down_sample = None + if stride != 1 or self.in_channels != out_channels * block.expansion: + down_sample = _DownSample(self.in_channels, + out_channels * block.expansion, + stride=stride) + + layers = [] + layers.append(block(self.in_channels, + out_channels, + stride=stride, + down_sample=down_sample, + base_width=self.base_width, + groups=self.groups, + use_se=use_se)) + self.in_channels = out_channels * block.expansion + for _ in range(1, blocks_num): + layers.append(block(self.in_channels, out_channels, + base_width=self.base_width, groups=self.groups, use_se=use_se)) + + return nn.SequentialCell(layers) + + def get_out_channels(self): + return self.out_channels + + +def resnext50(): + return ResNet(Bottleneck, [3, 4, 6, 3], width_per_group=4, groups=32) diff --git a/model_zoo/resnext50/src/config.py b/model_zoo/resnext50/src/config.py new file mode 100644 index 0000000000..c1a12aa14e --- /dev/null +++ b/model_zoo/resnext50/src/config.py @@ -0,0 +1,45 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""config""" +from easydict import EasyDict as ed + +config = ed({ + "image_size": '224,224', + "num_classes": 1000, + + "lr": 0.4, + "lr_scheduler": 'cosine_annealing', + "lr_epochs": '30,60,90,120', + "lr_gamma": 0.1, + "eta_min": 0, + "T_max": 150, + "max_epoch": 150, + "backbone": 'resnext50', + "warmup_epochs": 1, + + "weight_decay": 0.0001, + "momentum": 0.9, + "is_dynamic_loss_scale": 0, + "loss_scale": 1024, + "label_smooth": 1, + "label_smooth_factor": 0.1, + + "ckpt_interval": 1250, + "ckpt_path": 'outputs/', + "is_save_on_master": 1, + + "rank": 0, + "group_size": 1 +}) diff --git a/model_zoo/resnext50/src/crossentropy.py b/model_zoo/resnext50/src/crossentropy.py new file mode 100644 index 0000000000..a0e509a51e --- /dev/null +++ b/model_zoo/resnext50/src/crossentropy.py @@ -0,0 +1,41 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +define loss function for network. +""" +from mindspore.nn.loss.loss import _Loss +from mindspore.ops import operations as P +from mindspore.ops import functional as F +from mindspore import Tensor +from mindspore.common import dtype as mstype +import mindspore.nn as nn + +class CrossEntropy(_Loss): + """ + the redefined loss function with SoftmaxCrossEntropyWithLogits. + """ + def __init__(self, smooth_factor=0., num_classes=1000): + super(CrossEntropy, self).__init__() + self.onehot = P.OneHot() + self.on_value = Tensor(1.0 - smooth_factor, mstype.float32) + self.off_value = Tensor(1.0 * smooth_factor / (num_classes -1), mstype.float32) + self.ce = nn.SoftmaxCrossEntropyWithLogits() + self.mean = P.ReduceMean(False) + + def construct(self, logit, label): + one_hot_label = self.onehot(label, F.shape(logit)[1], self.on_value, self.off_value) + loss = self.ce(logit, one_hot_label) + loss = self.mean(loss, 0) + return loss diff --git a/model_zoo/resnext50/src/dataset.py b/model_zoo/resnext50/src/dataset.py new file mode 100644 index 0000000000..9608e3c790 --- /dev/null +++ b/model_zoo/resnext50/src/dataset.py @@ -0,0 +1,155 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +dataset processing. +""" +import os +from mindspore.common import dtype as mstype +import mindspore.dataset as de +import mindspore.dataset.transforms.c_transforms as C +import mindspore.dataset.transforms.vision.c_transforms as V_C +from PIL import Image, ImageFile +from src.utils.sampler import DistributedSampler + +ImageFile.LOAD_TRUNCATED_IMAGES = True + +class TxtDataset(): + """ + create txt dataset. + + Args: + Returns: + de_dataset. + """ + def __init__(self, root, txt_name): + super(TxtDataset, self).__init__() + self.imgs = [] + self.labels = [] + fin = open(txt_name, "r") + for line in fin: + img_name, label = line.strip().split(' ') + self.imgs.append(os.path.join(root, img_name)) + self.labels.append(int(label)) + fin.close() + + def __getitem__(self, index): + img = Image.open(self.imgs[index]).convert('RGB') + return img, self.labels[index] + + def __len__(self): + return len(self.imgs) + + +def classification_dataset(data_dir, image_size, per_batch_size, max_epoch, rank, group_size, + mode='train', + input_mode='folder', + root='', + num_parallel_workers=None, + shuffle=None, + sampler=None, + class_indexing=None, + drop_remainder=True, + transform=None, + target_transform=None): + """ + A function that returns a dataset for classification. The mode of input dataset could be "folder" or "txt". + If it is "folder", all images within one folder have the same label. If it is "txt", all paths of images + are written into a textfile. + + Args: + data_dir (str): Path to the root directory that contains the dataset for "input_mode="folder"". + Or path of the textfile that contains every image's path of the dataset. + image_size (str): Size of the input images. + per_batch_size (int): the batch size of evey step during training. + max_epoch (int): the number of epochs. + rank (int): The shard ID within num_shards (default=None). + group_size (int): Number of shards that the dataset should be divided + into (default=None). + mode (str): "train" or others. Default: " train". + input_mode (str): The form of the input dataset. "folder" or "txt". Default: "folder". + root (str): the images path for "input_mode="txt"". Default: " ". + num_parallel_workers (int): Number of workers to read the data. Default: None. + shuffle (bool): Whether or not to perform shuffle on the dataset + (default=None, performs shuffle). + sampler (Sampler): Object used to choose samples from the dataset. Default: None. + class_indexing (dict): A str-to-int mapping from folder name to index + (default=None, the folder names will be sorted + alphabetically and each class will be given a + unique index starting from 0). + + Examples: + >>> from mindvision.common.datasets.classification import classification_dataset + >>> # path to imagefolder directory. This directory needs to contain sub-directories which contain the images + >>> dataset_dir = "/path/to/imagefolder_directory" + >>> de_dataset = classification_dataset(train_data_dir, image_size=[224, 244], + >>> per_batch_size=64, max_epoch=100, + >>> rank=0, group_size=4) + >>> # Path of the textfile that contains every image's path of the dataset. + >>> dataset_dir = "/path/to/dataset/images/train.txt" + >>> images_dir = "/path/to/dataset/images" + >>> de_dataset = classification_dataset(train_data_dir, image_size=[224, 244], + >>> per_batch_size=64, max_epoch=100, + >>> rank=0, group_size=4, + >>> input_mode="txt", root=images_dir) + """ + + mean = [0.485 * 255, 0.456 * 255, 0.406 * 255] + std = [0.229 * 255, 0.224 * 255, 0.225 * 255] + + if transform is None: + if mode == 'train': + transform_img = [ + V_C.RandomCropDecodeResize(image_size, scale=(0.08, 1.0), ratio=(0.75, 1.333)), + V_C.RandomHorizontalFlip(prob=0.5), + V_C.RandomColorAdjust(brightness=0.4, contrast=0.4, saturation=0.4), + V_C.Normalize(mean=mean, std=std), + V_C.HWC2CHW() + ] + else: + transform_img = [ + V_C.Decode(), + V_C.Resize((256, 256)), + V_C.CenterCrop(image_size), + V_C.Normalize(mean=mean, std=std), + V_C.HWC2CHW() + ] + else: + transform_img = transform + + if target_transform is None: + transform_label = [C.TypeCast(mstype.int32)] + else: + transform_label = target_transform + + if input_mode == 'folder': + de_dataset = de.ImageFolderDatasetV2(data_dir, num_parallel_workers=num_parallel_workers, + shuffle=shuffle, sampler=sampler, class_indexing=class_indexing, + num_shards=group_size, shard_id=rank) + else: + dataset = TxtDataset(root, data_dir) + sampler = DistributedSampler(dataset, rank, group_size, shuffle=shuffle) + de_dataset = de.GeneratorDataset(dataset, ["image", "label"], sampler=sampler) + de_dataset.set_dataset_size(len(sampler)) + + de_dataset = de_dataset.map(input_columns="image", num_parallel_workers=8, operations=transform_img) + de_dataset = de_dataset.map(input_columns="label", num_parallel_workers=8, operations=transform_label) + + columns_to_project = ["image", "label"] + de_dataset = de_dataset.project(columns=columns_to_project) + + de_dataset = de_dataset.batch(per_batch_size, drop_remainder=drop_remainder) + de_dataset = de_dataset.repeat(max_epoch) + + return de_dataset diff --git a/model_zoo/resnext50/src/head.py b/model_zoo/resnext50/src/head.py new file mode 100644 index 0000000000..a7bd85c906 --- /dev/null +++ b/model_zoo/resnext50/src/head.py @@ -0,0 +1,42 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +common architecture. +""" +import mindspore.nn as nn +from src.utils.cunstom_op import GlobalAvgPooling + +__all__ = ['CommonHead'] + +class CommonHead(nn.Cell): + """ + commom architecture definition. + + Args: + num_classes (int): Number of classes. + out_channels (int): Output channels. + + Returns: + Tensor, output tensor. + """ + def __init__(self, num_classes, out_channels): + super(CommonHead, self).__init__() + self.avgpool = GlobalAvgPooling() + self.fc = nn.Dense(out_channels, num_classes, has_bias=True).add_flags_recursive(fp16=True) + + def construct(self, x): + x = self.avgpool(x) + x = self.fc(x) + return x diff --git a/model_zoo/resnext50/src/image_classification.py b/model_zoo/resnext50/src/image_classification.py new file mode 100644 index 0000000000..d8003ad200 --- /dev/null +++ b/model_zoo/resnext50/src/image_classification.py @@ -0,0 +1,85 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +Image classifiation. +""" +import math +import mindspore.nn as nn +from mindspore.common import initializer as init +import src.backbone as backbones +import src.head as heads +from src.utils.var_init import default_recurisive_init, KaimingNormal + + +class ImageClassificationNetwork(nn.Cell): + """ + architecture of image classification network. + + Args: + Returns: + Tensor, output tensor. + """ + def __init__(self, backbone, head): + super(ImageClassificationNetwork, self).__init__() + self.backbone = backbone + self.head = head + + def construct(self, x): + x = self.backbone(x) + x = self.head(x) + return x + +class Resnet(ImageClassificationNetwork): + """ + Resnet architecture. + Args: + backbone_name (string): backbone. + num_classes (int): number of classes. + Returns: + Resnet. + """ + def __init__(self, backbone_name, num_classes): + self.backbone_name = backbone_name + backbone = backbones.__dict__[self.backbone_name]() + out_channels = backbone.get_out_channels() + head = heads.CommonHead(num_classes=num_classes, out_channels=out_channels) + super(Resnet, self).__init__(backbone, head) + + default_recurisive_init(self) + + for cell in self.cells_and_names(): + if isinstance(cell, nn.Conv2d): + cell.weight.default_input = init.initializer( + KaimingNormal(a=math.sqrt(5), mode='fan_out', nonlinearity='relu'), + cell.weight.default_input.shape, cell.weight.default_input.dtype).to_tensor() + elif isinstance(cell, nn.BatchNorm2d): + cell.gamma.default_input = init.initializer('ones', cell.gamma.default_input.shape).to_tensor() + cell.beta.default_input = init.initializer('zeros', cell.beta.default_input.shape).to_tensor() + + # Zero-initialize the last BN in each residual branch, + # so that the residual branch starts with zeros, and each residual block behaves like an identity. + # This improves the model by 0.2~0.3% according to https://arxiv.org/abs/1706.02677 + for cell in self.cells_and_names(): + if isinstance(cell, backbones.resnet.Bottleneck): + cell.bn3.gamma.default_input = init.initializer('zeros', cell.bn3.gamma.default_input.shape).to_tensor() + elif isinstance(cell, backbones.resnet.BasicBlock): + cell.bn2.gamma.default_input = init.initializer('zeros', cell.bn2.gamma.default_input.shape).to_tensor() + + + +def get_network(backbone_name, num_classes): + if backbone_name in ['resnext50']: + return Resnet(backbone_name, num_classes) + return None diff --git a/model_zoo/resnext50/src/linear_warmup.py b/model_zoo/resnext50/src/linear_warmup.py new file mode 100644 index 0000000000..af0bac631a --- /dev/null +++ b/model_zoo/resnext50/src/linear_warmup.py @@ -0,0 +1,21 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +linear warm up learning rate. +""" +def linear_warmup_lr(current_step, warmup_steps, base_lr, init_lr): + lr_inc = (float(base_lr) - float(init_lr)) / float(warmup_steps) + lr = float(init_lr) + lr_inc * current_step + return lr diff --git a/model_zoo/resnext50/src/utils/__init__.py b/model_zoo/resnext50/src/utils/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/model_zoo/resnext50/src/utils/cunstom_op.py b/model_zoo/resnext50/src/utils/cunstom_op.py new file mode 100644 index 0000000000..cbe89a1610 --- /dev/null +++ b/model_zoo/resnext50/src/utils/cunstom_op.py @@ -0,0 +1,108 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +network operations +""" +import mindspore.nn as nn +from mindspore.ops import operations as P +from mindspore.common import dtype as mstype + + +class GlobalAvgPooling(nn.Cell): + """ + global average pooling feature map. + + Args: + mean (tuple): means for each channel. + """ + def __init__(self): + super(GlobalAvgPooling, self).__init__() + self.mean = P.ReduceMean(True) + self.shape = P.Shape() + self.reshape = P.Reshape() + + def construct(self, x): + x = self.mean(x, (2, 3)) + b, c, _, _ = self.shape(x) + x = self.reshape(x, (b, c)) + return x + + +class SEBlock(nn.Cell): + """ + squeeze and excitation block. + + Args: + channel (int): number of feature maps. + reduction (int): weight. + """ + def __init__(self, channel, reduction=16): + super(SEBlock, self).__init__() + + self.avg_pool = GlobalAvgPooling() + self.fc1 = nn.Dense(channel, channel // reduction) + self.relu = P.ReLU() + self.fc2 = nn.Dense(channel // reduction, channel) + self.sigmoid = P.Sigmoid() + self.reshape = P.Reshape() + self.shape = P.Shape() + self.sum = P.Sum() + self.cast = P.Cast() + + def construct(self, x): + b, c = self.shape(x) + y = self.avg_pool(x) + + y = self.reshape(y, (b, c)) + y = self.fc1(y) + y = self.relu(y) + y = self.fc2(y) + y = self.sigmoid(y) + y = self.reshape(y, (b, c, 1, 1)) + return x * y + +class GroupConv(nn.Cell): + """ + group convolution operation. + + Args: + in_channels (int): Input channels of feature map. + out_channels (int): Output channels of feature map. + kernel_size (int): Size of convolution kernel. + stride (int): Stride size for the group convolution layer. + + Returns: + tensor, output tensor. + """ + def __init__(self, in_channels, out_channels, kernel_size, stride, pad_mode="pad", pad=0, groups=1, has_bias=False): + super(GroupConv, self).__init__() + assert in_channels % groups == 0 and out_channels % groups == 0 + self.groups = groups + self.convs = nn.CellList() + self.op_split = P.Split(axis=1, output_num=self.groups) + self.op_concat = P.Concat(axis=1) + self.cast = P.Cast() + for _ in range(groups): + self.convs.append(nn.Conv2d(in_channels//groups, out_channels//groups, + kernel_size=kernel_size, stride=stride, has_bias=has_bias, + padding=pad, pad_mode=pad_mode, group=1)) + + def construct(self, x): + features = self.op_split(x) + outputs = () + for i in range(self.groups): + outputs = outputs + (self.convs[i](self.cast(features[i], mstype.float32)),) + out = self.op_concat(outputs) + return out diff --git a/model_zoo/resnext50/src/utils/logging.py b/model_zoo/resnext50/src/utils/logging.py new file mode 100644 index 0000000000..ac37bec4ec --- /dev/null +++ b/model_zoo/resnext50/src/utils/logging.py @@ -0,0 +1,82 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +get logger. +""" +import logging +import os +import sys +from datetime import datetime + +class LOGGER(logging.Logger): + """ + set up logging file. + + Args: + logger_name (string): logger name. + log_dir (string): path of logger. + + Returns: + string, logger path + """ + def __init__(self, logger_name, rank=0): + super(LOGGER, self).__init__(logger_name) + if rank % 8 == 0: + console = logging.StreamHandler(sys.stdout) + console.setLevel(logging.INFO) + formatter = logging.Formatter('%(asctime)s:%(levelname)s:%(message)s') + console.setFormatter(formatter) + self.addHandler(console) + + def setup_logging_file(self, log_dir, rank=0): + """set up log file""" + self.rank = rank + if not os.path.exists(log_dir): + os.makedirs(log_dir, exist_ok=True) + log_name = datetime.now().strftime('%Y-%m-%d_time_%H_%M_%S') + '_rank_{}.log'.format(rank) + self.log_fn = os.path.join(log_dir, log_name) + fh = logging.FileHandler(self.log_fn) + fh.setLevel(logging.INFO) + formatter = logging.Formatter('%(asctime)s:%(levelname)s:%(message)s') + fh.setFormatter(formatter) + self.addHandler(fh) + + def info(self, msg, *args, **kwargs): + if self.isEnabledFor(logging.INFO): + self._log(logging.INFO, msg, args, **kwargs) + + def save_args(self, args): + self.info('Args:') + args_dict = vars(args) + for key in args_dict.keys(): + self.info('--> %s: %s', key, args_dict[key]) + self.info('') + + def important_info(self, msg, *args, **kwargs): + if self.isEnabledFor(logging.INFO) and self.rank == 0: + line_width = 2 + important_msg = '\n' + important_msg += ('*'*70 + '\n')*line_width + important_msg += ('*'*line_width + '\n')*2 + important_msg += '*'*line_width + ' '*8 + msg + '\n' + important_msg += ('*'*line_width + '\n')*2 + important_msg += ('*'*70 + '\n')*line_width + self.info(important_msg, *args, **kwargs) + + +def get_logger(path, rank): + logger = LOGGER("mindversion", rank) + logger.setup_logging_file(path, rank) + return logger diff --git a/model_zoo/resnext50/src/utils/optimizers__init__.py b/model_zoo/resnext50/src/utils/optimizers__init__.py new file mode 100644 index 0000000000..d4683959b5 --- /dev/null +++ b/model_zoo/resnext50/src/utils/optimizers__init__.py @@ -0,0 +1,39 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +optimizer parameters. +""" +def get_param_groups(network): + """get param groups""" + decay_params = [] + no_decay_params = [] + for x in network.trainable_params(): + parameter_name = x.name + if parameter_name.endswith('.bias'): + # all bias not using weight decay + # print('no decay:{}'.format(parameter_name)) + no_decay_params.append(x) + elif parameter_name.endswith('.gamma'): + # bn weight bias not using weight decay, be carefully for now x not include BN + # print('no decay:{}'.format(parameter_name)) + no_decay_params.append(x) + elif parameter_name.endswith('.beta'): + # bn weight bias not using weight decay, be carefully for now x not include BN + # print('no decay:{}'.format(parameter_name)) + no_decay_params.append(x) + else: + decay_params.append(x) + + return [{'params': no_decay_params, 'weight_decay': 0.0}, {'params': decay_params}] diff --git a/model_zoo/resnext50/src/utils/sampler.py b/model_zoo/resnext50/src/utils/sampler.py new file mode 100644 index 0000000000..5b68f8325e --- /dev/null +++ b/model_zoo/resnext50/src/utils/sampler.py @@ -0,0 +1,53 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +choose samples from the dataset +""" +import math +import numpy as np + +class DistributedSampler(): + """ + sampling the dataset. + + Args: + Returns: + num_samples, number of samples. + """ + def __init__(self, dataset, rank, group_size, shuffle=True, seed=0): + self.dataset = dataset + self.rank = rank + self.group_size = group_size + self.dataset_length = len(self.dataset) + self.num_samples = int(math.ceil(self.dataset_length * 1.0 / self.group_size)) + self.total_size = self.num_samples * self.group_size + self.shuffle = shuffle + self.seed = seed + + def __iter__(self): + if self.shuffle: + self.seed = (self.seed + 1) & 0xffffffff + np.random.seed(self.seed) + indices = np.random.permutation(self.dataset_length).tolist() + else: + indices = list(range(len(self.dataset_length))) + + indices += indices[:(self.total_size - len(indices))] + indices = indices[self.rank::self.group_size] + return iter(indices) + + def __len__(self): + return self.num_samples + \ No newline at end of file diff --git a/model_zoo/resnext50/src/utils/var_init.py b/model_zoo/resnext50/src/utils/var_init.py new file mode 100644 index 0000000000..51fc109990 --- /dev/null +++ b/model_zoo/resnext50/src/utils/var_init.py @@ -0,0 +1,213 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +Initialize. +""" +import math +from functools import reduce +import numpy as np +import mindspore.nn as nn +from mindspore import Tensor +from mindspore.common import initializer as init + +def _calculate_gain(nonlinearity, param=None): + r""" + Return the recommended gain value for the given nonlinearity function. + + The values are as follows: + ================= ==================================================== + nonlinearity gain + ================= ==================================================== + Linear / Identity :math:`1` + Conv{1,2,3}D :math:`1` + Sigmoid :math:`1` + Tanh :math:`\frac{5}{3}` + ReLU :math:`\sqrt{2}` + Leaky Relu :math:`\sqrt{\frac{2}{1 + \text{negative\_slope}^2}}` + ================= ==================================================== + + Args: + nonlinearity: the non-linear function + param: optional parameter for the non-linear function + + Examples: + >>> gain = calculate_gain('leaky_relu', 0.2) # leaky_relu with negative_slope=0.2 + """ + linear_fns = ['linear', 'conv1d', 'conv2d', 'conv3d', 'conv_transpose1d', 'conv_transpose2d', 'conv_transpose3d'] + if nonlinearity in linear_fns or nonlinearity == 'sigmoid': + return 1 + if nonlinearity == 'tanh': + return 5.0 / 3 + if nonlinearity == 'relu': + return math.sqrt(2.0) + if nonlinearity == 'leaky_relu': + if param is None: + negative_slope = 0.01 + elif not isinstance(param, bool) and isinstance(param, int) or isinstance(param, float): + negative_slope = param + else: + raise ValueError("negative_slope {} not a valid number".format(param)) + return math.sqrt(2.0 / (1 + negative_slope ** 2)) + + raise ValueError("Unsupported nonlinearity {}".format(nonlinearity)) + +def _assignment(arr, num): + """Assign the value of `num` to `arr`.""" + if arr.shape == (): + arr = arr.reshape((1)) + arr[:] = num + arr = arr.reshape(()) + else: + if isinstance(num, np.ndarray): + arr[:] = num[:] + else: + arr[:] = num + return arr + +def _calculate_in_and_out(arr): + """ + Calculate n_in and n_out. + + Args: + arr (Array): Input array. + + Returns: + Tuple, a tuple with two elements, the first element is `n_in` and the second element is `n_out`. + """ + dim = len(arr.shape) + if dim < 2: + raise ValueError("If initialize data with xavier uniform, the dimension of data must greater than 1.") + + n_in = arr.shape[1] + n_out = arr.shape[0] + + if dim > 2: + counter = reduce(lambda x, y: x * y, arr.shape[2:]) + n_in *= counter + n_out *= counter + return n_in, n_out + +def _select_fan(array, mode): + mode = mode.lower() + valid_modes = ['fan_in', 'fan_out'] + if mode not in valid_modes: + raise ValueError("Mode {} not supported, please use one of {}".format(mode, valid_modes)) + + fan_in, fan_out = _calculate_in_and_out(array) + return fan_in if mode == 'fan_in' else fan_out + +class KaimingInit(init.Initializer): + r""" + Base Class. Initialize the array with He kaiming algorithm. + + Args: + a: the negative slope of the rectifier used after this layer (only + used with ``'leaky_relu'``) + mode: either ``'fan_in'`` (default) or ``'fan_out'``. Choosing ``'fan_in'`` + preserves the magnitude of the variance of the weights in the + forward pass. Choosing ``'fan_out'`` preserves the magnitudes in the + backwards pass. + nonlinearity: the non-linear function, recommended to use only with + ``'relu'`` or ``'leaky_relu'`` (default). + """ + def __init__(self, a=0, mode='fan_in', nonlinearity='leaky_relu'): + super(KaimingInit, self).__init__() + self.mode = mode + self.gain = _calculate_gain(nonlinearity, a) + def _initialize(self, arr): + pass + + +class KaimingUniform(KaimingInit): + r""" + Initialize the array with He kaiming uniform algorithm. The resulting tensor will + have values sampled from :math:`\mathcal{U}(-\text{bound}, \text{bound})` where + + .. math:: + \text{bound} = \text{gain} \times \sqrt{\frac{3}{\text{fan\_mode}}} + + Input: + arr (Array): The array to be assigned. + + Returns: + Array, assigned array. + + Examples: + >>> w = np.empty(3, 5) + >>> KaimingUniform(w, mode='fan_in', nonlinearity='relu') + """ + + def _initialize(self, arr): + fan = _select_fan(arr, self.mode) + bound = math.sqrt(3.0) * self.gain / math.sqrt(fan) + np.random.seed(0) + data = np.random.uniform(-bound, bound, arr.shape) + + _assignment(arr, data) + + +class KaimingNormal(KaimingInit): + r""" + Initialize the array with He kaiming normal algorithm. The resulting tensor will + have values sampled from :math:`\mathcal{N}(0, \text{std}^2)` where + + .. math:: + \text{std} = \frac{\text{gain}}{\sqrt{\text{fan\_mode}}} + + Input: + arr (Array): The array to be assigned. + + Returns: + Array, assigned array. + + Examples: + >>> w = np.empty(3, 5) + >>> KaimingNormal(w, mode='fan_out', nonlinearity='relu') + """ + + def _initialize(self, arr): + fan = _select_fan(arr, self.mode) + std = self.gain / math.sqrt(fan) + np.random.seed(0) + data = np.random.normal(0, std, arr.shape) + + _assignment(arr, data) + + +def default_recurisive_init(custom_cell): + """default_recurisive_init""" + for _, cell in custom_cell.cells_and_names(): + if isinstance(cell, nn.Conv2d): + cell.weight.default_input = init.initializer(KaimingUniform(a=math.sqrt(5)), + cell.weight.default_input.shape, + cell.weight.default_input.dtype).to_tensor() + if cell.bias is not None: + fan_in, _ = _calculate_in_and_out(cell.weight.default_input.asnumpy()) + bound = 1 / math.sqrt(fan_in) + np.random.seed(0) + cell.bias.default_input = Tensor(np.random.uniform(-bound, bound, cell.bias.default_input.shape), + cell.bias.default_input.dtype) + elif isinstance(cell, nn.Dense): + cell.weight.default_input = init.initializer(KaimingUniform(a=math.sqrt(5)), + cell.weight.default_input.shape, + cell.weight.default_input.dtype).to_tensor() + if cell.bias is not None: + fan_in, _ = _calculate_in_and_out(cell.weight.default_input.asnumpy()) + bound = 1 / math.sqrt(fan_in) + np.random.seed(0) + cell.bias.default_input = Tensor(np.random.uniform(-bound, bound, cell.bias.default_input.shape), + cell.bias.default_input.dtype) + elif isinstance(cell, (nn.BatchNorm2d, nn.BatchNorm1d)): + pass diff --git a/model_zoo/resnext50/src/warmup_cosine_annealing_lr.py b/model_zoo/resnext50/src/warmup_cosine_annealing_lr.py new file mode 100644 index 0000000000..5d9fce9af4 --- /dev/null +++ b/model_zoo/resnext50/src/warmup_cosine_annealing_lr.py @@ -0,0 +1,40 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +warm up cosine annealing learning rate. +""" +import math +import numpy as np + +from .linear_warmup import linear_warmup_lr + + +def warmup_cosine_annealing_lr(lr, steps_per_epoch, warmup_epochs, max_epoch, T_max, eta_min=0): + """warm up cosine annealing learning rate.""" + base_lr = lr + warmup_init_lr = 0 + total_steps = int(max_epoch * steps_per_epoch) + warmup_steps = int(warmup_epochs * steps_per_epoch) + + lr_each_step = [] + for i in range(total_steps): + last_epoch = i // steps_per_epoch + if i < warmup_steps: + lr = linear_warmup_lr(i + 1, warmup_steps, base_lr, warmup_init_lr) + else: + lr = eta_min + (base_lr - eta_min) * (1. + math.cos(math.pi*last_epoch / T_max)) / 2 + lr_each_step.append(lr) + + return np.array(lr_each_step).astype(np.float32) diff --git a/model_zoo/resnext50/src/warmup_step_lr.py b/model_zoo/resnext50/src/warmup_step_lr.py new file mode 100644 index 0000000000..d8e85ab610 --- /dev/null +++ b/model_zoo/resnext50/src/warmup_step_lr.py @@ -0,0 +1,56 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +warm up step learning rate. +""" +from collections import Counter +import numpy as np + +from .linear_warmup import linear_warmup_lr + + +def warmup_step_lr(lr, lr_epochs, steps_per_epoch, warmup_epochs, max_epoch, gamma=0.1): + """warmup_step_lr""" + base_lr = lr + warmup_init_lr = 0 + total_steps = int(max_epoch * steps_per_epoch) + warmup_steps = int(warmup_epochs * steps_per_epoch) + milestones = lr_epochs + milestones_steps = [] + for milestone in milestones: + milestones_step = milestone * steps_per_epoch + milestones_steps.append(milestones_step) + + lr_each_step = [] + lr = base_lr + milestones_steps_counter = Counter(milestones_steps) + for i in range(total_steps): + if i < warmup_steps: + lr = linear_warmup_lr(i + 1, warmup_steps, base_lr, warmup_init_lr) + else: + lr = lr * gamma**milestones_steps_counter[i] + lr_each_step.append(lr) + + return np.array(lr_each_step).astype(np.float32) + +def multi_step_lr(lr, milestones, steps_per_epoch, max_epoch, gamma=0.1): + return warmup_step_lr(lr, milestones, steps_per_epoch, 0, max_epoch, gamma=gamma) + +def step_lr(lr, epoch_size, steps_per_epoch, max_epoch, gamma=0.1): + lr_epochs = [] + for i in range(1, max_epoch): + if i % epoch_size == 0: + lr_epochs.append(i) + return multi_step_lr(lr, lr_epochs, steps_per_epoch, max_epoch, gamma=gamma) diff --git a/model_zoo/resnext50/train.py b/model_zoo/resnext50/train.py new file mode 100644 index 0000000000..29ccd9b00c --- /dev/null +++ b/model_zoo/resnext50/train.py @@ -0,0 +1,289 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""train ImageNet.""" +import os +import time +import argparse +import datetime + +import mindspore.nn as nn +from mindspore import Tensor, context +from mindspore import ParallelMode +from mindspore.nn.optim import Momentum +from mindspore.communication.management import init, get_rank, get_group_size +from mindspore.train.callback import ModelCheckpoint +from mindspore.train.callback import CheckpointConfig, Callback +from mindspore.train.serialization import load_checkpoint, load_param_into_net +from mindspore.train.model import Model +from mindspore.train.loss_scale_manager import DynamicLossScaleManager, FixedLossScaleManager + +from src.dataset import classification_dataset +from src.crossentropy import CrossEntropy +from src.warmup_step_lr import warmup_step_lr +from src.warmup_cosine_annealing_lr import warmup_cosine_annealing_lr +from src.utils.logging import get_logger +from src.utils.optimizers__init__ import get_param_groups +from src.image_classification import get_network +from src.config import config + +devid = int(os.getenv('DEVICE_ID')) +context.set_context(mode=context.GRAPH_MODE, enable_auto_mixed_precision=True, + device_target="Ascend", save_graphs=False, device_id=devid) + +class BuildTrainNetwork(nn.Cell): + """build training network""" + def __init__(self, network, criterion): + super(BuildTrainNetwork, self).__init__() + self.network = network + self.criterion = criterion + + def construct(self, input_data, label): + output = self.network(input_data) + loss = self.criterion(output, label) + return loss + +class ProgressMonitor(Callback): + """monitor loss and time""" + def __init__(self, args): + super(ProgressMonitor, self).__init__() + self.me_epoch_start_time = 0 + self.me_epoch_start_step_num = 0 + self.args = args + self.ckpt_history = [] + + def begin(self, run_context): + self.args.logger.info('start network train...') + + def epoch_begin(self, run_context): + pass + + def epoch_end(self, run_context, *me_args): + cb_params = run_context.original_args() + me_step = cb_params.cur_step_num - 1 + + real_epoch = me_step // self.args.steps_per_epoch + time_used = time.time() - self.me_epoch_start_time + fps_mean = self.args.per_batch_size * (me_step-self.me_epoch_start_step_num) * self.args.group_size / time_used + self.args.logger.info('epoch[{}], iter[{}], loss:{}, mean_fps:{:.2f}' + 'imgs/sec'.format(real_epoch, me_step, cb_params.net_outputs, fps_mean)) + + if self.args.rank_save_ckpt_flag: + import glob + ckpts = glob.glob(os.path.join(self.args.outputs_dir, '*.ckpt')) + for ckpt in ckpts: + ckpt_fn = os.path.basename(ckpt) + if not ckpt_fn.startswith('{}-'.format(self.args.rank)): + continue + if ckpt in self.ckpt_history: + continue + self.ckpt_history.append(ckpt) + self.args.logger.info('epoch[{}], iter[{}], loss:{}, ckpt:{},' + 'ckpt_fn:{}'.format(real_epoch, me_step, cb_params.net_outputs, ckpt, ckpt_fn)) + + + self.me_epoch_start_step_num = me_step + self.me_epoch_start_time = time.time() + + def step_begin(self, run_context): + pass + + def step_end(self, run_context, *me_args): + pass + + def end(self, run_context): + self.args.logger.info('end network train...') + + +def parse_args(cloud_args=None): + """parameters""" + parser = argparse.ArgumentParser('mindspore classification training') + + # dataset related + parser.add_argument('--data_dir', type=str, default='', help='train data dir') + parser.add_argument('--per_batch_size', default=128, type=int, help='batch size for per gpu') + # network related + parser.add_argument('--pretrained', default='', type=str, help='model_path, local pretrained model to load') + + # distributed related + parser.add_argument('--is_distributed', type=int, default=1, help='if multi device') + # roma obs + parser.add_argument('--train_url', type=str, default="", help='train url') + + args, _ = parser.parse_known_args() + args = merge_args(args, cloud_args) + args.image_size = config.image_size + args.num_classes = config.num_classes + args.lr = config.lr + args.lr_scheduler = config.lr_scheduler + args.lr_epochs = config.lr_epochs + args.lr_gamma = config.lr_gamma + args.eta_min = config.eta_min + args.T_max = config.T_max + args.max_epoch = config.max_epoch + args.backbone = config.backbone + args.warmup_epochs = config.warmup_epochs + args.weight_decay = config.weight_decay + args.momentum = config.momentum + args.is_dynamic_loss_scale = config.is_dynamic_loss_scale + args.loss_scale = config.loss_scale + args.label_smooth = config.label_smooth + args.label_smooth_factor = config.label_smooth_factor + args.ckpt_interval = config.ckpt_interval + args.ckpt_path = config.ckpt_path + args.is_save_on_master = config.is_save_on_master + args.rank = config.rank + args.group_size = config.group_size + args.lr_epochs = list(map(int, args.lr_epochs.split(','))) + args.image_size = list(map(int, args.image_size.split(','))) + + return args + +def merge_args(args, cloud_args): + """dictionary""" + args_dict = vars(args) + if isinstance(cloud_args, dict): + for key in cloud_args.keys(): + val = cloud_args[key] + if key in args_dict and val: + arg_type = type(args_dict[key]) + if arg_type is not type(None): + val = arg_type(val) + args_dict[key] = val + return args + +def train(cloud_args=None): + """training process""" + args = parse_args(cloud_args) + + # init distributed + if args.is_distributed: + init() + args.rank = get_rank() + args.group_size = get_group_size() + + if args.is_dynamic_loss_scale == 1: + args.loss_scale = 1 # for dynamic loss scale can not set loss scale in momentum opt + + # select for master rank save ckpt or all rank save, compatiable for model parallel + args.rank_save_ckpt_flag = 0 + if args.is_save_on_master: + if args.rank == 0: + args.rank_save_ckpt_flag = 1 + else: + args.rank_save_ckpt_flag = 1 + + # logger + args.outputs_dir = os.path.join(args.ckpt_path, + datetime.datetime.now().strftime('%Y-%m-%d_time_%H_%M_%S')) + args.logger = get_logger(args.outputs_dir, args.rank) + + # dataloader + de_dataset = classification_dataset(args.data_dir, args.image_size, + args.per_batch_size, args.max_epoch, + args.rank, args.group_size) + de_dataset.map_model = 4 # !!!important + args.steps_per_epoch = de_dataset.get_dataset_size() + + args.logger.save_args(args) + + # network + args.logger.important_info('start create network') + # get network and init + network = get_network(args.backbone, args.num_classes) + if network is None: + raise NotImplementedError('not implement {}'.format(args.backbone)) + network.add_flags_recursive(fp16=True) + # loss + if not args.label_smooth: + args.label_smooth_factor = 0.0 + criterion = CrossEntropy(smooth_factor=args.label_smooth_factor, + num_classes=args.num_classes) + + # load pretrain model + if os.path.isfile(args.pretrained): + param_dict = load_checkpoint(args.pretrained) + param_dict_new = {} + for key, values in param_dict.items(): + if key.startswith('moments.'): + continue + elif key.startswith('network.'): + param_dict_new[key[8:]] = values + else: + param_dict_new[key] = values + load_param_into_net(network, param_dict_new) + args.logger.info('load model {} success'.format(args.pretrained)) + + # lr scheduler + if args.lr_scheduler == 'exponential': + lr = warmup_step_lr(args.lr, + args.lr_epochs, + args.steps_per_epoch, + args.warmup_epochs, + args.max_epoch, + gamma=args.lr_gamma, + ) + elif args.lr_scheduler == 'cosine_annealing': + lr = warmup_cosine_annealing_lr(args.lr, + args.steps_per_epoch, + args.warmup_epochs, + args.max_epoch, + args.T_max, + args.eta_min) + else: + raise NotImplementedError(args.lr_scheduler) + + # optimizer + opt = Momentum(params=get_param_groups(network), + learning_rate=Tensor(lr), + momentum=args.momentum, + weight_decay=args.weight_decay, + loss_scale=args.loss_scale) + + + criterion.add_flags_recursive(fp32=True) + + # package training process, adjust lr + forward + backward + optimizer + train_net = BuildTrainNetwork(network, criterion) + if args.is_distributed: + parallel_mode = ParallelMode.DATA_PARALLEL + else: + parallel_mode = ParallelMode.STAND_ALONE + if args.is_dynamic_loss_scale == 1: + loss_scale_manager = DynamicLossScaleManager(init_loss_scale=65536, scale_factor=2, scale_window=2000) + else: + loss_scale_manager = FixedLossScaleManager(args.loss_scale, drop_overflow_update=False) + + # Model api changed since TR5_branch 2020/03/09 + context.set_auto_parallel_context(parallel_mode=parallel_mode, device_num=args.group_size, + parameter_broadcast=True, mirror_mean=True) + model = Model(train_net, optimizer=opt, metrics=None, loss_scale_manager=loss_scale_manager) + + # checkpoint save + progress_cb = ProgressMonitor(args) + callbacks = [progress_cb,] + if args.rank_save_ckpt_flag: + ckpt_max_num = args.max_epoch * args.steps_per_epoch // args.ckpt_interval + ckpt_config = CheckpointConfig(save_checkpoint_steps=args.ckpt_interval, + keep_checkpoint_max=ckpt_max_num) + ckpt_cb = ModelCheckpoint(config=ckpt_config, + directory=args.outputs_dir, + prefix='{}'.format(args.rank)) + callbacks.append(ckpt_cb) + + model.train(args.max_epoch, de_dataset, callbacks=callbacks, dataset_sink_mode=True) + + +if __name__ == "__main__": + train() From 9cebb1630b711f6eef2ef7e1a2c65107383cbd70 Mon Sep 17 00:00:00 2001 From: huangdongrun Date: Mon, 29 Jun 2020 20:29:05 +0800 Subject: [PATCH 146/254] IndexError to TypeError --- mindspore/ops/composite/multitype_ops/_compile_utils.py | 8 ++++---- mindspore/ops/composite/multitype_ops/_constexpr_utils.py | 4 ++++ 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/mindspore/ops/composite/multitype_ops/_compile_utils.py b/mindspore/ops/composite/multitype_ops/_compile_utils.py index b371b839e1..86100818d1 100644 --- a/mindspore/ops/composite/multitype_ops/_compile_utils.py +++ b/mindspore/ops/composite/multitype_ops/_compile_utils.py @@ -207,8 +207,8 @@ def tensor_index_by_slice(data, slice_index): """Tensor getitem by a single slice""" shape = F.shape(data) if not shape: - const_utils.raise_index_error("When tensor is indexed by a slice, the dimension of the tensor\ - cannot be 0.") + const_utils.raise_index_error("When tensor is indexed by a slice, the dimension of the tensor" + "cannot be 0.") begin_strides, end_strides, step_strides = const_utils.get_stride_info_from_slice(shape, slice_index) return F.strided_slice(data, begin_strides, end_strides, step_strides) @@ -217,8 +217,8 @@ def _tensor_index_by_integer(data, number): """Tensor getitem by a single integer number""" shape = F.shape(data) if not shape: - return const_utils.raise_index_error("When tensor is indexed by an integer,\ - the dimension of the tensor cannot be 0.") + return const_utils.raise_type_error("When tensor is indexed by an integer," + "the dimension of the tensor cannot be 0.") if number >= shape[0]: return const_utils.raise_index_error("index {} is out of bounds for axis 0 with size {}".format( number, shape[0])) diff --git a/mindspore/ops/composite/multitype_ops/_constexpr_utils.py b/mindspore/ops/composite/multitype_ops/_constexpr_utils.py index 519546af0d..88a900407a 100644 --- a/mindspore/ops/composite/multitype_ops/_constexpr_utils.py +++ b/mindspore/ops/composite/multitype_ops/_constexpr_utils.py @@ -44,6 +44,10 @@ SET_ITEM_BY_TUPLE_OF_TENSOR = 1 def raise_index_error(msg): raise IndexError(msg) +@constexpr +def raise_type_error(msg): + raise TypeError(msg) + @constexpr def check_equal(param1, param2, msg="{},{}"): From b01f47a092f77c39d140cab74671f2e7f6fa21b9 Mon Sep 17 00:00:00 2001 From: panfengfeng Date: Mon, 29 Jun 2020 20:53:04 +0800 Subject: [PATCH 147/254] fix mobilenetv3 scripts --- model_zoo/mobilenetv3/src/dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/model_zoo/mobilenetv3/src/dataset.py b/model_zoo/mobilenetv3/src/dataset.py index ae5070e359..d2fa12f793 100644 --- a/model_zoo/mobilenetv3/src/dataset.py +++ b/model_zoo/mobilenetv3/src/dataset.py @@ -62,7 +62,7 @@ def create_dataset(dataset_path, do_train, config, platform, repeat_num=1, batch resize_crop_op = C.RandomCropDecodeResize(resize_height, scale=(0.08, 1.0), ratio=(0.75, 1.333)) horizontal_flip_op = C.RandomHorizontalFlip(prob=0.5) - resize_op = C.Resize((256, 256)) + resize_op = C.Resize(256) center_crop = C.CenterCrop(resize_width) rescale_op = C.RandomColorAdjust(brightness=0.4, contrast=0.4, saturation=0.4) normalize_op = C.Normalize(mean=[0.485*255, 0.456*255, 0.406*255], std=[0.229*255, 0.224*255, 0.225*255]) From 0e1c21d6b3735683ca613909660807681c9ca3a2 Mon Sep 17 00:00:00 2001 From: luopengting Date: Mon, 29 Jun 2020 17:48:11 +0800 Subject: [PATCH 148/254] fix batch_size, collected batch_num before --- mindspore/train/callback/_summary_collector.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mindspore/train/callback/_summary_collector.py b/mindspore/train/callback/_summary_collector.py index 41bda092a5..4752e4500f 100644 --- a/mindspore/train/callback/_summary_collector.py +++ b/mindspore/train/callback/_summary_collector.py @@ -587,7 +587,6 @@ class SummaryCollector(Callback): train_lineage[LineageMetadata.step_num] = cb_params.cur_step_num train_lineage[LineageMetadata.parallel_mode] = cb_params.parallel_mode train_lineage[LineageMetadata.device_num] = cb_params.device_number - train_lineage[LineageMetadata.batch_size] = cb_params.batch_num ckpt_file_path = self._get_ckpt_file_path(cb_params) train_lineage[LineageMetadata.model_path] = json.dumps(dict(ckpt=ckpt_file_path)) @@ -673,6 +672,8 @@ class SummaryCollector(Callback): batch_size = dataset.get_batch_size() dataset_size = int(batch_num * batch_size) + lineage_dict[LineageMetadata.batch_size] = batch_size + if cb_params.mode == ModeEnum.TRAIN.value: lineage_dict[LineageMetadata.train_dataset_path] = dataset_dir lineage_dict[LineageMetadata.train_dataset_size] = dataset_size From 49c9bd769bca9fbbeec17a421d63ea5742fdf985 Mon Sep 17 00:00:00 2001 From: zhoufeng Date: Mon, 29 Jun 2020 17:46:32 +0800 Subject: [PATCH 149/254] vm no process empty graph Signed-off-by: zhoufeng --- mindspore/ccsrc/session/ascend_session.cc | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/mindspore/ccsrc/session/ascend_session.cc b/mindspore/ccsrc/session/ascend_session.cc index a8626a202b..288fe0bbb8 100644 --- a/mindspore/ccsrc/session/ascend_session.cc +++ b/mindspore/ccsrc/session/ascend_session.cc @@ -289,6 +289,12 @@ GraphId AscendSession::CompileGraph(NotNull func_graph) { std::vector all_graphs; auto root_graph = ConstructKernelGraph(func_graph, &all_graphs); BackendOptimization(all_graphs); + // empty graph dont entry to backend + if (root_graph->execution_order().empty()) { + MS_LOG(INFO) << root_graph->ToString() << " is empty graph."; + InitRuntimeResource(); + return root_graph->graph_id(); + } // split switch SplitGraphs(NOT_NULL(root_graph)); // insert goto labels and label_sets From 03c57a1e8b72d0996b9429da92dde829c3f0c2ad Mon Sep 17 00:00:00 2001 From: gengdongjie Date: Mon, 29 Jun 2020 22:01:52 +0800 Subject: [PATCH 150/254] add warpctc to modelzoo --- mindspore/ops/_grad/grad_nn_ops.py | 2 +- .../tbe/basic_lstm_cell_c_state_grad.py | 4 +- model_zoo/warpctc/README.md | 137 ++++++++++++++++++ model_zoo/warpctc/eval.py | 65 +++++++++ model_zoo/warpctc/process_data.py | 71 +++++++++ .../warpctc/scripts/run_distribute_train.sh | 62 ++++++++ model_zoo/warpctc/scripts/run_eval.sh | 60 ++++++++ model_zoo/warpctc/scripts/run_process_data.sh | 20 +++ .../warpctc/scripts/run_standalone_train.sh | 54 +++++++ model_zoo/warpctc/src/config.py | 31 ++++ model_zoo/warpctc/src/dataset.py | 92 ++++++++++++ model_zoo/warpctc/src/loss.py | 49 +++++++ model_zoo/warpctc/src/lr_schedule.py | 36 +++++ model_zoo/warpctc/src/metric.py | 89 ++++++++++++ model_zoo/warpctc/src/warpctc.py | 90 ++++++++++++ model_zoo/warpctc/src/warpctc_for_train.py | 114 +++++++++++++++ model_zoo/warpctc/train.py | 84 +++++++++++ 17 files changed, 1057 insertions(+), 3 deletions(-) create mode 100644 model_zoo/warpctc/README.md create mode 100755 model_zoo/warpctc/eval.py create mode 100755 model_zoo/warpctc/process_data.py create mode 100755 model_zoo/warpctc/scripts/run_distribute_train.sh create mode 100755 model_zoo/warpctc/scripts/run_eval.sh create mode 100755 model_zoo/warpctc/scripts/run_process_data.sh create mode 100755 model_zoo/warpctc/scripts/run_standalone_train.sh create mode 100755 model_zoo/warpctc/src/config.py create mode 100755 model_zoo/warpctc/src/dataset.py create mode 100755 model_zoo/warpctc/src/loss.py create mode 100755 model_zoo/warpctc/src/lr_schedule.py create mode 100755 model_zoo/warpctc/src/metric.py create mode 100755 model_zoo/warpctc/src/warpctc.py create mode 100755 model_zoo/warpctc/src/warpctc_for_train.py create mode 100755 model_zoo/warpctc/train.py diff --git a/mindspore/ops/_grad/grad_nn_ops.py b/mindspore/ops/_grad/grad_nn_ops.py index 00b9e3051b..107de1768c 100755 --- a/mindspore/ops/_grad/grad_nn_ops.py +++ b/mindspore/ops/_grad/grad_nn_ops.py @@ -716,7 +716,7 @@ def get_bprop_basic_lstm_cell(self): def bprop(x, h, c, w, b, out, dout): _, _, it, jt, ft, ot, tanhct = out dct, dht, _, _, _, _, _ = dout - dgate, dct_1 = basic_lstm_cell_cstate_grad(c, dht, dct, it, ft, jt, ot, tanhct) + dgate, dct_1 = basic_lstm_cell_cstate_grad(c, dht, dct, it, jt, ft, ot, tanhct) dxt, dht = basic_lstm_cell_input_grad(dgate, w) dw, db = basic_lstm_cell_weight_grad(F.depend(x, dxt), h, dgate) return dxt, dht, dct_1, dw, db diff --git a/mindspore/ops/_op_impl/tbe/basic_lstm_cell_c_state_grad.py b/mindspore/ops/_op_impl/tbe/basic_lstm_cell_c_state_grad.py index 440b1ce2c7..1e42c1d6fe 100644 --- a/mindspore/ops/_op_impl/tbe/basic_lstm_cell_c_state_grad.py +++ b/mindspore/ops/_op_impl/tbe/basic_lstm_cell_c_state_grad.py @@ -29,8 +29,8 @@ basic_lstm_cell_c_state_grad_op_info = TBERegOp("BasicLSTMCellCStateGrad") \ .input(1, "dht", False, "required", "all") \ .input(2, "dct", False, "required", "all") \ .input(3, "it", False, "required", "all") \ - .input(4, "ft", False, "required", "all") \ - .input(5, "jt", False, "required", "all") \ + .input(4, "jt", False, "required", "all") \ + .input(5, "ft", False, "required", "all") \ .input(6, "ot", False, "required", "all") \ .input(7, "tanhct", False, "required", "all") \ .output(0, "dgate", False, "required", "all") \ diff --git a/model_zoo/warpctc/README.md b/model_zoo/warpctc/README.md new file mode 100644 index 0000000000..cb941255bf --- /dev/null +++ b/model_zoo/warpctc/README.md @@ -0,0 +1,137 @@ +# Warpctc Example + +## Description + +These is an example of training Warpctc with self-generated captcha image dataset in MindSpore. + +## Requirements + +- Install [MindSpore](https://www.mindspore.cn/install/en). + +- Generate captcha images. + +> The [captcha](https://github.com/lepture/captcha) library can be used to generate captcha images. You can generate the train and test dataset by yourself or just run the script `scripts/run_process_data.sh`. By default, the shell script will generate 10000 test images and 50000 train images separately. +> ``` +> $ cd scripts +> $ sh run_process_data.sh +> +> # after execution, you will find the dataset like the follows: +> . +> └─warpctc +> └─data +> ├─ train # train dataset +> └─ test # evaluate dataset +> ... + + +## Structure + +```shell +. +└──warpct + ├── README.md + ├── script + ├── run_distribute_train.sh # launch distributed training(8 pcs) + ├── run_eval.sh # launch evaluation + ├── run_process_data.sh # launch dataset generation + └── run_standalone_train.sh # launch standalone training(1 pcs) + ├── src + ├── config.py # parameter configuration + ├── dataset.py # data preprocessing + ├── loss.py # ctcloss definition + ├── lr_generator.py # generate learning rate for each step + ├── metric.py # accuracy metric for warpctc network + ├── warpctc.py # warpctc network definition + └── warpctc_for_train.py # warp network with grad, loss and gradient clip + ├── eval.py # eval net + ├── process_data.py # dataset generation script + └── train.py # train net +``` + + +## Parameter configuration + +Parameters for both training and evaluation can be set in config.py. + +``` +"max_captcha_digits": 4, # max number of digits in each +"captcha_width": 160, # width of captcha images +"captcha_height": 64, # height of capthca images +"batch_size": 64, # batch size of input tensor +"epoch_size": 30, # only valid for taining, which is always 1 for inference +"hidden_size": 512, # hidden size in LSTM layers +"learning_rate": 0.01, # initial learning rate +"momentum": 0.9 # momentum of SGD optimizer +"save_checkpoint": True, # whether save checkpoint or not +"save_checkpoint_steps": 98, # the step interval between two checkpoints. By default, the last checkpoint will be saved after the last step +"keep_checkpoint_max": 30, # only keep the last keep_checkpoint_max checkpoint +"save_checkpoint_path": "./", # path to save checkpoint +``` + +## Running the example + +### Train + +#### Usage + +``` +# distributed training +Usage: sh run_distribute_train.sh [MINDSPORE_HCCL_CONFIG_PATH] [DATASET_PATH] + +# standalone training +Usage: sh run_standalone_train.sh [DATASET_PATH] +``` + + +#### Launch + +``` +# distribute training example +sh run_distribute_train.sh rank_table.json ../data/train + +# standalone training example +sh run_standalone_train.sh ../data/train +``` + +> About rank_table.json, you can refer to the [distributed training tutorial](https://www.mindspore.cn/tutorial/en/master/advanced_use/distributed_training.html). + +#### Result + +Training result will be stored in folder `scripts`, whose name begins with "train" or "train_parallel". Under this, you can find checkpoint file together with result like the followings in log. + +``` +# distribute training result(8 pcs) +Epoch: [ 1/ 30], step: [ 98/ 98], loss: [0.5853/0.5853], time: [376813.7944] +Epoch: [ 2/ 30], step: [ 98/ 98], loss: [0.4007/0.4007], time: [75882.0951] +Epoch: [ 3/ 30], step: [ 98/ 98], loss: [0.0921/0.0921], time: [75150.9385] +Epoch: [ 4/ 30], step: [ 98/ 98], loss: [0.1472/0.1472], time: [75135.0193] +Epoch: [ 5/ 30], step: [ 98/ 98], loss: [0.0186/0.0186], time: [75199.5809] +... +``` + + +### Evaluation + +#### Usage + +``` +# evaluation +Usage: sh run_eval.sh [DATASET_PATH] [CHECKPOINT_PATH] +``` + +#### Launch + +``` +# evaluation example +sh run_eval.sh ../data/test warpctc-30-98.ckpt +``` + +> checkpoint can be produced in training process. + +#### Result + +Evaluation result will be stored in the example path, whose folder name is "eval". Under this, you can find result like the followings in log. + +``` +result: {'WarpCTCAccuracy': 0.9901472929936306} +``` diff --git a/model_zoo/warpctc/eval.py b/model_zoo/warpctc/eval.py new file mode 100755 index 0000000000..df62c7c755 --- /dev/null +++ b/model_zoo/warpctc/eval.py @@ -0,0 +1,65 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Warpctc evaluation""" +import os +import math as m +import random +import argparse +import numpy as np +from mindspore import context +from mindspore import dataset as de +from mindspore.train.model import Model +from mindspore.train.serialization import load_checkpoint, load_param_into_net + +from src.loss import CTCLoss +from src.config import config as cf +from src.dataset import create_dataset +from src.warpctc import StackedRNN +from src.metric import WarpCTCAccuracy + +random.seed(1) +np.random.seed(1) +de.config.set_seed(1) + +parser = argparse.ArgumentParser(description="Warpctc training") +parser.add_argument("--dataset_path", type=str, default=None, help="Dataset, default is None.") +parser.add_argument("--checkpoint_path", type=str, default=None, help="checkpoint file path, default is None") +args_opt = parser.parse_args() + +device_id = int(os.getenv('DEVICE_ID')) +context.set_context(mode=context.GRAPH_MODE, + device_target="Ascend", + save_graphs=False, + device_id=device_id) + +if __name__ == '__main__': + max_captcha_digits = cf.max_captcha_digits + input_size = m.ceil(cf.captcha_height / 64) * 64 * 3 + # create dataset + dataset = create_dataset(dataset_path=args_opt.dataset_path, repeat_num=1, batch_size=cf.batch_size) + step_size = dataset.get_dataset_size() + # define loss + loss = CTCLoss(max_sequence_length=cf.captcha_width, max_label_length=max_captcha_digits, batch_size=cf.batch_size) + # define net + net = StackedRNN(input_size=input_size, batch_size=cf.batch_size, hidden_size=cf.hidden_size) + # load checkpoint + param_dict = load_checkpoint(args_opt.checkpoint_path) + load_param_into_net(net, param_dict) + net.set_train(False) + # define model + model = Model(net, loss_fn=loss, metrics={'WarpCTCAccuracy': WarpCTCAccuracy()}) + # start evaluation + res = model.eval(dataset) + print("result:", res, flush=True) diff --git a/model_zoo/warpctc/process_data.py b/model_zoo/warpctc/process_data.py new file mode 100755 index 0000000000..567ad10933 --- /dev/null +++ b/model_zoo/warpctc/process_data.py @@ -0,0 +1,71 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Generate train and test dataset""" +import os +import math as m +import random +from multiprocessing import Process +from captcha.image import ImageCaptcha + + +def _generate_captcha_per_process(path, total, start, end, img_width, img_height, max_digits): + captcha = ImageCaptcha(width=img_width, height=img_height) + filename_head = '{:0>' + str(len(str(total))) + '}-' + for i in range(start, end): + digits = '' + digits_length = random.randint(1, max_digits) + for _ in range(0, digits_length): + integer = random.randint(0, 9) + digits += str(integer) + captcha.write(digits, os.path.join(path, filename_head.format(i) + digits + '.png')) + + +def generate_captcha(name, img_num, img_width, img_height, max_digits, process_num=16): + """ + generate captcha images + + Args: + name(str): name of folder, under which captcha images are saved in + img_num(int): number of generated captcha images + img_width(int): width of generated captcha images + img_height(int): height of generated captcha images + max_digits(int): max number of digits in each captcha images. For each captcha images, number of digits is in + range [1,max_digits] + process_num(int): number of process to generate captcha images, default is 16 + """ + cur_script_path = os.path.dirname(os.path.realpath(__file__)) + path = os.path.join(cur_script_path, "data", name) + print("Generating dataset [{}] under {}...".format(name, path)) + if os.path.exists(path): + os.system("rm -rf {}".format(path)) + os.system("mkdir -p {}".format(path)) + img_num_per_thread = m.ceil(img_num / process_num) + + processes = [] + for i in range(process_num): + start = i * img_num_per_thread + end = start + img_num_per_thread if i != (process_num - 1) else img_num + p = Process(target=_generate_captcha_per_process, + args=(path, img_num, start, end, img_width, img_height, max_digits)) + p.start() + processes.append(p) + for p in processes: + p.join() + print("Generating dataset [{}] finished, total number is {}!".format(name, img_num)) + + +if __name__ == '__main__': + generate_captcha("test", img_num=10000, img_width=160, img_height=64, max_digits=4) + generate_captcha("train", img_num=50000, img_width=160, img_height=64, max_digits=4) diff --git a/model_zoo/warpctc/scripts/run_distribute_train.sh b/model_zoo/warpctc/scripts/run_distribute_train.sh new file mode 100755 index 0000000000..3cebf6d195 --- /dev/null +++ b/model_zoo/warpctc/scripts/run_distribute_train.sh @@ -0,0 +1,62 @@ +#!/bin/bash +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +if [ $# != 2 ]; then + echo "Usage: sh run_distribute_train.sh [MINDSPORE_HCCL_CONFIG_PATH] [DATASET_PATH]" + exit 1 +fi + +get_real_path() { + if [ "${1:0:1}" == "/" ]; then + echo "$1" + else + echo "$(realpath -m $PWD/$1)" + fi +} + +PATH1=$(get_real_path $1) +PATH2=$(get_real_path $2) + +if [ ! -f $PATH1 ]; then + echo "error: MINDSPORE_HCCL_CONFIG_PATH=$PATH1 is not a file" + exit 1 +fi + +if [ ! -d $PATH2 ]; then + echo "error: DATASET_PATH=$PATH2 is not a directory" + exit 1 +fi + +ulimit -u unlimited +export DEVICE_NUM=8 +export RANK_SIZE=8 +export MINDSPORE_HCCL_CONFIG_PATH=$PATH1 +export RANK_TABLE_FILE=$PATH1 + +for ((i = 0; i < ${DEVICE_NUM}; i++)); do + export DEVICE_ID=$i + export RANK_ID=$i + rm -rf ./train_parallel$i + mkdir ./train_parallel$i + cp ../*.py ./train_parallel$i + cp *.sh ./train_parallel$i + cp -r ../src ./train_parallel$i + cd ./train_parallel$i || exit + echo "start training for rank $RANK_ID, device $DEVICE_ID" + env >env.log + python train.py --run_distribute=True --device_num=$DEVICE_NUM --dataset_path=$PATH2 &>log & + cd .. +done diff --git a/model_zoo/warpctc/scripts/run_eval.sh b/model_zoo/warpctc/scripts/run_eval.sh new file mode 100755 index 0000000000..659de6d72a --- /dev/null +++ b/model_zoo/warpctc/scripts/run_eval.sh @@ -0,0 +1,60 @@ +#!/bin/bash +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +if [ $# != 2 ]; then + echo "Usage: sh run_eval.sh [DATASET_PATH] [CHECKPOINT_PATH]" + exit 1 +fi + +get_real_path() { + if [ "${1:0:1}" == "/" ]; then + echo "$1" + else + echo "$(realpath -m $PWD/$1)" + fi +} + +PATH1=$(get_real_path $1) +PATH2=$(get_real_path $2) + +if [ ! -d $PATH1 ]; then + echo "error: DATASET_PATH=$PATH1 is not a directory" + exit 1 +fi + +if [ ! -f $PATH2 ]; then + echo "error: CHECKPOINT_PATH=$PATH2 is not a file" + exit 1 +fi + +ulimit -u unlimited +export DEVICE_NUM=1 +export DEVICE_ID=0 +export RANK_SIZE=$DEVICE_NUM +export RANK_ID=0 + +if [ -d "eval" ]; then + rm -rf ./eval +fi +mkdir ./eval +cp ../*.py ./eval +cp *.sh ./eval +cp -r ../src ./eval +cd ./eval || exit +env >env.log +echo "start evaluation for device $DEVICE_ID" +python eval.py --dataset_path=$PATH1 --checkpoint_path=$PATH2 &>log & +cd .. diff --git a/model_zoo/warpctc/scripts/run_process_data.sh b/model_zoo/warpctc/scripts/run_process_data.sh new file mode 100755 index 0000000000..56b89f1a72 --- /dev/null +++ b/model_zoo/warpctc/scripts/run_process_data.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +CUR_PATH=$(dirname $PWD/$0) +cd $CUR_PATH/../ && + python process_data.py && + cd - || exit \ No newline at end of file diff --git a/model_zoo/warpctc/scripts/run_standalone_train.sh b/model_zoo/warpctc/scripts/run_standalone_train.sh new file mode 100755 index 0000000000..22a16ef4c8 --- /dev/null +++ b/model_zoo/warpctc/scripts/run_standalone_train.sh @@ -0,0 +1,54 @@ +#!/bin/bash +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +if [ $# != 1 ]; then + echo "Usage: sh run_standalone_train.sh [DATASET_PATH]" + exit 1 +fi + +get_real_path() { + if [ "${1:0:1}" == "/" ]; then + echo "$1" + else + echo "$(realpath -m $PWD/$1)" + fi +} + +PATH1=$(get_real_path $1) + +if [ ! -d $PATH1 ]; then + echo "error: DATASET_PATH=$PATH1 is not a directory" + exit 1 +fi + +ulimit -u unlimited +export DEVICE_NUM=1 +export DEVICE_ID=0 +export RANK_ID=0 +export RANK_SIZE=1 + +if [ -d "train" ]; then + rm -rf ./train +fi +mkdir ./train +cp ../*.py ./train +cp *.sh ./train +cp -r ../src ./train +cd ./train || exit +echo "start training for device $DEVICE_ID" +env >env.log +python train.py --dataset=$PATH1 &>log & +cd .. diff --git a/model_zoo/warpctc/src/config.py b/model_zoo/warpctc/src/config.py new file mode 100755 index 0000000000..ed9c2968de --- /dev/null +++ b/model_zoo/warpctc/src/config.py @@ -0,0 +1,31 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Network parameters.""" +from easydict import EasyDict + +config = EasyDict({ + "max_captcha_digits": 4, + "captcha_width": 160, + "captcha_height": 64, + "batch_size": 64, + "epoch_size": 30, + "hidden_size": 512, + "learning_rate": 0.01, + "momentum": 0.9, + "save_checkpoint": True, + "save_checkpoint_steps": 98, + "keep_checkpoint_max": 30, + "save_checkpoint_path": "./", +}) diff --git a/model_zoo/warpctc/src/dataset.py b/model_zoo/warpctc/src/dataset.py new file mode 100755 index 0000000000..76e592b906 --- /dev/null +++ b/model_zoo/warpctc/src/dataset.py @@ -0,0 +1,92 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Dataset preprocessing.""" +import os +import math as m +import numpy as np +import mindspore.common.dtype as mstype +import mindspore.dataset.engine as de +import mindspore.dataset.transforms.c_transforms as c +import mindspore.dataset.transforms.vision.c_transforms as vc +from PIL import Image +from src.config import config as cf + + +class _CaptchaDataset(): + """ + create train or evaluation dataset for warpctc + + Args: + img_root_dir(str): root path of images + max_captcha_digits(int): max number of digits in images. + blank(int): value reserved for blank label, default is 10. When parsing label from image file names, if label + length is less than max_captcha_digits, the remaining labels are padding with blank. + """ + + def __init__(self, img_root_dir, max_captcha_digits, blank=10): + if not os.path.exists(img_root_dir): + raise RuntimeError("the input image dir {} is invalid!".format(img_root_dir)) + self.img_root_dir = img_root_dir + self.img_names = [i for i in os.listdir(img_root_dir) if i.endswith('.png')] + self.max_captcha_digits = max_captcha_digits + self.blank = blank + + def __len__(self): + return len(self.img_names) + + def __getitem__(self, item): + img_name = self.img_names[item] + im = Image.open(os.path.join(self.img_root_dir, img_name)) + r, g, b = im.split() + im = Image.merge("RGB", (b, g, r)) + image = np.array(im) + label_str = os.path.splitext(img_name)[0] + label_str = label_str[label_str.find('-') + 1:] + label = [int(i) for i in label_str] + label.extend([int(self.blank)] * (self.max_captcha_digits - len(label))) + label = np.array(label) + return image, label + + +def create_dataset(dataset_path, repeat_num=1, batch_size=1): + """ + create train or evaluation dataset for warpctc + + Args: + dataset_path(int): dataset path + repeat_num(int): dataset repetition num, default is 1 + batch_size(int): batch size of generated dataset, default is 1 + """ + rank_size = int(os.environ.get("RANK_SIZE")) if os.environ.get("RANK_SIZE") else 1 + rank_id = int(os.environ.get("RANK_ID")) if os.environ.get("RANK_ID") else 0 + + dataset = _CaptchaDataset(dataset_path, cf.max_captcha_digits) + ds = de.GeneratorDataset(dataset, ["image", "label"], shuffle=True, num_shards=rank_size, shard_id=rank_id) + ds.set_dataset_size(m.ceil(len(dataset) / rank_size)) + image_trans = [ + vc.Rescale(1.0 / 255.0, 0.0), + vc.Normalize([0.9010, 0.9049, 0.9025], std=[0.1521, 0.1347, 0.1458]), + vc.Resize((m.ceil(cf.captcha_height / 16) * 16, cf.captcha_width)), + vc.HWC2CHW() + ] + label_trans = [ + c.TypeCast(mstype.int32) + ] + ds = ds.map(input_columns=["image"], num_parallel_workers=8, operations=image_trans) + ds = ds.map(input_columns=["label"], num_parallel_workers=8, operations=label_trans) + + ds = ds.batch(batch_size) + ds = ds.repeat(repeat_num) + return ds diff --git a/model_zoo/warpctc/src/loss.py b/model_zoo/warpctc/src/loss.py new file mode 100755 index 0000000000..8ea4c20e94 --- /dev/null +++ b/model_zoo/warpctc/src/loss.py @@ -0,0 +1,49 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""CTC Loss.""" +import numpy as np +from mindspore.nn.loss.loss import _Loss +from mindspore import Tensor, Parameter +from mindspore.common import dtype as mstype +from mindspore.ops import operations as P + + +class CTCLoss(_Loss): + """ + CTCLoss definition + + Args: + max_sequence_length(int): max number of sequence length. For captcha images, the value is equal to image + width + max_label_length(int): max number of label length for each input. + batch_size(int): batch size of input logits + """ + + def __init__(self, max_sequence_length, max_label_length, batch_size): + super(CTCLoss, self).__init__() + self.sequence_length = Parameter(Tensor(np.array([max_sequence_length] * batch_size), mstype.int32), + name="sequence_length") + labels_indices = [] + for i in range(batch_size): + for j in range(max_label_length): + labels_indices.append([i, j]) + self.labels_indices = Parameter(Tensor(np.array(labels_indices), mstype.int64), name="labels_indices") + self.reshape = P.Reshape() + self.ctc_loss = P.CTCLoss(ctc_merge_repeated=True) + + def construct(self, logit, label): + labels_values = self.reshape(label, (-1,)) + loss, _ = self.ctc_loss(logit, self.labels_indices, labels_values, self.sequence_length) + return loss diff --git a/model_zoo/warpctc/src/lr_schedule.py b/model_zoo/warpctc/src/lr_schedule.py new file mode 100755 index 0000000000..a0ae6c886a --- /dev/null +++ b/model_zoo/warpctc/src/lr_schedule.py @@ -0,0 +1,36 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Learning rate generator.""" + + +def get_lr(epoch_size, step_size, lr_init): + """ + generate learning rate for each step, which decays in every 10 epoch + + Args: + epoch_size(int): total epoch number + step_size(int): total step number in each step + lr_init(int): initial learning rate + + Returns: + List, learning rate array + """ + lr = lr_init + lrs = [] + for i in range(1, epoch_size + 1): + if i % 10 == 0: + lr *= 0.1 + lrs.extend([lr for _ in range(step_size)]) + return lrs diff --git a/model_zoo/warpctc/src/metric.py b/model_zoo/warpctc/src/metric.py new file mode 100755 index 0000000000..d1060d0781 --- /dev/null +++ b/model_zoo/warpctc/src/metric.py @@ -0,0 +1,89 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Metric for accuracy evaluation.""" +from mindspore import nn + +BLANK_LABLE = 10 + + +class WarpCTCAccuracy(nn.Metric): + """ + Define accuracy metric for warpctc network. + """ + + def __init__(self): + super(WarpCTCAccuracy).__init__() + self._correct_num = 0 + self._total_num = 0 + self._count = 0 + + def clear(self): + self._correct_num = 0 + self._total_num = 0 + + def update(self, *inputs): + if len(inputs) != 2: + raise ValueError('WarpCTCAccuracy need 2 inputs (y_pred, y), but got {}'.format(len(inputs))) + + y_pred = self._convert_data(inputs[0]) + y = self._convert_data(inputs[1]) + + self._count += 1 + + pred_lbls = self._get_prediction(y_pred) + + for b_idx, target in enumerate(y): + if self._is_eq(pred_lbls[b_idx], target): + self._correct_num += 1 + self._total_num += 1 + + def eval(self): + if self._total_num == 0: + raise RuntimeError('Accuary can not be calculated, because the number of samples is 0.') + return self._correct_num / self._total_num + + @staticmethod + def _is_eq(pred_lbl, target): + """ + check whether predict label is equal to target label + """ + target = target.tolist() + pred_diff = len(target) - len(pred_lbl) + if pred_diff > 0: + # padding by BLANK_LABLE + pred_lbl.extend([BLANK_LABLE] * pred_diff) + return pred_lbl == target + + @staticmethod + def _get_prediction(y_pred): + """ + parse predict result to labels + """ + seq_len, batch_size, _ = y_pred.shape + indices = y_pred.argmax(axis=2) + + lens = [seq_len] * batch_size + pred_lbls = [] + for i in range(batch_size): + idx = indices[:, i] + last_idx = BLANK_LABLE + pred_lbl = [] + for j in range(lens[i]): + cur_idx = idx[j] + if cur_idx not in [last_idx, BLANK_LABLE]: + pred_lbl.append(cur_idx) + last_idx = cur_idx + pred_lbls.append(pred_lbl) + return pred_lbls diff --git a/model_zoo/warpctc/src/warpctc.py b/model_zoo/warpctc/src/warpctc.py new file mode 100755 index 0000000000..9669fc4bfd --- /dev/null +++ b/model_zoo/warpctc/src/warpctc.py @@ -0,0 +1,90 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Warpctc network definition.""" + +import numpy as np +import mindspore.nn as nn +from mindspore import Tensor, Parameter +from mindspore.common import dtype as mstype +from mindspore.ops import operations as P +from mindspore.ops import functional as F + + +class StackedRNN(nn.Cell): + """ + Define a stacked RNN network which contains two LSTM layers and one full-connect layer. + + Args: + input_size(int): Size of time sequence. Usually, the input_size is equal to three times of image height for + captcha images. + batch_size(int): batch size of input data, default is 64 + hidden_size(int): the hidden size in LSTM layers, default is 512 + """ + def __init__(self, input_size, batch_size=64, hidden_size=512): + super(StackedRNN, self).__init__() + self.batch_size = batch_size + self.input_size = input_size + self.num_classes = 11 + self.reshape = P.Reshape() + self.cast = P.Cast() + k = (1 / hidden_size) ** 0.5 + self.h1 = Tensor(np.zeros(shape=(batch_size, hidden_size)).astype(np.float16)) + self.c1 = Tensor(np.zeros(shape=(batch_size, hidden_size)).astype(np.float16)) + self.w1 = Parameter(np.random.uniform(-k, k, (4 * hidden_size, input_size + hidden_size, 1, 1)) + .astype(np.float16), name="w1") + self.w2 = Parameter(np.random.uniform(-k, k, (4 * hidden_size, hidden_size + hidden_size, 1, 1)) + .astype(np.float16), name="w2") + self.b1 = Parameter(np.random.uniform(-k, k, (4 * hidden_size, 1, 1, 1)).astype(np.float16), name="b1") + self.b2 = Parameter(np.random.uniform(-k, k, (4 * hidden_size, 1, 1, 1)).astype(np.float16), name="b2") + + self.h2 = Tensor(np.zeros(shape=(batch_size, hidden_size)).astype(np.float16)) + self.c2 = Tensor(np.zeros(shape=(batch_size, hidden_size)).astype(np.float16)) + + self.basic_lstm_cell = P.BasicLSTMCell(keep_prob=1.0, forget_bias=0.0, state_is_tuple=True, activation="tanh") + + self.fc_weight = np.random.random((self.num_classes, hidden_size)).astype(np.float32) + self.fc_bias = np.random.random((self.num_classes)).astype(np.float32) + + self.fc = nn.Dense(in_channels=hidden_size, out_channels=self.num_classes, weight_init=Tensor(self.fc_weight), + bias_init=Tensor(self.fc_bias)) + + self.fc.to_float(mstype.float32) + self.expand_dims = P.ExpandDims() + self.concat = P.Concat() + self.transpose = P.Transpose() + + def construct(self, x): + x = self.cast(x, mstype.float16) + x = self.transpose(x, (3, 0, 2, 1)) + x = self.reshape(x, (-1, self.batch_size, self.input_size)) + h1 = self.h1 + c1 = self.c1 + h2 = self.h2 + c2 = self.c2 + + c1, h1, _, _, _, _, _ = self.basic_lstm_cell(x[0, :, :], h1, c1, self.w1, self.b1) + c2, h2, _, _, _, _, _ = self.basic_lstm_cell(h1, h2, c2, self.w2, self.b2) + + h2_after_fc = self.fc(h2) + output = self.expand_dims(h2_after_fc, 0) + for i in range(1, F.shape(x)[0]): + c1, h1, _, _, _, _, _ = self.basic_lstm_cell(x[i, :, :], h1, c1, self.w1, self.b1) + c2, h2, _, _, _, _, _ = self.basic_lstm_cell(h1, h2, c2, self.w2, self.b2) + + h2_after_fc = self.fc(h2) + h2_after_fc = self.expand_dims(h2_after_fc, 0) + output = self.concat((output, h2_after_fc)) + + return output diff --git a/model_zoo/warpctc/src/warpctc_for_train.py b/model_zoo/warpctc/src/warpctc_for_train.py new file mode 100755 index 0000000000..d847f47c62 --- /dev/null +++ b/model_zoo/warpctc/src/warpctc_for_train.py @@ -0,0 +1,114 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Automatic differentiation with grad clip.""" +from mindspore.parallel._utils import (_get_device_num, _get_mirror_mean, + _get_parallel_mode) +from mindspore.train.parallel_utils import ParallelMode +from mindspore.common import dtype as mstype +from mindspore.ops import composite as C +from mindspore.ops import functional as F +from mindspore.ops import operations as P +from mindspore.nn.cell import Cell +from mindspore.nn.wrap.grad_reducer import DistributedGradReducer +import mindspore.nn as nn +from mindspore.common.tensor import Tensor +import numpy as np + +compute_norm = C.MultitypeFuncGraph("compute_norm") + + +@compute_norm.register("Tensor") +def _compute_norm(grad): + norm = nn.Norm() + norm = norm(F.cast(grad, mstype.float32)) + ret = F.expand_dims(F.cast(norm, mstype.float32), 0) + return ret + + +grad_div = C.MultitypeFuncGraph("grad_div") + + +@grad_div.register("Tensor", "Tensor") +def _grad_div(val, grad): + div = P.Div() + mul = P.Mul() + grad = mul(grad, 10.0) + ret = div(grad, val) + return ret + + +class TrainOneStepCellWithGradClip(Cell): + """ + Network training package class. + + Wraps the network with an optimizer. The resulting Cell be trained with input data and label. + Backward graph with grad clip will be created in the construct function to do parameter updating. + Different parallel modes are available to run the training. + + Args: + network (Cell): The training network. + optimizer (Cell): Optimizer for updating the weights. + sens (Number): The scaling number to be filled as the input of backpropagation. Default value is 1.0. + + Inputs: + - data (Tensor) - Tensor of shape :(N, ...). + - label (Tensor) - Tensor of shape :(N, ...). + + Outputs: + Tensor, a scalar Tensor with shape :math:`()`. + """ + + def __init__(self, network, optimizer, sens=1.0): + super(TrainOneStepCellWithGradClip, self).__init__(auto_prefix=False) + self.network = network + self.network.set_grad() + self.network.add_flags(defer_inline=True) + self.weights = optimizer.parameters + self.optimizer = optimizer + self.grad = C.GradOperation('grad', get_by_list=True, sens_param=True) + self.sens = sens + self.reducer_flag = False + self.grad_reducer = None + self.hyper_map = C.HyperMap() + self.greater = P.Greater() + self.select = P.Select() + self.norm = nn.Norm(keep_dims=True) + self.dtype = P.DType() + self.cast = P.Cast() + self.concat = P.Concat(axis=0) + self.ten = Tensor(np.array([10.0]).astype(np.float32)) + parallel_mode = _get_parallel_mode() + if parallel_mode in (ParallelMode.DATA_PARALLEL, ParallelMode.HYBRID_PARALLEL): + self.reducer_flag = True + if self.reducer_flag: + mean = _get_mirror_mean() + degree = _get_device_num() + self.grad_reducer = DistributedGradReducer(optimizer.parameters, mean, degree) + + def construct(self, data, label): + weights = self.weights + loss = self.network(data, label) + sens = P.Fill()(P.DType()(loss), P.Shape()(loss), self.sens) + grads = self.grad(self.network, weights)(data, label, sens) + norm = self.hyper_map(F.partial(compute_norm), grads) + norm = self.concat(norm) + norm = self.norm(norm) + cond = self.greater(norm, self.cast(self.ten, self.dtype(norm))) + clip_val = self.select(cond, norm, self.cast(self.ten, self.dtype(norm))) + grads = self.hyper_map(F.partial(grad_div, clip_val), grads) + if self.reducer_flag: + # apply grad reducer on grads + grads = self.grad_reducer(grads) + return F.depend(loss, self.optimizer(grads)) diff --git a/model_zoo/warpctc/train.py b/model_zoo/warpctc/train.py new file mode 100755 index 0000000000..651d2a73a4 --- /dev/null +++ b/model_zoo/warpctc/train.py @@ -0,0 +1,84 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Warpctc training""" +import os +import math as m +import random +import argparse +import numpy as np +import mindspore.nn as nn +from mindspore import context +from mindspore import dataset as de +from mindspore.train.model import Model, ParallelMode +from mindspore.nn.wrap import WithLossCell +from mindspore.train.callback import TimeMonitor, LossMonitor, CheckpointConfig, ModelCheckpoint +from mindspore.communication.management import init + +from src.loss import CTCLoss +from src.config import config as cf +from src.dataset import create_dataset +from src.warpctc import StackedRNN +from src.warpctc_for_train import TrainOneStepCellWithGradClip +from src.lr_schedule import get_lr + +random.seed(1) +np.random.seed(1) +de.config.set_seed(1) + +parser = argparse.ArgumentParser(description="Warpctc training") +parser.add_argument("--run_distribute", type=bool, default=False, help="Run distribute, default is false.") +parser.add_argument('--device_num', type=int, default=1, help='Device num, default is 1.') +parser.add_argument('--dataset_path', type=str, default=None, help='Dataset path, default is None') +args_opt = parser.parse_args() + +device_id = int(os.getenv('DEVICE_ID')) +context.set_context(mode=context.GRAPH_MODE, + device_target="Ascend", + save_graphs=False, + device_id=device_id) + +if __name__ == '__main__': + if args_opt.run_distribute: + context.reset_auto_parallel_context() + context.set_auto_parallel_context(device_num=args_opt.device_num, + parallel_mode=ParallelMode.DATA_PARALLEL, + mirror_mean=True) + init() + max_captcha_digits = cf.max_captcha_digits + input_size = m.ceil(cf.captcha_height / 64) * 64 * 3 + # create dataset + dataset = create_dataset(dataset_path=args_opt.dataset_path, repeat_num=cf.epoch_size, batch_size=cf.batch_size) + step_size = dataset.get_dataset_size() + # define lr + lr_init = cf.learning_rate if not args_opt.run_distribute else cf.learning_rate * args_opt.device_num + lr = get_lr(cf.epoch_size, step_size, lr_init) + # define loss + loss = CTCLoss(max_sequence_length=cf.captcha_width, max_label_length=max_captcha_digits, batch_size=cf.batch_size) + # define net + net = StackedRNN(input_size=input_size, batch_size=cf.batch_size, hidden_size=cf.hidden_size) + # define opt + opt = nn.SGD(params=net.trainable_params(), learning_rate=lr, momentum=cf.momentum) + net = WithLossCell(net, loss) + net = TrainOneStepCellWithGradClip(net, opt).set_train() + # define model + model = Model(net) + # define callbacks + callbacks = [LossMonitor(), TimeMonitor(data_size=step_size)] + if cf.save_checkpoint: + config_ck = CheckpointConfig(save_checkpoint_steps=cf.save_checkpoint_steps, + keep_checkpoint_max=cf.keep_checkpoint_max) + ckpt_cb = ModelCheckpoint(prefix="waptctc", directory=cf.save_checkpoint_path, config=config_ck) + callbacks.append(ckpt_cb) + model.train(cf.epoch_size, dataset, callbacks=callbacks) From 5cd313635583ab9bfa034e41126c0da387b17c26 Mon Sep 17 00:00:00 2001 From: tinazhang66 Date: Thu, 25 Jun 2020 11:31:35 -0400 Subject: [PATCH 151/254] remove local defined mse and add missing mse/md5 validation --- .../dataset/golden/cut_out_01_c_result.npz | Bin 0 -> 644 bytes .../dataset/golden/cut_out_01_py_result.npz | Bin 0 -> 644 bytes .../dataset/golden/equalize_01_result.npz | Bin 0 -> 713 bytes .../dataset/golden/five_crop_01_result.npz | Bin 0 -> 644 bytes .../data/dataset/golden/invert_01_result.npz | Bin 0 -> 713 bytes .../data/dataset/golden/pad_01_c_result.npz | Bin 0 -> 644 bytes .../data/dataset/golden/pad_01_py_result.npz | Bin 0 -> 644 bytes .../dataset/golden/random_color_01_result.npz | Bin 0 -> 713 bytes .../random_color_adjust_01_c_result.npz | Bin 0 -> 644 bytes .../random_color_adjust_01_py_result.npz | Bin 0 -> 644 bytes .../random_crop_decode_resize_01_result.npz | Bin 0 -> 644 bytes .../golden/random_erasing_01_result.npz | Bin 0 -> 644 bytes .../golden/random_resize_01_result.npz | Bin 0 -> 644 bytes .../golden/random_rotation_01_c_result.npz | Bin 0 -> 644 bytes .../golden/random_rotation_01_py_result.npz | Bin 0 -> 817 bytes .../golden/random_sharpness_01_result.npz | Bin 0 -> 713 bytes .../data/dataset/golden/rescale_01_result.npz | Bin 0 -> 644 bytes tests/ut/python/dataset/test_autocontrast.py | 4 +- tests/ut/python/dataset/test_cut_out.py | 105 ++++++++++++++-- tests/ut/python/dataset/test_equalize.py | 24 +++- tests/ut/python/dataset/test_five_crop.py | 26 +++- tests/ut/python/dataset/test_invert.py | 23 +++- .../dataset/test_linear_transformation.py | 32 ++--- tests/ut/python/dataset/test_pad.py | 36 +++++- tests/ut/python/dataset/test_random_color.py | 45 +++++-- .../dataset/test_random_color_adjust.py | 40 +++++- .../dataset/test_random_crop_and_resize.py | 2 + .../dataset/test_random_crop_decode_resize.py | 42 +++++-- .../ut/python/dataset/test_random_erasing.py | 31 ++++- .../dataset/test_random_horizontal_flip.py | 3 +- tests/ut/python/dataset/test_random_resize.py | 30 ++++- .../ut/python/dataset/test_random_rotation.py | 116 ++++++++++++++++-- .../python/dataset/test_random_sharpness.py | 52 ++++++-- .../dataset/test_random_vertical_flip.py | 7 +- tests/ut/python/dataset/test_rescale_op.py | 25 +++- .../ut/python/dataset/test_uniform_augment.py | 6 +- 36 files changed, 565 insertions(+), 84 deletions(-) create mode 100644 tests/ut/data/dataset/golden/cut_out_01_c_result.npz create mode 100644 tests/ut/data/dataset/golden/cut_out_01_py_result.npz create mode 100644 tests/ut/data/dataset/golden/equalize_01_result.npz create mode 100644 tests/ut/data/dataset/golden/five_crop_01_result.npz create mode 100644 tests/ut/data/dataset/golden/invert_01_result.npz create mode 100644 tests/ut/data/dataset/golden/pad_01_c_result.npz create mode 100644 tests/ut/data/dataset/golden/pad_01_py_result.npz create mode 100644 tests/ut/data/dataset/golden/random_color_01_result.npz create mode 100644 tests/ut/data/dataset/golden/random_color_adjust_01_c_result.npz create mode 100644 tests/ut/data/dataset/golden/random_color_adjust_01_py_result.npz create mode 100644 tests/ut/data/dataset/golden/random_crop_decode_resize_01_result.npz create mode 100644 tests/ut/data/dataset/golden/random_erasing_01_result.npz create mode 100644 tests/ut/data/dataset/golden/random_resize_01_result.npz create mode 100644 tests/ut/data/dataset/golden/random_rotation_01_c_result.npz create mode 100644 tests/ut/data/dataset/golden/random_rotation_01_py_result.npz create mode 100644 tests/ut/data/dataset/golden/random_sharpness_01_result.npz create mode 100644 tests/ut/data/dataset/golden/rescale_01_result.npz diff --git a/tests/ut/data/dataset/golden/cut_out_01_c_result.npz b/tests/ut/data/dataset/golden/cut_out_01_c_result.npz new file mode 100644 index 0000000000000000000000000000000000000000..ce8c3237989efb6fe253511775f26d1d079193d5 GIT binary patch literal 644 zcmWIWW@Zs#fB;1XiEBy5e;64UK$w$3gdwr0DBeIXub`5VK>#cWQV5a+fysWMz5$Vp z3}p<}>M5zk$wlf`3hFif>N*PQY57GZMTvRw`9&$IAYr$}oZ?iVcyUHzK`M~1VW^{E zq^YA&t3W>BYG6*zE6pva)Jx7UO4Z9P%_+$Qx;L?sE50Z-IX|zsq^LBxgsYGNqKYdo z1tMF>=*`et$mGnJRLI<3$P!e@s^QHDgiLLPY*3{sC6xuKT!rirOh9k?TNHA5Gk7z$ z7jgy_a%p%oM}YiaQ^@V-=jZhw2*8B5eMuotQfF)-uSNtbP(Cv^F+H`AFSd|hBZDCW zY^6X3&_2PSLLrcSEUksY84wH7ObSIH7KjEFioqyOd`y< f5)RP+P|yHQEAV6#;LXYgl3@fwb0BRF3Sb5R;WVhc literal 0 HcmV?d00001 diff --git a/tests/ut/data/dataset/golden/cut_out_01_py_result.npz b/tests/ut/data/dataset/golden/cut_out_01_py_result.npz new file mode 100644 index 0000000000000000000000000000000000000000..036c3b20bdf1308207e2103455e72414d20fc6b0 GIT binary patch literal 644 zcmWIWW@Zs#fB;2?E=`frKa30vAk4`i!jM>06mOuHS5V2wAOIEwDFjJ^z+}Hr-+)L) zhBAg~^_0}&LuqFrRwFD=9FXt-J4j+6ix}*!soX6ExB+^qENo1 zPywz-5u`^cs8AWIM+L0M>u0gd_T?S1hVyRU+H=3)OrdH?p;}UEYDuAbl3su}Ba;X- gu7m^hKNK{8(+WHp1$eWvfn*qg&>Tpcg94ZV0HaB*K>z>% literal 0 HcmV?d00001 diff --git a/tests/ut/data/dataset/golden/equalize_01_result.npz b/tests/ut/data/dataset/golden/equalize_01_result.npz new file mode 100644 index 0000000000000000000000000000000000000000..8636a3cc550e8d76f5def85174db9faff065c6ac GIT binary patch literal 713 zcmWIWW@Zs#fB;2?OF|njIWjRYfG{V62t#5~QM`d(UO^=zg8*0%q!1(t0+anheFGvH z8Oj){)l*W7lZ(`?6x3_{)pZoq)AEZ-iW2kU^NUhaLBei{ImM|!@#2icf>a=1!$?QL zNK;3lR)KuL)xeybSDIT;sh6Bzl&Y6onp2VqbZ=rMSA0=wa(-TMNl|HX30ENlL={(F z3PiS$(VL;Qkja@bsgSw7kR_;)Rl}Rno5`E0t&k0>G^M1nAeF0-Jp$xYe~Us6Zw7D1 z_Cn5}LM{z&<_JchXiXuvpP!%Ce;@!8-u5MhJV~9gg}fROtU&pk#H7@mLcZ8SevJ%< z46u~~89@64g9?Q}_OY}U3THqpNHZxEfmk3KR44|sz@|_f)dGf+LW!hKXMvBG|F<}6 zmK(L{ev&=%v?!@ivZPQd31*2j$P$^LLRqLKav5US^vIVKDkP8(>1*t&3hM|sv zk*1D9tpfRgtARN=uQa!yQZG5bC{-`FG^ZpJ=-$LiuK1$VQXzADAxlsptA;lt5HhtDvO$%mlvEa^auu>iFaf>kZ&AqM&EU=0 zUdS0#$fe=U90BrwO(D0RpP$!%AOI8I_9cZpNu9BUyc!X#K>5tv#PrlczSu&3jSPki zu$2NCK>GxP3WY%Sv9uNnXFx1SGbt2-SRfiyCgU0x_zQYe)KvqTzXiA+$TEYuRY3^Aadc5zqcyRY7~stGhn#_ZBtQ7B(h zr~uca2-2ezRHzKqqXO3RwEh44iOz=(PjYQ4PM0h5E>tZkR7*-tEh$t_(hKlrWD;S< gm2iOmhk^!hT7f5{0B=?{kPIUbngeNbPyjOk0QpU=ivR!s literal 0 HcmV?d00001 diff --git a/tests/ut/data/dataset/golden/invert_01_result.npz b/tests/ut/data/dataset/golden/invert_01_result.npz new file mode 100644 index 0000000000000000000000000000000000000000..fc549c16af725de5b1c7d41c95548ec6bf1a12bf GIT binary patch literal 713 zcmWIWW@Zs#fB;2?DrYlUMa*?`~f_ja=x{iW+T7FSUQDRg18ARMP@xdWK9<%(;S7icX(ojt5DP?u3dLX+*c6JRTEI|JD3R3ZEb#I2{}yM> za-%lgPqIgz79|x*mJ~`Q!7PymSt1iuC=0bjE<+5P9{G|&g``xlOEYs5(^Cr-V+)nw zE>#8@q7qc73N=IxY)HrJGq2|IeSXVg=lAW!g#CPl>LrC5aEmoTdbENHwV`@+u#cWQV5a+fysWMz5$Vp z3}p<}>M5zk$wlf`3hFif>N*PQY57GZMTvRw`9&$IAYr$}oZ?iVcyUHzK`M~1VW^{E zq^YA&t3W>BYG6*zE6pva)Jx7UO4Z9P%_+$Qx;L?sE50Z-IX|zsq^LBxgsYGNqKYdo z1tMF>=*`et$mGnJRLI<3$P!e@s^QHDgiLLPY*3{sC6xuKT!rirOh9k?TNHA5Gk7z$ z7jgy_a%p%oM}YiaQ^@V-=jZhw2*8B5eMuotQfF)-uSNtbP(Cv^F+H`AFSd|hBZDCW zY^6X3&_2PSLLrcSEUksY84wH7ObSIH7KjEFioqurob9aG zDO7;#Q3UBx3My2F>QMpf@e}m^F!9Du=J`)n3$9HMOKrJxMRXn~_O` h8CSvq`X34!z-a}Zi~_t_*+4RkKxhu6%|QXo003=`vF88) literal 0 HcmV?d00001 diff --git a/tests/ut/data/dataset/golden/pad_01_py_result.npz b/tests/ut/data/dataset/golden/pad_01_py_result.npz new file mode 100644 index 0000000000000000000000000000000000000000..095257f847e31328065fa03f8f2021872f1a565e GIT binary patch literal 644 zcmWIWW@Zs#fB;2?3t4xj|6ycc0AWrB5r)K~qId(nyn;$b1_7`jNFhiP1Sb21`UXTY zGL$h?tEZ$ECl{$(DX7=@tLrGJr{x!w6eZ@x=NF}8(>1*t&3hM|sv zk*1D9tpfRgtARN=uQa!yQZG5bC{-`FG^ZpJ=-$LiuK1$VQXzADAxlsptA;lt5HhtDvO$%mlvEa^auu>iFaf>kZ&AqM&EU=0 zUdS0#$fe=U90BrwO(D0RpP$!%AOI8I_9cZpNu9BUyc!X#K>5tv#PrlczSu&3jSPki zu$2NCK>GxP3WY%Sv9uNnXFx1SGbt2-SRfiyCa=1!$?QL zNK;3lR)KuL)xeybSDIT;sh6Bzl&Y6onp2VqbZ=rMSA0=wa(-TMNl|HX30ENlL={(F z3PiS$(VL;Qkja@bsgSw7kR_;)Rl}Rno5`E0t&k0>G^M1nAeF0-Jp$xYe~Us6Zw7D1 z_Cn5}LM{z&<_JchXiXuvpP!%Ce;@!8-u5MhJV~9gg}fROtU&pk#H7@mLcZ8SevJ%< z46u~~89@64g9?Q}_OY}U3THqpNHZxEfmk3KR44|sz@|_f)dGf+LW!hKXMvBG|F<}6 zmK(L{ev&=%v?!@ivZPQd31*2j$P$^LLRqLKav5US^vIVKDkP{{lTe)>d!By)>w{kqGA>LrC5aEmoTdbENHwV`@+zXsDhC8eg86zV7G1$Z+ui7?|zeZWA3f(B54Bhq4kH!B-R OfDs7IfpiTxJpuq}-^3mO literal 0 HcmV?d00001 diff --git a/tests/ut/data/dataset/golden/random_color_adjust_01_c_result.npz b/tests/ut/data/dataset/golden/random_color_adjust_01_c_result.npz new file mode 100644 index 0000000000000000000000000000000000000000..bcb318681fa103bb2c428bb5b52302a455f7455c GIT binary patch literal 644 zcmWIWW@Zs#fB;1XSF!7V|1dHzfG{V62t#5~QM`d(UO^=zg8*0%q!1(t0+anheFGvH z8Oj){)l*W7lZ(`?6x3_{)pZoq)AEZ-iW2kU^NUhaLBei{ImM|!@#2icf>a=1!%#=T zNK;3lR)KuL)xeybSDIT;sh6Bzl&Y6onp2VqbZ=rMSA0=wa(-TMNl|HX30ENlL={(F z3PiS$(VL;Qkja@bsgSw7kR_;)Rl}PR2$|Xn*`P{ON-7IdxeD1Mn1J5&wJ)_&t`Pai(0+Gpis4>P%SAnwWLrzNiV>gkx7IZ gSHc1M9|{`4X$78)0=!w-Kr)O#Xbz;!K>^GF0OZrG@c;k- literal 0 HcmV?d00001 diff --git a/tests/ut/data/dataset/golden/random_color_adjust_01_py_result.npz b/tests/ut/data/dataset/golden/random_color_adjust_01_py_result.npz new file mode 100644 index 0000000000000000000000000000000000000000..7855123b6712d346ad6e77ba7a28b9f7ec346527 GIT binary patch literal 644 zcmWIWW@Zs#fB;2?xoc)#`oqY;0K%LMA`FQ|MezoDc?Fe>3<6+5kV23o2u$`1^$mz* zWGG{(R!>PSPA*cnQc$n)SJzQcPs=YVDN4+X&o4?z1qr()<`ky_#fvi%3sQl64MQCT zBTXHJS_Sd}R|9i$UTJPYrCxG=QL0{UX--Kd(7lP3T=7Mz$@zK3B}JvlC0vCJ5LH}x zDG=F0MsJ4JLMCU%q(bKQLYAOHRt;}PAY^JQWP>VADXA<-VkKmaDZ?Mn)Ik~(7xc{L(ff%2KTiRr0@e6fZ68W{{3 zU@HYOfc6On6$*juV`(iE&VX2uW>P2uu|PDaPz+{)O`$ld1q>yH5=ouT0!RM+-Eg;c zz3gi#=@j1O-!~LWmJ~`Q!7PymSt1iuC=0bjE<+5cr-5B?>NE3w+4rjR^>+82KUFAS zQm6peqX^QY6jZ1T)uRH|Q&MVtA-IKo=B$Rvd-+diD;BDj6sjeqrj`_{C+P)vGct)V g<4QO{|3g6oIIX~wQGhoq8%Tx`2+e`CIVgY`09xtG|6ycc0AWrB5r)K~qId(nyn;$b1_7`jNFhiP1Sb21`UXTY zGL$h?tEZ$ECl{$(DX7=@tLrGJr{x!w6eZ@x=NF}8(>1*t&3hM|sv zk*1D9tpfRgtARN=uQa!yQZG5bC{-`FG^ZpJ=-$LiuK1$VQXzADAxlsptA;lt5HhtDvO$%mlvEa^auu>iFaf>kZ&AqM&EU=0 zUdS0#$fe=U90BrwO(D0RpP$!%AOI8I_9cZpNu9BUyc!X#K>5tv#PrlczSu&3jSPki zu$2NCK>GxP3WY%Sv9uNnXFx1SGbt2-SRfiyC#cWQV5a+fysWMz5$Vp z3}p<}>M5zk$wlf`3hFif>N*PQY57GZMTvRw`9&$IAYr$}oZ?iVcyUHzK`M~1VW^{E zq^YA&t3W>BYG6*zE6pva)Jx7UO4Z9P%_+$Qx;L?sE50Z-IX|zsq^LBxgsYGNqKYdo z1tMF>=*`et$mGnJRLI<3$P!e@s^QHDgiLLPY*3{sC6xuKT!rirOh9k?TNHA5Gk7z$ z7jgy_a%p%oM}YiaQ^@V-=jZhw2*8B5eMuotQfF)-uSNtbP(Cv^F+H`AFSd|hBZDCW zY^6X3&_2PSLLrcSEUksY84wH7ObSIH7KjEFioq>#cWQV5a+fysWMz5$Vp z3}p<}>M5zk$wlf`3hFif>N*PQY57GZMTvRw`9&$IAYr$}oZ?iVcyUHzK`M~1VW^{E zq^YA&t3W>BYG6*zE6pva)Jx7UO4Z9P%_+$Qx;L?sE50Z-IX|zsq^LBxgsYGNqKYdo z1tMF>=*`et$mGnJRLI<3$P!e@s^QHDgiLLPY*3{sC6xuKT!rirOh9k?TNHA5Gk7z$ z7jgy_a%p%oM}YiaQ^@V-=jZhw2*8B5eMuotQfF)-uSNtbP(Cv^F+H`AFSd|hBZDCW zY^6X3&_2PSLLrcSEUksY84wH7ObSIH7KjEFioqtVqr| zuYczV3vP{2Jy9rGQYe)KvqTzXiA+$TEYuRY3^AadH+#9{u>jIaA7OIvMswJhSmK3Td=>>Q*GKnzb fN;p9OLqP*Lt-zB}fHx}}NQMyz&4IKzD1aFN5dW-s literal 0 HcmV?d00001 diff --git a/tests/ut/data/dataset/golden/random_rotation_01_c_result.npz b/tests/ut/data/dataset/golden/random_rotation_01_c_result.npz new file mode 100644 index 0000000000000000000000000000000000000000..794d6b714e8ba6d11ae9535275a674cd8ecbc738 GIT binary patch literal 644 zcmWIWW@Zs#fB;1X&SUxu{xC8yfG{V62t#5~QM`d(UO^=zg8*0%q!1(t0+anheFGvH z8Oj){)l*W7lZ(`?6x3_{)pZoq)AEZ-iW2kU^NUhaLBei{ImM|!@#2icf>a=1!%#=T zNK;3lR)KuL)xeybSDIT;sh6Bzl&Y6onp2VqbZ=rMSA0=wa(-TMNl|HX30ENlL={(F z3PiS$(VL;Qkja@bsgSw7kR_;)Rl}PR2$|Xn*`P{ON-7IdxeD1Mn1J5&w3<6+5kV23o2u$`1^$mz* zWGG{(R!>PSPA*cnQc$n)SJzQcPs=YVDN4+X&o4?z1qr()<`ky_#fvi%3sQl64I>=| zBTXHJS_Sd}R|9i$UTJPYrCxG=QL0{UX--Kd(7lP3T=7Mz$@zK3B}JvlC0vCJ5LH}x zDG=F0MsJ4JLMCU%q(bKQLYAOHRt;}PZzgZ1wn8?j(v*_Qf>f?T_6U$q{VfVPycxV1 z+Y32^3b{1AnIjm1qBVuwetv#l|A7EZc-xm0@+5V}7V>IDuma_C5|dJM3i)CS`86^a zGQd^}WB~0G3@Q`?*~ijaD4YSYAkCyu1Y&_`P@x#i0-Hi{R0|kN3MG;{odt}~zBzMl zUxU$n`P=X6BT7yeN|qE#CBZC_23aB#R45CzL@q-Ns3-GasrA(#qKr2KjV^S3a7ihY zFDX=j>rn*hQ3@(lhU!rP>#5a^KhahUpQAyt>B^1C``0V;x-apT> z8VfB;3aye-Q%ef1lk@_-8JR?waiwx#WI;g#C}I(5KERum4J5z_gyul{2{_#Y0OS+! A2LJ#7 literal 0 HcmV?d00001 diff --git a/tests/ut/data/dataset/golden/random_sharpness_01_result.npz b/tests/ut/data/dataset/golden/random_sharpness_01_result.npz new file mode 100644 index 0000000000000000000000000000000000000000..27fe0fbfea6981dfd0166742f2d7a50cedd74d68 GIT binary patch literal 713 zcmWIWW@Zs#fB;2?m3ObGIx;aZfG{V62t#5~QM`d(UO^=zg8*0%q!1(t0+anheFGvH z8Oj){)l*W7lZ(`?6x3_{)pZoq)AEZ-iW2kU^NUhaLBei{ImM|!@#2icf>a=1!$?QL zNK;3lR)KuL)xeybSDIT;sh6Bzl&Y6onp2VqbZ=rMSA0=wa(-TMNl|HX30ENlL={(F z3PiS$(VL;Qkja@bsgSw7kR_;)Rl}Rno5`E0t&k0>G^M1nAeF0-Jp$xYe~Us6Zw7D1 z_Cn5}LM{z&<_JchXiXuvpP!%Ce;@!8-u5MhJV~9gg}fROtU&pk#H7@mLcZ8SevJ%< z46u~~89@64g9?Q}_OY}U3THqpNHZxEfmk3KR44|sz@|_f)dGf+LW!hKXMvBG|F<}6 zmK(L{ev&=%v?!@ivZPQd31*2j$P$^LLRqLKav5US^vIVKDkPXsDhC8eg86zV7G1$Z+ui7?|zeZWA3f(B54Bhq4kH!B-R OfDs7IfpiTxJpuqC`op0B literal 0 HcmV?d00001 diff --git a/tests/ut/data/dataset/golden/rescale_01_result.npz b/tests/ut/data/dataset/golden/rescale_01_result.npz new file mode 100644 index 0000000000000000000000000000000000000000..09c64d9fcb5e8dea69db1a302f14ee3ffb58fed7 GIT binary patch literal 644 zcmWIWW@Zs#fB;2?X?^M%e;64UK$w$3gdwr0DBeIXub`5VK>#cWQV5a+fysWMz5$Vp z3}p<}>M5zk$wlf`3hFif>N*PQY57GZMTvRw`9&$IAYr$}oZ?iVcyUHzK`M~1VW^{E zq^YA&t3W>BYG6*zE6pva)Jx7UO4Z9P%_+$Qx;L?sE50Z-IX|zsq^LBxgsYGNqKYdo z1tMF>=*`et$mGnJRLI<3$P!e@s^QHDgiLLPY*3{sC6xuKT!rirOh9k?TNHA5Gk7z$ z7jgy_a%p%oM}YiaQ^@V-=jZhw2*8B5eMuotQfF)-uSNtbP(Cv^F+H`AFSd|hBZDCW zY^6X3&_2PSLLrcSEUksY84wH7ObSIH7KjEFioq9SXE%6F{j4qHvg3;L&O+6aLbasS)RIE=B)tG{MkWzv gTnPv0e<)}GrxkcI3h-uS1IaJ~p*fH?2L&(#0EsxRWdHyG literal 0 HcmV?d00001 diff --git a/tests/ut/python/dataset/test_autocontrast.py b/tests/ut/python/dataset/test_autocontrast.py index 648ecf5787..d212994e6e 100644 --- a/tests/ut/python/dataset/test_autocontrast.py +++ b/tests/ut/python/dataset/test_autocontrast.py @@ -20,7 +20,7 @@ import numpy as np import mindspore.dataset.engine as de import mindspore.dataset.transforms.vision.py_transforms as F from mindspore import log as logger -from util import visualize_list +from util import visualize_list, diff_mse DATA_DIR = "../data/dataset/testImageNetData/train/" @@ -75,7 +75,7 @@ def test_auto_contrast(plot=False): num_samples = images_original.shape[0] mse = np.zeros(num_samples) for i in range(num_samples): - mse[i] = np.mean((images_auto_contrast[i] - images_original[i]) ** 2) + mse[i] = diff_mse(images_auto_contrast[i], images_original[i]) logger.info("MSE= {}".format(str(np.mean(mse)))) if plot: diff --git a/tests/ut/python/dataset/test_cut_out.py b/tests/ut/python/dataset/test_cut_out.py index 483a939f65..862b1f33c8 100644 --- a/tests/ut/python/dataset/test_cut_out.py +++ b/tests/ut/python/dataset/test_cut_out.py @@ -21,11 +21,13 @@ import mindspore.dataset as ds import mindspore.dataset.transforms.vision.c_transforms as c import mindspore.dataset.transforms.vision.py_transforms as f from mindspore import log as logger -from util import visualize_image, diff_mse +from util import visualize_image, visualize_list, diff_mse, save_and_check_md5, \ + config_get_set_seed, config_get_set_num_parallel_workers DATA_DIR = ["../data/dataset/test_tf_file_3_images/train-0000-of-0001.data"] SCHEMA_DIR = "../data/dataset/test_tf_file_3_images/datasetSchema.json" +GENERATE_GOLDEN = False def test_cut_out_op(plot=False): """ @@ -34,7 +36,7 @@ def test_cut_out_op(plot=False): logger.info("test_cut_out") # First dataset - data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"]) + data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) transforms_1 = [ f.Decode(), @@ -45,7 +47,7 @@ def test_cut_out_op(plot=False): data1 = data1.map(input_columns=["image"], operations=transform_1()) # Second dataset - data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"]) + data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) decode_op = c.Decode() cut_out_op = c.CutOut(80) @@ -74,25 +76,24 @@ def test_cut_out_op(plot=False): visualize_image(image_1, image_2, mse) -def test_cut_out_op_multicut(): +def test_cut_out_op_multicut(plot=False): """ Test Cutout """ logger.info("test_cut_out") # First dataset - data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"]) + data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) transforms_1 = [ f.Decode(), f.ToTensor(), - f.RandomErasing(value='random') ] transform_1 = f.ComposeOp(transforms_1) data1 = data1.map(input_columns=["image"], operations=transform_1()) # Second dataset - data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"]) + data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) decode_op = c.Decode() cut_out_op = c.CutOut(80, num_patches=10) @@ -104,19 +105,107 @@ def test_cut_out_op_multicut(): data2 = data2.map(input_columns=["image"], operations=transforms_2) num_iter = 0 + image_list_1, image_list_2 = [], [] + for item1, item2 in zip(data1.create_dict_iterator(), data2.create_dict_iterator()): + num_iter += 1 + image_1 = (item1["image"].transpose(1, 2, 0) * 255).astype(np.uint8) + # C image doesn't require transpose + image_2 = item2["image"] + image_list_1.append(image_1) + image_list_2.append(image_2) + + logger.info("shape of image_1: {}".format(image_1.shape)) + logger.info("shape of image_2: {}".format(image_2.shape)) + + logger.info("dtype of image_1: {}".format(image_1.dtype)) + logger.info("dtype of image_2: {}".format(image_2.dtype)) + if plot: + visualize_list(image_list_1, image_list_2) + + +def test_cut_out_md5(): + """ + Test Cutout with md5 check + """ + logger.info("test_cut_out_md5") + original_seed = config_get_set_seed(2) + original_num_parallel_workers = config_get_set_num_parallel_workers(1) + + # First dataset + data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) + decode_op = c.Decode() + cut_out_op = c.CutOut(100) + data1 = data1.map(input_columns=["image"], operations=decode_op) + data1 = data1.map(input_columns=["image"], operations=cut_out_op) + + data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) + transforms = [ + f.Decode(), + f.ToTensor(), + f.Cutout(100) + ] + transform = f.ComposeOp(transforms) + data2 = data2.map(input_columns=["image"], operations=transform()) + + # Compare with expected md5 from images + filename1 = "cut_out_01_c_result.npz" + save_and_check_md5(data1, filename1, generate_golden=GENERATE_GOLDEN) + filename2 = "cut_out_01_py_result.npz" + save_and_check_md5(data2, filename2, generate_golden=GENERATE_GOLDEN) + + # Restore config + ds.config.set_seed(original_seed) + ds.config.set_num_parallel_workers(original_num_parallel_workers) + + +def test_cut_out_comp(plot=False): + """ + Test Cutout with c++ and python op comparison + """ + logger.info("test_cut_out_comp") + + # First dataset + data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) + + transforms_1 = [ + f.Decode(), + f.ToTensor(), + f.Cutout(200) + ] + transform_1 = f.ComposeOp(transforms_1) + data1 = data1.map(input_columns=["image"], operations=transform_1()) + + # Second dataset + data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) + + transforms_2 = [ + c.Decode(), + c.CutOut(200) + ] + + data2 = data2.map(input_columns=["image"], operations=transforms_2) + + num_iter = 0 + image_list_1, image_list_2 = [], [] for item1, item2 in zip(data1.create_dict_iterator(), data2.create_dict_iterator()): num_iter += 1 image_1 = (item1["image"].transpose(1, 2, 0) * 255).astype(np.uint8) # C image doesn't require transpose image_2 = item2["image"] + image_list_1.append(image_1) + image_list_2.append(image_2) logger.info("shape of image_1: {}".format(image_1.shape)) logger.info("shape of image_2: {}".format(image_2.shape)) logger.info("dtype of image_1: {}".format(image_1.dtype)) logger.info("dtype of image_2: {}".format(image_2.dtype)) + if plot: + visualize_list(image_list_1, image_list_2, visualize_mode=2) if __name__ == "__main__": test_cut_out_op(plot=True) - test_cut_out_op_multicut() + test_cut_out_op_multicut(plot=True) + test_cut_out_md5() + test_cut_out_comp(plot=True) diff --git a/tests/ut/python/dataset/test_equalize.py b/tests/ut/python/dataset/test_equalize.py index 85cb25d9ee..0a5f2f93d5 100644 --- a/tests/ut/python/dataset/test_equalize.py +++ b/tests/ut/python/dataset/test_equalize.py @@ -20,10 +20,11 @@ import numpy as np import mindspore.dataset.engine as de import mindspore.dataset.transforms.vision.py_transforms as F from mindspore import log as logger -from util import visualize_list +from util import visualize_list, diff_mse, save_and_check_md5 DATA_DIR = "../data/dataset/testImageNetData/train/" +GENERATE_GOLDEN = False def test_equalize(plot=False): """ @@ -75,12 +76,31 @@ def test_equalize(plot=False): num_samples = images_original.shape[0] mse = np.zeros(num_samples) for i in range(num_samples): - mse[i] = np.mean((images_equalize[i] - images_original[i]) ** 2) + mse[i] = diff_mse(images_equalize[i], images_original[i]) logger.info("MSE= {}".format(str(np.mean(mse)))) if plot: visualize_list(images_original, images_equalize) +def test_equalize_md5(): + """ + Test Equalize with md5 check + """ + logger.info("Test Equalize") + + # First dataset + data1 = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False) + transforms = F.ComposeOp([F.Decode(), + F.Equalize(), + F.ToTensor()]) + + data1 = data1.map(input_columns="image", operations=transforms()) + # Compare with expected md5 from images + filename = "equalize_01_result.npz" + save_and_check_md5(data1, filename, generate_golden=GENERATE_GOLDEN) + + if __name__ == "__main__": test_equalize(plot=True) + test_equalize_md5() diff --git a/tests/ut/python/dataset/test_five_crop.py b/tests/ut/python/dataset/test_five_crop.py index 61632e3989..ef2e376c0f 100644 --- a/tests/ut/python/dataset/test_five_crop.py +++ b/tests/ut/python/dataset/test_five_crop.py @@ -20,11 +20,12 @@ import numpy as np import mindspore.dataset as ds import mindspore.dataset.transforms.vision.py_transforms as vision from mindspore import log as logger -from util import visualize_list +from util import visualize_list, save_and_check_md5 DATA_DIR = ["../data/dataset/test_tf_file_3_images/train-0000-of-0001.data"] SCHEMA_DIR = "../data/dataset/test_tf_file_3_images/datasetSchema.json" +GENERATE_GOLDEN = False def test_five_crop_op(plot=False): """ @@ -63,7 +64,7 @@ def test_five_crop_op(plot=False): logger.info("dtype of image_1: {}".format(image_1.dtype)) logger.info("dtype of image_2: {}".format(image_2.dtype)) if plot: - visualize_list(np.array([image_1]*10), (image_2 * 255).astype(np.uint8).transpose(0, 2, 3, 1)) + visualize_list(np.array([image_1]*5), (image_2 * 255).astype(np.uint8).transpose(0, 2, 3, 1)) # The output data should be of a 4D tensor shape, a stack of 5 images. assert len(image_2.shape) == 4 @@ -93,6 +94,27 @@ def test_five_crop_error_msg(): assert error_msg in str(info.value) +def test_five_crop_md5(): + """ + Test FiveCrop with md5 check + """ + logger.info("test_five_crop_md5") + + # First dataset + data = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) + transforms = [ + vision.Decode(), + vision.FiveCrop(100), + lambda images: np.stack([vision.ToTensor()(image) for image in images]) # 4D stack of 5 images + ] + transform = vision.ComposeOp(transforms) + data = data.map(input_columns=["image"], operations=transform()) + # Compare with expected md5 from images + filename = "five_crop_01_result.npz" + save_and_check_md5(data, filename, generate_golden=GENERATE_GOLDEN) + + if __name__ == "__main__": test_five_crop_op(plot=True) test_five_crop_error_msg() + test_five_crop_md5() diff --git a/tests/ut/python/dataset/test_invert.py b/tests/ut/python/dataset/test_invert.py index 8bdf63dd72..f366553c6e 100644 --- a/tests/ut/python/dataset/test_invert.py +++ b/tests/ut/python/dataset/test_invert.py @@ -20,10 +20,11 @@ import numpy as np import mindspore.dataset.engine as de import mindspore.dataset.transforms.vision.py_transforms as F from mindspore import log as logger -from util import visualize_list +from util import visualize_list, save_and_check_md5 DATA_DIR = "../data/dataset/testImageNetData/train/" +GENERATE_GOLDEN = False def test_invert(plot=False): """ @@ -82,5 +83,25 @@ def test_invert(plot=False): visualize_list(images_original, images_invert) +def test_invert_md5(): + """ + Test Invert with md5 check + """ + logger.info("Test Invert with md5 check") + + # Generate dataset + ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False) + + transforms_invert = F.ComposeOp([F.Decode(), + F.Invert(), + F.ToTensor()]) + + data = ds.map(input_columns="image", operations=transforms_invert()) + # Compare with expected md5 from images + filename = "invert_01_result.npz" + save_and_check_md5(data, filename, generate_golden=GENERATE_GOLDEN) + + if __name__ == "__main__": test_invert(plot=True) + test_invert_md5() diff --git a/tests/ut/python/dataset/test_linear_transformation.py b/tests/ut/python/dataset/test_linear_transformation.py index 80153902ab..0dd25a4da1 100644 --- a/tests/ut/python/dataset/test_linear_transformation.py +++ b/tests/ut/python/dataset/test_linear_transformation.py @@ -73,12 +73,12 @@ def test_linear_transformation_op(plot=False): if plot: visualize_list(image, image_transformed) -def test_linear_transformation_md5_01(): +def test_linear_transformation_md5(): """ Test LinearTransformation op: valid params (transformation_matrix, mean_vector) Expected to pass """ - logger.info("test_linear_transformation_md5_01") + logger.info("test_linear_transformation_md5") # Initialize parameters height = 50 @@ -102,12 +102,12 @@ def test_linear_transformation_md5_01(): filename = "linear_transformation_01_result.npz" save_and_check_md5(data1, filename, generate_golden=GENERATE_GOLDEN) -def test_linear_transformation_md5_02(): +def test_linear_transformation_exception_01(): """ Test LinearTransformation op: transformation_matrix is not provided Expected to raise ValueError """ - logger.info("test_linear_transformation_md5_02") + logger.info("test_linear_transformation_exception_01") # Initialize parameters height = 50 @@ -130,12 +130,12 @@ def test_linear_transformation_md5_02(): logger.info("Got an exception in DE: {}".format(str(e))) assert "not provided" in str(e) -def test_linear_transformation_md5_03(): +def test_linear_transformation_exception_02(): """ Test LinearTransformation op: mean_vector is not provided Expected to raise ValueError """ - logger.info("test_linear_transformation_md5_03") + logger.info("test_linear_transformation_exception_02") # Initialize parameters height = 50 @@ -158,12 +158,12 @@ def test_linear_transformation_md5_03(): logger.info("Got an exception in DE: {}".format(str(e))) assert "not provided" in str(e) -def test_linear_transformation_md5_04(): +def test_linear_transformation_exception_03(): """ Test LinearTransformation op: transformation_matrix is not a square matrix Expected to raise ValueError """ - logger.info("test_linear_transformation_md5_04") + logger.info("test_linear_transformation_exception_03") # Initialize parameters height = 50 @@ -187,12 +187,12 @@ def test_linear_transformation_md5_04(): logger.info("Got an exception in DE: {}".format(str(e))) assert "square matrix" in str(e) -def test_linear_transformation_md5_05(): +def test_linear_transformation_exception_04(): """ Test LinearTransformation op: mean_vector does not match dimension of transformation_matrix Expected to raise ValueError """ - logger.info("test_linear_transformation_md5_05") + logger.info("test_linear_transformation_exception_04") # Initialize parameters height = 50 @@ -217,9 +217,9 @@ def test_linear_transformation_md5_05(): assert "should match" in str(e) if __name__ == '__main__': - test_linear_transformation_op(True) - test_linear_transformation_md5_01() - test_linear_transformation_md5_02() - test_linear_transformation_md5_03() - test_linear_transformation_md5_04() - test_linear_transformation_md5_05() + test_linear_transformation_op(plot=True) + test_linear_transformation_md5() + test_linear_transformation_exception_01() + test_linear_transformation_exception_02() + test_linear_transformation_exception_03() + test_linear_transformation_exception_04() diff --git a/tests/ut/python/dataset/test_pad.py b/tests/ut/python/dataset/test_pad.py index 7b66b6b36b..a3038a4b91 100644 --- a/tests/ut/python/dataset/test_pad.py +++ b/tests/ut/python/dataset/test_pad.py @@ -21,11 +21,12 @@ import mindspore.dataset as ds import mindspore.dataset.transforms.vision.c_transforms as c_vision import mindspore.dataset.transforms.vision.py_transforms as py_vision from mindspore import log as logger -from util import diff_mse +from util import diff_mse, save_and_check_md5 DATA_DIR = ["../data/dataset/test_tf_file_3_images/train-0000-of-0001.data"] SCHEMA_DIR = "../data/dataset/test_tf_file_3_images/datasetSchema.json" +GENERATE_GOLDEN = False def test_pad_op(): """ @@ -116,6 +117,39 @@ def test_pad_grayscale(): assert shape1[0:1] == shape2[0:1] +def test_pad_md5(): + """ + Test Pad with md5 check + """ + logger.info("test_pad_md5") + + # First dataset + data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) + decode_op = c_vision.Decode() + pad_op = c_vision.Pad(150) + ctrans = [decode_op, + pad_op, + ] + + data1 = data1.map(input_columns=["image"], operations=ctrans) + + # Second dataset + data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) + pytrans = [ + py_vision.Decode(), + py_vision.Pad(150), + py_vision.ToTensor(), + ] + transform = py_vision.ComposeOp(pytrans) + data2 = data2.map(input_columns=["image"], operations=transform()) + # Compare with expected md5 from images + filename1 = "pad_01_c_result.npz" + save_and_check_md5(data1, filename1, generate_golden=GENERATE_GOLDEN) + filename2 = "pad_01_py_result.npz" + save_and_check_md5(data2, filename2, generate_golden=GENERATE_GOLDEN) + + if __name__ == "__main__": test_pad_op() test_pad_grayscale() + test_pad_md5() diff --git a/tests/ut/python/dataset/test_random_color.py b/tests/ut/python/dataset/test_random_color.py index 8ca0071e42..45847ba653 100644 --- a/tests/ut/python/dataset/test_random_color.py +++ b/tests/ut/python/dataset/test_random_color.py @@ -17,13 +17,16 @@ Testing RandomColor op in DE """ import numpy as np +import mindspore.dataset as ds import mindspore.dataset.engine as de import mindspore.dataset.transforms.vision.py_transforms as F from mindspore import log as logger -from util import visualize_list +from util import visualize_list, diff_mse, save_and_check_md5, \ + config_get_set_seed, config_get_set_num_parallel_workers DATA_DIR = "../data/dataset/testImageNetData/train/" +GENERATE_GOLDEN = False def test_random_color(degrees=(0.1, 1.9), plot=False): """ @@ -32,14 +35,14 @@ def test_random_color(degrees=(0.1, 1.9), plot=False): logger.info("Test RandomColor") # Original Images - ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False) + data = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False) transforms_original = F.ComposeOp([F.Decode(), F.Resize((224, 224)), F.ToTensor()]) - ds_original = ds.map(input_columns="image", - operations=transforms_original()) + ds_original = data.map(input_columns="image", + operations=transforms_original()) ds_original = ds_original.batch(512) @@ -52,15 +55,15 @@ def test_random_color(degrees=(0.1, 1.9), plot=False): axis=0) # Random Color Adjusted Images - ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False) + data = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False) transforms_random_color = F.ComposeOp([F.Decode(), F.Resize((224, 224)), F.RandomColor(degrees=degrees), F.ToTensor()]) - ds_random_color = ds.map(input_columns="image", - operations=transforms_random_color()) + ds_random_color = data.map(input_columns="image", + operations=transforms_random_color()) ds_random_color = ds_random_color.batch(512) @@ -75,14 +78,40 @@ def test_random_color(degrees=(0.1, 1.9), plot=False): num_samples = images_original.shape[0] mse = np.zeros(num_samples) for i in range(num_samples): - mse[i] = np.mean((images_random_color[i] - images_original[i]) ** 2) + mse[i] = diff_mse(images_random_color[i], images_original[i]) logger.info("MSE= {}".format(str(np.mean(mse)))) if plot: visualize_list(images_original, images_random_color) +def test_random_color_md5(): + """ + Test RandomColor with md5 check + """ + logger.info("Test RandomColor with md5 check") + original_seed = config_get_set_seed(10) + original_num_parallel_workers = config_get_set_num_parallel_workers(1) + + # Generate dataset + data = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False) + + transforms = F.ComposeOp([F.Decode(), + F.RandomColor((0.5, 1.5)), + F.ToTensor()]) + + data = data.map(input_columns="image", operations=transforms()) + # Compare with expected md5 from images + filename = "random_color_01_result.npz" + save_and_check_md5(data, filename, generate_golden=GENERATE_GOLDEN) + + # Restore configuration + ds.config.set_seed(original_seed) + ds.config.set_num_parallel_workers((original_num_parallel_workers)) + + if __name__ == "__main__": test_random_color() test_random_color(plot=True) test_random_color(degrees=(0.5, 1.5), plot=True) + test_random_color_md5() diff --git a/tests/ut/python/dataset/test_random_color_adjust.py b/tests/ut/python/dataset/test_random_color_adjust.py index f79137e14f..3eb55043b4 100644 --- a/tests/ut/python/dataset/test_random_color_adjust.py +++ b/tests/ut/python/dataset/test_random_color_adjust.py @@ -22,11 +22,13 @@ import mindspore.dataset as ds import mindspore.dataset.transforms.vision.c_transforms as c_vision import mindspore.dataset.transforms.vision.py_transforms as py_vision from mindspore import log as logger -from util import diff_mse, visualize_image +from util import diff_mse, visualize_image, save_and_check_md5, \ + config_get_set_seed, config_get_set_num_parallel_workers DATA_DIR = ["../data/dataset/test_tf_file_3_images/train-0000-of-0001.data"] SCHEMA_DIR = "../data/dataset/test_tf_file_3_images/datasetSchema.json" +GENERATE_GOLDEN = False def util_test_random_color_adjust_error(brightness=(1, 1), contrast=(1, 1), saturation=(1, 1), hue=(0, 0)): """ @@ -188,6 +190,41 @@ def test_random_color_adjust_op_hue_error(): util_test_random_color_adjust_error(hue=(0.5, 0.5)) +def test_random_color_adjust_md5(): + """ + Test RandomColorAdjust with md5 check + """ + logger.info("Test RandomColorAdjust with md5 check") + original_seed = config_get_set_seed(10) + original_num_parallel_workers = config_get_set_num_parallel_workers(1) + + # First dataset + data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) + decode_op = c_vision.Decode() + random_adjust_op = c_vision.RandomColorAdjust(0.4, 0.4, 0.4, 0.1) + data1 = data1.map(input_columns=["image"], operations=decode_op) + data1 = data1.map(input_columns=["image"], operations=random_adjust_op) + + # Second dataset + transforms = [ + py_vision.Decode(), + py_vision.RandomColorAdjust(0.4, 0.4, 0.4, 0.1), + py_vision.ToTensor() + ] + transform = py_vision.ComposeOp(transforms) + data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) + data2 = data2.map(input_columns=["image"], operations=transform()) + # Compare with expected md5 from images + filename = "random_color_adjust_01_c_result.npz" + save_and_check_md5(data1, filename, generate_golden=GENERATE_GOLDEN) + filename = "random_color_adjust_01_py_result.npz" + save_and_check_md5(data2, filename, generate_golden=GENERATE_GOLDEN) + + # Restore configuration + ds.config.set_seed(original_seed) + ds.config.set_num_parallel_workers(original_num_parallel_workers) + + if __name__ == "__main__": test_random_color_adjust_op_brightness(plot=True) test_random_color_adjust_op_brightness_error() @@ -197,3 +234,4 @@ if __name__ == "__main__": test_random_color_adjust_op_saturation_error() test_random_color_adjust_op_hue(plot=True) test_random_color_adjust_op_hue_error() + test_random_color_adjust_md5() diff --git a/tests/ut/python/dataset/test_random_crop_and_resize.py b/tests/ut/python/dataset/test_random_crop_and_resize.py index 8ccbb98c2c..de039e6d82 100644 --- a/tests/ut/python/dataset/test_random_crop_and_resize.py +++ b/tests/ut/python/dataset/test_random_crop_and_resize.py @@ -331,6 +331,8 @@ def test_random_crop_and_resize_comp(plot=False): py_image = (item2["image"].transpose(1, 2, 0) * 255).astype(np.uint8) image_c_cropped.append(c_image) image_py_cropped.append(py_image) + mse = diff_mse(c_image, py_image) + assert mse < 0.02 # rounding error if plot: visualize_list(image_c_cropped, image_py_cropped, visualize_mode=2) diff --git a/tests/ut/python/dataset/test_random_crop_decode_resize.py b/tests/ut/python/dataset/test_random_crop_decode_resize.py index 4a46851f9b..c6125d4b69 100644 --- a/tests/ut/python/dataset/test_random_crop_decode_resize.py +++ b/tests/ut/python/dataset/test_random_crop_decode_resize.py @@ -15,16 +15,16 @@ """ Testing RandomCropDecodeResize op in DE """ -import cv2 - import mindspore.dataset as ds import mindspore.dataset.transforms.vision.c_transforms as vision from mindspore import log as logger -from util import diff_mse, visualize_image +from util import diff_mse, visualize_image, save_and_check_md5, \ + config_get_set_seed, config_get_set_num_parallel_workers DATA_DIR = ["../data/dataset/test_tf_file_3_images/train-0000-of-0001.data"] SCHEMA_DIR = "../data/dataset/test_tf_file_3_images/datasetSchema.json" +GENERATE_GOLDEN = False def test_random_crop_decode_resize_op(plot=False): """ @@ -40,22 +40,46 @@ def test_random_crop_decode_resize_op(plot=False): # Second dataset data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) + random_crop_resize_op = vision.RandomResizedCrop((256, 512), (1, 1), (0.5, 0.5)) data2 = data2.map(input_columns=["image"], operations=decode_op) + data2 = data2.map(input_columns=["image"], operations=random_crop_resize_op) + num_iter = 0 for item1, item2 in zip(data1.create_dict_iterator(), data2.create_dict_iterator()): - if num_iter > 0: break - crop_and_resize_de = item1["image"] - original = item2["image"] - crop_and_resize_cv = cv2.resize(original, (512, 256)) - mse = diff_mse(crop_and_resize_de, crop_and_resize_cv) + image1 = item1["image"] + image2 = item2["image"] + mse = diff_mse(image1, image2) + assert mse == 0 logger.info("random_crop_decode_resize_op_{}, mse: {}".format(num_iter + 1, mse)) if plot: - visualize_image(original, crop_and_resize_de, mse, crop_and_resize_cv) + visualize_image(image1, image2, mse) num_iter += 1 +def test_random_crop_decode_resize_md5(): + """ + Test RandomCropDecodeResize with md5 check + """ + logger.info("Test RandomCropDecodeResize with md5 check") + original_seed = config_get_set_seed(10) + original_num_parallel_workers = config_get_set_num_parallel_workers(1) + + # Generate dataset + data = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) + random_crop_decode_resize_op = vision.RandomCropDecodeResize((256, 512), (1, 1), (0.5, 0.5)) + data = data.map(input_columns=["image"], operations=random_crop_decode_resize_op) + # Compare with expected md5 from images + filename = "random_crop_decode_resize_01_result.npz" + save_and_check_md5(data, filename, generate_golden=GENERATE_GOLDEN) + + # Restore configuration + ds.config.set_seed(original_seed) + ds.config.set_num_parallel_workers((original_num_parallel_workers)) + + if __name__ == "__main__": test_random_crop_decode_resize_op(plot=True) + test_random_crop_decode_resize_md5() diff --git a/tests/ut/python/dataset/test_random_erasing.py b/tests/ut/python/dataset/test_random_erasing.py index 842b4a15cc..3265ac2a63 100644 --- a/tests/ut/python/dataset/test_random_erasing.py +++ b/tests/ut/python/dataset/test_random_erasing.py @@ -20,11 +20,13 @@ import numpy as np import mindspore.dataset as ds import mindspore.dataset.transforms.vision.py_transforms as vision from mindspore import log as logger -from util import diff_mse, visualize_image +from util import diff_mse, visualize_image, save_and_check_md5, \ + config_get_set_seed, config_get_set_num_parallel_workers DATA_DIR = ["../data/dataset/test_tf_file_3_images/train-0000-of-0001.data"] SCHEMA_DIR = "../data/dataset/test_tf_file_3_images/datasetSchema.json" +GENERATE_GOLDEN = False def test_random_erasing_op(plot=False): """ @@ -69,5 +71,32 @@ def test_random_erasing_op(plot=False): visualize_image(image_1, image_2, mse) +def test_random_erasing_md5(): + """ + Test RandomErasing with md5 check + """ + logger.info("Test RandomErasing with md5 check") + original_seed = config_get_set_seed(5) + original_num_parallel_workers = config_get_set_num_parallel_workers(1) + + # Generate dataset + data = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) + transforms_1 = [ + vision.Decode(), + vision.ToTensor(), + vision.RandomErasing(value='random') + ] + transform_1 = vision.ComposeOp(transforms_1) + data = data.map(input_columns=["image"], operations=transform_1()) + # Compare with expected md5 from images + filename = "random_erasing_01_result.npz" + save_and_check_md5(data, filename, generate_golden=GENERATE_GOLDEN) + + # Restore configuration + ds.config.set_seed(original_seed) + ds.config.set_num_parallel_workers((original_num_parallel_workers)) + + if __name__ == "__main__": test_random_erasing_op(plot=True) + test_random_erasing_md5() diff --git a/tests/ut/python/dataset/test_random_horizontal_flip.py b/tests/ut/python/dataset/test_random_horizontal_flip.py index b6a4fef00c..1272148e4f 100644 --- a/tests/ut/python/dataset/test_random_horizontal_flip.py +++ b/tests/ut/python/dataset/test_random_horizontal_flip.py @@ -49,7 +49,7 @@ def test_random_horizontal_op(plot=False): # First dataset data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) decode_op = c_vision.Decode() - random_horizontal_op = c_vision.RandomHorizontalFlip() + random_horizontal_op = c_vision.RandomHorizontalFlip(1.0) data1 = data1.map(input_columns=["image"], operations=decode_op) data1 = data1.map(input_columns=["image"], operations=random_horizontal_op) @@ -69,6 +69,7 @@ def test_random_horizontal_op(plot=False): image_h_flipped_2 = h_flip(image) mse = diff_mse(image_h_flipped, image_h_flipped_2) + assert mse == 0 logger.info("image_{}, mse: {}".format(num_iter + 1, mse)) num_iter += 1 if plot: diff --git a/tests/ut/python/dataset/test_random_resize.py b/tests/ut/python/dataset/test_random_resize.py index c581712ace..1ac790ed19 100644 --- a/tests/ut/python/dataset/test_random_resize.py +++ b/tests/ut/python/dataset/test_random_resize.py @@ -13,16 +13,18 @@ # limitations under the License. # ============================================================================== """ -Testing the resize op in DE +Testing RandomResize op in DE """ import mindspore.dataset as ds import mindspore.dataset.transforms.vision.c_transforms as vision from mindspore import log as logger -from util import visualize_list +from util import visualize_list, save_and_check_md5, \ + config_get_set_seed, config_get_set_num_parallel_workers DATA_DIR = ["../data/dataset/test_tf_file_3_images/train-0000-of-0001.data"] SCHEMA_DIR = "../data/dataset/test_tf_file_3_images/datasetSchema.json" +GENERATE_GOLDEN = False def test_random_resize_op(plot=False): """ @@ -52,5 +54,29 @@ def test_random_resize_op(plot=False): visualize_list(image_original, image_resized) +def test_random_resize_md5(): + """ + Test RandomResize with md5 check + """ + logger.info("Test RandomResize with md5 check") + original_seed = config_get_set_seed(5) + original_num_parallel_workers = config_get_set_num_parallel_workers(1) + + # Generate dataset + data = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) + decode_op = vision.Decode() + resize_op = vision.RandomResize(10) + data = data.map(input_columns=["image"], operations=decode_op) + data = data.map(input_columns=["image"], operations=resize_op) + # Compare with expected md5 from images + filename = "random_resize_01_result.npz" + save_and_check_md5(data, filename, generate_golden=GENERATE_GOLDEN) + + # Restore configuration + ds.config.set_seed(original_seed) + ds.config.set_num_parallel_workers(original_num_parallel_workers) + + if __name__ == "__main__": test_random_resize_op(plot=True) + test_random_resize_md5() diff --git a/tests/ut/python/dataset/test_random_rotation.py b/tests/ut/python/dataset/test_random_rotation.py index d399dee00a..a6efd3ccec 100644 --- a/tests/ut/python/dataset/test_random_rotation.py +++ b/tests/ut/python/dataset/test_random_rotation.py @@ -21,18 +21,21 @@ import cv2 import mindspore.dataset as ds import mindspore.dataset.transforms.vision.c_transforms as c_vision import mindspore.dataset.transforms.vision.py_transforms as py_vision +from mindspore.dataset.transforms.vision.utils import Inter from mindspore import log as logger -from util import visualize_image, diff_mse +from util import visualize_image, visualize_list, diff_mse, save_and_check_md5, \ + config_get_set_seed, config_get_set_num_parallel_workers DATA_DIR = ["../data/dataset/test_tf_file_3_images/train-0000-of-0001.data"] SCHEMA_DIR = "../data/dataset/test_tf_file_3_images/datasetSchema.json" +GENERATE_GOLDEN = False -def test_random_rotation_op(plot=False): +def test_random_rotation_op_c(plot=False): """ - Test RandomRotation op + Test RandomRotation in c++ transformations op """ - logger.info("test_random_rotation_op") + logger.info("test_random_rotation_op_c") # First dataset data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, shuffle=False) @@ -58,8 +61,44 @@ def test_random_rotation_op(plot=False): logger.info("random_rotation_op_{}, mse: {}".format(num_iter + 1, mse)) assert mse == 0 num_iter += 1 - if plot: - visualize_image(original, rotation_de, mse, rotation_cv) + if plot: + visualize_image(original, rotation_de, mse, rotation_cv) + + +def test_random_rotation_op_py(plot=False): + """ + Test RandomRotation in python transformations op + """ + logger.info("test_random_rotation_op_py") + + # First dataset + data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, shuffle=False) + # use [90, 90] to force rotate 90 degrees, expand is set to be True to match output size + transform1 = py_vision.ComposeOp([py_vision.Decode(), + py_vision.RandomRotation((90, 90), expand=True), + py_vision.ToTensor()]) + data1 = data1.map(input_columns=["image"], operations=transform1()) + + # Second dataset + data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) + transform2 = py_vision.ComposeOp([py_vision.Decode(), + py_vision.ToTensor()]) + data2 = data2.map(input_columns=["image"], operations=transform2()) + + num_iter = 0 + for item1, item2 in zip(data1.create_dict_iterator(), data2.create_dict_iterator()): + if num_iter > 0: + break + rotation_de = (item1["image"].transpose(1, 2, 0) * 255).astype(np.uint8) + original = (item2["image"].transpose(1, 2, 0) * 255).astype(np.uint8) + logger.info("shape before rotate: {}".format(original.shape)) + rotation_cv = cv2.rotate(original, cv2.ROTATE_90_COUNTERCLOCKWISE) + mse = diff_mse(rotation_de, rotation_cv) + logger.info("random_rotation_op_{}, mse: {}".format(num_iter + 1, mse)) + assert mse == 0 + num_iter += 1 + if plot: + visualize_image(original, rotation_de, mse, rotation_cv) def test_random_rotation_expand(): @@ -71,7 +110,7 @@ def test_random_rotation_expand(): # First dataset data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) decode_op = c_vision.Decode() - # use [90, 90] to force rotate 90 degrees, expand is set to be True to match output size + # expand is set to be True to match output size random_rotation_op = c_vision.RandomRotation((0, 90), expand=True) data1 = data1.map(input_columns=["image"], operations=decode_op) data1 = data1.map(input_columns=["image"], operations=random_rotation_op) @@ -83,9 +122,50 @@ def test_random_rotation_expand(): num_iter += 1 -def test_rotation_diff(): +def test_random_rotation_md5(): + """ + Test RandomRotation with md5 check + """ + logger.info("Test RandomRotation with md5 check") + original_seed = config_get_set_seed(5) + original_num_parallel_workers = config_get_set_num_parallel_workers(1) + + # Fisrt dataset + data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) + decode_op = c_vision.Decode() + resize_op = c_vision.RandomRotation((0, 90), + expand=True, + resample=Inter.BILINEAR, + center=(50, 50), + fill_value=150) + data1 = data1.map(input_columns=["image"], operations=decode_op) + data1 = data1.map(input_columns=["image"], operations=resize_op) + + # Second dataset + data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, shuffle=False) + transform2 = py_vision.ComposeOp([py_vision.Decode(), + py_vision.RandomRotation((0, 90), + expand=True, + resample=Inter.BILINEAR, + center=(50, 50), + fill_value=150), + py_vision.ToTensor()]) + data2 = data2.map(input_columns=["image"], operations=transform2()) + + # Compare with expected md5 from images + filename1 = "random_rotation_01_c_result.npz" + save_and_check_md5(data1, filename1, generate_golden=GENERATE_GOLDEN) + filename2 = "random_rotation_01_py_result.npz" + save_and_check_md5(data2, filename2, generate_golden=GENERATE_GOLDEN) + + # Restore configuration + ds.config.set_seed(original_seed) + ds.config.set_num_parallel_workers(original_num_parallel_workers) + + +def test_rotation_diff(plot=False): """ - Test Rotation op + Test RandomRotation op """ logger.info("test_random_rotation_op") @@ -93,7 +173,7 @@ def test_rotation_diff(): data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) decode_op = c_vision.Decode() - rotation_op = c_vision.RandomRotation((45, 45), expand=True) + rotation_op = c_vision.RandomRotation((45, 45)) ctrans = [decode_op, rotation_op ] @@ -103,7 +183,7 @@ def test_rotation_diff(): # Second dataset transforms = [ py_vision.Decode(), - py_vision.RandomRotation((45, 45), expand=True), + py_vision.RandomRotation((45, 45)), py_vision.ToTensor(), ] transform = py_vision.ComposeOp(transforms) @@ -111,10 +191,13 @@ def test_rotation_diff(): data2 = data2.map(input_columns=["image"], operations=transform()) num_iter = 0 + image_list_c, image_list_py = [], [] for item1, item2 in zip(data1.create_dict_iterator(), data2.create_dict_iterator()): num_iter += 1 c_image = item1["image"] py_image = (item2["image"].transpose(1, 2, 0) * 255).astype(np.uint8) + image_list_c.append(c_image) + image_list_py.append(py_image) logger.info("shape of c_image: {}".format(c_image.shape)) logger.info("shape of py_image: {}".format(py_image.shape)) @@ -122,8 +205,15 @@ def test_rotation_diff(): logger.info("dtype of c_image: {}".format(c_image.dtype)) logger.info("dtype of py_image: {}".format(py_image.dtype)) + mse = diff_mse(c_image, py_image) + assert mse < 0.001 # Rounding error + if plot: + visualize_list(image_list_c, image_list_py, visualize_mode=2) + if __name__ == "__main__": - test_random_rotation_op(True) + test_random_rotation_op_c(plot=True) + test_random_rotation_op_py(plot=True) test_random_rotation_expand() - test_rotation_diff() + test_random_rotation_md5() + test_rotation_diff(plot=True) diff --git a/tests/ut/python/dataset/test_random_sharpness.py b/tests/ut/python/dataset/test_random_sharpness.py index 8689ae8ffd..d8207ff099 100644 --- a/tests/ut/python/dataset/test_random_sharpness.py +++ b/tests/ut/python/dataset/test_random_sharpness.py @@ -16,14 +16,17 @@ Testing RandomSharpness op in DE """ import numpy as np - +import mindspore.dataset as ds import mindspore.dataset.engine as de import mindspore.dataset.transforms.vision.py_transforms as F from mindspore import log as logger -from util import visualize_list +from util import visualize_list, diff_mse, save_and_check_md5, \ + config_get_set_seed, config_get_set_num_parallel_workers DATA_DIR = "../data/dataset/testImageNetData/train/" +GENERATE_GOLDEN = False + def test_random_sharpness(degrees=(0.1, 1.9), plot=False): """ @@ -32,14 +35,14 @@ def test_random_sharpness(degrees=(0.1, 1.9), plot=False): logger.info("Test RandomSharpness") # Original Images - ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False) + data = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False) transforms_original = F.ComposeOp([F.Decode(), F.Resize((224, 224)), F.ToTensor()]) - ds_original = ds.map(input_columns="image", - operations=transforms_original()) + ds_original = data.map(input_columns="image", + operations=transforms_original()) ds_original = ds_original.batch(512) @@ -52,15 +55,15 @@ def test_random_sharpness(degrees=(0.1, 1.9), plot=False): axis=0) # Random Sharpness Adjusted Images - ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False) + data = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False) transforms_random_sharpness = F.ComposeOp([F.Decode(), F.Resize((224, 224)), F.RandomSharpness(degrees=degrees), F.ToTensor()]) - ds_random_sharpness = ds.map(input_columns="image", - operations=transforms_random_sharpness()) + ds_random_sharpness = data.map(input_columns="image", + operations=transforms_random_sharpness()) ds_random_sharpness = ds_random_sharpness.batch(512) @@ -75,14 +78,45 @@ def test_random_sharpness(degrees=(0.1, 1.9), plot=False): num_samples = images_original.shape[0] mse = np.zeros(num_samples) for i in range(num_samples): - mse[i] = np.mean((images_random_sharpness[i] - images_original[i]) ** 2) + mse[i] = diff_mse(images_random_sharpness[i], images_original[i]) + logger.info("MSE= {}".format(str(np.mean(mse)))) if plot: visualize_list(images_original, images_random_sharpness) +def test_random_sharpness_md5(): + """ + Test RandomSharpness with md5 comparison + """ + logger.info("Test RandomSharpness with md5 comparison") + original_seed = config_get_set_seed(5) + original_num_parallel_workers = config_get_set_num_parallel_workers(1) + + # define map operations + transforms = [ + F.Decode(), + F.RandomSharpness((0.5, 1.5)), + F.ToTensor() + ] + transform = F.ComposeOp(transforms) + + # Generate dataset + data = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False) + data = data.map(input_columns=["image"], operations=transform()) + + # check results with md5 comparison + filename = "random_sharpness_01_result.npz" + save_and_check_md5(data, filename, generate_golden=GENERATE_GOLDEN) + + # Restore configuration + ds.config.set_seed(original_seed) + ds.config.set_num_parallel_workers(original_num_parallel_workers) + + if __name__ == "__main__": test_random_sharpness() test_random_sharpness(plot=True) test_random_sharpness(degrees=(0.5, 1.5), plot=True) + test_random_sharpness_md5() diff --git a/tests/ut/python/dataset/test_random_vertical_flip.py b/tests/ut/python/dataset/test_random_vertical_flip.py index c09d9df224..2fc9b12774 100644 --- a/tests/ut/python/dataset/test_random_vertical_flip.py +++ b/tests/ut/python/dataset/test_random_vertical_flip.py @@ -49,7 +49,7 @@ def test_random_vertical_op(plot=False): # First dataset data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) decode_op = c_vision.Decode() - random_vertical_op = c_vision.RandomVerticalFlip() + random_vertical_op = c_vision.RandomVerticalFlip(1.0) data1 = data1.map(input_columns=["image"], operations=decode_op) data1 = data1.map(input_columns=["image"], operations=random_vertical_op) @@ -65,12 +65,11 @@ def test_random_vertical_op(plot=False): break image_v_flipped = item1["image"] - image = item2["image"] image_v_flipped_2 = v_flip(image) - diff = image_v_flipped - image_v_flipped_2 - mse = np.sum(np.power(diff, 2)) + mse = diff_mse(image_v_flipped, image_v_flipped_2) + assert mse == 0 logger.info("image_{}, mse: {}".format(num_iter + 1, mse)) num_iter += 1 if plot: diff --git a/tests/ut/python/dataset/test_rescale_op.py b/tests/ut/python/dataset/test_rescale_op.py index e6ccf17e82..a26f9a50f3 100644 --- a/tests/ut/python/dataset/test_rescale_op.py +++ b/tests/ut/python/dataset/test_rescale_op.py @@ -18,11 +18,12 @@ Testing the rescale op in DE import mindspore.dataset as ds import mindspore.dataset.transforms.vision.c_transforms as vision from mindspore import log as logger -from util import visualize_image, diff_mse +from util import visualize_image, diff_mse, save_and_check_md5 DATA_DIR = ["../data/dataset/test_tf_file_3_images/train-0000-of-0001.data"] SCHEMA_DIR = "../data/dataset/test_tf_file_3_images/datasetSchema.json" +GENERATE_GOLDEN = False def rescale_np(image): """ @@ -72,11 +73,33 @@ def test_rescale_op(plot=False): image_de_rescaled = item2["image"] image_np_rescaled = get_rescaled(num_iter) mse = diff_mse(image_de_rescaled, image_np_rescaled) + assert mse < 0.001 # rounding error logger.info("image_{}, mse: {}".format(num_iter + 1, mse)) num_iter += 1 if plot: visualize_image(image_original, image_de_rescaled, mse, image_np_rescaled) +def test_rescale_md5(): + """ + Test Rescale with md5 comparison + """ + logger.info("Test Rescale with md5 comparison") + + # generate dataset + data = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) + decode_op = vision.Decode() + rescale_op = vision.Rescale(1.0 / 255.0, -1.0) + + # apply map operations on images + data = data.map(input_columns=["image"], operations=decode_op) + data = data.map(input_columns=["image"], operations=rescale_op) + + # check results with md5 comparison + filename = "rescale_01_result.npz" + save_and_check_md5(data, filename, generate_golden=GENERATE_GOLDEN) + + if __name__ == "__main__": test_rescale_op(plot=True) + test_rescale_md5() diff --git a/tests/ut/python/dataset/test_uniform_augment.py b/tests/ut/python/dataset/test_uniform_augment.py index 26bca3bd0a..a26b647265 100644 --- a/tests/ut/python/dataset/test_uniform_augment.py +++ b/tests/ut/python/dataset/test_uniform_augment.py @@ -21,7 +21,7 @@ import mindspore.dataset.engine as de import mindspore.dataset.transforms.vision.c_transforms as C import mindspore.dataset.transforms.vision.py_transforms as F from mindspore import log as logger -from util import visualize_list +from util import visualize_list, diff_mse DATA_DIR = "../data/dataset/testImageNetData/train/" @@ -83,7 +83,7 @@ def test_uniform_augment(plot=False, num_ops=2): num_samples = images_original.shape[0] mse = np.zeros(num_samples) for i in range(num_samples): - mse[i] = np.mean((images_ua[i] - images_original[i]) ** 2) + mse[i] = diff_mse(images_ua[i], images_original[i]) logger.info("MSE= {}".format(str(np.mean(mse)))) if plot: @@ -147,7 +147,7 @@ def test_cpp_uniform_augment(plot=False, num_ops=2): num_samples = images_original.shape[0] mse = np.zeros(num_samples) for i in range(num_samples): - mse[i] = np.mean((images_ua[i] - images_original[i]) ** 2) + mse[i] = diff_mse(images_ua[i], images_original[i]) logger.info("MSE= {}".format(str(np.mean(mse)))) From 71cede81739b66b6aab94e92ba4ebb832123e261 Mon Sep 17 00:00:00 2001 From: Alexey Shevlyakov Date: Mon, 29 Jun 2020 14:01:50 -0400 Subject: [PATCH 152/254] remove de_error.h --- tests/ut/cpp/dataset/cyclic_array_test.cc | 1 - tests/ut/cpp/dataset/perf_data_test.cc | 1 - 2 files changed, 2 deletions(-) diff --git a/tests/ut/cpp/dataset/cyclic_array_test.cc b/tests/ut/cpp/dataset/cyclic_array_test.cc index 746e482439..55f75c403f 100644 --- a/tests/ut/cpp/dataset/cyclic_array_test.cc +++ b/tests/ut/cpp/dataset/cyclic_array_test.cc @@ -19,7 +19,6 @@ #include "common/cvop_common.h" #include "gtest/gtest.h" #include "securec.h" -#include "dataset/util/de_error.h" #include "dataset/engine/perf/cyclic_array.h" #include diff --git a/tests/ut/cpp/dataset/perf_data_test.cc b/tests/ut/cpp/dataset/perf_data_test.cc index eaa5e85fa1..0eaf74126d 100644 --- a/tests/ut/cpp/dataset/perf_data_test.cc +++ b/tests/ut/cpp/dataset/perf_data_test.cc @@ -17,7 +17,6 @@ #include "common/cvop_common.h" #include "gtest/gtest.h" #include "securec.h" -#include "dataset/util/de_error.h" #include "dataset/engine/perf/cyclic_array.h" #include "dataset/engine/perf/perf_data.h" From 88bb65768e6815ce1708aa91ee617d8d69dc6cf6 Mon Sep 17 00:00:00 2001 From: Eric Date: Wed, 24 Jun 2020 00:04:14 -0400 Subject: [PATCH 153/254] Clean up part 1 Removed extra fields in schema Fixed test cases Fixing CI --- mindspore/ccsrc/dataset/engine/data_schema.cc | 40 +--- mindspore/ccsrc/dataset/engine/data_schema.h | 196 ++++++++---------- mindspore/dataset/engine/datasets.py | 2 - tests/ut/cpp/dataset/CMakeLists.txt | 1 + tests/ut/cpp/dataset/schema_test.cc | 68 ++++++ .../data/dataset/testAlbum/floatSchema.json | 20 ++ .../ut/data/dataset/testAlbum/fullSchema.json | 32 +++ tests/ut/data/dataset/testAlbum/gen_json.py | 22 ++ .../imagefolder/apple_expect_changemode.jpg | Bin 0 -> 432358 bytes .../imagefolder/apple_expect_decoded.jpg | Bin 0 -> 432358 bytes .../apple_expect_flipped_horizontal.jpg | Bin 0 -> 439581 bytes .../apple_expect_flipped_vertical.jpg | Bin 0 -> 851411 bytes .../imagefolder/apple_expect_not_flip.jpg | Bin 0 -> 432358 bytes .../imagefolder/apple_expect_rescaled.jpg | Bin 0 -> 150549 bytes .../apple_expect_resize_bilinear.jpg | Bin 0 -> 115017 bytes tests/ut/data/dataset/testAlbum/images/1.json | 1 + tests/ut/data/dataset/testAlbum/images/2.json | 1 + tests/ut/data/dataset/testAlbum/images/3.json | 1 + tests/ut/data/dataset/testAlbum/images/4.json | 1 + tests/ut/data/dataset/testAlbum/images/5.json | 1 + tests/ut/data/dataset/testAlbum/images/6.json | 1 + tests/ut/data/dataset/testAlbum/images/7.json | 1 + tests/ut/data/dataset/testAlbum/sample.bin | 1 + 23 files changed, 241 insertions(+), 148 deletions(-) create mode 100644 tests/ut/cpp/dataset/schema_test.cc create mode 100644 tests/ut/data/dataset/testAlbum/floatSchema.json create mode 100644 tests/ut/data/dataset/testAlbum/fullSchema.json create mode 100644 tests/ut/data/dataset/testAlbum/gen_json.py create mode 100644 tests/ut/data/dataset/testAlbum/imagefolder/apple_expect_changemode.jpg create mode 100644 tests/ut/data/dataset/testAlbum/imagefolder/apple_expect_decoded.jpg create mode 100644 tests/ut/data/dataset/testAlbum/imagefolder/apple_expect_flipped_horizontal.jpg create mode 100644 tests/ut/data/dataset/testAlbum/imagefolder/apple_expect_flipped_vertical.jpg create mode 100644 tests/ut/data/dataset/testAlbum/imagefolder/apple_expect_not_flip.jpg create mode 100644 tests/ut/data/dataset/testAlbum/imagefolder/apple_expect_rescaled.jpg create mode 100644 tests/ut/data/dataset/testAlbum/imagefolder/apple_expect_resize_bilinear.jpg create mode 100644 tests/ut/data/dataset/testAlbum/images/1.json create mode 100644 tests/ut/data/dataset/testAlbum/images/2.json create mode 100644 tests/ut/data/dataset/testAlbum/images/3.json create mode 100644 tests/ut/data/dataset/testAlbum/images/4.json create mode 100644 tests/ut/data/dataset/testAlbum/images/5.json create mode 100644 tests/ut/data/dataset/testAlbum/images/6.json create mode 100644 tests/ut/data/dataset/testAlbum/images/7.json create mode 100644 tests/ut/data/dataset/testAlbum/sample.bin diff --git a/mindspore/ccsrc/dataset/engine/data_schema.cc b/mindspore/ccsrc/dataset/engine/data_schema.cc index 419816f30a..6c5f882bed 100644 --- a/mindspore/ccsrc/dataset/engine/data_schema.cc +++ b/mindspore/ccsrc/dataset/engine/data_schema.cc @@ -183,35 +183,7 @@ TensorShape ColDescriptor::shape() const { const char DataSchema::DEFAULT_DATA_SCHEMA_FILENAME[] = "datasetSchema.json"; // Constructor 1: Simple constructor that leaves things uninitialized. -DataSchema::DataSchema() : dataset_type_(DatasetType::kUnknown), num_rows_(0) {} - -DatasetType DataSchema::GetDatasetTYpeFromString(const std::string &type) const { - // Convert the string to a more easy to manage enum flavour of the buffer type. - if (type == "ARROW") { - return DatasetType::kArrow; - } else if (type == "TF") { - return DatasetType::kTf; - } else { - return DatasetType::kUnknown; - } -} - -Status DataSchema::LoadDatasetType(const std::string &schema_file_path) { - try { - std::ifstream in(schema_file_path); - nlohmann::json js; - in >> js; - // First, get the column for the type of dataset. - dataset_type_str_ = js.value("datasetType", ""); - dataset_type_ = GetDatasetTYpeFromString(dataset_type_str_); - dir_structure_ = js.value("directoryStructure", ""); - } - // Catch any exception and convert to Status return code - catch (const std::exception &err) { - RETURN_STATUS_UNEXPECTED("Schema file failed to load"); - } - return Status::OK(); -} +DataSchema::DataSchema() : num_rows_(0) {} // Internal helper function. Parses the json schema file in any order and produces a schema that // does not follow any particular order (json standard does not enforce any ordering protocol). @@ -399,8 +371,6 @@ Status DataSchema::LoadSchemaString(const std::string &schema_json_string, nlohmann::json js = nlohmann::json::parse(schema_json_string); RETURN_IF_NOT_OK(PreLoadExceptionCheck(js)); num_rows_ = js.value("numRows", 0); - dataset_type_str_ = js.value("datasetType", ""); - dataset_type_ = GetDatasetTYpeFromString(dataset_type_str_); nlohmann::json column_tree = js.at("columns"); if (column_tree.empty()) { RETURN_STATUS_UNEXPECTED("columns is null"); @@ -430,16 +400,10 @@ const ColDescriptor &DataSchema::column(int32_t idx) const { // A print method typically used for debugging void DataSchema::Print(std::ostream &out) const { - out << "Dataset type string : ("; - if (dataset_type_str_.empty()) { - out << "none specified)\n"; - } else { - out << dataset_type_str_ << ")\n"; - } + out << "Dataset schema: ("; for (const auto &col_desc : col_descs_) { out << col_desc << "\n"; } - out << "Dataset type: " << static_cast(dataset_type_) << "\n"; } // Adds a column descriptor to the schema diff --git a/mindspore/ccsrc/dataset/engine/data_schema.h b/mindspore/ccsrc/dataset/engine/data_schema.h index 9debd2d466..ce61b8952d 100644 --- a/mindspore/ccsrc/dataset/engine/data_schema.h +++ b/mindspore/ccsrc/dataset/engine/data_schema.h @@ -30,196 +30,176 @@ namespace mindspore { namespace dataset { -// A simple class to provide meta info about a column. +/// \class ColDescriptor data_schema.h +/// \brief A simple class to provide meta info about a column. class ColDescriptor { public: - // Constructor 1: Simple constructor that leaves things uninitialized. + /// \brief Constructor 1: Simple constructor that leaves things uninitialized. ColDescriptor(); - // Constructor 2: Main constructor - // @param col_name - The name of the column - // @param col_type - The DE Datatype of the column - // @param tensor_impl - The (initial) type of tensor implementation for the column - // @param rank - The number of dimension of the data - // @param in_shape - option argument for input shape + /// \brief Constructor 2: Main constructor + /// \param[in] col_name - The name of the column + /// \param[in] col_type - The DE Datatype of the column + /// \param[in] tensor_impl - The (initial) type of tensor implementation for the column + /// \param[in] rank - The number of dimension of the data + /// \param[in] in_shape - option argument for input shape ColDescriptor(const std::string &col_name, DataType col_type, TensorImpl tensor_impl, int32_t rank, const TensorShape *in_shape = nullptr); - // Explicit copy constructor is required - // @param in_cd - the source ColDescriptor + /// \brief Explicit copy constructor is required + /// \param[in] in_cd - the source ColDescriptor ColDescriptor(const ColDescriptor &in_cd); - // Assignment overload - // @param in_cd - the source ColDescriptor + /// \brief Assignment overload + /// \param in_cd - the source ColDescriptor ColDescriptor &operator=(const ColDescriptor &in_cd); - // Destructor + /// \brief Destructor ~ColDescriptor(); - // A print method typically used for debugging - // @param out - The output stream to write output to + /// \brief A print method typically used for debugging + /// \param out - The output stream to write output to void Print(std::ostream &out) const; - // Given a number of elements, this function will compute what the actual Tensor shape would be. - // If there is no starting TensorShape in this column, or if there is a shape but it contains - // an unknown dimension, then the output shape returned shall resolve dimensions as needed. - // @param num_elements - The number of elements in the data for a Tensor - // @param out_shape - The materialized output Tensor shape - // @return Status - The error code return + /// \brief Given a number of elements, this function will compute what the actual Tensor shape would be. + /// If there is no starting TensorShape in this column, or if there is a shape but it contains + /// an unknown dimension, then the output shape returned shall resolve dimensions as needed. + /// \param[in] num_elements - The number of elements in the data for a Tensor + /// \param[inout] out_shape - The materialized output Tensor shape + /// \return Status - The error code return Status MaterializeTensorShape(int32_t num_elements, TensorShape *out_shape) const; - // << Stream output operator overload - // @notes This allows you to write the debug print info using stream operators - // @param out - reference to the output stream being overloaded - // @param cd - reference to the ColDescriptor to display - // @return - the output stream must be returned + /// \brief << Stream output operator overload + /// This allows you to write the debug print info using stream operators + /// \param[in] out - reference to the output stream being overloaded + /// \param[in] cd - reference to the ColDescriptor to display + /// \return - the output stream must be returned friend std::ostream &operator<<(std::ostream &out, const ColDescriptor &cd) { cd.Print(out); return out; } - // getter function - // @return The column's DataType + /// \brief getter function + /// \return The column's DataType DataType type() const { return type_; } - // getter function - // @return The column's rank + /// \brief getter function + /// \return The column's rank int32_t rank() const { return rank_; } - // getter function - // @return The column's name + /// \brief getter function + /// \return The column's name std::string name() const { return col_name_; } - // getter function - // @return The column's shape + /// \brief getter function + /// \return The column's shape TensorShape shape() const; - // getter function - // @return TF if the column has an assigned fixed shape. + /// \brief getter function + /// \return TF if the column has an assigned fixed shape. bool hasShape() const { return tensor_shape_ != nullptr; } - // getter function - // @return The column's tensor implementation type + /// \brief getter function + /// \return The column's tensor implementation type TensorImpl tensorImpl() const { return tensor_impl_; } private: DataType type_; // The columns type int32_t rank_; // The rank for this column (number of dimensions) - TensorImpl tensor_impl_; // The initial flavour of the tensor for this column. + TensorImpl tensor_impl_; // The initial flavour of the tensor for this column std::unique_ptr tensor_shape_; // The fixed shape (if given by user) std::string col_name_; // The name of the column }; -// A list of the columns. +/// \class DataSchema data_schema.h +/// \brief A list of the columns. class DataSchema { public: - // Constructor + /// \brief Constructor DataSchema(); - // Destructor + /// \brief Destructor ~DataSchema(); - // Populates the schema with a dataset type from a json file. It does not populate any of the - // column info. To populate everything, use loadSchema() afterwards. - // @param schema_file_path - Absolute path to the schema file to use for getting dataset type info. - Status LoadDatasetType(const std::string &schema_file_path); - - // Parses a schema json file and populates the columns and meta info. - // @param schema_file_path - the schema file that has the column's info to load - // @param columns_to_load - list of strings for columns to load. if empty, assumes all columns. - // @return Status - The error code return + /// \brief Parses a schema json file and populates the columns and meta info. + /// \param[in] schema_file_path - the schema file that has the column's info to load + /// \param[in] columns_to_load - list of strings for columns to load. if empty, assumes all columns. + /// \return Status - The error code return Status LoadSchemaFile(const std::string &schema_file_path, const std::vector &columns_to_load); - // Parses a schema JSON string and populates the columns and meta info. - // @param schema_json_string - the schema file that has the column's info to load - // @param columns_to_load - list of strings for columns to load. if empty, assumes all columns. - // @return Status - The error code return + /// \brief Parses a schema JSON string and populates the columns and meta info. + /// \param[in] schema_json_string - the schema file that has the column's info to load + /// \param[in] columns_to_load - list of strings for columns to load. if empty, assumes all columns. + /// \return Status - The error code return Status LoadSchemaString(const std::string &schema_json_string, const std::vector &columns_to_load); - // A print method typically used for debugging - // @param out - The output stream to write output to + /// \brief A print method typically used for debugging + /// \param[in] out - The output stream to write output to void Print(std::ostream &out) const; - // << Stream output operator overload - // @notes This allows you to write the debug print info using stream operators - // @param out - reference to the output stream being overloaded - // @param ds - reference to the DataSchema to display - // @return - the output stream must be returned + /// \brief << Stream output operator overload. This allows you to write the debug print info using stream operators + /// \param[in] out - reference to the output stream being overloaded + /// \param[in] ds - reference to the DataSchema to display + /// \return - the output stream must be returned friend std::ostream &operator<<(std::ostream &out, const DataSchema &ds) { ds.Print(out); return out; } - // Adds a column descriptor to the schema - // @param cd - The ColDescriptor to add - // @return Status - The error code return + /// \brief Adds a column descriptor to the schema + /// \param[in] cd - The ColDescriptor to add + /// \return Status - The error code return Status AddColumn(const ColDescriptor &cd); - // Setter - // @param in_type - The Dataset type to set into the schema - void set_dataset_type(DatasetType in_type) { dataset_type_ = in_type; } - - // getter - // @return The dataset type of the schema - DatasetType dataset_type() const { return dataset_type_; } - - // getter - // @return The reference to a ColDescriptor to get (const version) + /// \brief getter + /// \return The reference to a ColDescriptor to get (const version) const ColDescriptor &column(int32_t idx) const; - // getter - // @return The number of columns in the schema + /// \brief getter + /// \return The number of columns in the schema int32_t NumColumns() const { return col_descs_.size(); } bool Empty() const { return NumColumns() == 0; } - std::string dir_structure() const { return dir_structure_; } - - std::string dataset_type_str() const { return dataset_type_str_; } - + /// \brief getter + /// \return The number of rows read from schema int64_t num_rows() const { return num_rows_; } static const char DEFAULT_DATA_SCHEMA_FILENAME[]; - // Loops through all columns in the schema and returns a map with the column - // name to column index number. - // @param out_column_name_map - The output map of columns names to column index - // @return Status - The error code return + /// \brief Loops through all columns in the schema and returns a map with the column name to column index number. + /// \param[inout] out_column_name_map - The output map of columns names to column index + /// \return Status - The error code return Status GetColumnNameMap(std::unordered_map *out_column_name_map); private: - // Internal helper function. Parses the json schema file in any order and produces a schema that - // does not follow any particular order (json standard does not enforce any ordering protocol). - // This one produces a schema that contains all of the columns from the schema file. - // @param column_tree - The nlohmann tree from the json file to parse - // @return Status - The error code return + /// \brief Internal helper function. Parses the json schema file in any order and produces a schema that + /// does not follow any particular order (json standard does not enforce any ordering protocol). + /// This one produces a schema that contains all of the columns from the schema file. + /// \param[in] column_tree - The nlohmann tree from the json file to parse + /// \return Status - The error code return Status AnyOrderLoad(nlohmann::json column_tree); - // Internal helper function. For each input column name, perform a lookup to the json document to - // find the matching column. When the match is found, process that column to build the column - // descriptor and add to the schema in the order in which the input column names are given. - // @param column_tree - The nlohmann tree from the json file to parse - // @param columns_to_load - list of strings for the columns to add to the schema - // @return Status - The error code return + /// \brief Internal helper function. For each input column name, perform a lookup to the json document to + /// find the matching column. When the match is found, process that column to build the column + /// descriptor and add to the schema in the order in which the input column names are given. + /// \param[in] column_tree - The nlohmann tree from the json file to parse + /// \param[in] columns_to_load - list of strings for the columns to add to the schema + /// \return Status - The error code return Status ColumnOrderLoad(nlohmann::json column_tree, const std::vector &columns_to_load); - // Internal helper function. Given the json tree for a given column, load it into our schema. - // @param columnTree - The nlohmann child tree for a given column to load. - // @param col_name - The string name of the column for that subtree. - // @return Status - The error code return + /// \brief Internal helper function. Given the json tree for a given column, load it into our schema. + /// \param[in] columnTree - The nlohmann child tree for a given column to load. + /// \param[in] col_name - The string name of the column for that subtree. + /// \return Status - The error code return Status ColumnLoad(nlohmann::json column_child_tree, const std::string &col_name); - // Internal helper function. Performs sanity checks on the json file setup. - // @param js - The nlohmann tree for the schema file - // @return Status - The error code return + /// \brief Internal helper function. Performs sanity checks on the json file setup. + /// \param[in] js - The nlohmann tree for the schema file + /// \return Status - The error code return Status PreLoadExceptionCheck(const nlohmann::json &js); - DatasetType GetDatasetTYpeFromString(const std::string &type) const; - std::vector col_descs_; // Vector of column descriptors - std::string dataset_type_str_; // A string that represents the type of dataset - DatasetType dataset_type_; // The numeric form of the dataset type from enum - std::string dir_structure_; // Implicit or flatten int64_t num_rows_; }; } // namespace dataset diff --git a/mindspore/dataset/engine/datasets.py b/mindspore/dataset/engine/datasets.py index b66fb70c1f..87701ffa26 100644 --- a/mindspore/dataset/engine/datasets.py +++ b/mindspore/dataset/engine/datasets.py @@ -4019,8 +4019,6 @@ class Schema: else: raise RuntimeError("Unknown field %s" % k) - if self.dataset_type is None: - raise RuntimeError("DatasetType field is missing.") if self.columns is None: raise RuntimeError("Columns are missing.") if self.num_rows is not None: diff --git a/tests/ut/cpp/dataset/CMakeLists.txt b/tests/ut/cpp/dataset/CMakeLists.txt index bfdc2b4cb3..337f42342c 100644 --- a/tests/ut/cpp/dataset/CMakeLists.txt +++ b/tests/ut/cpp/dataset/CMakeLists.txt @@ -47,6 +47,7 @@ SET(DE_UT_SRCS rescale_op_test.cc resize_bilinear_op_test.cc resize_op_test.cc + schema_test.cc shuffle_op_test.cc stand_alone_samplers_test.cc status_test.cc diff --git a/tests/ut/cpp/dataset/schema_test.cc b/tests/ut/cpp/dataset/schema_test.cc new file mode 100644 index 0000000000..2da61bc047 --- /dev/null +++ b/tests/ut/cpp/dataset/schema_test.cc @@ -0,0 +1,68 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include +#include +#include "common/common.h" +#include "common/utils.h" +#include "dataset/core/client.h" +#include "dataset/core/global_context.h" +#include "dataset/engine/data_schema.h" +#include "dataset/util/path.h" +#include "dataset/util/status.h" +#include "gtest/gtest.h" +#include "utils/log_adapter.h" +#include "securec.h" + +namespace common = mindspore::common; + +using namespace mindspore::dataset; +using mindspore::MsLogLevel::ERROR; +using mindspore::ExceptionType::NoExceptionType; +using mindspore::LogStream; + +class MindDataTestSchema : public UT::DatasetOpTesting { + protected: +}; + +TEST_F(MindDataTestSchema, TestOldSchema) { + std::string schema_file = datasets_root_path_ + "/testDataset2/datasetSchema.json"; + std::unique_ptr schema = std::make_unique(); + Status rc = schema->LoadSchemaFile(schema_file, {}); + if (rc.IsError()) { + MS_LOG(ERROR) << "Return code error detected during schema load: " << common::SafeCStr(rc.ToString()) << "."; + EXPECT_TRUE(false); + } else { + int32_t num_cols = schema->NumColumns(); + EXPECT_TRUE(num_cols == 4); + } +} + +TEST_F(MindDataTestSchema, TestAlbumSchema) { + std::string schema_file = datasets_root_path_ + "/testAlbum/fullSchema.json"; + std::unique_ptr schema = std::make_unique(); + Status rc = schema->LoadSchemaFile(schema_file, {}); + if (rc.IsError()) { + MS_LOG(ERROR) << "Return code error detected during schema load: " << common::SafeCStr(rc.ToString()) << "."; + EXPECT_TRUE(false); + } else { + int32_t num_cols = schema->NumColumns(); + MS_LOG(INFO) << "num_cols: " << num_cols << "."; + EXPECT_TRUE(num_cols == 7); + } +} + diff --git a/tests/ut/data/dataset/testAlbum/floatSchema.json b/tests/ut/data/dataset/testAlbum/floatSchema.json new file mode 100644 index 0000000000..a6856b4ed7 --- /dev/null +++ b/tests/ut/data/dataset/testAlbum/floatSchema.json @@ -0,0 +1,20 @@ +{ + "columns": { + "image": { + "type": "uint8", + "rank": 1 + }, + "label" : { + "type": "int32", + "rank": 1 + }, + "id" : { + "type": "int64", + "rank": 0 + }, + "_priority" : { + "type": "float64", + "rank": 0 + } + } +} diff --git a/tests/ut/data/dataset/testAlbum/fullSchema.json b/tests/ut/data/dataset/testAlbum/fullSchema.json new file mode 100644 index 0000000000..ca040f76b1 --- /dev/null +++ b/tests/ut/data/dataset/testAlbum/fullSchema.json @@ -0,0 +1,32 @@ +{ + "columns": { + "image": { + "type": "uint8", + "rank": 1 + }, + "label" : { + "type": "int32", + "rank": 1 + }, + "id" : { + "type": "int64", + "rank": 0 + }, + "_priority" : { + "type": "float64", + "rank": 0 + }, + "_embedding" : { + "type": "uint8", + "rank": 1 + }, + "_segmented_image" : { + "type": "uint8", + "rank": 1 + }, + "_processed_image" : { + "type": "uint8", + "rank": 1 + } + } +} diff --git a/tests/ut/data/dataset/testAlbum/gen_json.py b/tests/ut/data/dataset/testAlbum/gen_json.py new file mode 100644 index 0000000000..b7f1bf0185 --- /dev/null +++ b/tests/ut/data/dataset/testAlbum/gen_json.py @@ -0,0 +1,22 @@ +import json +import os + +def dump_json_from_dict(structure, file_name): + with open(file_name + '.json', 'w') as file_path: + json.dump(structure, file_path) + +if __name__ == '__main__': + # iterate over directory + DIRECTORY = "imagefolder" + i = 0 + for filename in os.listdir(DIRECTORY): + default_dict = {} + default_dict.update(dataset='') + default_dict.update(image=(os.path.join(DIRECTORY, filename))) + default_dict.update(label=[1, 2]) + default_dict.update(_priority=0.8) + default_dict.update(_embedding='sample.bin') + default_dict.update(_segmented_image=(os.path.join(DIRECTORY, filename))) + default_dict.update(_processed_image=(os.path.join(DIRECTORY, filename))) + i = i + 1 + dump_json_from_dict(default_dict, 'images/'+str(i)) diff --git a/tests/ut/data/dataset/testAlbum/imagefolder/apple_expect_changemode.jpg b/tests/ut/data/dataset/testAlbum/imagefolder/apple_expect_changemode.jpg new file mode 100644 index 0000000000000000000000000000000000000000..d7a0624a6f95f4e702c17ce26301388474cd6db0 GIT binary patch literal 432358 zcmbrn4}8`2eeeHGXiN*MnOy?At(v=Ty~_<~?pCczD+hEbcXq9$GyJQ-Eo=j^Rx3dZ z8V=4nw{W}MwNTb71w#>+ATa~9A{eK|Y)~Nq5e1P}788Vk#6y7mIl0f*`~CTz6V&a| z-~F|ZV?xe3-|y$o`_JqBdcWVF@8Ii$zlwhQrZ4~Lm!q7VD9Yjg(cq!z#wa&O|I>eS z^}mk|(|_|m_OW3fyD0CXi|m#EiH~2D|B3vIF8cV0kAGsg{>N{>HDdU0=^wg?e>FF6 z*s#3e`4{EeNB%$kXYhsSl8=A<&4L$(<$N~Ey(DMYB{_q?h>96!o(+$|`u}orhkY#X zqI`xif)8x@ZH7N=7=zAZ#C*D%_oI(plJ}|K`=bdL{Z7R<^FRBY!asiCA0GSo@88h$ zFGXK(d+!fEKl|>#{>1Q+qki|leEM_$?GH=-d+AkIf8m;IuN!;gpOk-j;{W{9oBsTk z%E?ovPMbdGFaGkc=FYoq{(`E7_uTv4MT;N&-a~)$@ZbJj%@2OKe8rDe{^O5-Qn&VJ z>mFbK#6Q*l^QO&Po_^-%&u)G3rC;oLx#eH~ZRg&7ukL^C!0T@u{Po*M|GoX4j?QE6 zp6-4>b*AU+x%0i6S57p{r^SCia$c7(uUzdxzUGyadoQnHmwYVm_x|XjPfe)E|K@jo z=d*wOz{d-3_=m@u{^b+D|M{=KS2X+Xw&5fH;HqDL?zE<6GyBaZ_SgSUGs{dYJFkP$ zZw<>~^M+j#C8B}5`sOwto7n&S{+h;~J;}XGKiQn>C@((Key*)&Pip3))$^j~`aW6Q zc}w-CBdN9N>r0}wCzc#}viMBjxr*_5W&PWekA0nA)*ou_=o|QK&A2yu5~&mXq4-;a z(aN%?T9X~UDZSsncuC=mH`)fHvdxon%0H7lw!0=OS)0D~iNR?7sl7{wm#pnto%(fe zYfgFLq-ZeeJ~eS*(UPkw#$R-#cL5)(-#)c(Zts+@E!lkOulYn(UVG8Z+SI{|?`kOi z*4crQzPYcT>svTw&4}XeuAbt(4f|^n?LA9JGpF*Xuwrteedp4Gr|(Ozn7QE&?l-n^ zFuL-VoY@UbF}Zicq?|>%+10bs;*~q@0QU{na%!B~e{XqV)-eRC`lN z9pmIx+T35gzvkk*cxT_8S3EIx7}HprUNY{`|Cl8z{Ot!4ELI3Www(`QLf>*NTaA8IIkqNt})~9&4@UoopkyR`c zqhKpoX)Rk+S$qEOx`x)A$>ZKQ)myo^B)6siS-rDmCQ~S%(!jO!xr*^c>6WhE{WT?P za%)PS8;t(!`1rg+_F&X<{!?ThE}qjfFi&s$tM*)9UD%E_nvDJtzt*N_#h+}PY^!m6 z*Xj7HgV7EAI*!}5`8^6N7e`IynqpKaiK(mY9q8v<-CD?H&+ROU>d(w(4w{+`R{LYG z{PEM;CNiP%7oF<8zs}ZOcPnA#S00R4KJ#;J*G~r{?&06Hp9O4cDtX}4uPfQbYgtxS zQ46Tol!l_+3mz)0m=P5wcXdsvd-jE<%Fep27ytE+qFL!#^LIp3beGZ{Q#RgZJKR)x zqBEZ3nhyT-;F`>WY?IWoOta=49*k~Yu4`6IL=RfdUd3h>?arC_`tvP=(YtjfAthsN zm6wezvFn^lj~k3W9hGjMiX0)9HI3u*rt01Z>eRZLMqR8tD(>zpHxbE=eS*~Zio`{v0A zlotlw(p8I3pDi!aK25pS?lPrr+0ICd<|W`T%+x}p26`*? z#?mnrNEYL(?9AO!+}%F$(Dj4SO^MbkJ0q9I&om!nwOIiS8Wt#OEH6G+P<}*~4_l)} zXJ@cWvQ5iweyY_JAbDWrQ7pjBic0Qza>I|$MsZB*Jebo!jS8JK; zx|a^uh8KRDTkVVz$6C(R)neV2OVG71O^mwpmWuKDuMb8YsmjG{TkejcnM0IXS8d_l zw#BDY^(nq*24Au?7!_AB8W~6pVQzWh!old7>1Gd*+P>mvj#v6xqt1JJzf-p_eSKnh zl)xy|)t&}#ar-CKJinHr3GD<!+{`5xqp7A4f zwSi~4R;Ra)=}dH=p3^%iC;dRr){?~gM;FgC8!4~Y-8b>|f#&H^qI=d=Gn2cTUr$bF zIK{^{Cm*X^_LZIHW}=GQdXA5qck}u0)}LLTGiM-oFZMFM;yY+Ll2y_3Wj1ksqP3oB ztQ|;nRmnSG@+am0yt0N1lx>;4JsRDn{YdU=?EmiR@(5KjVd`Ew;^wD#r);yn)7>{d zf8g5m=4tlXA)-)Np%pZ>9fQ|k@NAJ8Qj=8at*pzidk4l<+?#&*wHln5v~|J5^1!Q) zHNW3C`A(+w3GO8Mxbl`M*XH*>-&Wp^ehx-6XMMl0y7tf+iRh+=!W^@wQ*g^D$FV)! zx3Ro^)bgleQF$S={{jdBvCvg6!^1w`mVV$~W|WWpeEikHsPh*5uH3J=kqf=lFhc=B z6XeY0uUkwBM?acdV`|X7H3k(cOI;?}Zh5faFYvF3Bv!zdO3q_uMY(K_JhzO49PqBL zsF1ZEpWnZ@5|Nfm?il#u?4@I7x+>vz`K4T1im!cl2QFnrgnE_a?$EePcLa2UuHK#X zj;@(}>(F5IwS`zH*(Pji#;?dYNQ+!RbMX*E6*lw9-3_gIH6W11l{JyAWjU}%P6N+w zcQ2Z`bT}HnW`zG?$n`H?%EhmHQNF0tCBRqnqkl&_ieC#u*d~MFKFfv}n}fOsAH~Yr zRop}JuliKP>XplYPhB%S#mJ_b70%<2CNan>me(zEe(xPx_QK|GHJ$xg#@h``kLxt& z>Lxm%B$;0vJAxD?_thRZ)dRqEH|$^5oZ7u5Ogp#cchWQEo=P!O-xrvTU5dNa(;XNZ z#7tgA0+@XpTiY}~4;(cqZ?VrQ_`15K1#%1}QO3;jUAaQ@2n!yVWnjciXwb%}2qRR# zePoqDX1Oq>kPWu5J?m6srKwvwS~yUau|ep}y%0C6xgv3N-+Qi{!raKJM#5F+f*gPReGT9(ctz8mQ-sSiSa~C=c^m=Vu(lG z27y~?b7?T5z*BC}`<)D;7T~hAU8jzGrPN{aW01NS?;~8c&{At zYpIIc7NwWrh+mZ?4@SaDcKhOU=YL+;Fc>|)zcz)b1<97I8GiPjg>$$~)6+7!OzXzu z}~-zoecGKF-zOoRs(dBdPP1oidJ9%(wqJJg$Kd z!qOxC`2>dY6jQ-IqX`q9N6x2gLzoXD5@nmev*>n&Mj>2RGk)jH{$tr?c! zpA+iO%o#|Wz+XI?oF1Je*kO3{cT8ySZ|mdPN){IK3q}w>G`H z`6g{Jz-u=H7Xm_?Ge{-*c-!3asO00S-wl9$Z_P&9OkFPV!ojljKCU-++vZ-&$<|)_hmIIn$hO_44zurDo>(p0{@k8D(U;WUi zX zFdBEwGUvNJ~q9&=O&?GoK;gCjxsQSrWv3qAf;{* z+q2sM3>1+R`;8BVAjRn!of;2%qMar`1wgJd;Lp5eV*w1ixqNcXjMl0=SE|t# zM(cwD92r!km>>qOIUbpnY}BXlPd;&lB$%kMq2`~qh555Xit*(|*&j)SgTYvpgocG) zVtKN-)bNXk>#@}NPZ=Z>P0Gg0#1Po+8?tdJ5S)#PEWj(lRlz19U3dk8P4n5K+qB)> z5);RN4|w<;55;0kWZn>m-9q3~1jO^c5!_G0n%I~AH)t<7ur{$@80wb8QSlvp#e>nt z8U~QNV~T}p1iPJY?*{Wn>pIfOCvS^)H`bYv&-Rt|UN&W$jkE-K56oSYyKH9&?lQx| zm6$WAiEdVGcMtyStERoF zg&0>>B8GCAsq&)Ppyc)*3wP~Lt@SU%5b~wQMR-PA^SotydaJRRa9s0uC@zw-J-I>t zu>Q<92c!14SoH0_nZW@VgI%Ym%d0S)aC+2YqoheEs|ks2a#a?*tRKszuWhR zhdOg#hVWvz$F?vxg^WkOIUx6#I5)6%*%$E@gHZxYI&jIC`CLQMEP=MDSTMGDi$Q)| zpv-so`}{QI(pTI&W#*FQIkR`yG%j5I&p7Eyyhw9z-eDlj&;bGXwdp64sKc-z_(Pb6m zc&6~{tn7i_qR$Cz{oDN@+~(dQ#rhZv{JYGv-(@(4Qs`uI$#pN`*tbIu3n)#-XEhui zcsQMW;<}dxzJR|e87n>G&Z~SC*;v^%aimbS4R4_2$Yw$6AbRw1H$j3quM{|4hmFbcVG#UkHOwa znG~7JTIUAQ&Oekm_R<*-VA=r!DLyAfZWBV=UXLKcuU zPF9RKDF`Q_Grt$=$}5(LPD=XyepB;#b3dfDZ1c zLZ-`OYzw5ZObVnGaF0ae+#Wad0iudrWhaNT2pSLTV_U0OrQv{vQZ)P77xX59PEF$?mP!!^GPJ+8 zx4VD7ye;vosWkQjE*^@qu;SKV*?2W}M|9Y6c|_5t#dn_T;f69z=TiFR_}TY@_A4X? zleV90UGO6$)nEs5xNK+BqVYv%;qgLLcyId0uhld;z%%j}3P2i`p~K4CW$1hFK>5$c z?j`;yH-l|9*CODwv_P0UjKF>x;U(c%>%N@#GFuX^f{&Fv`eGXc6a3T0y9!vBdF}*s zZ8&|qw)36HQ|wA(F{d*#XJ@8G(@(_($JXO1SNAWn|}WonuNNrL;W5oJ_j978N?7vvpkWx0kD z1ZFsjI;9SqoGW9*CI+P~B+nMx5X(xTS(g@IkoRHF%SR=Tt(qrO5P^jsz>L3k!_rR@ z<`B#E`A~%VMZ8tQ2U`?M{Jf|f@|K)bytQ)!fNz4&4T*&iq6DN;NgRn?9Y`!$ZiKZW zLIZ^|OF{n#oY85MjMzE_4B_|z&m^Hz`UFIPtgIz4$6S{3u+6nw zulP<+V`{8MB}NwOxAI_A0^`|Gd~6E{XG|xX#b3Wh#8!JS(SAJrr9~OrIX>^mz<1vy zhBBoWn~0ehwPGeI8jW8g;;qVLs+22WfE_U9x~PjvUs}-rk8<;3nUUROXRlc=v`6DG zm%^V}8PVY)yf^f=B7MYdW-E%%U1H%J0nyB>XP(?MFcROq=k?@GLd9!xcZ-ENb$Ts~ z)Y#!?lV^^~{T+W<4ydP$5%xZD-HQjB`}^7!9!gigD^C6Bk!$vg&wOlB)hEiuaNpig zcxk&b2rR$kC$|~sEc5*Ar2N*x%B06g%w&567;Fdp4_v!M=|0&aK2-tCUhdj*Lvq&+ zo93wHmN>vipFVr8t^DvR(E~;A?g%8*+D`F@d(x}tnfM84dyo%qcMN9E{-xfEXVQ;$ z%7`h@g%<|=<2Kjum&R1-PZ_YREA$${vD6Y*gFPRK8JmE*H=jjXZ(Ag9gOth~ht5p%B@(*u=LSTWb;+%QmkB6+%9vKHK$ z{OWoD9rKONkKo^$VMd$9cJe>`>7c-wu*?+ehPc-3?@9vVrNZD9h0c}dirtMIgs?4t zvjmJ-D6F@oh01KcHg9KOkt0I!lb(F!zT%Y4oPID)&D;$%8?T#rdPi26ZCeNauK1R9psVEKoL zKrNYr1Q|d5%VHv#ArLq?8cS=G?yCH25i7+@&aiMhx#ELLm`F-62NPlMEO-beFnBgn zktmRM8HN)nWHj&&7K0(FS=>Wt8ULd!{GzZM0MVL>?HD93^8x`NJZDu{fQriFc$i=& zJ=`k`#xx^+lI!HYvy3JnXVw}B3vs*(_XZU)RV(hi@X`@u3BMRMs56tm$!uH3Ln;i8 z34o7soEb?*fEuCGEau&K7m;scd6nf&ho^8EKppI(scM`Vv3Y{iFw;)Gx?>HX7u@P} zjdZxON10skRE>5_(95_+i~8Dk8p92>6IQjBcyUW6G_(lJK%I%3@3t~6+I zPxQS}ac|G3eQ6V|SLiz0BALs1jzfkm+ch1T9qs1+Zv4%1pmRV9{S0-hE0VRED@LG@>Th63o(pBS9nZa%?J(` zBoES^_Q|S?v*?>LsU7i9!o8*2N8P*J(hET}p4-HRA56A19;)8~Cq{;qDj1^oHPf{U zm9t4Nm;C#T>a9Crb|D{4m+}ubAAMP>N77ccBRKQlPj7#M$m!3HBPI(LEu7fDxFY?~ zgx32WLitk{-@V;dQ2x#bjfrNP;2OY3V5_-Zt2?h!o*CEXv{!k4dfc~JDzW_Q7Iqw) zsBGq>oIP(OI}Qp{$oG#JYv%2}&j#bg_T7X(vecBxur#`h=sHBbA9=jn8MoQ zY~MJr;kZtDNNIx7o=19%r#D$m6+(|fziF$=isXNJDRhx0VAg$LR`?U%3{E=@@}gBP*fU@%lo zUjVF|s@WeIMA1yR^(Njibcr3bsLsqex0W4jo7i97%3cM-hz22d7rxf_SEm{N&;fn; z_e@og>VYP-`XxXcRlZE|a@o&SR#X+Pfx{q*X%K^BgTFofbDq3m`$C2Paaeyue7QIM1sV zMhoUH^!2kaFZu7!&l6_g@HNG=+zGdiDkJDceQoX37w{Yf}a11|wsS1MAcY=!CB9o1vOBf|z4elje)nDcR(p zra-eqv;KNJ;USE?wkom;RgnooN!UcJHYQEb(FT8HU|X9qqQ}#Ze5L1KxcpViLGE_bimS=ToA3-&lQg&$({p7i!PFzqRiZC?hWE{CBq3Ik2Si%#=mGjm7jVc_@HY zxFHuv9A~lUJfB+8EL=BOsbw!-gsdW8U0FK%(Gi-gIm&G9Nt2{mU`ho|Oo8Zr3chZG z6xr*P$cNrqk77D|6QWVL)7$6Vt<5-9t|?u@gPDs_N2q z=-NL`KSF-`mgmr`rTGZJVr80RUq$9V>zM4XmeQ>hxoy54as1GfXR^|AiyB-LSTQ3^ zv1iW?S-gth0FZ*Uv+^6!&Z64n4dv$-Q34=>Nr43mk75+hp&ah`p1oEtMY>{+a|lCr zB=5eRO#qAh>y%*|MxlK?2z+Oloy{^6>Y1o!KEDh*eRBR<(VS5Y ztcuuH3)^LGJ_x3W@qE455tlDdO3P`%b`mh-L^`)$OM*_YMA?;SEllPlXrNP=OyUZ1 zMRpBi_AmXlqV)%Z8Fu;P(t$$9POU73Sc7S!KFA_DX||UX=r{BZ;7XB$l}}Sxp>m1Z z^9!+WmphFsvev`6Zx8FjEqbYP;`IC9F>Z6R=sajED<>5iS7P3xO=a_JW%qV_o!6L7 z-{d%LH@C7gC)s{{;0x=mScoA@r1=V?r)J|bU&32FMnV%RQc^hLxx~{m+Ek?mqC0); z)?cBitb^83y2&0_AshRvA%c~mRfd;5FAg1+?p<*uif3f9Z2gfZJfm@TiKFkQSDE?& z4i$vis9goP5J_1tE8%;utlICiH?;0~T*qccFy*id=;Z$Ed zxqI1jiN1R&et7Ur+o8tsp$6P&Vm5Ibf=w-^WS=$(R!fOHjH|*p6X(Wz38|V>r+-qq zqxgK;Poz)VEI6_i5%G8i!`<*G;lSk4fkK|rphV?7@QS$Z{Qe~~&i0KbPfD}}5>`b- zN$!jArl7z{dDVolN`b(_T6U$g_%9lA-RRO%x7!4eA9wF9FY6E_JjgJ5i4|J~&02l_`h|1pU+a~R4)*gpRt zZyAM77*TtZ!DL&zf4g5rMrOYxOTbAYNXfg01PG!OC}X8dgU3 z_`@%y%<3?BTBJ-Z{g4k5Y=QRcVmXr>uXRBC2WVz z$bJu<FP4)u#G^Hv)k6w}yOPH;;3O$gN7EQNUn;k#4A9H)HM!Kl=A6YX_r0!G&pm zy;nkY2c2~Q7QzQACT89KXs#u*wCS#X?s($g4ZWHjw7A7)fj7|B;k*``^|aVbh)$h5 zjN0loG6Jv`F@(dh;GkUIph*A}1&fO>F5I-Ci8lZUCLar5#F}TZjzCBz>KzhJ$;o>a zZjs0aOWq>pZ5OOfK0;=+fI~Fv5YaWNR46-h(t;J8=HI1eLHvdkF-O!5OHz!Hj77~_ zYz)6*5#m1%`6)+@AZ47i4-uPaf3Zv~%UbDSX4Z90k?yTv`Uvj0PRwxgxxRj)VHHps z4AV+u7Nq)Ac4irLv*Pih0=VxEiLuB{9mzEX^=ReF^vvdcw`qFyr(`oJXQy7@2n*S9 z?I;$*5*r@MGN2>tGFpoSDtRj<#uQcF*L?o;c6`{=Ghq)OK2DO^sc}|cl|Wcrnq1b! z30dj+QWpyF7kg`UtQNNzAW5_zNPlU`D`p2Mx1-&|y4koB3!=s!3V-SwTIHNQeG}xs z5`I8KzkbPDh*PT2$z8i|Jt^{jlFB09gXXG!u5179fKybIaRYcS&P9M+-TUnX8Ypds zCA%S$sU!p@x7;JKucj_yj&dV)gFF{m#=OrrO0^Z~yo zEERSzB$HK%%h-?ff4nzHZO6d4{r5FLI#Z?MAxbNy@ z6Cj-}6XDXDzYRwrE1FVcA5v?ee_3;HTDmP`7zOw-?v3=aeYH}%Dm+gdy>PFc*=yN%_0k|&|Q9P&bFZW;i#^FhM z_l;F{am@(J^rHMn29lwkl)Y3FrgGC645=FeDv7N?30@cjd#L+Z$)`8q&_lyYIU35? zLruZ8kl#r6?u4Wi`zuRO5^bR@nAG@k&8)gjI76d^nlY-KMYS-Il9`VFMGUfNHjs9a z@qr5E<&WbV3F*qtmHt zandp_s#X6A@9EEqyFQ9ruG%v4l(|)MjCC1;cS%3 zBnomPHpX%(m1htn(~xNXDD!6vdR@pAv4_OyGOVDA;F@HyL^}q;F)JRK(zv)@FyyS= zs)&@SJOw9Zq+j?J6MT*u`r5@&!EhH92_tHS%} zzN%%+lNNVY&R#ltVAH_p^$8MBU8~>k#lo4Aj_EWr6zds33wo>j7P3hgk)2qZ&_$iq zE#1~dh!{#{!r@~`!)H;bysitlu(|EBwJ5JXe$Y^HsX|}l*k(P2S(G^>!84iaSt`k& zd#6`af#K)cn

;X|HE63+9Z8lvU6jvOC>fm(F0o}-$A9&$bEAd)-wUPlv1 zB`XYhIU}>y=r(2F{1ijI2toANYa2uXQm*1e0x*H6vGlTOCww z3eF)5UchPhT~GDRz7tnLV@&ok#iA~i-&@AaV8IQ_!Qi9l+msW7)f=^(77X)VH`J%5 zwcYwFseQ~f`0GuV=TkU}9+Y0}#xeG5om$ODj0Q{1P%2IOcJ+yRT)XvB0W z!E&J76=8@Iq`7lb{aKKnBnCM!PdJoEG(n6)2hB}LBDDjJ693#C?S<3YkOXp2>Y}k_ z59KRn^9_RNH}V)+OQE?n8yTm9&0s22Lb@XX-Rfsf zB-2})^qbXui(Ro9$A>E1OZHaHw_0lJxVf}l7)FlSV^vGi*g+6ZlBF)onfv7>Z=@fz z9(ki{!-8R`Lz)RZA*POSLZwzVDU&o`*h*?l%$2Z_Y_E)93@oz2zx9m_*;B9P5XRv% z&%U67jjh@|T#zX~Vjy7Z4lOXBR=G%Hi1XukOZk1ktSY-gBfwN@wXMo&_}S%JVVThY zljPSV*5MokDzrmn{_j_YW*}`0{=z)(W(4@Qr3y0IrTvd@hh3% zV=U+LtmXmY&>*$KcMKEX!Mmr@%};ocO$6?4%El)>P~cnpbh0jWG?T%+Zqrs9TQ`}V z#Q7z1Ii|u;1Hv&B>0!XGyhVL9WW1rR8_2*sz8};#F8 zQ&y6Tc3Q$4yJ2ZPt#9$d*?3Bk;CL|^<{Q*g778p?Zb8uJ{VE4Im7*Y3@XxTRt#@9* zG#bm!oyGZk6-PPNf7b153g!jdesijHbnA65)(#XiA@TYsv_OiTo&u1RmLDas6cuc# zG4X`P`FgrRw%PbVo0lBq@KlDiW<^w`MzJp*2N|-`-Z5l0bOyuS8}Fj`_XNc`>g4E> zQG}owwSs$DcL!d^@sN^q93AjbCDkD~F7=Sc)lqYxDU#>!Rj|misE6C)JvigD)}O=V zk5D;{yE4+)cE~)FJnA}uN{h5*BZ;7hRU~6t1?F$Bnfzi)E~&9XBn66*D}WUY2&>6E zy4C?#jRqLaZk*MTKy7v|&QV2pLr)7)_sbZyBeMeZ!brfG3_%_4pV*;!SHK5t?O8R=v<9Rf<^)%Ua7% zDZCMp4oDNdNV`o@OlPr(`@Ql)qRtz0(zJCdRG3C|0^%3q(6R*wEkogE++l4d>MRI; z0{7ZMwh9OT!GxA9k%F^~7sy1Q?p6L14=F%$fg7$=QV86latsQhK>_$9;!#yESQS{P z;A6OezB;Y20uSX0O$N-xYr8CU9B(r*y}^b}nZ3_d$%0{$cGguAEwa-bi=2z&_$nj+ zFp|~o!Yr^UwbjVXOFm4+DIv5ydP$eVxcv};H4c_l|&l`9=1w3 z39G0aBb`I){MzWOw9#TkWUbT>Ly^t8YZImmsya@|30TNH%qGNk_><8S-C9$QjkKA< zTa0Lo-J<_A+rsMWlpu6(qk%mueley;UGvs(7<)tg6j97r@oy}-rTERRz^g5W@-U_c zG$T)D0jX!t;*nfB82t<|^C|+X;>U>t*H!-wt$o@(Lx9=ZUc=*#%{Ul>@c+7RlcoirFZdxynDZ6ZW!AAR*8{dfT~FwRWl$(o-koRPa@mTKgey``Rx{Q& zvqRy6iS(b%rpwGM5ztX9#V~zjM0Pn1D_vAP=lY_oUYZW1cqo(_U7&ZMBOVP6yd%n#L*?$?U`56p2F%2MMTDQRt9~GY=gLLlK7g zGA;CGcmo45EM@(6v#}^q{Ta-rWk`FfR6DX5_y|Bo2S&=_13YbmZvB<+({3n~UAL=S z+YHT2f>9oL6NM+OT`2T3TSC19BjyD)O)3-^S)~}qnY9fSa{*hKk|kudPFv<%Tiy<6 z$JMojz)4!|ls*j`zgX(MCx^%w}6B{fsjh?K}Ve zTPL506E0r!m71iIE0riU#nu1_Z&tQl;ypAds3>pu!Wu*t%k8vKCZ3T&cbMWqohVt= zcA<}|)@4s1S7kHIPg;2^H^9MJAD{ed$c31Cg{m!Kc@87gpQ`%dC8Y-+?4VMGOvq^PeA1E@i4tZGFsz#*6=mRt5nW2HW2i%;8 z5CE+Uy{y2Dv|mYk0|rtb@+_K}SLi9`OPO&sMS@8XY{<+^ZpE(cJijpJ%LtVY`mqYl zurOL2mDafd@dLMv<CY$ihay!MF<6I|yg|R3iR04Y$`>+9B#)fq%VV_CTluQ8mN_#x0K&|8 zaaHxE6ZbTq9vFW)7`fsgBA}5`jF(0CO1W>%IGvawXxo zKpBft0|u6*7t9r3v?ypKR{HWI-|YLUR7aLUNL@z`c$SucjJd56tEfHH%RBsBu_ZW$ zFRcvGw%OtvFa=$ks#735F(d_K8L(`Y1%)tR3PQ8#mEjy^!`y@B^HX^^W!asH38p2` zI|KuEk_4A7rBh><+13ySnFS=MWn{5wfk#kXmnz&deF3N$QL;2&h(~~j!SF%utdGls z3+Eiv7kQhBV-j7zlphu$CS%>if*1p)LP6hQZAWL;8R(jV2(qL41~0CL&(sJ!iyg$^ z0y4^nGIT*c{5uI}-vrMsmeu@G8o=pRY_!FcB1deGvQ?D)0Rw&c3UeC{ zN|-zhP;9Tv;o>`(90rlIB-BKi*N$qK5yN>8^(OYa3lCFrry>rN{pU^yj&Lcz@o$yp z5Al)`XCf#HVrazLH(x}iOFDxx-l&TNiAkjepta|^S+`GHH9-|gO{uE!r0^cuoFt`b zgoRwz=og#|y+VohLnjjJ1mPs5`4cFf=!A@=KtZJ>iXI;3%1Avzt<5&j;7->)x#)J5 z1EO5MGwf9HUl1CU8bi%IG?(B7qG=fL6*O%OMh{>q(vMi5gx78c_~DlcY=mi;U4V<| zNHanW$*IT@Eu9+jL*xW{SM{+=B{!8gj9Q(7v!~#|cA{P46ug+IflN3CWU+aLu|mrz zGlielQcwI{NZF>o>#IUkZl_t8fHbNWlB?6Dr;5t6;(vzBoTg`=C#QjU&8JYGk&!R z8Pr^IMeN0_3(Vevhp3@<9&$qKRL=5b7-YVfmkRjHwm*ST7?D&RRPfzV@ z&9&vh;2=zcQP*2QMKOU{)l;nMhY8rI8Id?W^K zqz9mcBAST2u9c@@$j!pTOA%5j%Nysl+(@=hX(&LBQwF*ioaQgA)MUU3J88yCmc`f+ zd>9%VY$czA7PJCcq9RbMv4OI_l>%r)3BzNP%=e0B+xi;l7goxei5k=ca>TR6;gexE zm@Jz?)38@WaTjxn?4C`_otg1zQhD=tApF=rsjvs{hrw34yOP-%63kGuG_~B$NFT4zwiv(%X(uFPmZS^#w_ zry@ryQcrIwZuW!phpR4|Op$En9MvOW3cbVQZqAHqKHB#aBAjhF4EDuIPDA@H!rEOO zBxk7bcaf-tn3r;)7oU)Pffd%L9}$yfOD9=Ze!gt8Eu_T%vK$Z+V-Sfhy-uzw$@nrQ9_1BvM2d|uoTKeBy4DKyso3yS7gP^mU`94RC(N}0HGFbDP%x*ZPStaSje?-r#j$S2E}~nZHs2GUS{wzD1zH^W{;B7F@>kz zv$nYXKxLtx?IMYVx9n59VUsqAbABj(mt%{X?oB_0f3BRlfv8`)D_d`-%&kNo@fQPt z`IL5;HRXFWe6>jU1;Rt`s)iPdX54d_`$|6)I-iI-EIo!l6ZwJ=*coW;q|e$Bh^wra zHte0z*FotGP$QBWHl&(Xmm(flSE1*?S<@q6ZFiVjYk_i0k8bo3mhcZ9p61XLL`Yzu zgHR2Up#c_ca5_LN*NCKQ;Y*0Lp}H}7KWX%jESHM zJ`lg=^n?_MfpM+^Z^#BXZ(&e$xUZojH`Rdz3P=~?SC$tO4WyQWdwLMpZAHwy$K~aP zh zj28Y(zFobce`0>c16-m?E0~hv#s>#W!=mKTgKwLw^3@287XQdN9b-OGGzY1KWl~4L z3n!jzd^B#pG9)n$iY@VY+>Is^obj_zGaU*HN`qFK4x=MZ_BT#8s!0|%_RIVXM%UoR zW$xYNB(O2G2J?&<;@5+6v~hUFu3Z+u5~KgYdDVa8!%M~vM^;IyEPpP>_e;lcy^Vv5uwks>iUU!CoeN|*F8_O#euHKS-tSy|v zWGB7=bIeX(^!}(>DpSR#*+E*sW*W9)O{qeJ6vsUKLJVE(u&)`6M(1SlBOFd+5us5d z3x32ttVKj7RTZ5G?gwizMq{dga^oii5OWO2z|W?z{P1s$Vo( zkYq`-;vD>wyrJa``IB_=LgCT}EeIgi%tJyNQwMLR{D>fdvLY&X?EJa%^*Yi*!8}}K zj-4v@bw#xncA3`6N^xkx?&Y?Hxiay7K8G3b_&6?o$9#p^cif0h*R|C$%YXs(&Po6# zdZWq(S@<)vEiWQsL(z)ND`A2_sC~j^SFBxjI4q`xhRAuSsgN(gOxYipGNGAEJBCz& z8TL1JUW)1PE_4LKADHZhN)m(P=IhK=x(|~d(@P&j8|a=0pLVHZvE{SbMQ(~sH=wdb zej-qn86#mXLo+jkikDWL#@ddtPk^_>1+qFCbCj%v4Iv(iay|iOcny3w7bFG-`YF=L ztv-^-Z*s|=0h1vNCogAbm;)=HnY5iRt{uQqgPP}C_~kV{v0F0(Ev`#$4zd_*y-BVL zryv29%E}(&ILw`UH*5)INS0)A=n~undH-wxW9QcNBzlu}D3Zn}of|qoYdzraSpl># z)u4bk!$Ns@jg~5$@M$NFkg+Fxq0)Nz71d?%RmempJ?Y|!jghi5^bfw+0g+DTxtybx zJN0B7d=yTMHD_Zxv6t}(8NG0hSx{qb)e|ObXOip>bMS|=#DImxlku;$lln>t2-}Uy zkh8d))A>2UOLgRGSvR8A*4Ggw>A6thVmi%um1BFQ$kFpIh{CZT z6S7*nPB(~0)=B?7Q%GkXAqjT;2a&60VrQ7KdF93LpI$b+B=>oh@J@XF)b@Idm%RDy zNcv-521r(o{&URaNeeYph&OX-(Fw>AJtP5y;g$u?=->V~4WUFpEmUic!;L5hgb@^L z95symQIhQ8%6Fv*RvLoi<`f{dvE#&Y4rIBV>G*0{2t!K>aTQXF5;lxns1p`gdn}d0 ze3pK!d!yM__-m~L`RTu1`^XpTl^1bgMXe7 zqw39`%fM;J5+!H?B)q+lgxTsPF9nk2ip0J|`^yS2d&splk$%EN%YE^(B5-a-{9!PO z>VANmKxiJTGM>ZBcvywbFHvo6&MXC-ia1=9)q8*taW)5qyO=NH#gzvQx_x!c$JEeo zc2P`i4+On`m_PAL&{cgND9*RNt@M2o$4U-RVqxdgyGGZ6KIM`8oNViO@<45f)*FXv zGMnKFx2c(fA~3KK5s)`{vw`6#ZLliBjWxleTOo@3Gp8vO!|~`0i^AF*&tmm>FF<)N z>4so6=fqHzPd`8gvK#?gXC1qzphWQ6KSHdQ^G(Ss)U(6Z?Z(w>*x+BDeDn#&f zo7q1=eWnY=uSa{5+^%LSW1^sr-3|Uz;6Rb3GhF(VmDpo)aL|UKH8=z(YmUvo%MA1S zaQ%N4vwzrrn>`;|{vi7jdr`TzcvER`m93Jy^yQE=z?xW;u4Pfu@_7VVCIHzF4k&Gf*ebHa>31YF`wA^y_ zrh!rGi>Hr%R8hkPAJ^B!Y+(LWFs`HAiA{+jtBTlB+Nt$lHP`v?Q}mRB3(%FVjpptT zLa;MGiA*IhI2#)wdu#I7g?@)n4g{~VcQ#(ZB~VtxiBv1MXbV4 z80i~4B%ry$p33Pby+U+M9`o*YC^^kdSBAxwy^v29+OqTc@ndRuGz=Q|aVBW=y>-6V zpeb5OiJh{cD1b6CfS|K|vAt1+j?o?Mnjc^Zmuv*6?Ndw|P%q|Uwy0e0)+T~UVfuTT zUwFrz3K>9s{(MK(i9syv?xlzPQ$iPHFSp-D2R;n9bS575U&TmyeE;lRt$tus2A;qpTdD1V%(MYGy}yXHml@@p4FgpL3$TLbRbNyRvoIx##*8Vht+KZQ5dp%!rQN zdSD9}5j1rZwZ`f`Qb`h?L=TuzY8|sjO3adSZYyP?cWa4dr)bSwe^5IK3IaOkty@(% z&D?>P1KH+_6;w&h{p1bp%uIos%a`YHq?SF-%L^3<4D}FM+oISO5^DruaN}xpQ~W38u+;BlVH$sOj?pN&f@FAC`96gFtNNh`N4=+k{trPZeVgt7d+c>9LmFQK5w-KK#RGV8fEca`U|YQu!54&u5MN^J6fetv_*2FDP5v(K+LKr&aYKepyasm#n8C`% zQ<3`NXDqs=@kA86TSXk?R?Qf@zl;M>cRsjA4|%9haXOY;SQ9FZ*fPKxxBBhF1VW3l zvc=AIvLJQ-`cO~#@RYA{2Y`)-BYQbcyhmDCSwF5kvWowNx)R2WFAH3OonWE@NKXaw z$X!2Nf%}=Y%}mH~kP9A3AWXQU0_~F3lY|tfuZS7X!RQ5F!A8>Tcs`FV^Sw}50F&tm zJ#tG{*SHI;yFjUzTZm_2=PzCgA%vA0ss09fW!F@1)PhdkBt-z`2i;}y}xMo?oB(_5aQ>??` zRfU}27?+5N{gpoug=(2dhUJeCweB1;&@%6=i1CMV#JYo(Yk2Xdxo4ez0cNrXUChIQ zS>u-bgc8Vujg<79V$w(8ovz*u$|r=FD&4aq-E8qDUztAAGhMu*_Tm5sq|co=t8d{N z-1;j!yhPF$!a@X$v7I^zQ0cq#^}%Rhp&d*@8VU=(pZpL)+8GlG5Io~$pWSfy z9%?W}jqXctn-_G0hi&NT0nKetJ0Lk6KO))i7J(0E1~UM9x>Y(sOD(-$r;mvWEN`b$ zlc5k%O`)QZf-8=KZ>9_Hq4A$6zkD$IcHh;OEpOqAD)&I2zQ;!C*nfJSy{AKs9u&~k zHa>+xZx1XA8qKkYJoQhH{@yW1LN9C%J=G;=&2D%xd5VUg)2jR=rP(SeQsGN zpKBhdXV23!cy<7089UyVDH1XM!>i|o0zSaU#fC~hxtZrPjJd_^G*u!)0Mw>H50obY2q&x(97IXdFIm|BxdYt+-DHn6877J9E)be zb`BkB_ILU6z-C%T$&^{LV`bu?TVWVXt#VmrWwe~me#LjtPZrp2FqEFm23grpS|ZA~ z7ssiaTUen`?*cx~P1c2_Mp`b(CfX>r1EN6>jN^+|QI_GWtf@*dMrK|-%hVQ|r3*O9 z+c3;KFl^&IT{9KAyuqLB&j_^#{0AR136oNp7jpS#>z1#jJcqXEg z@d6}5!68gAdJoFf+mT-2@R41LsAD&&G5Rxj%}xs8sa~h3HGV3<1Uov#B_z0Uui=1; zG*&ETlH)D1y~kMvCBhEt@Y{gRaD|hN3<_>In0tNJAaGD3U#4Mf!*=II6b6iuS_?v2 zPnCj9@6mxt2@573j(_Onn7~Syn1-ZnrU&22I!T)rTf*aUh4C`RrlOzbd@S;^4}>rt z%Sr%Gm5L67^$fRBCl;Anmd(PhjA$0gM7!Et$- zPaAFyIy9_V76~As0SV5@bE+MU{&=sqWos?yGW7}a!YG+j14jwl(>8<&cCFxn6FNU> z73Gbx0@v;GdPd9BkbOJRN&Iz{juE)KzGpQ@&1%u{1D9Ip#=1CpLyix3v)OVIo-E-gQcSYo>)+uil+T6AJ zbevf>eGvE7ZIyaO$U6ac*ng9 zUKywCC~OSA&+;=Q2Di!?D9W^F?*#wAViYeF9wL4O%{2v4)W>8v%M!LUg-uC2zsC@VcC)c0|o0WoRg(sZKxSR7)S8< zIngX$EhpDHe}^bHiIM0w_YJbf4Ki%_6X)ZmY!b!GI$23pu*_N6llbxqUeTRI>^Mbh znwPNzYDM`3d^)Fd=3Fn5#VZZLQf}k2t}Qc`cueeL3_asdf5gveZ6EPR$#Uimj|;+p zO>yk^Y#V}?9?OsW`TbG|E8c`Sv@ke$So_38G)0Rxa6dfd!xHgfk2P$soiD-<*7u&pQ3WYH31@&*g};djtFW-b5Q3kiq6Pav z^42VEfpX8&+O*tS_~hfQI`s*{DVrqaSFeM`M$5$#g39G8wvz-%%M?FLBw!$pzs#`| z=TEO6Yn)*pJJfwfsdSV(DTs$YjvoVfR`SC!%^i>osydj07*r)Y}8vHys zwzY3mdYI*IuSgsxyAz(Ex~THxsj}@>?XzL#EqLahpF}qgTv9%jWE56TwGDPrPF{+5 zKD6km6|B8dI%w5H>_X6N; z+8(DQ$k`atR=z4SHliiYOASkpy90~i#$&`-k8@)CXj0iExFV1fpUEh%W3uC48jgp< z?GQ9sO~3gg(~Y1SER@9~!lABzazUOyG+N1Apm4p2FKiEf3p=TR*bI5*>t>W>n117z zQT=`qOXm#zqAdgEJr0J3z7zk>2pEUFs7~awa_Kx;D>6@WN_BtK+s(h;pPSSSQFxpE8fF?qVT_*$oz-NnUL+Yv?nwWcz9a0Cc4)T@r%E_!iNvQPV(&z{~Vh>p56z}0g z^vWp4hY>q=8nSSqpokfmYxLrdLOaCOjE`I}FWF%`%!`Mnm)IM4F1f-?6$`?sUTRKm zDR_F;eZ&J2JV)X&Q1g7E;E2<1lDAOcVPa~59{Zy1Q27ra=~Os) z%TYQ@R63=og8mP42ul;EqbF`B$9IKVo-n#o{UKvVj(Sq-IXx+ijnY66KC2Bdp8-GQ zbau>RX*YRi^rHpy;*|HX!RRj<$6Lf~Wz6c^VGb&5IPz_t{^~IUO%TJINYIF@O|eBP zqdO0YxE=`B(p!C4Ch>RZfaXLGW^#>;f|Z%7Ig|hgLQ`M?vofUM-O>(TqCcdAM%CXX z>RD)2WLc@)s+Ev)&f-RGnnct-9xR(#tvuE8AnUCj`!>%hPQCx<`(<8Grk%Fcp@uiV z|GsbR7cpkBgE1(|O(7fs>o-`m47eir*%_c|e1w5XESQi}Bv306Eq~K}yz5md2~g3F z%8zsgiSY1%N(n7jYZj9Ct#2c_hTlR~SiO=$v4&*GzgT6&8P@O3Hsf(*LNNV~OlbfK zE!SlbraZWvaWcZhT#3@(_gJ4Ztf;^Y*4RqJZnns^5BPY^Lz9Ua%!$>X%4 z50=EFxC?_L$D3JA;&>1eReH;(p@vAe_vY}631X8`A$i8f>)FegA+jWlBw#ytN)uYm z{ql9pYcxH{6C&AA6(A$Kz(jFSnF+OnJkeQRvPeKP)r6rZEO#J)_38Dm&oh$3kc2`} z=nXz7qtb<$8*4EQkW1r}oh9hBe1SN37PJ(0yLx}{OgMCMKo}LkN^;kM{^SDwf@z8s zEB=WI8eAhbpTw633S}hrFiFMx;;dDspuX2KEwnKbx0rWcZBqQBNrp7;L>6IANon^ zGW#Ye@1Uoy=^=*lDR{$kP&J40&_gmqs8&dr@lTD)fH-dqHMZF&&wg+WA+Z!Fe?_fM z2IA!8=-Txk**^ANOpg`h>Do|eD#p^{EpnD#1v%y_f~QbbGePV&Yk4`4PW+sBrAo^U zYr+ZXG1iI=t?r>4BI$IC74pb3Yz9;rRa~OzMxgI zS3~Q9AMqQ{Ewov=i?I_jG+9eXo6i#9io;P-k`g9%&{xX2=P{jJ&=fXg33OkkE7zq$ z-uYQW9G8;`X<15I7c(kSo|S7@M#JSv2Lpx_0N$m}uI9yo7$qy*#Yv5!pwAlY&_>6H z)7ntfESH8UcW_eKxlQ%Ylp+v*WwhcxtGeU>0cC1=N93*ggXpR?)M!loJlwOD(ODij z_SNR>qiKCJ&K~?Hnh&R%2$;|spvfFADI+KciCyHYmaW(N?3>?AE9{op@oIX!U>d1A zuNk3}=Tv`uE1a4MxmBgHIke(plKCF>kG8i~o^P>tcp3s5mpKAOiC{UL$6GnLNmX2B zn=r-yY2bJ({>1}JwQw$ z2<-=3`#uhqv4Z!3VO`V^9A5B17sOl*J;k{cadC;CZ+qoH?;RGn^OS7sfHv!>8B24i zf3wH^c(J%W8MTut*-j!)OR8Cy1SRhGw|Ztw53ckYb)Agp%E!>1XhrTkE&$S(ZBg%cR3HY1XWvvn-q?Z~E!kXZFvu3>r(PgE88yFyJwIUf2e0t))A4 zSgW3iP6yO%Z?Zd*VybMpo)#w(Tq#A$+&sx_Xb)Fe3M=%N2`gANk9R68MyI9M86+62f`@{( z4n?zL_A~}q{4_Ke+o<3CXIu<{?Zj-x>3W$V@QEyd7ieS^3s?{I#SF-0m}NmAVkhGc z*3~*&?RFs`%!Z1YnZ`5zO@C%lCPawW#j(nv;>qfsA?4$AnbVIMj}t4q6h3>Xo-}>v z%WbGF0R?)QOuvOwMz_YM+J8FK4x(bTt3amHIdRdLZpRDNGl9ejc)5&6FJ7t7y@Vyf z9KM4VjD<)DLoXsu2sP_JE*!Qc3Pgj)t~JPxe2LO3V{=z_{b*cjg;-^*3h8M0S*YI5 zx=g~-=F@3gN=QI_Fl6TyW*W`Nf2Ulls!oQc?F`BBA^YhGOjm|*@^6r8+CBUt`XWY) z97W}QoX2+EF8*M=>v(mKqSK@t|SIA>j=fyS$sR|qD2#p#qq;8hqKp`_ zu9AK#Tu}rl@&j!rg4Y9H{(YWn{?Wb5ct$0-=RS?BZSiTHlEkO$y0=GTOdCP#ob9L0 z42Q~^0vHjiHIyB`r>}rRIK(mR20WQ}ky$Abxn~SKj7$g5ll0bumczjRuMO6v+ca0qsChL#0#?(CUvrI9mOyH6hG?f7 zG+7yzGi#B(wpJqjn;2d(L&VP3Z4(q$Mp+|t9f+66)X<%yp(7<0pb*5tTZ|&3sj+7J4<^Z zll2I&No5Gt+&W6#czIlhg&6|;vvcbhQa`|7Wy&T@;KSDWkJzcxJ-cbNo)MpiYOl+z^2?k%{+7p%iU4W{2#NBs#Ok z0*(r!^FPwdJjjEtsLOe9P1qPRDA%K|o&^`)s=djcu0{EC7^h)DC(mMU72KS>TakJa z{q#^9tnZQ{D&-%Xu{Pk&VBI7s8nW^o;vJcy4511gh8;H&3%MhQXf&s_WQUqNb8AM0 z6QJt38?*t|Elc#mVuT1+R$QOT-M*TK00#Lr4{0_N`9?1@cH1M{IPS8#0F12*3-9S- zQUJRnUWoou0Y_Df`DtyOtmCCbEL-B-0L9l94qE0+Zjz>5$R~{^Y>N^zGUV||j}n)6 zcj#awjGUi@FoPnkBS(WoW5%$57MB>JojWiheSiHHmqH}b0uxt6yS{=s)!sGbGhL+`@*GN2kB&+(EK&AW}!av%&+A`^oIC4F&^8jk(u z&7!^(Ou6@Bh>28ns2615tD9K-J?HCRr39WH0ZZyrUSeT@_8Tsi|LXL6Jw)0b4t-Mo zhyw{FaCJN=WMcnw$z4yXq=d(=vjse$zH+vOd&bL#BGzH)M6#WSjj6F1LG!bTn2&;G zSuA4$w3+xgeA)_&ZbF)I_*1G*k;8|L%sxkfLuk!;FgGxqD(+-!>2F)BY6Crw+R_o$ zppStx=1VO(p)^07-eh*&ivEL?81rfsfeOd> ziOmc71O?rpjh0`BW^goz?CXP_qFJXVI3W_ss5eC(Fr-+}tS22(({0`?xF$<&d7lz^ zMpoA8PL&s63bwK@#>JXG!-a(-of@f9_L8a?7bpb+!+y64J`77QDlg(35;!r1Bbj51 zE?5M!w8(#}dunqT&AenW0M{=Urw5^2q!W?nu1y* z=<7C2PaaVBsD~Q@xBAR}<#OQ7qWpp1I(j0}DbVn>S>NVewp$N9(b9$|<(Y+J?d(X; zV;xsX_Ao$-WN`p7&uMO;Gn9)@Q{d!jT$z0?s~hvO3P8T43*c}vWM4T?e9oE3qdjWI zbc(Cu5l~F*t1P)ct&cI_J9Tv^7nv0pFgqyDN`Y)f+KgNlc`Dz-Nvhci@T0hjjbX?h zwQwB4c&XVVi%U3QA_Np>5>%aU!#7h%0HdO>;w^Yl&T3G_mzpo3iQQ&5mS0-^NP7D` z72Rm{wLCN?Xss->uM8Y&NtFC zofn}Cg@bT?Wtmt!~1Oou1b6Gh;t z+C0FP3FJ%$&K$#ni#8P%8yK{@@MK0=bX1S$*?2N$6J_V$78CoJ$|O1LmL@csd-QY^ zRZHlh@{#p0HNUKtFdDV5xIfKbS?8_toBZMpp(5QaAcqU{;(M6_c~|C_IJWD=Rb(UQ zmUB9V9?|?PLV5WLah7&gQm`O#HBwfnV&Sr%+3E;(UXu2ozYGov| zXg$%)MySf6D5#_wMUv;{Y+Rf~Pzk%G#Ldd}mb>>5^gweQc?I*m%Bn5?gNX!5fJ zd#ASNi?dQ)cOPsx{ML6b-;++#MvKUjV7 zor!nmwT;Wsgn!$5$sm z+;jC{)PIiD<&6k#$BX}X;+Km@&rM&_aO(VfS0!J(CfS)N&Hd%ebx&Wp`(fZ-@z(Zd z(icrBd%O7itMl(Ke&y8PC-x?e?cM$C$q}DeFh{i>#dDtT*;xFGSx@Kx%i-c>R}_5X z)#P#B?`uBv!yC7Cyi}b3?c~WT7WK@ZHT{34?mGWhfArI)fqz(Z$}IV5J~Zdnmi`|; za{HZq_ixZwWOM${<$rg_t%K3QYtg}P&RKoqn+a5M_xu*BcVD@GPs5V8ii^fTv`_}K zbYbmaRPbNVeP1Je=CuRY6sL|&?D_lOPL2A+J1wti-qj5!68~!}ZHM1O`&*Y(c9->k zY17+fECBUId4J;H+7mvpo}guJ7TTl|eNK?!NtK-_7SrAFgTcc==HJaOs}@ z>u+RWt4Gu)HsAF?`k&tEI4~H!G3uvhW}&6;Jg?JE|N7Bc$z3O(1^6#r_!#XgdDk}e zmG!@reDC7gu1o)Gk~^eYddfSt7r%b?wdCo(SzkPsc<<|bh7UBHxMu6+kI?D3;;H1X z>V9(B>i#ADm%X}i$t!5=Ju6q=wzhBL#}~}L_V#ODOkKZLb;VPDsBacL(A(0# z>db@bhu(hj3mxh0C0oCAs{ZKje7^I^5!;Iof|chCMvw0MZ@=7Aw_(N`Z@ppF6-tp{ zd_L+WOe;i{H4as%Y4Q28pO~M=2^t)_Zl_pn6*6$FLKfM2(|`IA0%^xZ&Wm$|iIvJ4 z#@bz@`bcB%V(gIp37k{pDx(2KA8eEbE`gG@!MWceT;pRgB%PrTFL?CV&h*3)wBg2p zdn`LIco-jOAe$!ZBT}5eP<0a+T!o_21g_0Vz=B#-G+L0rWA91rcV@%+s6rKV>nrrs z7z;+d!k&L;u!s(|C^bq1Y%`{l0Gj8dU>J{WzOcs83;$zu7-)K}1ISc?R1IexxCeu+ zlETKkW=(0x^=gmuN+_x9?jgWo{MO#iYI*ijB#wD4c?Q)TI{7` z?i+Zn>uTf0^z;wV7gVRa01#gj3rM-ZlGQzDU_L#GXUVl^_4H&zYU9zzT}Vf$&I=oi zn>HyDDWhL3Rhp`{6Sw2jbHH{Ap?Nsia8(Y2K!AE0JaCU16gk?E<2G2*!D$q{&5b<=02T2VJW2vt$-My5)NV6*nB0i@~M%*q6WvJ7h(MY@RU&@=ZY54Kp(}hpL{` z_3i1pstqYE0eaaTt;EKNS`+W-NzoI+NRo5)eyJuN*y2ss0*kgt5@YE?fA!Oh-sBhd zcV>l==w)w|J-=%r595A(ujnAehEUaGU!henQcQZe)(Y!#wgn2#b3hf5h@79pY2Qa% zWHfZ*e+Iyqxe$6m4^B~%UtRa?3wR1 z_0W+Ezw`h2@2pMxFrLcbvC3s%8QXN*TB>_*o%gXvFyysOuXpw|fBUt@C!cIsdAgvb z_JMQ(8*l_spZ8{R&$@LB25!4->bJ%;{kN9a168(fVozh@^r)@}-g{?Hdi=rSblzK2 ziW`<3D(-CVoO0x^2BT-jj@$Im#HPRb*Z03Q<;Ko}b6;+Jepk(&{r7+6SIJKfM*n*I zj_G$iGw}SBe@%}W_*vrVac$N24Xj%*7~Q#|qxYGX?_d1>ErZd7re_>dEl{eMZ^XzD-SOGOijm`h)xt)6vs?l12A-Q+Lt zC5`dI&hx9&BiGk}##%RQs5*FM`hVR1qgmZY4O`5u`}rUK>)B=7XZNM&%q;EuYRka& zZ%t12-&}TO(_dEKF|%Od+cz#+*wZz?Z{V{VruQ@U?;+8)v0I^_9=;Nqy$fsmg6Ave(=WEuI;OSWnS{d ze;tgT*bQuXU`6-9`dR^?e;@IUk*hyd)<{Floy~ho()&jB*46&(igBgYwdq+K?>Iht z-_e(!<=&UiEPeIKE|#PFc@D+-yOT`|dybbCRsZDe%~yOW|MZL92Uh>>!8^*HytMe; z9~CsETKdasi;}xm{^E{1fsdnaIMDMw!$emv`+8quV9|kJ4o2@iaISv$Ye&+bO4j`7 zwKKE#{o7t+F0~)&tqru66=aSCtkasrpi;)>5}|kW;420)kQt2!bJ9 zt)e2$r*q1x#dxf(oQ#|VgsdW@m1;PQvk-wHhY*e-goHePulxN3Yxn#7e&5&c4-|zY z&+~rXhwHlU>%Q*#L!B@5r3F?wH11W7Ey@x58ov#_fP-Yxv~ItqOK@y{kFLoSeKa$@ zq5M_6t9r6EQf_|Nw6N$Czd!Y-i4C6)???(N&o-}tIq}7Nes^)VXEY%J`m7`+Y$vK^ zZJBquy0C`M>t4SoaU6{w)l~X@0XVZ$bz}5F@jfoYQXJy_(B_#G^NMp3SF|L?4+Y z+FjPPG+7No8CN}^c}duU(USxpyj(R$kQd~y^%eC^;Dwo|0WX%}yWMXpmu6lSL_?hZ5gD4Qp$<Mcmrld^4$ls>lzENO~(@m(McdSc2()pXa-B@;pN(}t;er%G@9GtYQlD1vi_t3^5 zG9U)swzKk5Ppl+-Tzaj$S)aDJ%j;;CPFizg&o_a7+d79^*Q(}wHYnTTsx#Ji+z{6L z+wVnyG$7&cW}Q#{MBd$7|M7SBUOQ;}B-bHag46RqbGp+uws9FtwT&K&fw*5ZeiIhS zoen)8DVlPL5ubTzgH7z;5yDrSmM7&F^_BFtWL}v2g}~)V=fYo`_H>E>@V^}{u%vWJ z4@wm|PIwb@K=KdeuGojx7{B?d&2pLhq+v?jIV=;j*(f087~wLc|H$qH!GgqQRbs4a;hTjD#Fipkt+=`$UXXKb+|d%u-_ zHvdGC#XTUz?`py;h7pUxLl9|L6uGu=)J|#fY+Ca}?fCUGFH~*wE)-pJ^&DVPni1*G zrQdE^zDlDP9MQ~L;bPy-CF`7;aj|Rm|IGL~cc4>xxHY;})LSr^?;bSMQRSU+B{l`p;&L0K-q&H>UfaNYAOy6W!MvLu44U1C##A9zJ z9mZngRI(h4HBM1JRD)PY0HZ+I6TSt6By!QLf!BAiY9&Gpd<>98iqJ`j!;w-UzgjA5 z0Ls9v^Pq3m+fYJKohBKjvJm7GtmF&fJko99y75JP37-IXPPc&1V{$>f5$j`%YA|pr zz9|r8lMp`*g>JwU&PI@h!4T^LCZ>UFFg&(l5-w^3i+dtFV$z86eO-O57@EzN5NU5!<8eG*t6l~%U#oL9 z`PTCR^e378Kwx~1fD!>I9=Q`0Vg3vFlT62cMomg7Yf6D(`n5+%`93}@Of#1c6Jv57 zILjh7OlV5}q_4YEYLRyhg*gN%-C*OuUljvSK*OY=pePg{)CKr_M8sqz!WKhEj3Lpv z4-sPzWseEZnU5Sk*^Eq)w9llc+$!s7fi|5K@uZXhF(1|dgF$TDDefBMytF`G6zDbi zkeUpm6ICe=Ic1eq2?{pm*vM08bgE*`te7Y?zbT51(I?za1H~r|5kRCizgkR^dozt| z_*^I}(7@$Vbu6n-;C;Z^TN0Ef^#Qppg&|kE9PmGp)KAj09l??V%dr5N43s{W#U3#c zTM3ifQreV~Yy{%rV?cQaGF8HZ#RMeStw1gdjho~!tfe!{-cL~OI ztmOay6(~OnQ2w~n@bC2453Fzf2wLM~5VoYMB`h!O97rC`q%iS4zs+Dd{(mr!p9C6d zYvB7q&Mfx-{`;pvexBIOygUDD>n`!%FNB5Fk6IKO7Ju`ii5~y_+F>Og;d%-FIav_C zvoTR{NSEB+m89L}uHnuMJ5`teifa@6eE&1?2G?gT7jA)R>iaPr8`KjQ6W@v|dif~# zv;LK57MQ~lJ=SLJe|uz6&i;t5(~*f&;wrEL*DBYSq-TvwtJ>BTfzfRoOHw{B4hfd- zPB>lZ=jLE{=kO2SSE{c&IRFDE2(OPS!7y?w(7NlO@88xfe<@g68IdFX zPfyr@sp?=DcVLGkCMHPP<}B?oC?I3!;Q(=z$LDlQC&1}VD}u&HM9u2F}VJ(65qhjFGj&fz_hnW^3dk^iICwccZB8k z`3d5?P?=%LGM8;PqV~kUDZ_F(W?-aczSow_gbIrjC(0BKt4;}~mC}nreu8~?e^G%v zJ|PN`h^p4~MsF|k+*huGQH*mzQ8%0nJ8Sau$`P!2uPjGeq1kPAZOaYkn#-~H1p1xA zYj>rC`Ed@(9`q|RSN+cJ>9t*(WBg0Zl74JD4F$5ofP2zgV~9^WrI>e2PD>XI=Wm%f z8N2P;+OznOjYcFE?7=|cEP+|#f5_7W!n}? zt~Kdee>>T_OPhjEQ%{-tw@&VjN2Ibx;??-F+HzR-e*S3h^ecOn`cW@n#6oGctR?gN zp445;G-g7(A4hc=b=d*d}bvvwzne7sg$ z`0YL_AVlmKm5C%1YJDR1t*aRLy18`px1$)<&FAaJu?OB%a4izqRm*j3**G5@-sY}E zNpE`T5a2&48tvSHxgTX-l0lPPeL0t3!=@oQ%P0pP;o7DOM z1#QTAgMH__ObUhXa{1K`scP8dSr%$RN!LX@^8)WiZ_z-<`a5t0l&U8D%)!IcT%$zO zrtF!tyoR);>T+kW$Un@jAr=01I5iwsY|m_!l#fWy`5q*j`TpYG?HiGs0ty&VM1%zh zFwVUzGPu*&od4{&4~)uDJZ4$fp57+Xc4W4OYz^^SeX(hE_2GgaB#XV8JlsH)Ur;en z>zk5N{_6fek2IDUhK(C@c4cc>mH;>WsZix#u)IDj?@DO-O8h9 zo-8!SR}3EuNXH}XuNy0S@}_j;ec7=PTZ9@$FQpacMPu}(eGQ|c6lLMGTO7!gAyqo1 z&rTH`6KprHOB*)1StLz={Nao4?Fo^yYAi1q!OWhTWn9&5s~5}*meLUj4#X6^K1tmYek1a~Iz_^VUd1Q`?PYInoRF@cb?%&Uw=- z_3-^1mG)qV?6lZ83bKVhUw*x!3Qb#Rcgn{IOGMrHQw*;0y%WO%ThaZm<$a(R8TXSS z!L0IgPW-de_ROu3jyLBh^p$qi)1Hrb>40 zVQY(1Y`Bb zHPZJwuBnf3Fcb;^ALvRI3yF=jWXGzS=QX&5aw$fKFFrs|;F3zRt-c!nb$G%c(;ByD z{$T5RP=(0fIjv(-RWY>g+{gd((d8VqVS;jpe*{32O4=%%7vjE&2FMZ5YU7YIHm*AD*FD3w5pk0lekBh5K8) z6>ZVG19(G7!s7CfQ+ekG zR$a$OJCUYgdHyStd!_nrQaOPJA_!~EPm}E`GS8W?4+ACYt|R!2TsLDuASVPyS&D!u zaB{JKAZK!vpb$f)C;*#9TC_?cb|?lyCj$85;@24Gp_{?w^yg#BlCT0oWXcWEyJGUM zg<~ibnEfawS=bQ_RSbNYFeGFf+Lw2p#L;{wM>0@Na!0zEP^z;};-l&#yt~yOT!AN9 z06%VbI=6&ttn(S-1RN@Ye6)dx!|>L8O>KR|i?Xu9$#92_G1aj8SvAc7)O=ZuwU03t z6$Q1&h|$rMYQy@DrD!}=w@6r^-3A#5C>#Kn$>?^bKNlWU30};rdFUSkF1VkWZh!m& zOSCc>E(jXN10#bTQcuuz*iln>JTqW<;&iOL2NWLI==zR)rffsu*1@5V_#Y}D6qj$s)GJd-3S#lCDB+PyxwK@O{DI+`} z!l5aBe{GlgrTuRkqazK&^Aj>R*DJL{W2w#k{=Nn$wqDgb#ERYUjN3H&NU13)*A@W# zYV^|biV@eFjZJSzYXW_9-A2PIy$^T1;o#7*c+}^P8j&Zz48Zd9)d|rKslvl}$=f4q zJB!V>Z8>p^)^d{Nr`<;%+8h=aExr|5Exu}WT~k;KL_YSx4Uz9?PnvvMiPS(s1-u7) z45!8wZ=XzQ@44~9!GWm#2$*%t9Cylhj|z)pz6Qu_Zw~|1ksO2hhMy?D%Nq`{T!u{F zHZWSAFX?dj&9|NI^92B>U<~JAg^Ml7RoU-mCo^ai4{b~XsItew&T(2s8}kz zNAS|(V3`dxTodgg?KWi&XvS==U(hbzc(YkGYJ1E%O6xd$D;@JR{q#=7ML>f~LDE~F z_2TRvpWMnh=NV(}Tat8*eyt}j?1{rfPM41ALRKz*KFL4G<+iC%A;GGeGN2mj6)1AdG(L!gXQ6pr!#>ScVdk8Z|1=f!On?`N8zmizR7}@QhxC(B ztcKJNigsev&F>MW&OLNhl7sN&_Ih2@smsk#S5(b|(My%}S&^X|Y9HEkF2KI_KY|vn z{+)jYw^$I*&yd|CxKbd=P$)Q*G7uqSga*!UwoJWJDn8dZuf~G$tP-^J)*b} zDy07dO%yYYNw-e*9+()`sg3Jm+jnXPvBze@I$?p@IA-3=RwJlN$o&Z%&_9 zzAx}y{?wUW()vizV0;ti(6#^pp8GnjsxEJ6kfdUcewvnlyHpJP%Q-7bZQ3?sBv9 z(5~U!vC_th$mLu@5h=x@!9NZdiMZ<+D0WKm=HF4q(CdPHZ!*%2AF^1m44ztgdp^ zifCg;Xd`6#&(4LBTzcJQY7)1wCsjnx>IC5M<}CB3YS$@Ad3Ne1bzy*|{1vKdS8kHoHDhGQ;$?ET1WrqAl-JO1h2fUh%jjJWE45FtTUp9fH|85 zXjtHVNvdki=@5c9nHZB)70SOm~3}wA#TMN`J(wc#?4jH(HUX!uxzZjxKH!^*_iLwnz+081hxgLc@U40|>u3o!h>4;QP~ps8OGcrndaR z;pv~}zBA&b``hTBI)fXOhNZ>d1hzFDw_IZ>x=5Nrvetf6ZSVGvez1SlytXra+U_IM zxaG;JlPg=(l2U-WkKebIxg=L488Ym<9w=RhCQ=tvqa0{{&B){tlm_dyPObVkT0S3Z zNq=VBGSN`);QHeaZGKU7YKpm7Y#IY6`vI zJN@D{Q~f5c!gwGwrprk-WyTfl)PLCWcj24MI&XaNqUit~<9<2fPLVERB)Rqc)T6K1Fd*V|aGQ=6gW0N6ABjl@&RCxSOZgq+H-C;l@Qe?1Wlh>_$%xaKv_K-{3 ziwk>T?S&KE+SJ9EVBgN}(cAd4X%85TPP7~@efv==Db zVnC{i3|_3qFkyHVqSwSx0JjKT%z!8+3Id3D9JdpqB`nq;)!P~|lyj`~f+A@%DpVQ( zxIW|fr!p|4y;xLgQ4}sUQH?geLQ3X-~oiQ%%(;f zKbDYCCIn$r(pC^v_J>AiofD#E2Os5}EZ0MUsv zARVk7{aVaCoZ`3Ybv0C8)0XB(doX3x)zAD9#drXwJWgVJ#m zfTAs2aXyP0`WoG82%z-=n^H|8Tl9FeR@0+*o=)n};2EJh!Y5>$3-05EW>hrt5%4=) z%x5wug?K({S!+0mG0lToTs+f~;Wlf8-!m#o1dW3I42Tyqe*>byw$uCX5qE@-4FOR~ zet=EjciHP{B*!^9Q5iVN*`ezjuZ_n=xfT#Tvo%f?@wze8`yz2bvx+iZ@O+=Z`*~mn z)2)G#!c7-m-Ykkv)Ce?{YW$z#%~okOuqi#~8N4%7Cs82jF1(=uRaqVc!%87;tSMH9 z>7)IU&2rMql5{15S{qVLDP}kc&)6(twG1<}{?W|0`uFxc`hiujMx=G7JdId7=)|Lc z(`;bOodwG2Q)oIs7tfYG|6C}$$CnbYS%%?aJQ8&T7#gUe$X60qitAuPB({}+E?VWt z#Gi44{Qu4X>Yx1d5ySo|cK(rhv;T(+0Wu#}TiM5GIUmHIag)ND!EetjTD#CeX;eve z>gM?>FTk+IB12^2)5oLN3hTYfZ**ntY*qb@ThQk=zc($enqub3+V8n#T$5qt_U@ZX zocH;0wnhU7K$IRCFp^ZdYWVr!ph)=(`+Z`v1_b&^OoWEs?R!=VGo3qs2yr#`e!YJU zjJVa$%Z=-lVePhDS8G8=LYd~!a6xKjT1;2((VV#KwcMG3s@!f{g%roQs-Wyc?$@Ke z-ui)21i(!p8!9%xmVML2pdOAg%_ z&VEH7RP?I<`@&bt8=tD$sWM5Y5n#Rm#mnD@$}6}uYv8lOFz}#wFst~rIQ#UI!hg^z z$dbIKMhMvwV0TyGv;JUsaJWIVlhXY$$gA7Cd*+&=TQX&%ww0R}tjclyCU}v5P(hdM z52d5M2eXE^Ok&q*uK_d6^7&JlJnnQBvo=d z{2y1_=HzxuMs$cY6G#Rk55v(K^A6c`d2^#Mn@fo_XxrwG?^)pZS>Qp_H}d!-2&;=i z8a>+)2L^5gcxp{869hFby^j6K3*v+{P57j^&sPNfQf|ou;itI(ao+g>7?%18zMFXP z-wZ)gwR+lwFq4O87bo(M`)-urL7&3g zdn>o)d@}s)eW)Y~-9!`pKESRj?vwC2|0IkaQ8^JdfMCAgwA8nf%J1xCnbM|-S?>u# zMPv)p6Hb{-KrB(gtdK*(>@ct9a@@f{_MB4pc`^yfWU6R|g{qHq_G=cstvadRjt`&xfE>62q(0>y)5M?0{OZR=tsM)JI z^Ew9}^)_TJx;u_q%j^TEwXb)Bc-Sfc_vG??2%JdAoEH&>T!)}zan^VjL`qXnoIW-8 z2XmQ#^Q^Q#xqVDK_=@GVt}MwhOQvO|2}I1iP+DD>@y<)8GI=xRkq>={<3pQW*|v)2 z4qH7k5eu5hvh4P5uo`#Xp!GTFPsEU6vtQqA|`w zrB!~Oc}GK30Cx4}NojvUNS&G!UjfL+dpW+Mb6dW!Gv2TexZtw%0{qfa)kaHsF+w*} z__4ST5OaJmc3~&}0lZeklrDw-=6?f@mK!e7XI~cRrKa(-_AjYQZ`2G9Df>HMJvNnW zj*9{V%Y%?Ezn1PsP^9^wXo7WD#%pXF;Q_kEuu70vgX`=M3*BW%ocEEEFRMf=7yLdV zg8}xZ@KoOpsRvfRVfKSb*9bn)>r2goQuD|O7MJybkwqI*B;E=IC;cA9vTR*eP}nt zJ=4+BE-AIl2U@==u*d=hQRNV^3j0hoFX3UD1rKd<@VmB*j^ z!0RP`9;1Fg{Kok*yCXJr(ZZ(q070LR%uua$R~V{+v8VPGnq6YEHr&OmFPcswp`G)P zvOf7v03KN69)lJSV@jW8_P*M&&~oVGG-PN2)umbu}g6*_Cdr zsf&JE5&TxvpM*)j|Nf1H3lpK8>AEc=@m-fqwr$G10^2El?mMNmS;=>xnA0DfmW{}d zc=%jk@@nOvT;~j9a|oO z=NS;q!v13i&dx}zDL>DGSY%wxm-9Wc#$s>F-tMUxE8%ri5Qj@*fbVr$=L~TBwPC@@wMkvnXIl_qw*J!__90h(>HS{C-{;YQZj)Oz%H*uU5Hs~RzF;Qg%ZHlM#$#EA8iMs#X{~h zUkJ>i7a6l;@#)LQ`zxz}kQwr~mKI8y-m)ilwS_{|DsuXyHA`-BW7B9lL)?n+JqRCt zkyr+)WlSv`a{u7db&nN+vbl&Xiyh12Flde)gDKI#k!onB5|d{nfhRpI#K*4(s{@An z6BMzhwBV0L=A@ygN6m8rnj())d-|SD{h5KF!Juv`0RL7lN|^;~RGfv#p{c`C;;HF> zFX;cvzd@V$he$Jqeepz!KAeap3S0mC7w8u;h)k=58^NSlQh7<bZLZZg%$)ygSQFFF;gW86o+qOp&ntLf=lI2OE_5&m)+%U7b5-6UZt~V;rYqL zZ*RB>#zrx`8#ITicl#lrAG=DepCi&O>79rap%Wg|@hbWBbqjh#e+J#k`m4qIx1yHZ zys)-&_z(-FbN9-!Q%kE6w8zO^1W?^ORE%6gvc*X_GzNjlwwyO@GlK?QJOq+`SEQ=R zR*|+YZ+a^-fFkr63~lc-xO?WvB|9YC=NK6IUlDD^=U$eS>Q+wN75J`382vUMPdyi5 z7@qn_K|2f}tZevd&F-UylTvIG@8wifbS%*BcM^2!deY96?McD`&N4Q#O(jDv%cTm- zdDCt4Ahhk+jW+f^yS`GU7hD^7!eDB%EKv**ihse|_?5HM+6}l30?PB7^YLA}?NmZe z%y*{hWpuAiD?W>s)@F))^8jJXD|T?7W9;8Mq3F8v8zUmhlR>zjR!NFZ{FjF|>(6SQ z-D4_hQ(hZ#RrU@*3WLTWPFF|EdYgxo-71S?v1PI4_u4_xwwwbk6@&ICrK3*fxw_B+ zgy@GYX`yU1PSgP65U;ezEl-#HY&V6%Wfu2IPPnA}@6y5*!+MvG9&~iHW=TVvwGO@y zO41g4)eP#D#!*20oDXkB#m@IWq!&FOF6s-=sSPO>Pk@1jaLM1Rx&4`<3eyTKH`x-! z=>^LS0*g~lOz#0-+A1-6SCKvn3x)O6lIBt{gB<6AJ8nR%7TMZLm7$~95P9^;xy4vr zl_@QIg09!B0n|%KXRfL2+M`eS^57nnii6N!=yKcZ zZf~2t=W3)E#asQ`Ah>|njWN^yWPH;}<%BL^zPI7QRcXDnZmHARXOl|Rk*9i|7 zH7`X8=4wBI)T*f^J5#`UP?X)K)Eqx;yw{Mn*zzVIM=y4DzXnC8XXx6-rP+f8nF77c z6nnP!&4IE@J(HKB60Y67eOHS({pfH|nZ~rJDZr*VztK4gCF;pil`f2$GdIZi;_`R6 ztW-osg03jp5n59h`Q-y!qXi=|AkvOoev|LAJZZsgkpiVa%9^jA6qy(3zO)?5ch*bw zfl@sf8rCiDqq{5B`)W&jXh6AnMT{kUT(Rn;?IFdLKueKjKKQ``#f?nIUyNv`U$aM! zV>?y^##&yYh&O0gWu`6|&Xb<6ow!hSYx}zoZ3dItM8>fdato3W6oKz!D2R}oEPLTY zo2_qK0@tIacAwPfJbvP#w)xKp7X;=X`56_7-8@ z5C_qisNo#l0tidf#k^OJA*+E^?*nyq6ejWJoaCXqW=@KS;F_haso1`I))qP|vm)Hi z*flQI={?eVvNM8HX!gIXfc1l@{YwavdDEZuc&2pRA?A z@O6{NEJM;Fhe|&YjtluFrT)}ZNWBhaH@nSq!y`o78Yy+v+Euo27-am&Xa@MWM#@4vX#C4Y*q;v&}i^5T>jSXMlFFtPamU;Nx=hB({7D)qsA*{IJL}?%WO(9dK-RdS}x7A6dKyL6y?5;q|mC7 zcvZTu>E$Mud>vwENPA}1`X9e_tLL9T{H_LQ$?lv5$rT|M3t|*HQ`2Vks$E1R5F()H zHjf8L$w0}F+edi-iQ-e15rDqI=`vdAB9i=H?W76R@f-`zQ55k6WkJ>g-~MkNk|csA zEcInBQ{W>rvU;Y~G>gSQvSeYk3|32OVmFk2c?!B7Mlu+@ex!Gz5FOYWBm@9fZP@%{ zw=iiJ?t*>RS*EukTL|mB^W_@|e!hJZbnH=uGewel=LG;uzTpvXFaoNNG$(IR;>s#Q zx6oZ9<-ouwc|im8k!>$#-$vwsAn~?l7Od}%>=CB2tmLuJ0;heNnkw;a$DR4;_Y>%w zW#-#wv$taD2`1L31PAL;MA2@pg*#C%&nC$=Oxej!nrIv^x+ktojD=@L3UY$Q|MbGj zmpvB%PnPBQS;+99UZ!Z|KLUvXu!U?9>+cFUG>REW@5Iwlz2tnFL0A+1XMcf4&6e4JCo?YG)NOif^z8*^ zTg)g1%>-*D5cy!iewZ;Ej4|)O!JZ;rK5s`ubdrb%wKT`YF)L2=@_gzNvM=-_29KrK}fOilNis@_#D4X-m%k>i!enIK*(mpJ^Has=WI%QGP(|z z6&6_lznG+GROZ5#c0@g#mGzmpEB%*tr(Qa^0s+b0G z!bPI47f(b~R}%BM6(o6%Ss4evr14swZL%c9V|j-kD+^|FD=dUEix2iAb=HkI{dr zi%rtr?&V(F{}auTbFypEc9l*%KDx(ud4@nI?|X$4I^DgseW}E8fy+(1>9t`Ns;_MWIf=fClIuS4 z=xA`t;b@UUUovJ9`9;YV>Zsc2E3gL8~FTAPsX1bZrN8ev@3JLna>2zI{td}khCUB*+Xcc&^1@oqvATWYX=hrN8ZA99CdM~ z*tSp;o`c$BJH33CZqwj+5-P;ltmIG*5Zr<#lpU0uV<9}RRckisq4g*(`luWKQ zyEPs_vqO`h)p>JTfyz=2ITxOfW*b8O8f>Qhuxv4}D7)h~GE6vsr+n0%;lRr!2zWby zN|-}q5ZTS+4q}6wxR;Pr;VT-qG-f5Fe9PI@>V)k{p@!2(dj$}*6K29Lc5F*LSk`DJ z98)wqi_E)Z(-L2}+{90wx>TM5%(eU%x%bmR76jXfVpNbTU=`>Go_cFrL+YkgTG7;= z;=)|f^J>v^N8X%>T(zKDtaqA2+AvG`T~fLrIdoZ6&aZ~m+Ijp$*wD5S?xxPv9?lPPfM{62#sWM`L6TnlVB0hqm0Zt1dPvvHY zGd~%_&UrCw>Miq{pl@rC-&Gir2Gm@0lXFgvb8-IlI2SC4_YI+Uv+9E}N^y-wx_e2PZzgtj**@|yA%aL8uq&qawH0dm*ZrJ!g}9uSe8-ud3^uvtnsS6qsYr*QqT_3Mn(t zE^pR;IsU|u+Tt*{LLOmQcn>})r}F1M*d?vWbQ~TOjhRDNe{v6X+`42@cWWLtBc$kl zVL4T}bl53<^r@c1C1>@GRR=GFC!z{?AEu_DZ+XbpX6awrG?YQC&LxKKRhV<&7@mP^ z0j0zXuszXcC3D0@a-O2uy>Bi1ZIU~F4k?m-tl!&RT^mhGL(g!_aMlgL#N8*h-@R)V z>|VKCKBnbPZ;IGwKtk>8ediD;vN1NRC|uj#QuCBdCmV;ksCvq>LcXrOx6SGN{qTTD z!T5iqbu6yzuSaDxh|8O6#uSM8=mU^M(z@ryXZ6L)M@>t2W!6?~f2Vh7aG>gVlH>zK z`0@0$o`15;HnsNBfdR^iD0fI*{1Yn1cah=43ZD;9pS4w92%R4n^1F`%5-J9zVdZ6_ zu$&7O9CBMZ&JbOSFv(Y6F{GzA#`}~ncz)YN!EzUzXn&U0ol&``YW4xA5a59!}kJU3DEVHMtZlNE9ikR8YB!bDWL> z-MbQ~2`^H9kb6<+$la4inLTC426yE^1k^iJ_Uy{Pq7)}(Swd(B!df+^+O6CR#&1S? zRVT04##Kk>!(${WxNmZz-}#SWYL7Q>o)|#6mqRm-bf1utd4D<2RGSQMcGngE3;2k}7D|@Dt4RNFT?^9>tpp{h=>0_#st846yHNZF;a9zH(+q0^tyeF>_7iBOV{G7o$)L~KycFxbaeyt2LuFAZP5UPOC@($0Fe=1(cwE6V8@^9l$WF^(giLbl?Xw zU~4T6D~%r~H{io4%LDX@U@t4@v0+Im2&b&93QQCgPGbFC%9nr+;U^TZ`PgH|P)i#u7#VaX8fzyRc3RUU0@gJ#u1wQm4dLGy#TAGuM}rR3xvA$F0i` zQtbgf9IK8EwwY%i&%cs@&OVl@^{DQHckpJ_|3p98Vws^Gd`k!F%FLzK%fQQ?V);nW zWfG^c=fY>1$j|#MrGxm~4}xn{3C$u1#P6>d;_lkOQoFciyo@Q{tZFw33;g7Zcr?W|;A??vN4+<0iX;$h4N zUScJ3KlvNH4KHFkT|BYJEQCqsN98xx-Hg<7G9*()EP5W7vkYaH#$JsQ8Y ztem@Bi;Y3OVIoJ=6@5!)eAfeMMoG-q0|L_~e^Wv4^z~GP#YgwN{kcgqcO<}@PYa;gKhgI$7`{98^Z zy^s_W(_G*R5ZpLPAVbksn=;*<)htRx17|*^|}~C ztM~JxDSk)Q5Ht$hoE8Pq6)Jc3Qxg*}tUAlsD`=m|PtqZS=lY{h`u#opj1x zUz(~#Q9y0{2P@K+YJzIBcJIo83rd#syD-bF6*UQqJC#UN)kNo(V?EXt?@nAN9ZVX& zh7hC94Z_dnmhq%OheQ$Smv)5ubHC;)1fv!SbUiUh7gddO9$CAxL0zX$q2N0-h>~@i zFu>1~z9i>>Z|yJ)nQ97dtMf-BI=RIItNPNcb+syl-%*L1-cqhMsPk(L%M6*aLXB@_ zhb$Ba`*$bgREsh1lW)y?WmwT-qXX4d6}m`!dpQjnA(dD@mFGkAB?+rcH?6PDYUJV{oh&3fup=s6f>k zK5Bc!!}h-Aj~e*)HsG_6Ep19~t@;LhXIP&k*PohqY(=%>ufYvUijZFT)R*Y3Vj5)V z*XklT47G5`Z;MiQw`6Xu zY%A6-Q3GG$ucaWY(Sd!*bDD&;TnhJv4=j4-atBVKEnP2~BP4BFCAfCaznc?Ysnl2! zPzpp{BHf?G|1T0;N<(s~=L~ABuR_L&%yn33Kb&H;oT&Khysr9Nht!*jXk0hsh*W}Y zfR$}XphQAbmUDsfiATiEn?sSh%hNq!K<4j&dfD`IVTM&%r%(w)n$hYszq(i{23K_= zP7r0@kzFQhmez@oNagZly7nw~?b6IeL~=orRy{Qi+qh>)Mz~smXCe!juzhZ;Msnpk zWP@wLS>J2S3OCjE^;|#GJ1^QfxmsQ%`3fUk7=V>hxfk|PnE0y6L$29MZ9NwfDIB|N zx~tVS)J`0)HGMC`66jxt1ie$JdBI@t6$BnYV4iVm-D>$P3q@Vk7NLR53ZRlHk+i!b z)AHFZ)Y3QCJY9LYS$d6YHw>t8NmM3-tlVBV)EY?)JCQ}gl=>|UMgn^;Q0UY`z&>dGsEjK|fL^X+jDys!!;g z%^eM?rS2=hx6-*YOUm%E-6}5m>$iSLDnm>HqP30?sagk3A(f$haQ&f;y>xeo^z8d{ zK2oN-3RQ8HmN4QIREpC!f9W8ns?<$jxx@BCKO8*k;?=<#bC^NZq8Fje2E|I+QzB^9 zvdh(5x!;)Q_T;u@DQo+N{Z|DWI9x z6=e#ur~jSLgov^(owysb{llt5$CE6dM;hkWlp-&7DTezs$kImh9wf~Ukb=;by6*&c zh!cG!F=yf%zF>3+USUuMqj)6KKK&o~Ouro@Gde+%t*4p1qy3@Hea~1_rKn3u#8Z=H zzCB#AdkVcIPpR5OHPLw3c^m|dSM*o!G|!z=s-B*8(Htymy^_+FjPcU)$RW|` z`KJ$dU@nTNIc(t`L=^<64^-x9PgM^tPzbbx?cOIZ>5zV}k_a@7mCukW?k;mu@S|os~D5 z(ma$s9{iMC=Bn&5AVjwUOz^|)A6~AVzq$8doLs*slJmr*80I7hczr6W3|RpGv-IhA zV@+dH+l294Q`}z17e;$@R20~BUg3reB15@GR!P?&ve69GUEX zy7<1uZ_ur%X@!1Pg==d}P#%;g-opBjy9<6VbX(S`&J?`6B}n;+&?7-`OWrphf!iTb z|4`|iO{*5vLa1q+%-ym5xG?^Og9(2u!};0KQv2?M*0uP>dA@ zvhS@iX|N@hC?bqFqc-FL{Ue6N!(tNf!s>@25+Mi9A_c@!b&RlCL+4L`#qcE`g#^c41NHc@7Q)Y5Af8kS(`~{qo{Gn<(R9{`3ZZj` zsjc9Y?j9(a+w=f^CFu!UFZ9Q3*JB`-I>&&B`CuF69|ckZD8E2^Z2G5cqrzI2#Cj2v z^lYG#x+5qaF;x0AD*NYoDjWHCUY`7iwFxCTLxkaCVWoyYsRUmOMr>7e1yer;s6bg9 z3{k_&w<&euNrVwVnpjyLXc;wKV!)e_wv}3;Ip>cZg9%zcnlDI#AfW?4;8brYRnTGx zTevVEyoX8bFydk=4F{v*d;3u9Kjhn3yH_?H!O`;0XThb5G%u?#IL<0!dH6v^hr zU=*wp1+!+SSWtj31jEJZTNTWm9$qWcN`~k`s&OZevM*pq{5XSZq78tJu+o6j#)sYz z(%BG3&WI$iq#v6SI3>OUzoM-S|4(ynOuJ& z_E7+Vl4+RUc#AdX?;`5p>pZ}kGu0vqjI<`?f_!HOI0*D%2=0558?ksNc3sM3oiF}Z zE-k6;);L8h5;+ZQNbqdw2V$h!sMsHfq&}Q%0a#;ql5Mi&vvr=+Q~{l09PFpnK{MCM zFJDZtANd%FBjz3IhiD3blfCm_E1q7;ii>7qKkS`}zOgTs;*LY$fP|MwjDso7Fe?6O z(exrA>`XM_r+@#*9jyVN?bNu$I^iGto2Ap~Gm+5ZGpsEC&fN6qLQp!yOKJF=Ey)+} zLT<lD}yL|*Mu9295lw4w@xM_{-~KwXiQ06Aya?~sVsX8 z%Ki+e-aT)fmS5O*^($2?0J>o?FzBA2$e>;*RT&C$0s~oTb)k=}P%KHud7%lvp0zaT zt-l`So*C&ll0^nzXAHfT1&xU2BBq+UPLU=3B|wLGd^o_-5w#NtA4*CGDTNUgYo9J} zh97ZRuAA}TK1tTjbccpMN4G}T{7CUtON!3Hu9sr#)rS)u(zg{XBZUS;WIXK8Y2 zva)l*s?ERe4y?Tq87-PN1xWtq{aDG4g@`M+lPYs^QM~;r&oq2kbgQ)PiZUgCe><{f zWs*;BS6#hzCVBDm7+3?QW)tdLP!Z3qIcb$_WzOH6k=8q{WPvdhjtSgH1BUh($BH78Sa=IS8UM9TKaG_r-y`3O*=8$bI9Wi>SBiHh}$<<%0HCd9S3?)+MUJj0L)R ztj_CYkon7L6Ivd#a?Op zt~@_HVr9H$R9&8p+yz6KdPpa69tb&}TOUbg5c_`_LudTvC1(kJZQG)A zRB*qQHpdq=YQnN4fmH&CJmX~3`+cOgQdPFn&@{dO{TRtsf1>zYs$qa>3Wt<6%I>aW zGkgkA5chFu87P8m)% z`Wt;@@nL}u+f{?g@lj;)<)JwTF0PR3g@r3nJ~nTlD|OSi5Jl1S=3hu<7cjns{ zr8k|_?1sj9^LET-E;v)3-zcgTB9Y#$9yEwX$D>!@Ief3V6yjab{&vK6hfTgPgQK-r z&3S5t^?nLUM7L#r`M{+3cR-}$L?|ah2bp5%9%ITt(d-#@X`BGkX_5i6D-ZZX&?=1Z26TbG4CgxRx+B0>a|dysEo$WnH>@mxwH(BD~UTV5l;B{Q0@Oo(;qjj;hCd&|E1nL%TmRF7xR*&N1+vvM=x zHUa?w(*0SMKwYZ{ZBs0#N^Az4r#Gcu8SwDz#tK2HtkS$4*>_#zpJ`mmU+PH+l>Z?i zzKYzubD-(=Q0C!sWdl=Ed|#ed080vaTjKNS0N*q<_D?r zqJ8DZvtnR9$H@Zi&%8mY!dS=nhEwh!ym~QE=EV-!()YI_(&~bB4-51O$ad8Q2>;*Q zCb`faXgO;7I3e;(=Y8ohQ53=~2uR3|PTKxFx#mX#mimu$b?SAhqcv1c357%x5#1$?~fO+Jw$5E|D-fJ?1jNNDy>z!?6(~?ROC!2Qpef29!r~TxM_#ae_+X04|hLJf) z@IY?{@LB(URILtt=tn9&ikDN`17B31!V-3mSR833XF%QKg;|vE;~AVC>T8O({8UFp zl`V8?5haC46Jc2N&j3IL+j;EcQDTFhtU^{&=VcKkBj`aDILwe1nMsz%Azn##wLOm@iH zvCefE`hP(7O37Wc$fG2k4TCJH?n@E9%VgOUd=&4BVL;AL_z%X&wDHlY^Yu2)eYgNI z-S`tAolW5Ul$(P}lackabBNV6Ky*s2i9wwy#fqm(SwnW=2u0AqQrZu3jF%JcAIi#(@}IK=Z(aMM&HM1h*4M(u2ef|*8{?lrFyIt|KjGfMwC}7P zJO+i?xPY{?1~1^EtOFv;RL*|g3>#mn0@KmJvdt3Df!gBoER_FTT4N_7bvSW&7nsZ3fWxNPOyqRZN{4Y>WlKX)w z`J%XvCa;d!Am@QzzK*OpmZbFta3icuJ3GltNB z_Hl4@8X!iShgfy24eR3*Fn_&_q6K+F6B)K%n2Bvj#UqLj9`(I>4V>T3GW!Q`Lw!!v zd%xZ0y~(9|S%S)ZAhcVmO0)e}VsKT;d6cL4eb2eUt4$HN$IxcVMP*jxsmi9d+-P7T zCwu_%^!nDRAl2`S%|)|6JmcW(7q%KFw?MqRJac}I!r&B;8}WfNiSvTWIuwT6Q}^C` zt+%7|YHJ@)gYGdk2;MN+954 zJC*i(3%>u_4p}*8z>QpKUF6s1MY5J;M2&?7aN)?cD?6_^0PtwRuC)=h6^(MIFH@Ck)`jiE1er~yGqS@ zWIERC3{*mw7CftV+ZWXXts)a*2c^>b_ORB3-KbqdQXmBvUP~rckw;(mv^%P(22OCp zBq_8D-H?Z5Y;IKKT3>{NH^NtJ86c{w<^vc9*UB$q$D=xwHAuix_NMNN>XYTsipHvJ z1k5qgC-xb?+%Vpp<JB zjeRX<(${xod~kAoG+21EX%S`8CN0U5oD#SLLDqJBkAx;f4P3}KADv9Lb@$Bv7CFOp zN{hR1ayIgHwpUwx(!|_q`EC$Ix!MS`l0aQkkuP#=qQ1#LFN$9-RD~KclNPi#=W`V* z)2^^q>W&2~6NfoUWNb(=)}>-}*8d7u5l_BbY`Zt%zJ41tH;gV|7%$xAcTKPbtG4y0KHA`#1 zHVfKZBG)wem!msNbkwdZ<;V9SE9}43zsaj{YoDDy!_kQ99OVc06Y{uD7^K4i4g5OA z*TvQK#tOty{EQmT9^Ha}bTj8Ry@wQIKzV{iHNct}xs zSq*n5Iue;ApL66}zI5fE%dft4Y(pqMnOJXDB%U!Gr4!y8Z=1+Dk9pE;3a%1QxB*ihY?5RKf|%Br+LOeCUJ3r;Pa-XOhzq|_iJ0Jt#b`1t-u_b5<>@X~8C zh2>gf>L!Svv1HFSd=|a*>nG)-W%B8bI<1`Q{o(|_%yrGu!$r>3HA=58dOy)W{u^+o zq%Z_2aRAj;OnSgh?%7it=mJMSYgPlxs5tO0PDFI#;QU8;eWBR&fZ+n{mp@mf$SZ)%+W&^x z0q`%L`=YT}?HVH*+$$RGHkeO|;4N6zp7Eq*&Y=D!q!0PVf<8Kt)&v_r1lr&3-u_`F zHFLKX>VW*=&Y)UCt!M*EeDu>x_}D3&79|1d!FW($kiC*Qw|j<^{kN;2Exd0Bg@9{* zu%#rkr9)ahs>pq3jsUara;p$$gXpWg@)n5>`qEHX0b}b(&(;6ha0;-m*HG0vI054P zrcTf&`T#cHCP~hT;=3^r&g0jd28h;bEUU&5@%k1=1G)F{Q*Xn?AkOdC@Ra8RCzcBGHPyflG$<2O3#>pcBpx#7&6$*eW zlkrNL^Q#qSXB$qb%KYo1U6n=3t~mBVF#zG)upqQc)JV53iha8G>jdwvOOpo*#8(k9 zQk5lEUOu*5^b!(xdcLi>Llc{exR$KM1tDQ9 zxQ-fRf>rqm(pwqYtB58Py6ocGLTr(q*-)~D0k?n{r% zw>9)0TA|K|M!!^O@EH+(I=!JQca~POdqjLRCw4f`@85!1q9_pBu>s9Z@9)`|!z(_67_ zp-eA`ltJt)w{#!8l%KeK!ihX0esnnNn_;rlRvuYQU_E5|W^#9Yc^>1gjBH`%R6{ZV z49L`^|A(+Q4`}jC*T!i*mNRvfc3PkZluQ?^M^xIWXk8#Khg$6@_Ec7ZRB|i`D7AnH z7}C~JDikvvt4vi4idM^NWRo?mESZr?SjsL?V2~w52qYn9>+iasK--z`{J!rGDus}| z@AE$I^FG&oEw^Pe`7X_I+So3c!;cnJM@LGVSKvw_6d*ADr+qEZ5R;EuHs8QMtP{HB zjajoGNI%(Y&`}Z8&{pc>%_2~sBxHhA?3pc}Of=pszQ(^;LiUmrk5N3oIswZjIU0L% zuXyK63!Z0n81ULgt#gT&LWy` zmJVJ>36+HBEMiQcJr|u$HkECEF6d)t6M@wnKu_Bd#*VQ*rU5^qxeVHtAstsfWwC$J z&=T4B{1WI?2_q74MZyJQg)L|SZK&9HbE9qvoMF3i;P567d!fceuS76CI=y2@)RKy& z3y%)KG1Idi=@_Coz=LHlf?DccBVagZ8?u*vs|E5v_V^q?Bw$PBh(#ZgF)*3zf)Wg{ z7cQ*CV+f#l2}L#nuAqXubea*>fw=wqSfeF&gshM-1cCMjc-!AKMxEN{O&n+Y*V z72FagqdLcO92khjuv)q)kwy9|jt=92#srM^fisdqOMgL%v+0{SzGO=uEpU@ey*m5< z%uZ{$LqO$YmG2_)>}8ZD{x!4Ef+2skLj1g9(18C!e9xsD5_O1Mgu0i_7KWB%(pd^t zHY9HNmul9({6Fak{;#k2mCVzN<-E|Ege)Hn4NxGh&frZk7tw3sBOq9qP=zq8hSiZB z6ctMhgph=z(&fx7!?~ zkQ=wDZX58>Jk6mTOE21q|Uj>Y$Ldmqjr!>%ekeBIAo$qskmR0|iF%Cs_b zg_1iwRx!T&Raw!=`t|9_=9Q{dqH5yXv<9U^?fv6@z?48`d+)Tnc!VC`r5c(MxhkM| z6wa*r$oyj4N{F%U3Y*OJyp3N4dv+gt=Tw_*05VOsdv99SJ(-t?-A9_&mF%EC=$8J( z^N}A;OultpwD?#Z8KE|~z%u1xPDi`N10Fp?p~8$2Jqico0M(BJ}=yN1C2bV{ldOwP2I^S}`qA&2gu$Ai`2qXMW#8Q=4 zC=&$eR+PgFF)ctr|I)+i{dqpE`4N#+#K*}en7<6{6+&;8JvIY;ZitEIc$~U>vacI8 zaO4<;3eMb}9v}8Fcu>iOdt68dh>Nj{~hUPrY$J9o(aIAG~)qw_BNT zgB2vU{DAehdlk1a3z9?gImEm{ae_s`XMKZqEv2@_XdO8^WMEu&Xm>$Ep%+M6yfq)a zd}yN~>xo9ifNyX+86{qy9RJnKWry0Q!XYxaHHO2#O+u#@#8q;QQe17@hNynYx;GOL!r|8yrM-|J8v^?!hH}8id@@Lg z4NO3RYP<26>1rR+>PNSX_ovvQ*v{KuN!m{>IRUYm>S)c8Aa>dVIgO@MbtkS4+hiZ) z;6=Eq@n9lOV(aO|J(!y!RW}r&YEviw%W}8HYnN@U6&uSR%BJz45?{kU^)wERf&tUI z7A2f}G_K%!8*EOIUKrUa{`TiY*?tHX|LTkz+HGk2gS{B0EA z+<+&_I{PnJ8Yt|EN}msDt70HqSl_Odsbr`~Aqfq!ntS4~f^Y|^ z>PFi)Y{0NLLDCzyDgKBiV^}W802PYnHcQD1I{w55VW{w4_j=euJr_ zX@_DiHm&zu@Pz4&+|WQgSEh&jwBiwA=zZ$1^EanQ#h9YSun)Hhs3m}F<5P8K<3RJF z?Hu}7C2qt!COY3AtG!NFz47My#2mr1{oLRADy*R|uFKUTO%rS3HK<@jx&*%}d)|nT zX3XyLCKPbaw>?mJ!=qe8Yh;%*5CXNyt>ULM~)_K(SrX`N#1;~ch&J7+TOM{O~!6?wlyW; zvAF}2DQ#n!O0cZXmi{{L^R=(aQj+hA_Z)XLWx2|&dmwCIR%LF&%`?~yTdez`@X^#| zH5WLauPGNqd9<&}1KBwwA3ygPIaXXJ-(F!x`?ce4x^GmD{XxX1As&3=A9UaJ^HQLMu6P|OU8tdQF6`8+M<_=p_ z0w2t32nF50?l4Jbi7hWl2s04{R5X_pGRuE3$s%*C;XU)WG!i2DS@cGzgE$FL48;9G zpE;{YC0>n=mBF$C?K=}id!B^S6OfmBq%a(62QUOchxhv>DsWB?M{pGaOqNg&?ub5R zX!fIi{y2Iqu|%Y^tF%QW|K?n*1PIbR63&5J^|mm#3|q2@dDg~;d}QnsNd7tn^m_#y z$L9IIBuKyBq106JsE_W;#I5* zmgN;9792pA%ko7Nhf+x?35M3lKjFw2MBI$<0{BPpquKdufyY%%EW?zgocfUx8y3o) zE6|StHwTGa*)_RJ1S<3hSJi_G68W+#Vg<$(qe1f#plXOw>?}EyW+y-sQG+x=$f=1F z2T&!F8>GM$cp;WjI1fT$4uN6=i3ye&XGYFj%UO{JGYdf)0)j#W*y%+yRZD@)X>Q}1 z4+H13{!j&3YV-~JRXnBhCCZ@(^*CD%0RP6CPT_`_egCr;KZn@7G{qN5B=sSwHA^K_Ctc;kfKj(Ib`WYgu(PbpuR@g!c<_|Kta|=T;;}H zx^MVD{$!>5LdPHP>!OX@w_$5H+`1E)ds|lHa@vl&P?=bL6%P}IJ@>J55H|z@$H4at zQT~Qq_oJvrAAUiW%)MoFfq84m)?xEDhaxPp?MDv6paLo8<74?7pF1tlz8O%o{|mnI z1aehkCIHn^jC|r6Ass=iI+pjA+!)St$B#uI1<;&tkF;-~up@cG`pP22xa^p@wEi!` z@3*bQq34YueNUwdkk&ZYJ8qQ1y7hhxbe~G0VyvQ%)1}vdX~6xf8Gr+6tn;u#t;j}@ zv>hIY1V4QhXi14V9pjfO>Po!trHL$w3b1bKkWZpJk&iw^zbo|E?P=wO>+{Cf2E!-> zht1w4Pn{GScPKyf)O$ogrW;|WMjT1VxcG2oUiIC?dFBr0GD}Mcn?CY_IP3DO&d+J~ zOwOE|$QKUYgKMfh#dF~d!q`KxQ{lc3ubdkkZ`eB*;n|qKL3PANvAu2}%DF(KT9~d) z?N3PQhKvhuhPFSB8~v5$-;cFle!1+#e{ob3huUuXALQ^`+`A84neG}Zy+0{0l`Cm^ zSqUahTSyBfZOaV@a_f?Um%F{ZZAS&l^aI7Nk$80O6P&u2t&9IYR6oB~WULUJfe)(g zdx&pc8{!OKp*K`D-2dD4n$kp{;cr`>qYmpR|rG+Q{2&C&|rqNt|=RiPJRhn!UAv<(nWJC4S^g9{Ac-8#Kb<%Z-5-zRI- zVWRk)({%Bh_8yY{!}^dm0X1gafETkCPSzo2CUJ!-$#|+rUa$|-`TBLJX7x+b^E)D{ z?xQn7VFjKCY26O;*1~X((z9ewv}yk(Ge%Dd!Z^v~B%z%sPVE6~rh@YWIv*QcL^Nmu zdUSqc!kIfRR3khPN27JzATKm59A4ve=7oGq>oY}gi+%B(+lX4MC5lLXCQO=C)f2L% z6x(+4EOClNad99wKRG~g?Kq4sx#9hSR>;BlX3-X3!RNwfkk=wMS&t)arG6ASk!$ik z4iZL$cbx=mOO$KxK`sc&`SYmOxKhwrFft{ zbrbjNh4$44!*%(Qd8^L`w;7{Qkfd}4 zbm&lEG8C_KMS#b11SdMM)XVN$uXr$IbPExNXis3MxnBk;aEtaSr_jCOc` zQ}McT$0KqMhgDujTLPrBgy~g^mo&ux=qIed z59STd()+X)Oc{}OdE#@6?#Q1Hj-ZOG)(u2H7!zj@~0h>;a3J9y&Y}> zgiI0Ze{;jrScnicb96;jZX8M*x+#kA#Z9@%hntOB2~TCx##AgpA zKn~j%$$CZx8XgTQi~3H{oA9NWPR{7U4VE2zI3l;Q9V*jPFMks$$i*>V^G^Cqcx&QN zVh=y?_x7uT%tSBa8OGHaIlv1 zXl#DFpeP4jot5#|eVSuZ#ui0wtATL)0F6BoQc1yayA2xu59;K54GXQ%vKdwePLb(T zfJpuXGTlaHQ@y&f9YHv$m{MryQZy#LI{Id}x)SE4R( za9CDngldCeU+%XnYSTlSh=DF^bmSZwG;igMz-Q?#u#mQ40D$^cui2Uy*c{WDv9d)`83PRzJ&Nk0CZsD!c$GTlv9za*1u!ml3mYTD zsOeismf%y|Mc<*v0Sa~Y@iJ{K9Q;EAh*EUUz}24`La&OM_6xl39N|zG3UJe?yPChi zK%C>>O(dMj4Kf^tXai^arPZ+nDlH(@s>0{UHN|2m;6yroW(^m10o93j%nx)z!J!-B zcG9WQfS;_Aay0cDFQzW2fqp@CXv@&$PY%Yhj6WalwQ2K-1hhgogur~MqNjX=^_FYB zf`aQ54aCp0NPR;EP;DsN3!MaK;7=_>*JWfA2MMV)pJ(fpgE9)g*oO&A;is=y+VC zd4<0qX&;K@v7fC10&SMAGd`=@;|Ji2JnLJ>5@c!ZNuE`P*v>rsy5Uh(%7Ww{bl=yn zbe}9QU?GCIk$+sLT!1LnleyAQ%)jq5iTB{9+9-Xj&)-9$V8PQh4|ET9JC5bZujT~3 z`$rEzw;RUzMB>m;{RiAcldPZNre!}%R;6Du_vW_!1Z)VR0=~=!Wnjbl>%Aq}BkB3T z_PDcjrH5^B>9+`uB*O3Kuh~z=&UBt7-X3uef>w*05s(br2ZhPZ879$U0ZuyLGbaNy4g$sm7Z zRuTL2iQ+e;$V2;E!c5PTC7Z?huvkt6+5kEj@FB#P+Nf=AzpHh@<5_<*lXlHkkOzz+ zf|L0x=d`eN(?hLym3q7p*1%CT;?OL-C~o-Z*#iZ5^{}FC%?eAmGmAWP&fi#z17a{7 z5RR%v1o6?&A)P%*ehGRlDkPnchO%Z(>27&@78BX^i0Pk~^rEN(_ zt&-jTrZplr${{5suOU&1)U$imyGkE4@srh)_!&&-^g|nFvBWqLuLt_^W0HX)zIB*L zW6G*!MFDv0ya?2jpmR$Ddo8h5BZTjn))ekV*2;m#Ibl8q-5cgg=O$cTZ18BN5iXE*PpOI%@FPv?=Y9Tn3njhNKlvf? zk-wCOeu)>g)OpxMWp8e&x1bmUHrz5HX;S{5T7qBy@_+lDH5i91Caad3rsE_wTN6|7 zhq>TV2ukQiKT*7>P6wbv1cYYcYIPl5CWZYCR6<%gvPI$(Qk(dD`~8&u5{p?e4WSf*CEJB zi{bgGPR@Mp>N)QA95drTDii(Nu}Fw8M|dM`TB+)B(CTH+Y0f8}JpYfBl^&wsjOaPl z0+^wIFEQ)9s5`-eRjUW|`fOc69XFzNHFvnT3)ukm6TibA3{~^Y?rxv-(>3Dlr|OIF z&>L9zuykNof~O&}>rkO-H@th2*i7MFPKN2!y|R>Yu7uiv*|k%nF%efElgF)_`D@Xg z{Pdau&+>b%IcGSvmtUzW&q8+qe&eC>T|R~PAAV!LNY6P*?NE=*_FKV2$AwprcR^%B zdw1$gXO>ipM{{c(Hlhzxuys(6N&0tjlxQEEh+PTC{mUi$=67IHLi&n?Epvx~3#p;G zKG386lc&x$+!i{p&Fc-tGMBcB>K#xE^6-a2O+SjPfeTVhzr{u_p#7o_tZ1v|o#Q@@ zPD=_Z3yba_>$v10|GYuvAy_s}{Y@7t;VyNTZT~DrRD3LW&}5BFg-SY%?nq(t3h8Ij zf{&)fkjIvaQY$-$v3Uf`(B$=K!X_=UWJ!ex1$n>|oH{|lA6jR{Mt`IU{Z4pYa0(fSD1L>fwdRaR#sr05 zdae7f<`r?H5fk>|}OYnSF z$2UJ~Bk}WlU@GURG-uQcepkQtg8QkMcSOawX0ldn4*zu6upQaLh%LzxM))cm0xQoH znNx2HI2#oMc>^!7%}P80t0vspCch!M|mOWa{|dC!w03oT#}9zJjqnq1A`C@plQmE zJP@QYh*CL{F;aaP-SXZye|IM*aizLEzR|$xcvOZoRNq!Ky;jXz0C|H~t&eIEOaN%+-kE*eeR#_-`dN3ck*fP1AHCX)lpz2f^kFy4Vvg_=w2^uP z5{C7|ozRUyy^nd)T=@(PrV~}IwW*k!BZ9EnzV51a^T|BHiy#FCq`Kd;i3T-07q{wGx>A|deNgveGY5>B7j>x-fn54P zSqD&SI$GQG%y#_-o;8Gw2P&K zz|?xyn_Iln#aV0!F};~s7^twZ4?NQjRJ70<4MRp)NG+Iz%cAGXTG8Uru zDvt{G+Rgd2Ioiz1b zLB22>Es7$E@36M=<^GA96dV0x5Wlc@WK3#r;pQ5=SB6~My4wAR1yBn+Id<<$aGcQ1 zz%SyjKcW55bYo?VsKEH9E}gYUS{+|vH2DlmKiw3!KNHbgSoJF7=ifNJGCCm}G;dXD z>8~iLh-R zw)&PYKw)g2ne_J8$bMG1NvPKH#gdRpzh!GKv8eRX=)H%c0v2~*cQRfQ@krfC$kRBF z+NKk@{YC!EG>M4r#}d10Y0JU}zsthB z<3=b$;Pc%1!U@dmvkYmIv=-*MpXUNot+7f0U0Wi7>9o@4&dupN0@#2wbbT2}VSMq= z5SBbBgkiVqh18gu$^pbe5?^VO`Y)#mf~Q}R@aq98(=^OUZhx|R^>vFr7z8Gvd_>za zGy9xLHI8U=CbRaXf?X5t_uYGOV0JjhE11+Q6O~ls)LK57QPGx3fG0BgIkGy5rD#Lv zhLS}Spz_FGKzn*EYCnY-B+t7aFOWZu1zr1*rC5kT|GEkJ1X!w5;iqRWVYX!A&XGSs zAILaU%)HWfH{*0g7n6yp4y4vpB*n%{#*d>pE4bG=rMum7vNOzCno$v$=)P%*Fe2;@ zjy6lZ1PTF*YM%A9Sbt9wY>8D^P6@ws1=#1$O<_7-P^|3kWO83jVWQ0$PA0TWRR-#Q zK6k(V?8W}2j>X1q_J)|7D6_=~c02tnu3)pCtgYU-VScu7lG^vzepW)yxce>TGfOe+ z=aRv4s{&S$`PC1w3a~_F{FwwKHwnEF?@DzeOiVm}E%zRo$Y+bVA$phH?uA$oTZrdo zEIlp=(e6a@23n!UL35AJDlaO7wJTg9Wr=Q_A|6nsWD=nu(xE|ffO2_da<%ga2VDZM z=(z7-yTSP&|10y0$Npldv*#VIMCyNKjMF;R(it?jP$L{6FHvq@X79Vq@F)j+8uFZ$n}b$o*K5K7ma$o|?;ExE z5v%Zsa4&iYyj(3fc+tVrxO+9FhV-`O;6eI7&102)+NJ5ORVVvJ>J+HZ(DWFj>_T`B zp~?KP?RDj*9VWCA!xZ=7sQA8y50JM4b-yO>F7wz~D_j48bc}9$ujxCf^78uE{)W_5 zy?2r^4twn@UJ2!%<0;t*i}wkxK9nn6dESQDHEpuyRZr`SexzKF{H(^w%OU}`y>>-n zv`RpKHtMGUIPk~3!S5WnTK6<)Z1+AoT@(Bc1gfeDK@J{8`@EBql-8eVxDr2g769jX z+eC>MFS+Ex zs;@q+HXQEi#nvJ(zAf#uO>zAGa?fJ1Au9h1^Q*l)n`C>=vxz;3?}Y%Y?P!{CO0m7| z-RaV!j{lKd`#_VrZ+xv#t-_?j6Ws&%vXlEcalgV@aes9$nH9N1Q#n5CHKzr>khgx- zma@-f9OA@{3Yd0uMu3m~If(N$m7QqzKSu$L`8snRuQp8_0SQN=unBp9&8hngA5HJ2 z9W9CvjT2~PAZ!|O`+TjF%;unK@v*2s-f0|{4h`SvQm%l_RHzQ*j(e};jugi<9(124 zDJZ=Is@Buw)^-w&5EAPnP4HnwJ~n2I`2H!`{Ni``935qwED2r|c`54tY}v5pbr6bim> zC_rWl_F-_RA|?qyi>2#!-`ph69`kKJzT-+S)Np=H?)NFz_6pj9>r_pVRV}{SGPhK* zUf7+ReRDE>xEr%3^tW5zKstv6%?JD?WE;$cpKfX>0BnzDIC(X?-Z!DY9qeh%L})MD z!fM36$jyN#7Oqt=VyH~*9NsjEh+KDBmhg(;La9NDsO27E3&S~i;Slc!Uu~&AYDY%o zEme*S`P-o5Lpm|~C0$P^4!L$6032?ryc0VrCBWJMBRB3n6{1VRlO=7zihfNvi8a%V z$GuOsNkA=&)JTLLGIu6u^Q5ECJDqY|WDo8Ni_!Ce*yr8o_lA;D?GGbchL(6E1x9L< zmq=|W%a6mTaf^UB49cT`Qs_upH(!XHFdG(fI?jE#0I!AHR;zRx8#F!Dm5t(w0_{7r z@lTb!0g%yzyE_YAFS=K)`VQ9uOnZSL#8Z7SS*BWu8@dD=#P~&kz|9SH4GT3Vt+s z!@j%Eu&WZC8NE?DAT|tHx$*O!m58!*hpTVP>l5=*=N zg<4G-G>BC#0mHA0;`*hOACpq~Qr-8avSRlR9Ei5A3&vJ7lRuO@!f9J1oYhwddZ3zs zn>Z)2C2)REdMZjF&)|kD?&!qPDWfi)U$s)+8I^#6whbBs;TgaaW;R{xL22cNA66x> zqiVRKqt2Pz5GXM`oy{2`?XMaUejzOy4n1@RtH#tKI;L5K( z4>;MUx3ykhKh`&zj!r^r(P2oZH+q2F^w~1HU+aujLzsya7wrZM!6WOjzJ{Q(^lEbrxn6@bh9xA^61->o=(r?WV*Y-y0fi8mwELpjLtExd9QUv+3|DAd)RY;%}+^_&lEfx?vWPT*UmRR7Cmq!qB4$=g#US5W#=dR>1muOJblc$y&|W=DZA> zVqr^?!Xw@1YD1ZoB*foAVt>8}(~W8iBN}*zEwv5QA$JXH-C84pe7OFI{NraVfQTU+ zU0buT=5bHQE8m1o|INa-MUwDjNyz5^z=nWhjPpwIRM!Uq$;y>! zUSc3}Zb#n7YvvClzuVl6a`F#;t!ya8v=BL|pFnz&;C5Ti!40*ke?_`+4nuJJzg1z6v9&NXUrEYqCRKm#R!POhO;zETZ=+4%7|v{V9K2g{t+>)5(_ z4fi<^0Ih;$HAEgCzMXfXW}SHQawr%170(o5bK2R<6J%NJ&fII*fOK_t;GSPTZ)@`u z5+&Sa0ke%((U5id`i8-`0k_rd8OJ}kp5SC{uisjIyUF!3q8Q*3++F_Z&_eZr@8mks7uIa9;H zqFk?0#5t*bBX&&w_0qalKQZD<-+P7_EK9e03(B6uVsks{fBu*51=3^!5g7M39Kg7Ql*Dk%=oJAJnGt2ma+rkm}fY~DQo@xh_e>>|Ar4Sw9Vo@;$TNGwF z70l(`L|)Dzd<$_%c1r3z2}%-mOC`ur!XhZ!q4gsEM`ZkKEkjl?W}?(#X|%#fYZA?> z@peP({i3wI$6BiM4I<(ga3mSsCfh;%N-a$dhcG0B@+_Ceay%Gp9-3rHuQS#)@56tJ zMJ4P(XT#tyD%eSCcJ(2(#R=`CvPwS03oNDwP)cvhfE9BSbM!A+Xk}nzRy_I1>MxHW zFW`l;zr@?nz#y;zz3ya`U9r6AFq@`laz$x2f0c#O*0 z5D$#Dq0n!mq7+7w_Yj5!IXW8)Tqo!y93hCt>Usp_DS8&~gD(M={wK^|vMWJyXhiSe zLeCx}=>sq&X!6XuIaBUYq%MN$FnnyXSgIuWbNi#w`0uPKgwP^kMpm3)M@$L>dLMMK zv{84lNkYnIP^OHR1-~b3&Tb0XY>hWYkY6*^duUtMl^;iDXQT$BYc1&u*!ckoBYD>+ z6Xl!6sPnk-uo0)=DF=2j>58*JFU$GPCg~DUO@Hly&xLw|s9ek2(P(YA%nah!kt5}{ zqBa^8HPkrR7|o3p{VV*D4K6ib0T10c`+Js;h9hDwf%W&)bAN%^OjDdr4%9FPj3zY> zL^As(vzLk;6&=~^7aPT2Eih=rx+RE-B9^1RGmD&|LV%78xRO*ePmmuLtDN4QB>H6Z zved4Yvnfl<31$}*FAV9oVoJ&pKp)C_JZ0mV&_i-{s?w3^xo1dpN-Zr3x-h)KuH060 zC|LHLK8QOwFfdlZ8?YB~i#NTCp5YU^zk17RMNYiVE?8u-m^vV<6(Bx0&V(i(KW^kH z%bT?6TgNH_WWN(kEc}Of`q|H%;oZV>*;UeS;^6Tn*Tpv&d|SDV*%`W%1Mj$4k8tzke4wN=!i>WO`ml_Bn$M8pN`46hg1giYt{ z$cG&VPQI07cNBzO@4;Ay9^XQUl9yI0(bvQ6Lj*v5Qf|*d5b5?3=oQ*8o~}vv4OrLq zl}Ke%$LY2!;Y=wb0}{__0b(+1)>#~lnz$Q1`#HFn2&^)ybO!EAaY4-s4$SMOzciHFU>C$(pBVMuKyg|w9hS!-$ruMb`x9Wite5Lu;~d5R@mUQoN(3elS+!=avq(+|a%7l-niOTQ}v1o;<*3a>gbh=}ga5E4iwEf4lQB z&D@*^GFnyDT{4wDxRw3rbsiUPPDi*7k~HD`70Se&(x#igUAyePq;;Qyj@krK-S!S} zV5-s*eOr?8MePPRO6iiK^G1vEoV4Hcl6I;isL!{1PXgCZ^0zRp4+l4Tcl1#h<_s`rlRRe3-KxkwZ5 z2VKcVQwpG8pVoR)L4d=Jj-moBn$43DcjWei9d2zRzUih*6*Z7_LQ9}CqOS!>9&7tT zN^`Iy33qrXGM@ld0+Yh9wk)5F+Tw%*hhS)JUq5=$;a!PJa6`LPIn$}Jc^p-ais9kO zzIDaikQ~FtdIdna!w7dEE+-W1$&n+Qt&^~-7q2dT9{$OOHc@ft(WAvW0M)niK9z58 z)TFwYo`x6u#P>DljLS0nf&^Ff_Y5YdGW7Xe{7_p8u1DuyzrDT4#unU@P}Z(r*Xewd z*l6(zq}Bcoy-cvj^leK*3wC8tE;}teL8<&!8;R+0=heceA^PwUgI6BR&oxlSyIox34j59&M0I%Mt?0LS>X`R zMg-EhaI_9>)n&TDnhQhet(77z$5h$4X{{r4!!39;Lx{>cStB#7dsUcpZLE)d-;^q9 z%Xp4!GB_YRcgHfrZzKi}pYo>s{MA)G`<|N8PgfRIN=TcWeNH!jTUpYGkBgX6L-vbYcb%eXAAK5qKYZL8Dq;1lnhtsA- z=xiSZlW%cdf$=c6*!7Q4X_Oe_8eL1zY}%pa{>MJWdE9B*dyT3PJ+BKYnTG|?Zj*Uf zDIIC$o1c>_Q{Nl6mwte9THYJqMEFi>6aPM|Y{9JphKJLNZXLyl)523KlQ?n^>xaN( zZ80pgelZxbhS?GG>l4<)KhjF3`|Wlp4IA9R1c43&O7wO=o7pD} zW9o)?2OR4a&Yo)Q-7yu572^Lf#6pSsrlUgcf>;JIw5R!Mlk`Men*Q$RFp3LmP>hw#2A{>5jtSyQ!CQDrNTLHFmqLe z59ulwuEG#uQRr>lkIu4qP;4){BQ-3%e;=t_{yrRp?Sx-`rT9)&TZMB}W&nC3_QZ1d zRTbFc1WR%R{tY@jdM5O4*6xnaL=kkV`{oGlBd5Lt*>wE!HK@1?M_Xrg^pRLhQo?NZ z2k%A^4a@A1zxS%gkb_Oz$gJFn$1#hl->^Bn|+b#}4eaGK594WK6Ol6iP z3d?1!7^vp#G@!%^hMX7}(Lzm3`-sj0sIxS|Ljdcp&B6!#NHw2(Z% zfz)u{Hus#!e4xpe$n0kKKgscolNXqh(KQv@$3o-LnpkvO;81eG~JV#Ce_1cZ%c(=L#J%$?Wt~#V}pV*E}L*)GO;dv-F z3NDvfQA?X8xl(lNTA~>l%f!$js~Ub-U&xC44i{5%D_Ob022q6O zv)hx#&E1Fv*wPS91+Oa_%)||=#}#M`hDfPRy~3;L!Ya4`If=R+r6$b5+=ad=^;8f; z8&%5P&`kkh;ncc0Bdt7dO!?C{)~*Qxw5xB8Ui^?*e6zPcnvyjmc^ULA-|#*Pipk>E zFB&2QaN~s<0@IA3SX z*Oi-l%s&_w#=nJ)jg$1_JRW56Z6O!otaV^hYHQ^;yVM^ts6 znsT=wthLII2@z=XT%=7%*2S5&)Os5Z%oO>NrMT59G!Q97q>LzHA*FamgqTkp*o@B8 zYWH)E^ybldLetv9H5cz$HCBLfrz;m?59JJ68DEcPJ8-083P*~9L`A)rG)gO9Ote3a zB2tiuCTQioeChbDe6ZG^c58bZ+Z^N{-jiiNj|$H<7`o~IhSe9Ph)@X+ZYXVdW1BfB zu}2R-(ohu3CxT}PYZVeuGKSaKAT}S}e^IXJv%M06_IPwRKJmlBb?4@aH1*0&@2}K& zve2&DqYiJuem1gRzGqSks|ydQBz0$Vc%l01anhtBlmIW6FLd@b9LkM;74kY)`AzqK zH@+`zGRw{Byy)Z4!ltKsZwrhI=9VK^qR&0%!tgER_xLn({w;A+_+=ZjynW5S37r;G zV5j!OJ7<5Kt|&ANOl3Ds&d4rs>TI6s)@A<}eo4i9j50s}s(t8n2*jy7t zodu`=Cv63UVjIjk{ zmOaI40@jup@1F7kvy+6WjFXv*sp% zCM3cp1k&^HP-3;^SWyiizIgk(w7nJ!5i_zjvFiLBNFnMUMw=n_7m$`f!i{3ac0D0E z%b|d8C(OgTKe9pvMv}&mFd`-VS{rd2*Rz4che6iBc zmdgbEPzg{TGlq&FI(|1hehfQ3O*MJ*N$=$ZwJGgc3OX`~;4g($N5jPHF~pg|s^+u_ z@6B*Oxv8@VsG6K_aQ#v0^I#2TEx(-C0p|T`jt>cbhX<8~@Vg?L7hMi#_4-tLV1jG{ zm`F_`X#J!+E&+LhvGl?}145Xn153;m!@AfKj4+suOk`kfntH&X&yx!L=gJZE?!$xQ zd#M5g*>mx>#(w zJnC@0HRIpFIjAwxOk587Rm1|Y2oKJENg&K;2a33IK*bfvuq`Bc0}~NoRA=uko*9c@ zGk~^5{%s2_0}TMF+%Xxv;)$cDzQ%ODvXNeM4!0!kT$s#90FdGPU>a)xbIZ-K0+k-v zjfHB6Pl)}71&ZYI#GSKg`|EDTZ1DCx!8mpDNdI$hD-tRE|9)A*??^2GgOSK#3nVxEBSGE^b>-y?A2tGP0gj#g4p>UHQc0Cb+CZK3HW%hkzH9tQboyJ$3h*=)W3s~u?QHa~mlc^?rni%D09 zJMqH9n&4+g4|E>-$*Su=8lfn=+WxLdWKN0HcF86$pPAoUiS*%KUxj_F-CF-)y+?`O z_1K*g&0|E<84l%lz>M45R?%1c*NXDI>KZPBMI?C~@BB6T#Rb2$H1A8}3{#U!wWa7T z=~r^hZu032{VGJS)MG>0)egDvk|X-cVJ;Jvb*QQ&ravDVUzvm76{u4=W}%1m+Qp5N z9>X1)@dXa*6whaGPS%!IRzB>`jWR!nV!NvPFDAUpQtTWRP1ZQGxslu46ZPD zKqj1KOlXP$iG?1N;)3p`fMNN|EU?PAu*q~25dV*@uOA4U|20M{(E6>HfA`T(R-LT- z6jFOPbijEvb5uBG(mFg5{70}}$h7V&Y>S!rB(XK>qVr(nz6~&Z05ml!(prly{zzfA z8S2;cncUiVk28OENAcXn7MzA4L>mwNoaprGs_Iv!L(wa|dK9o_KOxr;64FSfLts>9kM&->7&Zs2v~9ISzUx$Ate%pa z@T90Of@BnODZSLR9;Kt)s8dPr9yBFUKAo2(HY%KhK19M0gKe}sSR7HNI%DuTFnCL^ z<6wC~2Zi8~G=Wd$qDM$tC&I(042JjoCZtHC3pV0Fq5T_S2?`(FG{h2sJG3t#jxcj7Oqk%irwdYfpYv zVmMriT8=&eE9f&9m7I65Peo-c4P-7qE=H@2z<3NHGRcm6pak3#r?4BheTjal^?z5} zKAM<+Jy+U+mqT&tGO&+^lFq=H`->k?xzi zjq$VeMQCHCkL?Iu-ypmXZrQDyZk89?W|b$^+B_>%2nBr zu_`_gd&&Cy@$-jG>%HA*a|64ln|l2{EG&4M*r0z+?bp!>`_6=+(QD%0Beh8J#aO0| zjcoWWFodCuJ4m&R972nDBaQu+OPr&`I5|oL#C{jJuz8>DqjBtb^e!D$)n*au(H#?{ zD&O1?1fi3=v$)(4156GY6(IvTiA^|Et(#^jSEg;9(ZOPkke;BJ(E)Ve0KeUKs{XxW9r`df-6Tx_@&-fXaa0 zOWKi-ve}>Atk++O9pHiGmZjb=>4HpUf;&B}y&x|)EPR+o7nvQOapDg-luHRuZC5J)K5|8<1uU(9O z2%8-_pjN}tzV@8xV2jU1LYA2`By

2@qSkJk=CnopI3G68XC@*dCXLS{mxO+Rt)Ko(fuuy}fY$%sL0*ez(`pw__xposFd?C_a;*e9Ei7E@?L z!3&<8r1Yq&QWs%T(1P}*17KeTmTxTl9OH`d1J>q-5|9bh5U8LQQiT1`nzND>4lKBZ zk@j;XfOO^`w-b-11%?4wAyyieY=+&fo`*euYJx#Dtc@&}n_(-`8Ni_nR0BainSNOl z88m?+91S!|Y$i^97B>9?7YVpoE*LWfSWX6_UPOO|m0&OQc+Y$>!XW4}LhVlksij6R zfmbe&gOO6Q`^o6#^QC{Mz(f-%Qif@a0=v(-ztl zHV1Cw>M@E5&+dJ9vr`iZ`&q{xJW6kDz|oNz06uamm9yc*>Cj#qi<$#DU#LDHz*S)v z3;5jrOGq?O<#>lyNcX)E{^F1XB9nbSXD7u^Q$*NRoGBiATz%~9H6H;Yv;b3F-jyTx z4;Lo3}yeHEwj=>&;#yKgT^uN#@AqMCyn|wNXZQrwvXMLqHJ-7E=sc z37*hyRNPz6o;$WER<@+WbU?8VRDz;H7n{SVY1sx(ee4WTD?4?Nv6zr})8t_*13&`* zF|BA+jiW2?;ke`pU+i=$_Co*-`V|?ufsknqfrA8R*=`dX2T=Xu{cvoZE+UL<^uQfn zF8{XzB0$M8h)0e+Z*<+cVSQ1w)5^B_N_b<_S|?UU;of6`uy0g|?+^DEV4)NNTqZF@oL}u?_Gysyx;Bs*+zd@AI z-O-kW=Tp-mg;*70)M3vi%Z&%JDIG!e*Muv1^Aez$_VxtGxcq;#x z&s|<{OGtTTm0^dg{LZHTeme>B>n7|!Bj6!!73rpdh<4?IAXA6QTeCB!&9^$mv04ZD zy1(KVTXSvHhMm~#OdQxUvj4&xnn;~yRC{3%3TKHa)AUs3JhmD{nO8z#eM4`X)~RT! zb~^vOtl2{a016r5GlJ3>#V6SIxV)^i{o&xzw34G=@wKOs>0D9u>(JCE*UfEl z=G;tEeh{`!8o4Q6aJ!J(4@vSG>G(#)lAXQ1?dG7dsRp`SsmXN)D9I@oj&H?q>a=~=MCN!?k_+I<|FfSlLAEM#dcRg zk^X2+?#o+SHJ*~tx_^Xm_=)=#yy}Q>`);+HFKekA=9iz4epIfI zR&zK6+q)|osN$b-3W)g z)V1;8hPso3^|{-&;d9Wj2$lj=*-{t5h_a^so=x~~fnOF5yGpKyYF}k47+T*m$-MU;fV^1vd=q5O!6d?(7gR zQoJYnH?*m6-#ek6c&y@vT%$eeIS}P=)$WJ+TGlEYRH$ruq8M;dK$z33Y#4xO~EwR;{gn(zUddOp#NSpR_hoY~@E|Q;xqtG_M#jvGu58ktV z($>EL&Zl$YkU;C55Aw4W$YH+V8V)l(6R1a?3!XM#0%g&GQ&0?)!OS2!h(|}HEf0$E zoYc^;u3A-@;X}$6FZTD@`XD?Clsyddlqwz8Du-16<{6D72z{m|NU;{u>yzjhl_5uC z2O zPom4hvEB6s{$8N!D6LpEs(bfZFAtvQ5gv>nhn+u&c6#x)gIjyY4)njmwS$;{OLjO1 zp=#|Yg0_oC_d2;dJ>=p+G&t}rrJ#$-I&uthbYnWfM_gKu`Thje@Z)eH`61!WC-2y; zMo@rOW~>^>5yH+z28Xs{D@GniBRm}KYRd(S?#WCqp}L_nO$c=A~FV%=Y)dMuL0 zR(#yX?09?#`m8BY{HFYLw!^NW8RIbpZ8mDQDXpd7y38cn=!rFChT_aZb+1GX$&!y zAjGk(5t{lHmA$Eg9ASGcoKby({PA^AA~i-minj*YuHF{~K7O}E8RU5lmJeD)@~U^X zxp#*%P!Gzyj)T??`Jt2I(-Z(dq0{AmVO%!AgEdV5a(|j=WrPwrIcFSycl_%ECrhEd zKW%thaI4$sthzn}I>s2+)81&m4UNLGr-jP2F1OsO(yM)f%Yy(+JMs&;@Uw4~4Sanf z@=7>w>KUw0Eb}dV;px)Jug*BguD$C`VKzhh?Dovkfs+LCX6@KS{;`%jY}3wgFp$RV zzrZM2QjopaLeB+yQ<|4G0=yaIxA1^bCqWkayopXp*q8tBV1b2DXWM@X%*UO4$@1I3BOFncKIR<_fiPt1kCI>TNSh$flThn%yX4=l#6D8#M@zZMwIp9QyGbN;pLzNJJ zC_s}TOAjd^X3A*8+riq8zsLt?j-+R*RQwa15oQSTaEjfykD>|YFp?Za14)QCgv9n= zJ|V_K3dbP9>JPG#E&!{LWN8}k5zGOPyfpSPID3Sx2UBT+5IeJFvvL5c7g@mibSOAV zOgGsK~grdQMr7!Wh{fJphCMJQ}D3Ytf z3FsREYPfcsXqqQ*S+Ht}OCj~nItZk`hQh+_qJiNT2l+SAIkL)#RQNQ@TFCp5XM%|Iv}>B}|*E&EozRVG6?imcW(>oKn)T53AOlJ=rZZ@iAVAs z&5)E{s=?>C<+0Y0A32yk8-uBAAYN(8V)%LHx(RJWXoQ=1OE@)Kh`;k z?QJ0gzn?xL3gHaTTGfrJGMW>LVYJ(RbTt@SEBrz4{FtbSs7c>H3Vt?m8$3Npt|6!Z#pC)|YFp3|J+-P|!3-~VEbwIHng zCC1q;EFrpHRlD_i_U=y(-`}Qm>&YIM49IkpLM7-M+q;&t%XpA8sde9R)Ayib)ZFhC z+RCejr%{EM_@LQ;=>5&ep*|GK6P5>gf0vNp>)V5k$X)WrQz?nr{d>EveD1_;dt_?a zUjQat9DXM@J<2>&3= zdHcqD0vtSBE2nknE^qB9A(Zg;We~sbReoT!JALzOeP%EZUiC=C1<5c;vmq*oe*y9e z^fapkikj!sh3-%2HuHnsjfX{fnp>mtcV{u@{jD$9)+&*qg=L_eX?p&%J)us06LV)r;yEE~ ztFN_6kUHxvc^}3`@2wpwA6@pIAg6OQ)37Qd{dT;a9e@zH1p{3tntncN<90Y@yg`1m zt+%r?0Dv(ZwgrC%e9_`W!lk}FQ){P$rjy<%wl$Jg<#NW$p|n_Tv_(Fx&M7fRO1 z)10_cT`f+H6-_5A&;v7B?nP2I6gO^1^_lU?W91xVF2myqI<=>E}QP4p!vPdu0@bK==g-(dcMH3C7- zu{P!@Q_=u)Deg{SC-~?YGv>E{{x@phbrTL~(dc2Z3@$}E^$*|2NM(VmO2pYv9JzL%rwrwCMq^s>HHX12?SV^Sj0OMBym!|KdlE!#D+wk#!JJM#_Xz130 zX__6P>^J&hQPjjRcK3Tet-$G3YTOr!UE?m^d!O2+VKER@df$h3iKGd(tI?05%mq@z zrM8>rhA>yDrr(!v+iF!f=`YWyXhe|C5>LNFPEY*r%DP&eL$uiKhf&3i+dx~9;Rg8^8zovnn-@= zIrTsioKojIgH$MdO6x~UtG3y?yqS;kjz9qwff5)NQzFut4BoD-V5-U3Cw4!L{#s-<4;aLLioNY=S`;aNvb;6sR`MHxPgt5eKPT|BaQ;j3>+^-lA@Gk6N1=d~}0dL$He1AeHd zR?xN;W-HWde&ZWyd>YK_^!V|K4^ocIPKEn6=lWz{(XNQ2v8!> zA?+OSa_9~#rW4Co|7yM;sqVO8U?ZRlD;w&5fg zQw>wt=#g7NvIrT75yt3g=-=^!@<0tS8ORE-D}Y~K0DZD$X(ytQZzFfH2a{;$c~S52 zK!EE%CjrWa+sY(BKN3nrp|!x?d6I7jnC(u2mQm!x0rlqe;^C+XOTgRs!fLAiiK1ad z8VKV#YiPl+@-Pxng2t@QLVKknV|(1bU1uw>oqu^Q?aJKctWL4ijfbLRGvuVUGrYb4hjexE8s7!U>SHu-); z4W45AoFNBaq$kdR_c%P{)9!*79(O}yS0r5Mxd$=I!sK(B?XOWN=ApYjV>>@uH@~Z| zYRuWm_~B1{iuJ^?tRR*%wFF%cf(Aq5Iyf5YwbmbPUFtDsFvgJ!{PTj`SjlVmXPN&{ z9qrrh+tYHYd0M(>sD)#BeD9%T_#a{#W`;SB&%ee0W#R=a+H%!KRQM;SFRM_!vV^oT zG|rS7B<&-~S^@V)c^P*ERnQHZKIul6Xa7*gxy*T__Vut}+wHc|g<<9DuX{q+!+;;uOOAtX%git7snmXI~p$~sS6C4bMHObn44M`$&ONx3{;S1T0Gv2^hdZ?kX zvSXHqO?GMK>pV=&W(yWG{Dz~j6hv=g-9rX|t5|?T03mDSbmT+d&?1lS{o!Ri{69L_ z3m<|vG)vi*?yzjO37Q2NIymcI$gw+M+_CM$R{8`9g!9BoYLESvPVx;!%|(LdM#?(^#7mz<_6gb+fv_y*KyLF-*TQYHhlMg)Z|_o#0quHlcKV9r`6Lg5h$j=tFl;#cE#Dgt%k zvC9Wqh7iJyrS79XfOUd*1ckS;be#o%2}&>rpvQAfC7(-T7&I&pSuX`@5)yRgJ4y;pb+#-XncjeY(8 ziiVW42^$dfiNB;k8NtOaa8)OVa-rQnaaY<1VZJ8b6FOD!a3XFPeJZbS2x7Hoi5=;R z;AN=}2`~B7D$y(LwmlyyEaRUyIfBi;Jn~plc1;!>pgiZF?W<~uJpWQ4|H>Dwfc6d|g3UkLgg3|E{pRByd~+hx<4E_Hx0uDr`J5`sgc(Mhw>vmiIXS8!vS=?l2u z`P9E>qRP&_eKk!3ea?e3<5@sNS+{1=U&WqUt?qLcb-O5n(x7fg?Niltn~pWqY2WB7 zbdwY$e05^Fdcri`keL;>c7sfcW15vHFG9b`FY}Fn@*mPe>s_|pzsanb+?QToG7F7( zm2zaIdRr+);e?dz{~GiXNrb@coqw&JsB==)%T(< z;}C`5EF1>&(HT1i{|Is-673msvV=c2F`A|Y zpQOB8i(&ipl_WQ%OCysS6g|0VH&B;MgD_?8`^F717n=Nh?sGpuaj;?5w4xF4un3lg9N$`u~m zyn*!IuKBji+3M5x`MKx|!zg?glpe>fvhpH>ym!CyhJzL;S&CAle9) zM20|1swu~{WaW`)xD&QSr<+%B=Vd|7A0ji)sgH}8>@jt>2&Xm}T*UZO&N(rV^T!lh zm}@Mn9S2C;PIXY1%9Sq7sZ@x*UR3GzqHEHeksISeF0}>%pp22Vt+)LQUB56`4&cFY z{_qwk5GGsAOauRIn-E!DkDXjohRCnNI&G>p7`|X4{B+ZqZghh;WpXD?r-!yoZa$mE z$fegRuN!R)o4N!;tw;k+#h2O-A%! zMKundTe8H8yKQeU^!LfK z2a~rcrs){$p3%0!J+ls1uFaegeTJ~VQ4tMSd3>eP1(D1d`W4GRa~$I6w@sA#D;xeU z7@YHhg`^0{{2z+`y05_;niH^;w1up*?fY`nTw0 z+A1eqHn$Z{-Dwl00z_yIE=dUN?464ay5mqP0afqv0aFoH_DP-eaa`b~5RMp~tCH_xwa;UDrsu>Ejo5T1#=S!O4P zhezV85M+E$YXp;nyu(=L1JiqrhsRTyFWkGGYQDp?z>07HSeerWr><$uabF1WxwK9t`z3`kQ+1$c$qux1n~L!fZH|?m$U>!6 zUw}D^7=F}Jhfs2UzA0F%%JRI(Qb1XJt=h3t@JZBd5ocsWW6xYPJu-Jd3x*SiHU@F< zC_e#u{VnDK)i0blNAS40v;&VHPw5GC|0IgGZb$3(#!+Ql$C-SW`eY9s*BECO8gwoD zoQ}xVg&7s?$3A3UEUvv)nV+O_a)7yI04(g1<^s>muM1L{EOWiZR4$0`(xkKpE?OU( zMlzXGQe?3fu>?gEMXI+bGeD@AGhs+XhPATp2zhF1x8rrr@J!HzX1#@95dVxy zgJ?hfiXz(iPLOzVZWhH=xMGtj#1U+Ku6g=6&|;#MI1Vh4Po*rsdv|1r**+8fVCE803ICw353;GPC@;T559zt#%}s)S`6*3Y zI$~UZgsX!#l=(UwD@t@t#TJiy_^CJbo4C56_@ImZt2j3*H=c+tOmM=6?r!9CO4f3R zl%tph!s&&Af&PzBVn_B(_)+3ZHPZ~0LMeh-J}qw*w4pYA2$5~1akRce)q?iZicMeRR(E3kMbUOY z-@G^lwbQ+laxZym=@+O6BZM+Z0j7U<9NvmEiV;$<1Ru)5Gq z6RFj2rR9w}Y9~F(aO2+oa@ zuN2g#7KFubvNmM%ANiNXe~isd!1+lm>}*nPYi^wv# z2$nx<@%1=<|0``6wSfh~KMD>2lrYx+^*R;-!P05plJ;-d;)7ogv_;5-1srt6Z1OSU zV(FWxi4V3GI*<4-DgPJ#0$>OQkC_nG${rL43v=`G*Xzf95L}04It%a#!xq)pq_9zd)rdEU*;5a|&YsE_X5&J0YCDPv3qN5AdYSa0|DV5z z6An{9#PYJg0STJ0vQM6RL^f^Qu@uDwFj^~Ead?kJVrVuA+6EWys40zQ-XOaSi3hOaPvVXkb?E-H_`CW0h1Po-=vC;uqKVE{ zS!fAhKh9cn0wBHld(unfSbjjy8D1Gk$Yyarp-=b#kmDeI>B4PrMo^T-dnlJ2O51dW^ zCZ~g#zp}*=tfygA0mui?c+eQ!!S>|oiWVv*EL-g|_Hq2(=rEiWo$>KI&|_jspzSQibY8AQFgXSJ7JM)d}X%l`_)wBX#0-Ue~7Vg*hfD&4Aoo@5NFsk zn-7-XNXa81W-GsH{AqGF;1|@Eh)z&V+9X>1-w;;=JRB59HU||0?vdf)5(w;)vVsl68VP8x&18v7`M) z&4$UZGhcgW=4avBYu6*tf*b%|w3`u9LPI4eXYy9*OeA+M1E(nAxk+J3T@=q$#20Iz zOZ3_voyZH#?e1h%%wYT)nZQxmCn@A$hofRvvj0{~w}YKR0vh3d-0mY}U9{n}EXOZ6 zaP*Iz>QI|6?dmmW>VEXfd@<$b;-mg6va`Rw{x6F^uAI#`I{&AwPOJGYly=n=k&t}Gg6`caKzLwS6-v-X#fF6q4Ux9=7yp#vv{bK#Io0Ta{;J|#b*hhNP1Mdp_{ZVD_(ARi zM&a<5Gw}m9H>xa8Ck1SMwYcDJcSpSOwWND4FC2g7{?#A-_oi_c_udtJE7HkIRdD!g z)8ejeoS)nf7rCSxTn_9~8x(V=ni{5tti8CQ^Y%QiYL+wm=dJgRCwqplB%7la_&O5U z1l*FCiX6sk>)jN8bh0#f0N?x02FiwdLCVyMm`Wn?Ds(G`wti6c4ItjEK3CoyhWc3W zd;(Jvh_jq}z>DQCOe+GR?pKrva$58ra~bVIjbIwORV8=m)(NSV_A&63f5hOx<2*j` zyf_(|J$l;eSRhX$8;%pg#<_H;bA6QnEzNqK3C3mt;%^S3 z8g&Ly>Ur{R2QChJZ4?;{c7+7r-q~Y1r9Y7aZ}kZo3V*=ffOm3u7$^HQWVsu_0#0d; zF@^plp+^;*c|V&ZVgR?D^ufb~cs_df;EYEl(vlV+Zbup7anNRGZO=2j%?H8!7&0-} zwc#{Bya|pQ$GC`+9xW5FL{69rG}_{m3RA`vNFapdHTGcq7Ne|)pDFGN3IVvlq1Fz` zUgs3#%Uuv_ccViZPHqbWOpra-w9TE)nz$Mm#N)pnHz=C|wgN8QaCDgdL0&rdW1;U{ z@P28H)cCf;m=JDGe7wG|kSgWD^akFKoP(3E%_n-hHi!9f%KS{%4iTmG$b7SN?a5M*culA1TA|GjfpOdvZc*EC;_5Ou;46sj z&^QXG!9Z?{7S+A!fWj0cQcs$&{y5LAdj2oBNlF)C=B&X;|6dlTgRpn;#aSiQMG!}o z>uk?kXC^?bU;{h1%uq1}tc zQsY{1b<0g#yt)0hTXjWpFAfj;CroeS6R|oovgYTMe1O!Z%kW5;i%IrnUIaMVLk@1o zYM}IX8B^WR0E*h1T|%m=#nuAdVBnIsE(n@yTVSKxq(p`AJIuM!!?gWCydDLd4mgn1 zfaA^|#Mc`A(4P}w%@<9(Hih=|!rT3p)9Th}2SF*fAAkcifW)u9c{7Mr5#~oIk8uxO zH!hn2#~iNjT;(o0^m&t{I?R`B$uAx`!fPS$T0%B)l}h<$+B2^9kwcwh9@xr3mZcDl zy$*UKo?G`8Ft|!T1v=(-@XVz+>!yf%nZg8h+LcY;M zS9gAlGrzO|Rv1~QL14@sn!gUa(uxH^KB(e=UZwIbS@T# z<)u~a@(yTO@E(H_T$8lwLKfs_Vgt*dsW{=R+#0s)OkEqQ@yu;_ii(56Qnnj+ zR}}>F&+mtyUf%ZC29{ZxUg}fa9J9^dhWg^hBnbR`!nblyM=>^cm)QhV+&Mfh;kK=H z#5Gh)@2hV{3g-&fR&R^OJHs(cFv1=29A4pZN^y#@8MRvj&%7>pl=P8N_8l>zCwi|3B|+5hxMDzNjg&0dj3wO zLc38AS)4Dh3^UkCBfs+rj?CH!qZ-l3#yAvrC%sQm+<4^fdqYMKX8ZW?A}Goj`J}tS zO4vEh9l_Bkk5bwb;i393i~FUN@`$JT!GEFhrYXeREI(`!*KCi3bU#EvLev}T+JuIG z27xjctR_e9G_jU2&Z_wY<;kmt(0LjET)!B)jOqE+k6nk`Aq<5pZ^<xFejY2f+y`Kg+C^)O(yX@b#s9efq9Hp_WoD z*$WE(6jAVx;=Da*H4M2B5b(p3@NfQ)eit2&sWk^s`cPm0YX~!os>vRkIj@J!@c0LF!5KDJIYCrKG###8sE zM%l0l{5TeJ{*Rw#X1#kk470sNd;(!Q!ZWdi`YCFR&hZb{{d(yC&8@;1{<{SzV7}*i z0#CSaI%lxN=EY{h742g<`#`fyUHUNsQULxaBH&Z0pN`>z2;PjWSTI7nI6BBwlU(@Q zyx9yud~n+>JX2|DvB7X+cDn>!K9=?pbDe-V{osSsYDNjZ#f???kY^h8CZpFcOoX?X zwaiC+`Q*q#8yC!W7>1#_M_Vq?`vRt8nF;15Iw4at*pV=Q)h`S&+P`h%!$fn&y+c$^ z+ItLLwpG)T5?}N+Ym!iYz+9Or4nRj}#v@Y?f_4;}rA)REp*Gl3+2nIL!=o6Rjn_Mi zjPSKn@PyFu3m44RCN8m36o5V{N;N1IQScxB3!I#GOBzV@3Hxsd zh#yE~{{ZVr@Hg;KkmCY^{>&WsX7lnka||dOjvbHx{eRY;{$ULUA`Z4b#y;H`bJlQ@ z>M$4x9KaAE+DteVc^T*`z!aOf95!?iYjIOWKbU0|`p7w?$_6cO8fk~&hrjK@0)(X< z@(7@WF(+QI&w!cBf%*!&ml6wst})wUWoHk{18zhnrJ@hM8$%5c909rp)nLemfCO=S zZ&ASj^8pV5YW4%aWx;3?WeHDY%E3qw=8zgRZK>YE~7iE%l%)n3#Z1uu!3bRWF!K2;r_ zsrGX2#oNHe^Zxa3-ZC(Oczplj5ELX(XhY}J%*%$yt-Utz>hxJ%HSP5?;9j1S&z-~t zIzC#VaDS3F7mqJlh9g^$4ZEPxn+$*7L3>yefs-gB553|l3bN19XYKm@em2?$$9k6q zt{mkauDJGN;qEp@kT1@;|MoIP_rAp)xd|QG>a~f<363t^ht5PUhKe%_2Vx9G#K?q` z&6}m=IFZQzQMe1?L6z5Qk!yXwtC$O?yvlw;_3-rf^{S`103ZtW5gdfe&sko)vC`B7 z_t;R;0|x%5uwhxlUsBhl`sA?VvRskqyhn-ev#y%JSB|ai{UX_PQeS27RN^|~2Row< zg_ZD^Y>q$mmittWDL4h&VPFdD-ZEM~xAy0CFMd0$-{{S1hv9m*w^Gp>nbR6IdUS~* zWYadwCn9Fmn9sw7AGf8ygTAk#Q(2!_=JLk6H#^AAXtNlu;JB(TFyaMQ5sFiUS;N84 z(smbbL0TaJH&f2n1{f~9M?D&4@jq776gt4w1`%9i+tAHJTNYwNA+sGX_sZq@E)6M= zJ1{5Egd?SQ91PjTyR5kp_#J!$xlxt>q=j)IdGq00hH|Lj2sz6J(EJ#!>6IR2gvYsq z_i(Sx+<_ZHo#i!eXT_m|UtEN@_8BGHh1=Q^PL3d)Qc-;3j(Wcfp#TkaPj>Ie~=&N8oNKyN;MT1Pc1D7?JT%3$d& z)YWaY-P$~9iMRwx1NL_6x7{}8yoU1doQIuPu7m%UX^l`C7OY!kB{IRrcaOZf@QvYi z4Sa?)u}7J6E^Xk&WuTkHYsL9I7y~rpB}W^t5&cXdBB=BdWJ=MITGmjuK~U7$rQ)*X zk1@oXuUdTe5V`-Sx?R-|f&zJ%H|K$Hp`@!F`=TOxUp875$h#xIxp(tCZQW#0lBLab znNft>h`Kp6K1^!f{`Vfnc6Q=GrDgvBDhn*$02blPTUC4;0qfoy%~072g$Hd6NnLP3 zqr+QY??w=$Syb(c0ttq=TUkHI>+)?*+I;xqHq&xtZ+?WK9x-Zl`W1xs`Gxc%RSyLG zQoateE+}=W0}a#p7vn0s(SF0N22ASQG)vZWvTF8AAkzK%RaQR>Kpqg>z&7-_XO=1X zK%(9Wn!fRAW5_5L4Lno!hSa9W;iy@O1ZQoQS3lVMYE7IR;qR#CQil1Pn9c`}b*p{% zFL6cDt~fVz4i)S*wP#^39TlWL(61;*n@;K3ba3V`JX2vKBZ%bIr@SmJHae$gb=I~3 zBPusEZu3v$JPEH5^n?^W34-~TFKn)E9pCPW80InSO8I<^>TO$IOovN%A*y6>sYGV< z7`Ohy>#FVUdHibrISi}G<|@bUuCC~(W=6#}FT|!a)QIvFAVGP*ol+u*611Is%tvY_ zbjPDJjXDY%$Bo)0t`}+NM*JP5*N9gq$mV8@vj7=1nCcipw6t(y&S(=A0CNg1b+^Nt z+H%v=MG;*Msh0&;6GQ>VOjw#((V59b-ihI4wWX2sB_!uh?r zZBSdCWl{>ow!dTw`Ioy!R}5+~23V(dg1_V?By?`O@LwY~f+N-UNZhST_gT1f_={p! z3A!U(79%1*_`$gj6<3UrP>dKi9J3M8mbwNss!)D%oajM(;WsYw9O)8HB3$I)@6l{N z4EoGwXb+zGTN7$AkTY;m05jOX#Dc!f`U`+qhP7b`hNpF8{#D>2?%zy$b$@aXGE1 z&P}~L^v3W)BspnJ#i$WWv@@-X>#ugpir1Sygc+D}mOD9wli#~=omHIM?wlW4`?Ap` zH5rj`wMx7z9*L--R}qdr6F&9r^`R{&K{My|E1Jo3A=h6mP~2X!CD^nvZL0$*#j&Y; z@i~zMQw=FFrSJ-_a5!8Et zl?{zTVCpCEkvV;CR=)}_kTC_*mDCGmNqe%#^D##+_NrqJ2QGQxPT*W!eTpsK?{I}Af>QwNqgF;DR$wceX3cq~9R|8luXG_^unmd7}-iiD?(8xWMce{pJ>!;At7 zzffF^FfIAqjHPA5bpJ+a&M>pNB^nLs;KQN%XwA6?|FU47u8Hw2k#`p!aRGRA%z)i( zh|#?gkAO6~P=HXDDG1N~rd?KFr_DtuEF&HL#a@bXo1FBVi7pz0KV4Zc*;Fh5HhL8b>}gtvAof}Zb)RA9N=HN4KbZ9y9*McU7DhXAC+}zQaY9QaN-4k zKtF9@%*}%3t3$W*7Dx!LNOLO*@I!VTQv1eK*4*i_-r{*~K4~b7aZw4W%xf#?jk4r| z;!TvTj_71XmiA`8}}v4HI0++ zFjlgLy~&8!oXZ*$4XZ@Dv0(e2xwFhuPA>qc9j-O*lXZ;U;r?@7PJjSA%H;ro{C0NI z=D?QeuQNHjbeFyI4ShH;VU=X5pHKPSh+%NZu=^t0@vcv{=MC6>1_|^#zlEp2{vN=5 z!Kogg`8PRc!@v0o9_+f<7Wlu1>BTvZvIbXr>w9`1x{y-?|BxEQ?vyLHyl1J5@j&?Qa$?1nuzZ$1!=9!8gh?#te_*OobYAl{$zhi;? zurN-S$V&xzX3lKr9$XAif$8S9R>qip^r-^DlBxLAm!uaW+d^l#&Qx45>%$YV!%?~3Fc-ii$xby0x(uU+8p+v7K)<%(gFpP^1OxXG@?B_l^=xJaBXH+@A41W@I9}G zT=pBMC2Xr3s_f@5#$>ezM@Z^>H_;3OUegT|VV|8CQcbXu34*2|jHAW@gS(k#n47Y( zk7zBoFz2a;g{(R#HbMi5u+hHYG076!bu=*i_R|vfD?BkU>@$oZB4Pi3dsDK7A6i~C z5dZeqf&c}Ah9q1%@l5DP(O|Rk1h(2^u9)B&sVc!PPxe^Q3`7l1I-Xyke)i7DlAfv? z_(dIWB77hgfz5m%8^Qeq!An?^LG#07cBCKBY_1UaE-X^nzU3vKTVA)9$_ag&5l=+4`*mWY;zG|iHroBQZ7Cg~O$b>qJ z?dkqL#>4Fm6%fZ(OTGmAeWb$9cJ%rm(HT}~c?t`dCWPdG@{H>Mey*%Y=vt2IO$MLAwtFYk6+QhMs{z_>1k zWQBBxNs^CB18sjWJgLfxihI2NYNvLsyWe!dt02bxSq&NSo$aq(HWlx=81};7J(lo< z#l^Kor*2ZIYDH6o;c-S1=TzPwoVjY+?tjOy_zPbAiPOu23&Pf3J!Mp4-hxAl(U|#T zJ%=Y{8jW#yPTwqyk>Bh;!X6nb$DnaL|6}yQWa<XhZrlia&iU-+gVM2)W;zM4>g;@LUUnGQJ4c+jAd0*`DTK6VZA{!24@-boXh? z?1W_775j4U!|)2bVR4z z_HuH@#y=`P-)Qfoo0xl$GwS!kYzBPkac&1vq1Y}OQP2pO)VOGV(EBgX+`l6DBp(t~ zPwjev&ZC^n6=&z?nqBJPp0i7k0qw^cga`!7zeY=E6IbD)VS(u_e4ZJY?MWFSmwf{y-y-Z zXYWQwjV2m^g33D>(W_{Qz<#EF5^42RK5^Ky<-RJ4a;Kq?)ok23qy=|{#dU)?AL1M- zh)zDl4}2^;aY<3pPmUU-KZkCN#%N z<#TUO2P)SdagqpH3YbnN)!z;-gngGWZVxLV?kW@fcj-KO!zpr5m~ve5W1o{nVPHgye3Yc_oHU`Y zV9{Q3+|N|m$OJ`-Caq8!)+Za0B41H>i6XlNIOhMf#I#(lX>X40D_e)gKDQ;WM$!#D z;!^K9iLyvhZwG{$=*=0tDb28Dc+$e)pI-O)k{=Vl10SA{6m8#gL-1kx^ z?p6GtfJYDOPn{VX*t`F++~Kv4om}a+VLD`Aa)pDlxg;{`zyA3 zu%uvxebi7on3TY8Vy&D|@|&E|uW+(=K|);cNmP8A(lW+1C|xnv?l zZHTwd5`|P=mv;$T!#0VgERA9K_Xwo6_V0xs08$4lF?82VRo3cIsJw&tsm6Ci#e9+W zOA*9r=l928MR78@p<;^7LUrdC6!B15H+UPOO||%9{a5OGx!`+g~}BZfxn!d)Ni$1 zIZ6aI{3%4Te~zEOiUSOtF_Y*F;|Zq8{yNy=Es;6XM_>CcLYc(*1X#KkKTF;YS72;0 z3Bj68vT&mtNK=xv_S_rZi1`Wbh~#?R7Z^P2XS)kiT4Dm1H1!J$xd(-xa_;|iemA+o zd?ol)22YLA+8;3CLh#vF%**Ev_3!Q8<`3o-xN!TjQ@%XzJb6-hQ<%C{a5`TwINDUR zecPP}f092L1`H+nySq$Kla)qWuc5$eHy|MZAQn^*_lA zGK*dKIz+7#c^3UY>M0T;rr%>M@L8q50s`y;@SMO6tbSCEr@MrI&DceBYm5eaN(VT7 z=*&hViV4~SS8i}u*>NSGEc(Ne1z5=95f?d=rouC_gmEq;*nlAdyaSt@E9Z9x%Fur4iU}ae%@C;rsp11~VXI z6=1|(@Luo(%7~#Z!Kve_PbMnPw|Cwfoi0?!fBQgf-xr-shap$Nzvxv|%YatJ< zdm?Z!tywz?2UzI+)7$RA4r><`)kyGfSfpSBBw@XbHQ(F&g48FYBQChIEWEdJ>Jq%jm-^@b`-YknmRT7<>QO?CTT4h#$kPErXg^gi!q zJ~MQnzgv2FF60;0H-16EN!}ilr}cApyNAmH$G$2l&KYhzwpH+iDNQ39;YTh4FAoS= z9HL@&SkI(OOlK7DlwWK4!5XiAt!sPg+$Z+D8^;8b>t_oE0GWe<6DQ}_q*=6UxBj@( z+j`BOHJbANs_O8}dlh6l81X>xy8@0INe&OXf{>x~0!1hrRlJZrHI(wf^-Q`qqWaFW`+A}NrKwVwF?U?W_rrUsEN zkNI`!5=g<>RO}M zMLi1ehQi}-4-Xs4Qk%&ms1gNkJ&)ulH-vIz9)t^uE8>)To z`0h3hY<9il%VqDM&?ejt>nxr4>v4~xmlD1l^0*@#Xd16{xmLw_wW#CgSDoJ3*``&A zhBp_$7&FWe+$|fphv%((%=sP7&U`{_H`|~&ZW+ry^3I)(sw$@AzDz%unHqEWD5g3q z?%r$?QnzaKwmnavds1T&s1EYyr)ky0Got=gzrWez-(f22>L2L+?CqakhtfNLfT`Dm zf$51oirgkjc{evAuonjAChOYxXPKYDP}s}1$NQ{KTS$;U^=jEIbSy`EJLw1aCG*@69Y5jW@!ygkN%GpZcUB4TO&6dro?)~F_=BW3G+T5mKBpe2*^U+YS&v-0)jk& z)1C$*KhDgEdGl_Knv8@UMj`*Ib_4A9>V^JJb903$ZBV0A<9WrfrTDR)zDr~hoMx}~TtAW*Um>DS1P8FQ_4NJ#cYRm8MS3EIiYIQ@$C zB3%dfPPe>Z{*BP*03)`c(w&kt}i^AK^2{z0H01cq4m{# zs4#8SsSd(f({$o!p|_Jb(nghP+Qc}6Td^e>hBm~Jf)h?4{2QYp)Ve`m^zk^6=(%C* z%q%5&Ph@N5GJlgLvmNCt4d>9sBe5%MdIqgUN2>d=hE3PTOV+LzNL79v!^<7Ch$2G~ ztQL^x0NJLsMSgzorY+~N5m=&7+zZr4Rp(}K1Qwt8hz;){IwVm(!ML4v9n7RN(ZnPy zdH-(Ilm?sWy55^#02`yal492o^F{?z%m}Ce!&|6_ZY?`fxBvt-8zKe}D(+G`a&UH$ z-4slIFudTeRy5<4iOT`S>=u-|MxZA0KXNt zNI?uH+Zg)8UxjaXv4l( z3w}|Mzu*eSrZf~>mH2mq_~mHcgIFGDL2c%1r0Hl3*1bf2VsMHhD%@XiuB6TI5=iWT z^fDc{*mQ>EYaZo9g_G{EW{dTx25nt`P6Ce`oJl(;G3UG*46E=ZzvsMx;;TTr*mM0h z4lJ-Oi-8OfF*8K|d_s4?b9{mbt9fcv1-?VU4%IKm4b3s`MQ&Yc^+A4qxL|Utw5(vM zKay{`*ev5ie>oZ_N>zGO33YlT9x0ehu?24`u?X@M%z^DB7|F&ixD0LqV0?kcPeB(T z^H606c3?5P$ihkbt&;E43MPB2$Gq%`4ir-Q!+i`M^GSF4YwA(a5acjc&50dnTVZgY z7#ZzPwMb!b5!B<(JS{R4Y6?$iQ#B|Xg2(+@4!PnKQytJ4it=aHoQKaQDDNaYTG3YM zHB?$|d+N-Iw~Wg!)Lcb@N7cqG@OuD*IC4T797Kd|>VL53{)ZTV zj^;oJfOT7L&9LrOL^%F$&a+}GVkOE_6xl>jVtKID4>IjE|A=*e4A$TPIN^rM9_gr?AQG#5z8vD(}{V|}(EGeygsYK05jba<9IW1y`?(*x%) zM-S%}|5wkWR6}D9WJC=~AR1z<(}%I6hJPw1Su3uK8tKGFUhoRRxw)D%t;MXcWrsoB zEL$YNFi@wJ4!ux+@r5RP33tz$3jA%_gz_T*8kdc`^GQn~wie9{?6B+e91#qTnKoX< zs|05Y!v|!({YbFs3A5UlMu|2LkAGR@(msKCBL6KG*g`lRa^tkcMr!lZam8&!t|Wpl zg^>|A7~mQ=LcVM$otSSAaE~V69s#b&49%i-EDlQGoS_I$yAG{zOK_4=Ib;_Zm}wbkng5^%9z^UofQjGmF|-2@t%@za#E9o*oZlAiHjW$5_8} zPzz96qHhKnhEz_)SVrV|W=4+ek|v`-06@ZdfX#y-$uIwyH0Q!S5|FtU$_4Z&vEKZw zC&5GeK^9U<-S9dBh94uIH{11Qry6)Un0_SggNWlZY&D1VBw(LD^q>R8WcKg%f20?f zBp}&`Yt8(BR`UFtzZ`l3L(Eok7PJ_cLogIDRzSxj>Zhi2k!WfXPYJ&zPK2^+ODJjE za~{R(2g8mVVoNAkCF~k`3@K+h0=Wr;MXi7K+t@EK>-Ew1W)`*J+fY_xx0RK>ff!X{ z%hR}=UXAk|?AJ2P4+A?c>>XTwZ)ufUlN?;`eHOxFo(NLQL?O*rYvg|7$u(WZJ*xMvT~q#K+;?S5fzy8o=Q0H$cI{34{|LPQ z{pAa)nM3Vy|Cn%;??FdG+SQ)!)zr}T?)7V%t_)~;;Tj5beWLY-_@$iIXctv3e3MSZ z;q?D@JMXQ_UOA-|qVUk{KlfJ6lqgHi2YkFtHg8Zp(x(ZWyVP4{6jya$)IZ&NLimT; zla|(ki_mwqhemXv2L88Dqs?xq(Ybcr-hpI=t}2=HKu=aMQSjRi7`V|_36U;(xMT2w zPEkcOC$RRWpD8@?R&_nR;c`Z9t|C7YJ@p$JYi{Eochc!#4RuiPYU5!iE_*Cn)$!f- zeNC6fC$vd62m%ZCQ+ej_~fwiwyaaTr8IAa2gL*B-~hQlHh;s+h8ga)grmDZ{l(b) zpA-5#Mf2JXRoNddvuj~8cNV9U{+RyIH9*Uc|>Z#hg`Qr>d+gq!N(`}*{# zn1r%v#c1T)Z7?J14v140I+i*FMP@2*{PR8U1G)YQm+q-dHg4BNqv2f>yBcoRdM5s1 zX$?;_$CMe}yzey)F}pI_!bM9b;@%VVy;4;6v8Ah*V}9)6SKC&&i45+M{%8FK?{@}F zGZlY3()i9-FhLIbx;_hxQ|vIFUwr#T=oOEG^PJ)#hP$R-bw(4H{Ee;s`Q+qtOZkOE z8h1_m$$ap--rhcDqbi5h7YY)=lS96S@@~mm$92>jm*$OFzZ|hxKdXTfE<+UFzjvwg z5^Z~bh)47?X;AC5(GD+o@@|~*8=Y#n@2~bf5dVHu!C^$+1~lArL9<%|$d2|TI$>^%DPYWZ`S7MrRSKlep9)6#9BQ$6cl{Fk#Q+>g7CU|9bO)zL)$ft%XqA3%<^dWE_)ug+qxWY`$GFv z)=eg4;%4Sn=W_TtyH0g{_C`g}G_^`V-*z{+g(f(TH@-F5^hH5$q$qrocc|4augO*V zLPtcrABhyU#tPWBNb`Wf z+musYLqWZyBXY>&vQOk!hJAx)q;R-(KRv&Rcexv58|=;1D>VH5KBt@8;c9(*joP%$ z^!pPnXI4tEmlGaT06I5~0iw5uO!yc%+=CTH6oogHA#boLDg1?52!%zHgI>^0Bd7TG zihbD`gg`ApQfIA?g{LZLj&4@bXP5JR>$7v~Huz17UJp#g$$;@Nx~T<^t# zE1*pdF^kZu@r4VQNeY?_=Z>YNRQPSLSE{~dsK%R-Q{=U{IWChVsi=SFqcV6EK9`Gj z-ALkguV1x8AeSLYy-kBIlnI{I;1+1SoO=e-@1eB`Mrw}3B!&PVPbiz*>4g!^#zX1Z zC}qMF{TEXV+P(4fRn1F3tf5+vY6F?8zmmJc3$*GP$OrS77f~ycL*X7Lw$hygmBQYSnNG0tz_QGtP&ZdVe&0 z?Y>9XbzH-Ax;A}?a}KWQGr<~4A=#wqv4@r4EfhWcnb)kEV_%rXQukJe<(vH7$>4CO zjF{6l@Wud;p`O+78(IgbMeRz~ly^+j}wZ+*gm^26E>4W&Tdz?+3gM38tat7!6WmqfXdB;k@681w2az4Z?y z#&8O9$>2dF(-|SNK2`!TswuC(j!6msE8FTWo{4}3xfHCoJXm#guLXYuUS_Dw6lSsWyzTbL5qR**1)zk8dn4?;8?q5&DB zB0&XTuF8%3>jmym!%r>Yp?t8K@1Pb9cce@h&<}O9IY_}p9W7#9SQc{uGEnvW9HOUY zM2P?*Bex950&7BtHG?64z?0d~T>xfY*1%L@N{jX+9#Q1ni{|Did;&swEUMTSW-j(Q zbC_qBN{Y^**mrRz-{{{sv8=~m9mXbCWxJpzA8H zFQCo=ch1C8FhQFepMpP<70pu~&W$*^Syvdn4TmhksEY)LBR^kt60syBVWr@I8o?Rd zR`D)vygoj77?L0mf0Rh5#m)|G(rxQl)-ZM~XJ|czg+(g!ebHNgtcRfanK8|Ck6Z-B zpa%s~gC}vG{!}`)!%FLfmJq--R)VmLJ>SNn&DNH|9VnQ0MH}-*C%-PCB$C zC&9|7NzZ`J-KRB9sLPWxKgp^qjVWmt@7`?}SVA-pi@Dle*waJQK>FxAOI?rUU*>ESEdywh9jw`#$)omT#9`#nUpD)&1e`Pu{&N9o9 z5uc>shGNV1cI_K$v~^Az*ahI+bTbo;GwuuFv#4=R#0C8onA)@k16iLL-?e3k35*0y zUN)bse!_a_zsBwCT95<-*PR04} zRBHU~Hrn%ayk9nb#_f9@7GiRPTWOrtv`phZ`TK_ZBl_P1LytDqenO4~@qNPdo8vg| zN&{be{Qwbt&4*-Sy8`Drp)I^}5NA~C`=QW9eLS2?^F5Q%9YTW{zv%dF#?;ubuWntv zbns>H9{FBrJjPf^s5;mv}U=rESY9#BpP5u#1DVPCS)28ITG zZq{+IPr}_@y+!3UkV}EFSU+LH==2LBU}}29?92=-ytg!)aUf3DEEdr_hH*j>Zt(;f}!{U^wA($ zC06$WWQ}E|rEESlT_uH|(Z8|Y+@u%(mHur8E#VoZC#kD3RaZNbWuk@Y84k0QtcNV>nL?v_N|&x5EP1lY=*S8ib65d zQM5%76mesT0V2wlR)ln*5*8sW0s=*rBF`}@OUC4?vUbKm!K zpYu7Nb2LY&PKSCs;daJ@%24K)G?fC#4q$sB9y4i%Eu;I_(d18)1CJe)A;()Vut@a6 zBkf`f1h=s=uF3Z`#gF`qArXu4zLd> z9DX$0oY!9+PYUz2SJ2Q2>uz$Loax;LY@+*1ziTB9b2CO#=+DNzCvJ|=sxb}oLL*Al z0_-L4YQ~@HRY;b?fXK*q9p#*V&Tuz|xlPsGnORIT87g3FJU@FLffd0^sj9seCx|7# z158MOh_Im9$J`d&8sjFYpMG)19A{W1vI0X%87-ttp!-b6^Q4Q_09>^&8|b<6ce$f0 z+77^i7(%`PWbc%?!Eh3DtJe;+xGnLfe@iksd zZ2-~cAR^-4((D0LPnb;IqSgRc2I2owN`K%;0# zTf@EQ7yf=9FnFM><4tKJXyhdhIY=^aI{Vv$j5D>%+`Kk43jIs~6k(`Z2>fK?p0dAB zmxu1ok$JtSm~bARP#k+$S-RlEM**=#7dlICpIFgg?I^mrsHz2%>z^K;32VIVYxE8E zn_d`s$3G9}O+^TE1xr;Q6lVXOXF4l$AoWNw3)>YZ)8ch4e1BV^hb)0Jc@VpPKv^Nw zoqHA;TM;W&CH&naz$9=$^)NlTA*({zZOIqHkO*hMirYBZcJ!%4CC@C^^qLO|IqeV3 z#p-iyqrBR|V28(uW&24LizDLcI2d06XPb$~I=xW8v#gTa#{d$IPT}l~NhYd}!EkEd#;QFK<;h%-E4x!} z*SB($a3%U7@8;dp9_fQHw|1!bZw^m`?}+2Xzx`Ez$y*Yab@F+UN+H|Yy}UV z&;`>?cN~v=`gnc?+GWA&=xWW-u5T)ToyFs;C$jj91~hFwZKY>8s^IY;abS-2VPT3a zE`L^6=swktgS=XOgZE-ghq(CiEz1}D+O6F3*O58eU@c3#II4pqPaZgK*;%FYkt%%N z_q@bA+&FkMBVeBHeC*NyReiz<{RO8@i}LP7hI%xJEaz-~_qRE*L4%n$wd;Q!mOoFt zW->eSSfw1J#y3HhOjSce%(1x67;D$RYmem2-6&X@THdleF~_s-LZC`}qUqR8WHQ}6 zzO{4cwekbc?5!lX)=rtN&z^pFnPqcXmW8R6y11u1B>dfp@E1l#ebWcJf{oPMUp)|D zRHCrnVO!}F$^Rei2c;#s!10fxBFdcJIQ~B)Mg?$l9SmA~NR=QKVY3zrJ}n$RTR`4) z{vG|SW^w%*&G7`2Fm^>f#K8Il)qK=NJFvy`!o0KY=0o6jeYkiVI^(v*vqR`_RK&)3-0N=4R<7oxB2U zW=NX;LTi0gkCMk3hZ7V^Mv{3v2gaoA1^Oq|-_*Ma^9r+G82u^ooOZt)7W&9+hCTRD zfmcJ~3Z02_|3{@+bU?kwy;h+jO}eP9@l+Dl6IvWTb&j7rcFYcD?5Gnp#Wvt+hWw>! z&FDafEtK^g{w)*bi9^#xt_f#0l+-x(;Bv<78zTpH0ZoD$=OvKc1|acw>u z8opQVyIal^qLY{Um$jyR6@MJnL5mkbxa(LgDVSmbgCqe=7~cp!e@dGb15Kuf zY-`oT2?rTveg8d^HT(lBmwT!*vjrR4VQ*ZjF9!IOzJd*Rx$(E$8iXoA0ZO9=oQ8Ko zH#&ONSwZQSs_q!yMtm!Uywj;(7==U4$mG`gQ&vcf+a=XT-a$ZNA3oHM=_p=R;%rkk zHm@#MmDXmQFWRevNmC%sYH&aY-zwghJ#WPhtshdYJ~d87%=8#WVo~b#=7Xpn6q=Z~ zyEUWWis-a=7_0A=US)@W&gC^VuwI}kDqkhZ%Q*qg!Ox?2M*a{KW@^=!*4zbV3~a}Z za4f6%`A17`QvR8DbHmqu2{;M@E5l&%)}~)I?0R}QXK@n{l6*%9q3pU#>~Rm!WcnmJ zD(OzaW!*utwFeFZpaf!5$D=!?<(pY=MOL8Iw8i!;&X#X=OvU6}8)7&VJg97?cru)Y zf?_}I**DG}1rkO%6m)K;(W6km$(Wxl`6Jo602k!_oYx( za(UIJhogA|kZAXgH}DvreeXY`=0ircINGxs3uo=$gQcZKhF)8{iSiR1FfERW_t7lH zfvSgtW^oi1vlCtQ_PkkMx+V|+;bEP8_JS|92Xsk|HvV|!D)==lJ2=plDckn+d0Bf8 z>a<{a@jP_C8U=<1$g1{DOgOZL15m@Lu;Hwg66Ha7*N3Net)AGQIUpX9tKYKJ-7`F0EGo+!oEC_EQpjPyr{(CkWCiii@?d0;^f4!6 zIF|9GKc>yqdAtNqaJ+X`Q7s7OEXB|O)>UWWXijB0RwH^Y-^Q_m4A@iQ))fUgU5b&v zd#&nM4;Uf1VblPNu*?Oz=$hF)^)$vA&U$b|v_myItrsTUdWkdLev$q^9Mzp~3g?Hi z$wLOR%Osy6%*tso?9Gw4q1NY|_|Dn}E|vKXty~qlee6@ouFy3kGcu?``mAI6eGi-F z9BXn;WA`_HwMmPv2Ip~_Ni|xQj;%ir7k9T_Gm+0Agr7qc!)DpSEmHc9578M9W~9F3 z4oOV)1}AsucjlQU|5Q6Dq@_tA9DA@xFh2i-xq@S_%H%|H#mRK_DC5B~m=E!o3`t&p zEPX@Gl3e*K)K1~jJkOS9_!9JFM~V>@$wD>8mz!L~H;F8JSZY!?F~__^{ln9MW4zJH zC`_9t7LndKpbaOsz^YQ;sx%exvCCv8ETIU7EN?Gv!oz!w4XLVjS;3Q8SFr`Mn;>d} zn1hdV51A}6j0H}t?kE|pFv$nz5dD+jF+U0LNBGhnjd zGtQ&M)D46#BAn423$o`ehqtA7d8jINWH~J;(`OMD3qZEWvWGJJ_-Di;|2S^*#G#x6 zC0B6n(`MPPdW(_etMRRG9NyC&ya7P;rr#pjQ)ge<;I*0_0$ykk5j##5>u7t}xk zvsMB6VRXzl86E`1w%p;4@+Fd!PAx&}w3nn5K-Omr%X5f6hRc`$j?{6x@ed@G2RDYM z0rJ;w`IJ^0YH)~9x(38L|4S;VGa_<8azb7(vLY}`n1`S?Q_4hD&CX!_O(X+;j0vHB ziQ@m)SCZROU{~6!OzoHeB7n5P;{ec@%SZkVAQ&NYLqmndd8+v$B-;?zREruv(31g< z&OjX#O{@-DbPoBO8Z5_WUO{?W051k7jT+bwgIP6Nnn}m}JJy@c6z>R$8o=TV7htei zX2!!HAjUs1AD1yJW1^R~8e*4?NgV(t1sx33YfWNaU?lux3#7e-{R#PxCREaYLEQab z?FV9~u3EYf08AeS&N4cCdb|?1F^+#{{#%Xe?G9S+fV)n-6}1Fqu?M z1eO3d$gq;JA8B?>T&q{C0|xDxnPNL;W)jAMxF?30p`QnVlR`@8DCYP3VHV;Y@hM1C zKu8!gCJ8V=H*v6lYnfL$qM-{rIJPtF;F#{jL*S{28B2e2q@eD=%dfyaLfU@~Jxc!6 zM$|(0d=*_JBLtX=#z!CY&r2s|KK>Q#+KhKtJmfbY&4$~_IIccrAv2*Er)&rC2@ja| z>6r&c4>}=;v726+CfbxZA5;|#cOJLD4L=w~ViY3VV5W8x z&qsXrO?ib-$20UatsXE245`AffiDRKYqng&tO+AuVUxBF>&xJ}mXA+oh^)M*zLdqw z@Z#O%{qkXT(Hy%`o;jZWIQQMenX-rRPjfIYVgw;zPe8xLCj&-e8r3FZi2=``MQHH(w}hRCu!e?>~w2kn781bm_q{st&f|nd~pmiY_g)^txYM z&90H(5*2CR&zfw)oc^S5)0)4IS#Y~8=h!lZT>;PD&uBu;PSd(c>yfcF{-j5 z=fA2;UuSS_2rY3}?t5!*uGOIpNl!d_$&y;iI5&DO**(;h&8rvO^1QtG9B1cn2UFpn zB;T%OM;i;v6Mxu<*$|Wbb#cVL>vOsHo_Dk~7Bxx|&XnyR+5W;PvDhc=XTK;;h@Ri? zRgq(Jcw!H`Hq%1xRx^+$+f|_-0g9(V>{!!1K!&td%Ec{AB9FYsL|AeDaX~oV&*koW+PRT5!iT&+daJp99A~+06Z` z@M>a!N}_wG$$6gV&c@;$fx*#MIqbSULJZ$6Gm z?sp5$wphR7!-xBVDp~W2*RKmNFg+FlhH3<-vrK!nfsc|KSJX%A?$j?9Y_++Q-Xr`l>T-^s`e3jAPu2mwdrCI` z=fQ(~(R84L{GnM@SD-Gq>>~VozUd3~*x+C;%pI|Z&5PAgUdT~U!sna|bMqhEFBQpE zPI)rRXp4#^t0z4#vrmeg3p^8pcib%~(C*B-w43#D<7)BUp~FFeAG_de^lh;2waUEd zzaHXt+0RrxFs=BN!xlM`x>%9T9MN`3vE?c0&tV>(5?OCaZ+ORYuV}A z#tn&K36@(`=4D_9#@aNd`p2Hak(*t6IzcruRHdH|3RfuJSTzi1!GP-5r2RR2p zjeI;hUq4d*!f5)q7q2QM6sg0A6x5&a`5FV#wz^BAfYp1M`rW?)XX(n%I!KYa-Qi-N zwSL8etB2A%8q05<8WNrLujAFO)|Yu>I*fA{?5N4HEjc);FR@;!5%%s6Surc>ncRGH zjzhi6YV4cc?f14g;+ebxqJ6eYUTGehgloigK`S+dv(Ky@#0$`l9h9B8VuvtSqzxgM zdq=Z}&sF(qBC`|N>WKo4?1TMlmfP20i<7VF9YnBDP-$BXa6?j4z)$WijrPCF1k{-9*oP2jC{|c?sU?*HR zHQh5&CI)LO;MraDTmwD&WbPdmK#eZO+jv}3kdsGcI?xL581{H;&Av}*o!Lyi0nU<96~yn9@}-pfB<(vmHmj`6Hni-re> z31|^k7QvR7joP^XE3kP)oxi3o77sWJTVI8rk{6t9qD-CA3Df4YyB7VBTZs8Pl;`ES z_fThERK`vd9!`q2k>`{ZqK9zd@<=cuR2PJjtOvtuY z`@^YMWgn`H(!r`eVlqzIF*@X8!|tyvG-?uo#p;LfG(45zC+O<1?jTM5gX_k|&WI>1 zYJI!Q$sTGL|0%w|Y{BU@N*mbYq0+sifHm5Uk#6vv`As+A5sC2>kn*0RD}btWXPPCYGQcHlX-~|q zs#ZH8nYv-$<7;-iYTwSIDLc@&C0Eh{ixU@X8)KDsIiIw7(hLOSMNCK5GCys!WVgRJ z@>IyTNl5Bxgdc{ZNoOr#3q;^(LgkR!SV1H2+HmU?aVyx^6L2VqCK}aYQ@VoJ?;WLa z#$mrI35DxROLc`r#px_joYy**xaO}%Gm)(!_hx=5qkiGlpveTP3cr4QON4JZyf9i~ zR0EqRY;REA?60c^nNpnub)3#LpL!I$jS#h=ci`7$Mds-pO4}%|%w4lvc@gK`y%^Xq zi(`KtH~9lO_q7MYR3eAfseGoj7K5|ehrI)WV7%fzydv_P-d+)WSgah(n|6mzM&Jby z1A18f&ZK_%1vN%2y3gS+Sbm3jk4B-6Tj_Z@GZdrD-z+G2Q^h|K`<`}Lbt>1#BqGaK z7BU=)pXT#5;l(+kyCZM*X}YzXkMkO=hs3=DrT@~M4Ns6)d2=2X6m*P_`C3lfk1T(X z73tF@fyYOyqN9J2l9jvSyLqYQ%N(@&SaGeuGTr3C$b$>ZQa>jHt4!zUai_}wI43SM znB34|%~)*!#WV1lYGS`P@G|%t3<7#wsVe`!{MM*058VQ8J=7>n*%oiWU?Cs_Lu&wm zq2G~_<01-sVtVEG?$Gd>(Ls13ZPiS37DYqjhv-)@Ad$cl;4u}0@vi!lK;`+x*NALv zGbj|$@5?Y6RF!`TbPOKWgqN5&F>s@_gB#v;5eEbw!JnCHqhCN^oxwC=Yf!WB2H1-|Hb=pHnTq3H#`jG^fmkWXI`0aERT z!|PAcppDNhPA4_Q9t@PBe+NmMHDLsQ4ZIBRD*QAcI>Y@mX~`Y|P%o|ZV`9T<0E8{h zV-=-?@i(5n452LEQklBM`wj$a4^M?I?U^? z0K-EXgIAy8V^bWb6>LA~PJ+;4B;H0=efZfHg0^JMPop#5Vzjaq7azv!)A;84axZ+f zfEiNYa!1Ozhr9wj5q<-)N3co5uoSZNM0YaYaYXMh=4l3s2W%3PE{s_D0vLArtDe0{;j4gAw;Yh^jeayb<@C7!rRPB?ALTvIvIA zAOi!eD+2V)%mvB9ycAo&fBCxjm87IVU;r9o(1byZ9O-lzw*q1Y4YQy6U1%C&KEb>2 zQ^x5o0IPxtUaR@0O;i~riP6#oUUyxG+{YeqHhJY{W({LMHohb2D_(J`rfK*GEybnBwx($s zXJ_;zUb(2h#KkTFnYL&7$($5b;gr|biC|&sowkI)qk-RJ`9taFssD4KXU=S7>9sje_VC6vME78Oz=aP;vXKPM3c_LhGWhZl_gD1}ApYc|1rF!t>HESpCUH3v zp6F0)o*FxQU3MAzP=^oi3)!Q}D4&}#U0U(%$n9ODza9){^?g(9RHiB@yjKzu%ju;7^@8V?}!1ee)uzu&vI1i)Bgk)$6>Vz$LKD z64z$SZ`7>2B+ty$%~xra>skNo8n_#l0;=n9=Eo0{%O^4#%FhTNAQF8=Xljx6^#}Kb zH~~aApd}4PI)-*Za*4f|S&0B#1v?}3%2_iG2Pf3G(IYBn_4D?j3d$*6GT-VJMER79 zHZ&EPwr)L+70vkEJ?$hx*X4++&#zXOyp1}C*6|A@wd>#7ZKJazgRIaSZo`vJhbO2d zK_YF*EU2^P29}9!pWR0p0V+Pz^pa5M{uI0dY-vJY*K1GvdLKEgzn5#esld#)KfLzj z)8Vl#PTQlmpI^sau?K-_?{ukp$JUkpwIR0t#v(I7?o<(HAmXCT<({TSe%o7j1<*w* z5(TzLMnBrh>k9l?^<7K!F~L<=5nmd%M=>Ud6VDq88Z7O|?(iDez#5I8No?$Sm0@Ix6jWn2t6oW z_k-gbkBpdfa!lr?j_cS-g8vyQ?^W3g7WI@`jWnL|^S3Qoth=e*rL(ARl$C^bxz}EF z>`IRHdlz(9?ET|WK~`C@(wn+-x(@~N_IH+uekvAj(tdQu^sB@V+ka{1-Hfj;%r6-p zyt#P}TJcoIUT(>x#W>`ERwV zuP8X*y*Z9985fW357^Ddn)w;i^3-{H)!XL+jX7ygiZ=xwy1f;q^L(RxrJF{7JMzM? zJ2q3v&pG09J0nr;yta#by&4dpCLNc{SyQ2c;02-2hkrPoU)a>EWOp5P=i!t zk;&@w#WD0|{M6H(DmSk^acs!FuOd`hB|(^wTYRSs`8^LZ1+v*Ll_bz z_!dwrKxUxndI)omz=wV*)#&)Zx#KZIuxOOMwqkDdM!#brlf2>c>T?w?`c);RXot5> zZO&36+gFOy;-pM`=s~uL0`Yg&R(gq-(0keC zgSbMq}J2{GDS;SjyCiAywtL$!QlChxZU%Jv4pXLpFb1 zOlP3}q$+tzvB&(grn3D(;raz5jiN(W`E#CeznXjSq6@Ph?a5(3jFyml)O3t4H&6eU z$cKX=rVm;YD^xeW~b=dIAK zb(VYJ+2t%}`sT18)l(Gg*3_|I>DZPs(tzxW7e-$tA@d-I7Yq+PN?q8vRG;nV#ymP? zd++Gtpc24(YYyMq*ZEyck(~eq_bQf%`z=v1IidAdK!;|^4$V$XAA>hh&IwevSQv(z ztNKuo&nvARypLNMa|c}(pmch&#hP#s0%?77m{^9Ao}@{j23~rtA)(-7m)shK5(4?y z`i~Y*^7=27_=5((Syz}RB2R28eZ9(g31{oKqWkhrw-Oz~;oh^+WRa2$t8(VB;?K>I zC`8gWY?jWx$S52Fx~&&y8~Fz`@(znAw@-1bVA0ixte8e_2b2u9HDUPZQThaTSj!*I zIq_&iN>*Yhej(=gsy@>7#uQ1%Jp#X6M;U!ia1^5$D~=q^Mbpu(x>e%!$HZe}-hb;g zQSXH1s6IK6<1zwC9(Lp2_WOovN`CJrT9c;0>mzG69@!1+M6_?gsOs+0x;LM-_iK>LaK9j-6#KvRkVzH)L1h`In$d2xxC6VUQ$%d4k!?6O*Hn04=q>o(%RvJ;HR>W?y4x9YgK)stfic< z@@)!$rUIjo>Sw4w+hop$gM4>IruvjrZ;z8N3+Hs&QEql_t^ERH5?l}~O7+0mz-^+g zV3YF8qGFg7q%M&n>Vm}oIT?4kZVXXm>jU%Wi`~xh>#qzcY<*B$dn)o6Z_3W(;JXg`ET%T1G7c>TGwWsklF7WrMh8T&#vt^UJjm1KbNp-B$vO2sIRS zIy-1uSnyiKQIuC7TkpcY7o3F=ecn#K`M%TQW*m_sqBuyX-@2c=b{xvUSOB%*P6SW#sxaTAwaD({ z!5kKO0JC{QUmu<%37MLiEZlnjcRV;t_tvZe_SItqYeG(Dq%ab39HZDlI)C!B;z@d; zwD5*{ck(>!Bcf5s?%|wq}+eh8e5cX;G@5qw%JzhosZ@b;peDbh(ffe?oyC87e!^pxxz z>_g!W@^P6AsrE;tt>T{1LKCwE4>)BV2tnuzZz7XP-TM1lA2Pl3o}E@&EYi-ieb8(D zEANlk%;2dISCHB?>B>5i&g>VYO#L3^)`x``BrX>m($r4fXby{R*LurZD<(ly{*Y?%g? z!maSrK>FHCk42{Co=Lcx@xnHo&I@r7X}!ciq#2s>fpIBp91vrsU)TVEfj|g_Fn_2# z4++T{hFq08sq0Rol^2+{c|MRx->6 zLjxFe3&ds^Lf05;F+s;erq7?D0|q2i$Rzw=;7c%E4EhFyVi*I$SQUZT5efOB> z8|fF^nb=RwY=Cxj!9LYMOV|{Wd`Rb=Pd@ilDx~#Fy%kXT1jDEt8z$`CaA)w21OjK$ z(u81M>jT8ou^^Zxr~?D2D8UYt!dE8KuU{L4>Vi>`7}-GNL3a{6^XkvVqZ9f|A1%65 z&a?&?G50ab5cbZRvPmWzb={kA+D1-U<&07T#p#>+Kj1Q1;XbHQ<7?{J1f0xfYR5px zuatLSe+Mk1dxN-~mm&-r2m+ym)?kA!ZDrR$7zRrT^m|J+N9kqAO{d2ZQ3`!1vCmi` z%#190?{RR4XSjh3GDL_oS4Q)cJ?guTqTfijU=RinklRaw4jNJ-gNQQ3JXvVu!E1dy zW8#k}KTIV92wAw-V9ZZB&Fubl%uvHaV0=bdF%t`5#_Qu^TLTk8L(25#0CLZ%#Al!FR;{38%0M;kqQ(j3B z74m$L0i$r2ptqcvMk}$0P>*XIv3~=Ri_5a5ornltfKN6SF!{~1q3I_K8 zv_5`2Ba|>ADS;vL5A7Lq7P7`Wzz~rvfuV+J5rBh%(*;B#u-u{9?Mw^IIht(zu=u6= z%eRQidYsY#Hc*m*&wvRShKdVF(@z+p0&{_xQU;Y#UfO(KSCCL;>B-ZC=LuMK!D|t* zJ#LaXiBOXLCaA_=YT;dN(cO<%7x(FS-wt;uPL(D-1A^IUt7LnXut(<>Yj-qT=8Vkl zXkMr0y_xYWK^T?s{Km+Qtb2Zni^u$ur^cEB)Q^s8CZ6f*OxdcDPa{3#+{fkpcl3XK z0JwgtJ7~AKF{__VQDCoUS@heKtl*7)_!yr$oDTOZ8mF?sNh3GftF_rZddx#} z;oP-;SC=L4r$H<(XjxsF^x%nb*W7<^;(cf48Me~?aR}Qa3+CTmzC39|7;Ev6%?=ee z@fnT-r+oLJwub2$@bh3yr1uA>Bng}FxDy17#gu%}+{U{h3qd>=D-Svg*GuYe6E>qh znl0V2dNcnGiYj`9lhFH?X(Ci=PEMErI!MaVG5dzV>iEu3mE6(>sHHw5dP96zk@=DN ziz*yy{I@8|Grs%Vn2D+S%Rary6_sjoQYAaq@lVmME%y4jBl;g4YF)M*$?^?cqAWj| zk|=Ff@_Gg{b##OZt$8@TBILT9HDZZLy+lBs%lXCZ1Cr^OTH0S~zAUMnj4q$_w6shX zMnz%6TJP+uOy#ipTpJKJ&4~(E4a_Th$Q!jN&>o4NNNMOjY5Vr$G-=C&@+khd<2-e; z7f;p`bF{CcrNwJ;t_3%Gv9~lqy5UmVQUv7>$5T6!&e5b#vdLmfb05bF7C-TQQH56xRIsN8bNY+426+?4yo&XsUAU7#W@&=D_NA zQ$_GvTCNyt5(4_x6?vNHgtV7zSu@s@VtV76mxu1e`nbuq%$=S~y}k9?#_zM&@l0BD z_vBy8y-dIF(k+nKoo*c8rD#nk9Eh=4>e}cjtW(PriJesm>x(5Vj*%TrOG+ap69WvaHD)5kq!m*R`;vjj;i4sr%cmtxXIeEiL%QrtiB2rg9* zcOV)U5}IR^5vRuf{^Wz$S~hlGIl96hr^pb?&R8=ib?r~#=TxnpTxb1yLwQM3(%IGP zq~pQ42TNGbCVH+2LSIYh`e-g|^!dD~*92YKdGGXULqVhP`JZSOsP^Su82Sk6aJSaf zyi|v}fxA^}*d7&g&iFQb;A*!lsD8(=D&4sG)I5v7)a{7R8XDL7jjlcPU1JLYyErBdf-4uRfJeqBR z3BR^Y?K1gO)4=5V$$Q?bdEJAslj8XKTAq1yLwc`lU)G88WVnIISg<`uEkzch%Tu*a zNYx1?QTO9Q>&o`ehZTi&3}Qy{HOD{~O+6Iq450^NIwvLD7p-z60R~5B=4- zR(U?WE^}~evb&M(P>lMvW={)| zO!EB0+k0(LMzYqoIxB4x7Nor}+NTri9d3=x>*1?rL$MU1N^Om5?EgDT^}Y;PfXAA zC6|HJRnoldIZp7|6jg_IMOcyW2tMCj3OI;N2FM%FxvZo9gVMYPcDF?D;Qt3eio;0g z+K}3$nM6upjBGEW_rmKQAl8m-4qq58_qSiV{%6$fqx30UDsxUy-f<1gH$8P9%3aI}Z63uz3t2`mN9EO?M3QhOol>zEU}HY@#hF`Jt{#V;IvF(1vQXwxjN zr(#P3vd^e_eOG0_=hu8g-OwxvJ^gr;;YXO7XO>%gsQ2Gq8AYQ@?f?^aipF}qeQcZe zpApVDFI5w+w8MaWOnEE`Jr=h%#b&t{)4{yoWn*4ZK^Lr|4lnY3v;i52-fmey zCQaZeCu|@1x~zfqou%L|;4=9g39DTZHpiwnyHVHHT>pv~g%fPKJk1W2I55mf zW=Ze?0O(+>|B{d}r$f~q!@I=;KY|yOz=jugfy@X&|7qC}QwcD!=^@1u5%lPBEzUWS zCZs|3-G>>#obDg|BrU!(aWx95@1Z9%di*`w;gEoOc}5Q!{Fwkab`78uDQFkE z<7{0X1~37xhsMQdWnK6s;L(wweZFn*TfTv&;*riHOx^U;&;{tL^oe<5b}XXfKvzZnmd#L!kXd7*$7sdq zz=_L}d@Me=`CxiJfpI2ByWLo0Z+0HXvCtF<20-%juDVECW3l?=*q_kyAQi&(&`Id7 zOIO}Vu&Y5UyC3$i&Q-aku({OpgkUym4*j{g%{kFW3d1_2j6$D*_60i9_IiHJDSPA& zQB`HhyPyj3#L1M7U)ntMiDJu&)?vX`s*lfM=YyZo9K0CpNMZ$OYqlXi zmgpJ~tP+{7l$)Gy$!5^xXRp>dShTZ5p0c}#>?EyWMKO5bUFYT&vwmjv@%lDgD=B*z zaj^r;-!?=$uG}&$ImoTm{sn1iZ#l)W`&?m0(=fm=FfyFK?gpQ6kO%@Sj7xCU$(nZz z`2a&@tT)a)B}iPj=L-f&F-Ltm?h}gA0l6`u$iE|KGH$1r1Gv&|!ocRW{qA{qcW4$REE8vf4}?HtIx6alI-w&-5H8vz9w2{mJ8WjK*j zU5`LDVw7G2B&NUG;4;7Jr6GXfdxQfSd|*Ymicm)j6#*H1$y^l*&nmHoOGQ<-;h(+R z(RgQzP-zDl29quF)v=XfgheSX3=Pz7cw{=A$A{9NM4i!9>&dGZ8G(tpxN1F?ONyb^ z$rPLEN>N&+8@i4_En|t2(=j&k5mLnAHUP-Mtj96bSW&#BCIa9D{3cge_r>GsfsS`e zl7iP79whz%1N3_sNDq_^jQW)oCNjXkkGNZsETlp=W55TqG$y+owu1(K0Dytsj(|1d zsR|jyiz{*rz1-=Csv-sydIIPQuwCGF=7vLFMZWor1u^?)WJBbO*`1omz+wt<1h|TQx>UB&81&~t_gc&eB8TZx>4rNr zbEAG=n+cT+4kw-j-5xXpiJmYf9Tadm3f-8-@l@$I7y>Rc5Ef*^|6c_D@=BPoz`)4$ zTcT*Qp)VZdph#NEP%Z0--<#?ALllD6Ok5thmNW8y@QjM_-Q4-Hn;`p)@bFjo0} zr?~0+jz~B2U0b0D6@6S$;I^PwDJd;5!x5>M*C3D_fk z8U$DIdb;(o@;XRTIessU{*tPm6gZ3+tKNTp6V0Z<`B4j(E%vTH@odlef+gPvdRm!y z^p|c5_{pu+-K48nyD2B|VMIacAJoMzN1lVYUr;B{JH-3En{IA6a zs$+-BA{sA(&t|LNK!{nOIeRKZ;8?$9o?*V1pH+xlSw*RiklRanIp6>qvz`5wr zI?&?v-2KYEful@|?>^F(DyB+dWm^^@7+C&c#b z+R-=Z+!Pt}H?SJC?d`#(1vCv$qHQx$wtexMjVbfLX(aBXt5Px|;0%A)vu@K6Hlo_+ z63yzsDu=}wpTm(TN$z`G@;F*mYMW@jO&f65F~-gNPgB)Vr~mQqv`{wZRiAHO7y&Aq z+49T^cS~DiB5oHu9nfVTnO&m@khF}11|QIu#>Dq|UWu^nzFpACo{B);MX_vOV7l(? zb7Ap<>plsY$CKOCx4S%TUhQKC<(ooi`2a*^o^fNz7^#uwVhh?zicYWt+meAf|t5IE;h+--cSHaDzq^_+e%>t@pPc|o_}6g($ji8_44Xp@ZzP@n@UFh zn5spoLm~TvE?pt&HS;kyO-`ve6LTU2$B)%X3XMqCRQe7|P=||0tp^Y5t7ot6vMEd0 zEM4FxPu`PoiJzM)!lwSqxZujKM?12zP0}Ae=fCk@>SsOTGo>anw0xJD+V-`1uDR=pbw}d~~J2^g? z4(5dGlE#j9K-q5g$WlD@~Pj0@d^l#x*~KTUeP1 zq*0I)D!zJ^0tp(eW6NSVssPD|7}e@~>#7ou7Q!KKwyp@$cs>}f#kKm{1KFtTq|3It zefC(nWv5Cy=c)gGiIUZyM0t=}i`&6~*EVLhGIdAUwZTR>N3l(Y${)e&P-SiT%HXJ^ z93l~rF{yfQviUDQzIS5dL02Zxsr-PR?`=H&-bHZ}yD7P0vc*~{?o~J^dj5=-+Ovm_ zO*VS;wgg^8>cXBADm31zbFf#Hd|2Mj=jv_;Q&hzzJ4ndQH+x3+j;Frm>l1K6vkYQe zcNsjNTDneTEG=o)9%z77qArRl|8h^=vkgQKb2oZxeJme83X0x%6-Y0Pcq@n`f2Fp7e*>l5gbW; z%o%9;xzPkG8z1y_S?eZvG5U^T0&hzj|FU%vNR=d6YR1gXCa8K1LVLGMg7KC<``Hip zndCrtIPz4LBEolK0$WpzxfCvLC+a=%=|EX}k@>&B zj0d6|+vFA(E~&F~GK~;_K3cC^p9u?>gk3mQ%Ha7Xfkk;@*etZSjZ9!<9VQ5v=g5)9 zhejXm7Z`(hNq()uGHuS1JvQtb#|bwCmq;+J8$hU|5JkzokPsf`-o;ar2n%6LE$q2H zzXlI`xLetP#)BcXKGWmU)C3rSaTpE@wd7ESm;|FYh9 zREMOsU7X7pbn?x(n2P;yYl$#R?xfb<*EnC=Rr!S(kMYOuNuAa=B)(_9`A*-aN+?FO zYBY@^Cr$SnHkBrTPFyccn$23-y3bKk;SPym0-<_gBtldAwE8)dm&5CZH~SWca+MS@ z8#pX_S}e%ZH>FD~Ot3KoevWkk`Onnx{IvN#)l4*baARgWYCDkT^Y7`fCSgB_KGY~s zHUba&xaaA}vxhy3JZw5SqOGz5T!67W*}k9^W_vL)cOV8Zq+O|u+@az7MLu|!i|oy2K8lWEtnJECnO;z-i}&b z5cLW zgeqC$iu`q#Dtk9onWpyn0zM6e8_vsX+bgL#rwf)G;Gpu0XS@RrDe>?UUB!fP%AZvO z!5=&!e3hAZhli-4Gfq6#9Y4XKf~8I9XDnV)4lhIHVKP{}uUkGQL!jpzHlh|au3LY? zzw$C%PJtuE;5sd;YDaQIn1w~d<5c~*PeYO|78Gk2$#gNUcO#wLP;uXY-#n)Du{jni z>9#z}?nZJP=UHN^N^){~FDDu)tHJ8A8wFdYT_g>xyyf`(uXj}JhR)u5Vlz8mCNqmd zf=E+l=L@3~$zsf%PGgxd-n{j-vGpKe6`f9rk1jB$XH0q6ZVhbO1xkTBWotsuH6Z<~ z&!9Yq${c}nfTcaC@G=1kdkMAYe7SBVPDvya=P=?R!r{BA1=*P~w4ggrVr$0g9A*KA z@;5C29M-u7@I5bN_z0%_Y$NhvUP(wD+F&Y_7vqG@Tpa^NGJ0MD5;}(icaypvx?=`; z5MaX}gTvk^AuzO{4Lc87gp4=s!_c&&wPoe=>H)HLH^h1Kt4gf`#K_2^0KOUiguu=#MR>lQ=95~Gk#fe=7x0-Fj{ ziVcl7?BRD{Bb2KvTF1ava6MwGT|+q!eX|$@C}ZouyV*j~fufGCV*nmS6{zJs@GO!PVs$xKzfo(uhH}#Qc*g5DT6$ z=r=59WU(YMrOPqfJ6{I z=oBDj585YiTv-JwgIkMryXSMJZ6ow6eC)T;TsDuPjaJ|S^mPsBY``;;%msj#q-XsB zjcJNNK&gS*{p5LPnoAH#RGMhaU}r!W8VDVP3C5E&F_WQ3Y77#YV<*S^`l|I^v?}OS1;bCaYLt}pP#=+w zp1Fs3R|q|r&I|+GhJ1p7zr|Pd%?#Jxvh?QOBblg4PXAWax>1#+Z^q*iSS$(6~u1yk&{QVn_*QC8O(WCYEC* zklWg9kTpQLiD?61aCu}&VTOCoz6C zzVOg4a|ADQz_@zdqCLT@cPpMs0VUKyKtJ{mU9P14-*`)AhGE^})wCT@yam%`2`fz8 zrcb$e*-ZmS6aO+S6B8DDqt@`R&?=58qUZfojxvrpc14fYCfWKG9D$W<#k-PexV0m! zy@AUu{m<>z=0?fsFy1&0;1v&w%wks*>}42@{tF!ge%{7S*F~P_IflB<%3Saf)FM>_lMH6rW=1(8zUMPOL6i zsP!|8iP^Fy!MakX=7AdpU6tDl>Ouc*56ShXIsbmQsL1T1>mnreozQ;v zX=w?pgZ;d8jqs4ljBgrFp~LOz{;l<=%Xqmv^T zGv~0^%oj|29;&x$+~S-bxxY(aiNkkG-t_xffywJn1ncr*LT^3gRrR#B^+X%1?30w) z6=$-Vx!2mC|GW2R{bINh#>mmW&I6N?t~(#HwQ|Hx5v=z8Twf`AZB#P3B=JVjfcftI z71b@zn%!Cv7k^a$m*N%9TH|R=ZA4T{WlHmJTxTEEI+!BqlY%)1^yi$>;W{)m@_-D_ zIN8kFKbvti^$% z3tDZBO|QDf#VJk{)4Eo7rr+oE31xK9YO!p|K-uBQrmPJO2V$Qm-uXiJ@8?-+UfGRb zom9dnPb^GXho9_Lc4UOrx4C=#{ljQkdfb>p=z`OSniu9M6o;qY5f*P%*@!#`b(RNd z5Bj{gE-dsnZdO=7{J0a}I6CkpYf16!m~QdxfhFMy+$&wr>veApp6j$zr*Bs~-|Q=0 zE{3P$-RA@Hl4CxW4MB=*=T^H75;Mk zqr%!Pj^U~WSiz`Q>nz_H9IcSFl@=qy zVobpe+y>|HYA@B@3LJ@HWYvIBIjL?{$HHar$9r@1TX~7TMR~%BJ8C?%!i`^f+6%4) zr3(d~%!srs$h5E;S~~J_B;`N$S;G zjl7@OlU<&qglTv5js%cLxen4CZV{xy|Iz=&$KA)9ch%KVy2JWOukyGrXE<-0g;{M@ zY;EvuURB#s(00`Gt93cqi?r86Dp)Y)x_Q2vR@6BA`NiP)?&*&K=gf`+M9Lp_{|x3L!T%o5UtCz!|2b^y#(jYqt4cyu7Y<5c z+!k9KvU9(CJ?AbBM~182hUw!W-j1*jlmrIj$hXL+eg{+c-hCrf=MOFiM4o?s-eW;) z=B*JEa5SCfBJGzijCP{P_;`++s9zP(^T@dIJ3Qo3OvmHy6PWr@d&T2WO+iyIySuWu zzNRc79`{6hAlobgP*(X+rf_i~eg~m;Ee^L4V}fV-I*YIS(V%c773G!q9;Hq2kadU( zv7K8{d8}7P^`bMw?b?vCW3-2>ie4qzaPf1C5T8MnMymJCyjTR)`HqR4^hb7*CS`0H z$157qy0#DcI=6xTsiVNRdh2NH1e|_UX#u~t!#yScn=ys&sMa0!IKV7>Cay;K>4X_5 zqoaNjI(fdadrlFE9qimz1F+-jFPlI>Qh%y{X|C;)v)`0==y)z{P}cn_upsE79xA#0 zwM1WYA$C>Xdd>5M7!^^L1$I9=jeI}y2((z#xQ?Z;S2u$@qF14@xOAY)XUchF3c5+Kpc>zx+wh3dzy?^PDGtBM#n+1+UPy+jR^ z#68q(uyW%Lxwh-Mu$y zaH_;Yycumob{Bh``P;LhPnYA+514{lptl?MD+Vb=;gfq%brVjYdUocmSJ@n#xwe%8 zdyPyoC3^!jn`P+caAmj{q+MV}@)D`KNz~TH$I$8UGBU`J;NwC$J~z1$fD*E%woWM0 zJys8`L|y&>!Zn-fJ@`ocKlu2%4_p9_9r9$E z+OTVpj^V{^reia6ZX*y*upWIN_x9pYp5QDuoICR5*v)EY^X~b&H=fEKdDm;i;v@jB zXbAp=atHJ!6CwG&KX8EAO*D3(aoUPx8DTiGkttZ{B@*59Kby!hVQdUCUuJFLv_@z* z%ZJR0DD9c(GAt7dnq++?ku=Fj2kqnD!M#IPc1#v^@0-L^7{ASdz^BBm6?Y-rXGpPb zmRCFOK&j0OBRipgY;&Nh)A0s;Ly<*7-3uJbVbauu{-`QJ5wHt~{2L8~s!$6z1UiEN zMau>H3mCFeTmkPkD4KS$?|Hb`yXmoLHj?`Sn9+PohdLw<7Ha1)=?AFI!w=mhPGj6` z$x^2Fm2A{BBrYG7D3WT7_ag9S8-T12pJ+k{=SRowYo)52t(o`fB-mf3h-^xHSg&{E z-Yc5-9)87vr#xwScF06c9F?0&D3O^v_B6F62KY$S{IptgfTtY`4L3<%Ib48HBDXzUPe?G0jW$7 z8Lw`mxJgX?muOBK4EikG2l8|xWwYNz#-uU9U>vUhPg}$Fn9^|cq>*F(YG~=Fvp#mx z8`P3Q;${*_h=4a3++aA*;}{S6>y>y0$fIj{M*a@G@Deis%7Y4_wi(6(V1`715E~#L zk`7$@?>}6!aA^->NgVQ>qY2AdanB;=9tz^Nuxs(Nf>_9@#59d^oOQZsW?qpO0@PcE zkDJ;g?y02ah68cfR4M=m>(FUUQy9Vu1Aqmbg5;SIhl9uq#;nkVc_U8-Z-Z*M-7EMH zR7RM&)tJ5vU^l{rWbtld@}G2_`GFa}2d~03&ICF`Ai$R}6UOEe?t@lY&QAsHh3QWzc}=f(uPyr_X74u}nGNV{`FFGCv#eiAFRK4~ErAtR&R-o{(O zcn}e|s^sN**h>JOvQD(KFZw?i@Ftlqo;L#D^fHIRpbQ{?25qqZRh>Tw#lIKI!3OvK zQZs7@V_ZiP4M2Z5?|9uQ0md}Y5xoHW{7-OS8OTob3v3e&-&(0<1*=tjs$A4^Gthhvt&Cch@^jo*Dg zP4G73cFHv1g{6gEk%7 zELP6^l#3lI*DF*4psBL3_92M}72Cb5z4THQM|9{&s{x`4j^6v~+3oe#| zIhp2}1Un)rR)0?W=xp#K?ltHwp{)ShE2JveZm@#&m)Rb@2$S);g-$J;a+1*pqxZRG zpkuL#GPET^C>Y96$JVjn)JnUvEtVzpLLg&M@E8!%9iG`vX-ob6&$g$1Z}v{RGrRX7vAl@?qe}HT#WP5P{WzC&Z4&=8rj!Ubu#a&n{4m1OJcS9 zXRRYzuwkeq^Btb%D_XU!tLA^V`@-Cx)F=Ip+wO|##7eTg+Q}AjFY%|w?&z&pqb8+T z8u2w!)yBt3#v7&|#x?27@sw%k__Z{5a4!_TFq-Z0^$qS_)>Dh5 zQuENqkBZ*G83Q4Yv=NC?;qt=^;C@qCAPU^IdG`W1N8BlNvV#yl8?Y<7OH-LQNJIiM zO18Me`#V3xOMlLft129m2G2WH)3@#t?6I;FmrEg@&SXK~zQ4K;BdZ4W zKgo|Mt;!3UE)0F(`lAQ1R>f^FGDCU&DpB9kL$4I;2gkMzxBsZJY#mZh6>E1#-9IOo zn`~$Dhd$WsHEMdNr<-Nz%LV&wuU@@U@XmeSse8AvLKjpyO-1G04huAYe5}KZJu2V6 zc&@euy3;RUw4=7A$g~w~MM$Skt3MGKN5QI)RQ(~X)>5a1uv4YVs;GdhA-0ZM zp_q1DXjP1gTiIe{lf4x&trWr{VG|H2J7EnWBqV*$bAvPUdq1D|{X<2CKhC$PXc{dMPKYAUd;}zlNLX(GXwm?;^66)wOOMaP~X3_|obQ&t|XIZj&w_+ws@hqDH!u zV_<_CwPq$A?NQ$ZdZhfcp{!ro&|~pdHfQXdm(d&eUt}ehhHZW*`cdd|zSGyXSo*BbCktDbd6y(N>7<8IVYNj)B0L9R zGODwTN_aH%==GtaoQVm`uNxGxLzuD%K59Fx4N(5wdoim*INo=7iqcz*dNNA_qV7#d z9wz(tyMq67WxpBqdWFaK6G;KzJVnis(iSERoDwRxT`j7u7GTe^g|dpc0L`H+q_eF= z3Dm6=Lly!hlSP32>}EuQmh62b{VZQ|s31?XE5suz#4w$266NQm*@d@?k#5VzlTR*a z3EnD$Aidwtm9sx9#Ln!~W_9(+N9f_RhP$S9Z0mF(whD4+9oUw z6Ki<)G;X;tR;<I{c0NT0aO@NSo*mRu4sU*a3 z)EI?bIr6j*J6T>tDaIf01j-(@y!}e-dj}Gjy-dc}Yv==ILMcfaq7{A4#H5r#yk@v3K1!pe(l^Xh2=j`nrsGt7%Qc)pDm#< zkGnGd*Zgh_t4xeb0AW zZG;kKDI)rOFt0OYfuO>n%<#GJWh(hROfT?pO+Lm<0yA}h#H{OyJ2#JdpqK|^Vcy0q zhAysPH#Ph3HN;j{nsFFi?#2H3_lvRNAMkJC--03sQ!Tt(lfwxrEzRPi_5I2V7J5)8sL z2X^=nXCRye@)Dd2^v(+pX)ww{ouLtgQ-Z5lfacqQWnw(}h3cVAm_S8QkBDszTz%@( z%9JodxLI;09}hv0Pcuw};?Se)p%#1hF;S=QvDpQoS-ir&-S-Rd>Kw)6@z^p&5s0Y% zjg_!$*hc}AXqo=69+fZOm$-TJXfDzYK%;Pi2#(s!Mj=ldvj13_7fHGrEs2o@v4!b zbH2}Egm{i-ybcGeYSPOD`y>=9c^+mwSfwE8eHU3?u4KCCkI&R1Q>ag_K%HJ1Ac z)!=9vUh=)$NbNR~`4&T~(c*)u5i(bPOS&IrtW@W$A%5`KE0~d4nBKA%xi{KP8N6@W zVN{c}a(KmOD9tY#ObK4Xg?K1x$wS#7zGBHKmw_cgn|4ojZ=rF@!-K&%$0D^!0MM;K z(UHyN+-QejqC$k2&FGet1&HHh)*yXUsgjUEVesLFXHj8U+YYjfL&G8xS5L%Zdl(dc zP8HIW@u`k(?;VfcI6h$X?xfBt{+e*+DEM5_RnF%&uer6yy?eh#hmFTZ7?>?e(Y?uk z*uC?9jdb?m@JMDlnB=bxsUB#YH0)Ecy0f8Z9|aAtKYfk&Wwaw2l!eAuVlT`A`&CyR zf!#1r0p!RahyQl&Gto;M0QS&7MmLWt+x1M%O_=MyU=0SKKoyjv0LL*DsEMAYqcHL3 z?*o9q;6oDE6aV3>U4&=+69%h7EaHqx92V(le(G}-3HUL^IMv74)!2GZ18PCe!4|A7 zX3n3+D;)bEbB(W=M-Uof2{>6~&lw1U6#;r& zlx$bE46y^mrA+x8TfesD@grg*@Z1%!c0hfFeNI>pUlLfU)y_?9Tbx5<;r1py#>0Tr zUPcyj_6=w!79_wflv(tQ9Y5aF(E$F5|CT=81MCA`V8GCF;Uo;-mA z8-yUXZU>Q50CmqX&xAoOeV?4cU6_=>7?ng|tG|APf9U)rulA)xfL&=%o?lE z#eWTA_k;$993NA-bRMCH%D^BMUp+xjU3AZfE*rjt?*V$lf6<)z^E3fw4j+^vIRSLH z13i+DEE$)A(=`DpUW<@Iq7_Io=7NnIqn6!EFyw_T$#)Ot)}iaOS*5?}f1;yo!D}<8 zWDZifWv3~Kabl(g?7g31V_HEykl=8RM5F$d;Jn zxUl8X?*gf6!OOXpnRGJ@9B&q+Z&b zM(bEuwqx#_1k-$ZH8E%ZMe;EjDKtI-1W&)_XdQ?Np!T8Nc@;vI@tO229L~~nlM3XR= zE&*R`#Xu90*o?ZtvB&M|%TrkESP|iK`YLcZSe5!0V$O-REhCqsZ32@>V3hQUlO#>S zU*@(p(x<@Y;6L&O=IgoTNaynZ{-Oi=w^PRR6G{69rLWSrf^%C1rf?va#zLem{2>|c zR~k{o+fs1L1X^Y64Y!z+i_TO(E{Ow{Ib-I5Y%DrQ^FZRlMFK{k4GhbLoLJbcfSbip z6N@!iK@CQ%2a;*bF)88yW(!h|O!N0(prJJDD)y$_io?_AdwczJ%Y~!-t$|5)cWPh<4#eug#A)cpkaz?&VfAFwS`ZM(Y6DtW~n#T|Q&?$pgzwM=`BS|sNZt$$&a#J758inJdy5vuy zvLM;lg!3QRsv5WSt-L1oR9?KBQy(wczcFLpQQOaLLN~o!Q^%>dKFF@n2|~T!I8Yt3 zqqGNe(pr(uzN~K||JP-blkv@RcNA&+e^i=TE-SmyX1qAScdxW%?Ur7L#Va0i1}@(A zfMo)%6C9_rsw>@08SSxCnzGO#8M%o?NSlue`S>L7X3A8bIwFaoK0ThfdwBBB)I%}8 z;zPL0q5cW~OUdr1QSbgXgsu1Zn8;TZd!xzS)3AJ>CNXGo^q;2u)}^f0$V6qF!sMw> zRGa;#%KaEksVli`E$Q3Xec_#A`;RfV|Mjd%^mW_sV3*{8C;QIatd&2zy4oevD1U|j zK%tBL;q<1Bf07=K7Up`tJTSZAreUx^+!i?{+Ud2Upli?Fu?L;Y4+adiIG(@$)aSdZ zs^|r`1X;n$GAqvHUTgL8MB$Rs7AyKjzWsdX_a&KYeO!W-Lz_Kghg$j`rW>z$*k zFU>oyV=oN8S7%_?@kQ3UPLSXu>y?JuuLJ(B%73Tn_qc5+%SntZ z6sl%L8;Zu?nd*F< zj@EqB4LLkG*bg*ZQ?6Bt?lL}T{ER?8`4|dfR3GhpqFx_!d$>(<#@c?Z;Hg*O+rP(0 zgmszxP=~PzcnLso)BA_PpNf5XW8D?3&+V3O;Pgj4iOPPowEeFz3zPF)y}$3|+zW|o zh_klxU)mowecl)QGe9Hz*GM>im}8*cAcH0MjgQ*wzZxXYsNZeKj7b;63x8GWd$ZyeLBnuW$aIqHy2Ke5-!+8jO}== zfs$YMB;VjWoXtNBbZLqs4O2Io-@EMjbN6c_ahJ#rWzjUEzXc{y6S8*QwARcSJyCOi z)yZnf!;A0+EB#p~aKf;LjLFIq#Ks0${3UhtP$usaaaUz4+yYM)#1?HHs`Q0vUY_-6 zd~<|ys09cZkgq9NIP}UuIU)k3*d(tuqsJeBvE`oq{F^0l1&f67=kd1#BWH$@b z-8=X<_bUqFct~`pv}wfhVoeJMVM?dmByGqQEb6(&f|}(Eh*vfM8UwGJR^wqh5dH6KyczV*HcOfXP|69DVD`>Jr@rkRU5T< za(e?(2R`aw&&gv;0-aiMw(!)DC&3GWf72xq05QxPhx*m%Fbl`MHPhJGr~j)V-AafD zBHXd_jqdYvnL4mcT{qaS2EHoIWHeSRj59E`nj2OMxP|0Fvh%2tgZ1EJjziL#4jOL{ zYXHqvkEQ1YU_Bzy(T04hfC}Ok>rXA;n%t%HWGEbGXii;HK%RYpAw*iPM%mYBoLQHf*9U$3`t$lZno&Nr2w$F=yB72JQDV#>m37qQ2H99FT_L^6# z=Tq!67Lj%;H~`;=M+D%prHHdb9Epo=&B1sNn664uKCcj#31G7H%y3KTTK0wDU?d?pPmWq^G@jX)F zf3hHkqd-3mM_9{16Sz~5m~9EaKDqS{3bxf106e-A2fLQQFcR z3U%bNA+4xJ=olqiDAU^jrLb+q}jdbW#Rm}kpI2EhlNG9SZ0kztU5X)cIVI9AHALasVnyn160jft~x8jOOKAD zZ<4K(6lz|=?vNm@i{JywEfW#oZQL^1sg#NbBK138))lGF;eampAx41S_mH%5y1R!X zp2RsI`gp8V7$4yIWfT84FL@XY@A#0voARP$_WsLF$c$9G9GP~Fl*O-lvQe`^y)Pp3 z%(|`BlRDp|q!_32w!OHwhjG!zPCoT@#}Lm^mT`QAGAlUw)5~hVPPSO-@Q?PA70Ufs zE}dqo+&gZ~_{6E84tHmZjK>`NV(Wdz$Ark_9ls#tK}CPQ{v4TA?oe zq7P;sPzRAvw67b}O#B?k%qHV+pmYQob2H_5VkXuLGXDmu0niu_O8OmxvN7PnI=}?B zGuZpr^Hi1-MM3cCJe_JN6ZFpXgo_v{04(3|&9F+O+TT#`l|}@GlWr%j(Z2-|bWd@5 zTH(1(t_zG}M+DbA^aPi8UQu$NFw+D*%nCGM_zgfzLMKkt+^qsZG28A9urQym=$O&o zyiWFo?=Qkzz+eEKVLe2lmZ;to@;`@zjw+h4AYGmxAFu@0*qjOhQlNf1vQ_%`1aA9x zua$=7X&i_~O2~zunH_PaTtuQK2&kTi^el*E-=@jucHQifol2?{_H^nD2=1D@e`X3n7)hAny$$ds>ORO`UFV{hYv z;D*Oc6re0M_yM>_(Y1#s0tnAXkLkSZ#1`Z`kn2I0Wf&h%PzVwctU;dvjX&yXRPZ;j zBvPQSI$Nwg1>lU6_;0zqW8Z8M7NPZL=-%Rj!@ve-h0-N}`Mukr6Chiq0NBn@tJVS1 z2`mJ}Z^C4d;jQa+jf@_kzJx*fy%R$aUO@Vx(5iR4$lt^HW+@WZFkB&Q0pZ9;Gh);e zw#GKCA;ixxTwqM*3`yWiNsoGrnuL@pfzzZ@{n%tMil#O(fPWOgUZore7yAw#F)-{i z#<45VqlX_Pge8D|O6iPtJ)|>3mE(932;uq(Lt?0Z0P;Z!9Kt@~n?$RPh&l5`#V!!Q zrzyBs@u8vO6A)tv?H&0-&ZCQjF^CsngpYGuOYegnO(|6$!YR>rt|8`_i=L!!hvNW4 zgpi8B6fPW48qv%GllZWhraHqBP5qpcgLSPQva+xh8sk4B{T> zG~n?e7b-f{pwM6f63mbggq5GWLmDfL@9lOpji`n}C2j(sY9xJ#V3epI`OuSFG#mu6 z5|fm;_*gnPLRdE7MiwYmW9X|IJ}h_|`100%<|ZcbCHadMh2*$nVM_y zUoR^dV#)e5a2L!K-wJ6eGbqC>2Id4&tm^APcXUhA-3OqLy+c2p0F44NK|u_yA-Mvx zeK0f&J{xwkVwwxVx+Z!^zul0M0XhTRxQsGuPEBKgd-VC4f)?pNC+7c6UFiP~na&-& z8<7rk-+;UKIf08i6k~9K{=6YPi15I|eT3VEmePDZsff=Y*a3pl0H6U;95~f*mIN_~ zktnb?_?QVuO46g<_8B`LEt_5WwDWIp)u%Ql?fmmXGRu_v|0 z@;9=|QFd>uIcO;z<hU_3mhIw^8$FV>jf+?SD%fW>x!c<)$l{+RUI`-{H6q<|4zgYe-j`zIW7#NQ3w9 zl@`G2O!v;Rlv0YEYj(3}H`}(YGo`66F5+-(>@W3>!ExdDYVDnV9GL#6_Da{YvEX^3 zoX)?$HX}}wX6kZ>{v*&gHb)7?-4IN_gr+i;@Yk5VsFg?((Q~G*qvrk8S#86(O zDc9pKIT}{Wy1;r_$@0I~KAjn^TvETw!X+|Q;}Xj;+CNxiSYn}S^Fj*L+wCW7Gcu+& zAC6XW*Oi-FuC;IV{p-+I<=a{JE39q_yQtXWuMO)pEX5^$whwg6Wp4x@^;l3cX282$S2TJ`t`!9z0GR*yddWA| zcV>!R-~8+NqC0^bE=DHFGtq(SWd*x!%5RO^5TVVF;lDgkx}?8+hr8sH;FH?Bxs@HK z*nhe|?s%eirRI}k@wW0-?YF9r)|a)GWE6fVD*6@J3B1H9M; z(}wTHoE|M>owfo+AP$w6N|hXyBCF%!(W9<@^7QtC=}r8$wS2Dlw?NLs5|Zm(s&M@x zD#)yr18T#p==}qkqvBt(QUi+W*^gFj9Oo9U9790h@we2jNjE#1R2g#X`r*BX@38LA z7Y!ag5Uu3j^Xb0mf9dO7Nz`$!#yrn4tnor}Vcw4i!*qfJo*RztpJ)V7gDZHU$<46> z(0t~1xFLU2CKL#}J-29+563F)f9G8(-ah*CwE8_nAEx(fV?TW5`fo3H&T?-U?)0&I zP(9I+xuT_V?WzgmSKWm(i&2VIaqDsgc`dS2Ux#3>=lo$@B#mi_?}feG2KTA&4}5%` ze8*g=&S;jMJX_J&6dY&$Xdx#_!Ot5!?^BOXD4sR@OYX2{kVgy?M6H)9rTX|*K#QFk zgVH>>;v;OV#vAqWGlwvBZgT6vvc9eYv6LKn7Ff&bx(V; zfd6MaU5CH zNf$RHvkg@N@j%6vfQ^Wg=Z&}nPkt(QLxVKf>R~^S)IT}8cYF^rAS#RRs^a=(1aJ;w zmHkv$(;%{yz7})=cw;m)*mG&}Db2!;ZRDrFuEj0WnNu{zpM8tpzahVaK$S&g1hU#U zz>8)oAgaAu<5qKEIACn5=L`%t%vz0S{{&lCb@d4d*A*U;23#2KVcqfQ;7c}Ğ~ zI649(m}0pq?&I6ZgDspAuI?8+(pj=1BVQ-YKL566?U}mb?B?>lXV^3lB3$9hK^fyQ z=Ms<+xZa%OZnV;=RLTj|IEDmna+q!r>&`l+gamTPl>IYVU&`q<-YPccGdLOXQZJn- z6|?c8*(g--ji1{l9RUuV(8z7YH`)VYB+bKouwE`t^6W=5cy42wqTetpf$h55A)wX6{7aUCW;{PyzI5+yY4b7k2nJ?Chox|tV zpz`wtk?=4j?a^6u<+Y3C4bnjw{|oeqZpZX(9A-rD!WMRGt1kWysXmhjr2x@wREF~F zsN97w#Qc)B9uuMRH|-TnzQf7udGC+39gM*p;BpfjHT%}{7*`ijpza66jZn*tEV`qg zOP?`^h^h8thDFRGDlM;K$P8K?_XHVW~LKuD-EEoLMr zHcp%lB_&`1Sq(_xHSw(_s)B1%xH=O(|~{K$}o)Why*~Vgh7} z)-J=tXE(z{AO+!>Q{C}zYQ{squ_@lcE_a*qqxntDW5cw7AAm)gAZ-}65#dF<)i`yS zG6x}n431;Q!@MJtmn_XLVd-l=FU|!Oh4U7q5d10bb8+qLqQjk@t!9^E6DIMT-zucb z6o&Lk{Uw3WJdP*OZo>2@ve1F~ul8ADYdB9D^IDv)a~{)p=NN-U>d@xFIdgs(sdG@m z!twL*Tn$dphUw-)0n2NMSsb<%T_ad;Q~j%iuZ(1pO0M~3YFKTOZP~^L3|uO~`T(JJ zyVDi@$m8EYT*OhN!6IPhNT%Uq4BmuVm$rE?x}qO}M|w-eO<~g@&S30nOf}=+$_~=@ zVX}7w_6B>>aJvUz0K!GKI#r@B39DXU*GWy2SqEVB_^Og69|e|-0mV{HE*{nj-ZxM| z79PKhXv6@uc~3)$5)4u3c_9)nI6ajn!f0q7(+sS2PxjWCNP=@VG-niL`L|PcTCK?FD@|=ciOwR zHQL;0=Vgr)CjIqNWt#uSA&VDfOv+VX7rZR_2a5~yL474B%#?NZ&GNi>!To~c$tRVA z$0ffx=ULeb^2jgfGY=nE{#T&jpSAK82tZ0VsfqhxZ^E1~n-N05-cV)ijbb3ck!hzdu$RDErGZN;61k9J{e-sO(nYzhBn%kbE4XC$X+7Mgx*a6)A zMaB_<#wZNj^b2^4vXX^Hpx@~GuLJetobI3>fNF)55T7hK1K`-L6S&X>5VJL0KenEF zoZ4(T+yN-@!8BrTc0W-N@0&6-|f^=?^CKZUN&M8&rbN1d9xStpw|AC>%GT8+?b}0XzZ& zeaN&Gz1MJtq+zln@kj+=BDP>o!3}^;@D0TYfgnAnumnkfZTf~M^AF6?GmTMzqB*2K z$8=zd2tNk2i4%yE1$bwpUZl0J`4a(b$1Ox9^nZ4aGBfYs`p_%^L%l2@2np(j(XXuH z&qU7I-_zhY0WZ)p2859yGU3;sVj?e{nsCBm#ufi94j={tY8X!Z`8lBr0cJQ7NRnso zW4E#X49AAUB8r0`75+xQI@IQkO`-f{6cu8C_fC}yCw~v19fg1-ouI{s=yyu=AeNN& zUZ7EWv+H%FIlu=6l}qO>woI)<)&XorP#@D=m2xI^*79&r=$$`?IQyX)bwkn`d zT2zQtCism^oSHk(x_}zfflhCr(fwgQO0$7lUK z#eyZfeU9Nbe9r#QKB1ZlGy~&ioU@zZASmmD8D(MQ75I3rh@pi~1buoe9%vxwl)U2iS!PU>4w>l<(fsTC2BD z!1~cAXR>f^R^i?!c?#JN7=R=J<$`M<6y(*+V|X(L1w;KYb5vst=?5ZsZg=Vx9QZ(s z@VSBa;Z%!R3lPx9j1^!KF&hQO%5W1*J!!b&5%VTQ&%w{}WyYlhkE3#OgJAqX9K%wu z$_H>>7!d|BA9I2iya0_W;Rxyakd$Wbqmp+7PLsc0BcXp4+NnU|;4IUbW`2=K#bEa~ zJR_f5-x420@O}>bJHp|@vH@&`ez*tc@W1#7=!$T1HsSsM`=6gnEcAK)He5M24>G`RZ^}EOz_|2Kuq1G0S`x^I1|8D?RLJj*V9Gf+eh(@e)SB6|Y@g zvh*CEGz$MbEwzwbFK4xE7wFzhA%vR{W^Ws#*{(K`~>B5PkMA1xC z7OUdIQML`M>4I>8wXeAMcx~l&)uM-+W%HEA+2guok7aq^!nnG%Ibgc!e!+6tE=*`D zMe@Riu^8V0{-4)GT>4oVdt}q69yw=*eG4zA7C1fhu+F~`B>Wr(T}dmcy|Y#1sG1&s z{ef=vxp#i-ZOpQ)&g9hSq)MBa=pP>{EiE5*jk|5`GD%9Z;z&eoVfG%m|*$Pz?%eqjBJR=G*G^lN_i z6J9XqL(NeE$h=go#c|t0E= z#-SoN?2626FUd+}R++jMj=Zjf^&m58aqMqA;qX)+>eP!j%bLpSf@{_cWAi^S9FOvj zWjnCvOi1Mc%XuHuqHX zP<3%=s+(%G%TCw@KW|~SyY@=SWhc#b&ZY;Q7prvQ8t3NJ^1PSAlp0G`c55SCFYA=X zul)?N1l6(M&uQ&%jP+bOb9aI7is9~rx@Bo?NjYO3BVYY?*k);x@|TLT!q&qVa|{~f z?`1{X*k*b@Y&6|))E9FJqeI^D$u|vs6?gw% zom117aFB3Xv_5&{mBg}qUK+1seAKHf;im?LDDO+y#PKp4il`|p3@PgQ!K`r&ib!RHbn241uj}My!(A)n7xzgL z5YO=)>UV33UjIv?uVS_RO0-m8VzsOZ5rG2oXsQc`pgyf{<8N>=O&q<~3t(0syv2;?g6k|?pT`=n(4bWvD+Owl%2p%0#=TvjExpGTm-VpJFkfL`A1{~#J_v5qLg0) z8GkvyNm+KVBrB}U=xc=MNyQ$mjfZiBmA?6VK~NdHqf5G13BiDJv&Z7N4+Rnae#$h- z`2G;Ne)hFD+TEIKkfnB&Dh@G9`J?ReeRw@W zpngA_0_dL?@+C>!OhZIPOamc3Hcs_i zn28d}4udF|*eGnuh^^$z!0Iy9s6c-?RO1<#&EQum*;UG*^-PA~Kz>L6l&@-i0iyrKcty%9A*9ec7f zVxPrl#ENQjHa9hj!$P)6~0e)8iCE z3W&s{QwtaH+}wnUq1B94T}D&_s=D|#cLfb(-{oD%9+?t|8xkZC&Htyfk5eo>fzb2g zm$AMuGpnqXRchZ&F9XZBr5X-^zA=RHBkCC2hw)T-8QyxTW}dP`4030P#g(xo9!n!H z*r7@8rY>s=wFLO7q!YFw%zB;spveWgI;nt^=f_?xkD&WsIj3DEWCEF0nf+(j6 zKb|hONF+6f8#80OOiF7{aY3>LA4fT~?l3Bwo^_YeP#=;vb`03=t($~;%{~ZD1X}wi zID9r-7<@2{k86E&jitA7uU?};$pR`3f-}oxP5oXSMDSyHcuRf~y2IC0^Ma6pO z`ZOhlL}KYxnvtXH;xLZRULN0}m%3ONx**&V=a)Z_=K}G{zhIz>uEk)QQaII(b9TxN zI$HL!X~W0TBD{3jqdWlN&FNroDQyHl_nG-Z9*7#$CabPbOzQXnjJAOaEl^?1cxhS( z8kUz_QQ8lI`)HwB)O8h{FNr{;@$&ZUGF6pe`_a@YJoNb{AqrYXe?@=|{Sj@jYbzUKcWSb>guC^*My!6NLsOWs9UH9hydq z59x6TRuR*Sx5;7n6x5Cr8LoQw38h0`p8XKSiCXDpjvblQ zy(>I;D{AKS^rCW0Ae9#nl!Ty6J_30mw$*2w`Rb#^g?U+sL^<~`gaO}FAZH!_o+@t#dsz1t_-WFWMnQxH#-u;cMaKs()C;d^!e9UD0Qi{O24nBzrD^11XXefB;ZK)PsI;5+GSK zbb0=nkdZr-B|=Qz18T1VFHo!8>yE+P=cE}C!aYn@9!MPg7sHN>(wh+|!w)T)^V}rR z!)$*LQs^-i&9_4(4~l?+G~}-zq$hgFSh0ruCNXHhB;YW>cr|_M7Z{=lM*{ekgLm}i zcWV+jV-|?r!QF+)1iU%2Y`C7W<;z@Vki%aVg>p~BoY1#-kqp01TCu=Srq!k z_{$D4eVVGC2BZ*}_!0=G{TkCYbZKhoYXgGIuFMn-Vg$fm`3C5JeonN6Pey1KIR4jf zXn7dU3218t>_r9~ngy6m9x-17k>q zBf~M^BO8dLDer`Vg!3!5;MmX^;(%S{&wh`Q{=ImdfstXX4_q0CQ|glkA_pQ=WEDil zt&HVKIKmL{nR_rvHizFZh;|MOn>8`1WYKT~DxQx9oMtsmx^wdsguV5=1N0Fw5Q;IW zDMT(Y;FCX4O?Qm`5|O8aJ|L8MAAM^a;(r3_Qr(p>g3-DY2*_qaM*%x&D`l`fLw3;c zzven6(=ZN>dXorzKU6Mukr*=3Q&do6K1+fJ^aXgp1&=I7SgLocgjMP%WS; zz$L+&ga#_`xtB1BobV*355$;2;hS;vSR2952WOb2!(_kESICuwa}xQb(s>Ive{TAL zn2_hMNk$UHNa)E78sVWQ(gStE{@+=*^oW@wR+zhsgbTQBo^KgT zc9VJJzsU;RypT!Im4`gzzb7L8zwkWs@3exkM9^bFERV>Q*p)Pe0raja&1^uLf`-!= z;sLA?&IHIyX4(dB9}8|W7yCA*An`$9_CT?`fy!k#+hBnl7?#+P;V|GjEkHd6MUNZ& zQqyZ0BUC7hn2*iu$&5(i`QI1&bxDv^q06m|iXTq-7z6j?i8==>WJuNB=Nq@X^MjB$ zS>5a3U%XM*>#u6B)=kK|O;S3wKEDlo>+933!_qjPI_}FsHN34Pi}UlF#bae!d)~cL z?yCjrci;3~=r(TD*MvJX$$#K|Gv9o7L3ZoY+Hpfz=_q&H1>OH*L12-+Pb3GQVl%4-gb&4z2Ro)lNLdst5$S&6SBpZ~l2A0}|SAVmk zi2I)LpUIt}OTz($C44OE9FnR&Tx45@IzPDEO#9j}?&#ZU^H1u~l#V!zhl|4E8=q5K zf&&HFXPqF2CDIyAAGH(4FLWoiKK)6-KY8@}=(eR7{B;gP_X`(}v^>yY@*R*nLK4R= zT(JMNCR2A^`5=Mev{DA z5J8L8lx5|m1exk==Vr4}BVK0KH+>e4Tb?~dXeHerD)7)UaX7qgx zCrG+b;*>Y=rDyxnBF%x@t-Ib9Z+g4oW>2tZyL(IKBP;tE^a5K76!~`fZ?*()uRZqO zdW)lPfAz1zy1gs`<{Gyzc^R_>|Hpz;!UmP)g>OURrfDG|6AOy+DGDR zUT%xgU-B{Byi=e)n$hu^r@QLZuFF>KTNGJdJ4#P&fAdb)NL+W9)JTx)#jh;!YXg|R zP_nJdRNxmIp=1Hz+~BezQ5jxu?1^so^bZ?7bpO(2t?*=3?{RGbD&8kiCt-GuiF%Ho z#xd!oQM-%1(c>Y!A5nQh^&$0==T7-ZdeoZ>d6-IQ8v+=c4M-9V-MY4EF zgo{F`y6jzx*0aJh0k5?P8d`Z{VV}0}Q_mEw5v2Z`IGAK8(%uu~`SQ}`Ee+GkqhLBX z5sipTc>tMZ*63*NrB{>IalQa=;QO^G>5Ksq2bir8?B3<5lrV3)O|sgJhA7i1CRE z)mjd3YKU1hy8?sE`K9+tHIC>Q_W~Z}eyMmbWXJZXV)0mms3{~QSXi8m+Ep;LdeDB` zGtKC5nOr&(nFS?9$e;pQY7NkCka|OeG}=bQ}V`Sev|Ly5ZZ=4 z;~Gb05gi zH{6;jjh7P{t%YGNt}x>4_uTD;WumgY3U)C~NTnrFK}IZ?sdUU4^dHigZqztyV;Z+~ zZmxvihN4AU2X<4(#)Sf((d6CY273r4fFungbkRm-i<9}%%GuZ^zNCcJ$~iFV<)1X= z$9ZVyr?Uq!g(CkU=W}|(DH78m&4f2*IVDq67v7a*U+BjSo-i8jc}Ll~LaLPu1=q_L zwZ`MK458D5dRoxJe6YqccrsEK^w=K;9*fUX%;!{w22rE=rJ2ZJu$S>}3e~58m$SdMF4y*O znZOaiSdL0X6(e^`O+X+Aj+%g~6H8`SWv82-LTt#vCG*%Q-B`X~-&qS!EJl)me>H(A zkI~*Cbl6-*Vf77RhA!>|mOc<^8}`cUvUvQEMcJ`TV+5?hG2;#NPF+AbuctZ!giYn> z4*iq`JBN4I9gZF#=<;ktR?&DsgO!NPfP19w3g~}VL`7aFyPlpHg4J-=8@0UplL$F` zgrA)sq$3Fem$iNql5Q-q;HS5^g zp{>?Rd48^B(BVGihm-6Qn&%sfsz1)5 zvrL}!=`a`+NDSSv!9CcJ#!FD<*stu`IchX0JV>`lVvuy)|8!{ahiQCi2|_*?D5fuW z0ptp4ay0|<*FY(lQ1ahF=nS1m0Rtme>kDOcgM zeH$8{`wcu`E*e^n=G{7uNRsqztyuEOV976 zP>uy$eFX@M3m7XD8p~r@s_pD@rLiJmaTp7D1fI?jdyl=$>7IZgXadY!l;&4{RLb|O zkFCRnLz$v{l-}+tNwirLYU`|Ew& zaKRU-p@iX&c4&;kEsnmGMW#PF>jjGH9|Cgg3y5b+NSg+EoRL0U6$H6e{jOUxU z$+};Xg&|`HB-(iH4ObOUdj4TZz;66su8@WkpkTeo^o@%%aQ~o@|H;BrbIJt9BUk@u z22h7vj6TgU#SOxJD)rxA#G34mVAvcg)2?DL=L%Y+G(do*S4a(K3Y3o?CxkI7ePy({SUzn)%;BnSMC4dxu_{@eL(x4;l|>kLIc{ONT|Wnl5*m1_&EK9l~=wskbsn70Dku z`zRd8#_U2p6QS3Li`0m=d z+yutX6MLOLS7$DbQym}5>5l-YcahZS{xf4r)&rI$f04@L0nIo-|EwV zhrrX~_G6e1V($rh1jnTmy>^CPh#`z<(-FZ&Sz9(#4bJ?GiYQ{(e^NIZ+o$nW=KjF8 zf<3t81w!Aoz^{uK%M>sS40*)dK@j8D0G~3K}`E)Lpx)Thxi9$f`DdBodSN(Uutsazv=m6gZz>RyBiWjzo zC&z7qc?TzqY2tX-(<=b&>tEqxN=w3*bFw8d7}x6=F7zNE>kQ%`@FSl=Q-r^O6?6jK z`21@eZVys2^euVf;)oZjCDQHr>oepXZe5X_a?m8o=q8A@_jC z|1g*b4vo=4Fvms5#{6Qa7n;|=^8fcgKL1bHPQZwyYXx2t#N?FrxqB|NhDF z5_2!UVClKJcf;({Zhtrz?M-iSuia9EKQRzNJjR?83Z6mM3akrzz@f%W(+J|F7&Dh8 z2WcH)2SD#I94`jlrJhN}Mh2+}@g}%2m(bCh;z&V5Kq|!OrRdhjzTr>OivK#qx!sW& z1-u#@maO^}+tH)@2P}1eRbR0&p5<}<#$O)uY4dt{0%~U{M+Zwk{X6HQPU-aDC67!^ zkGUs28k_29QgFmr*=-@SZaGVZ{v}eiLzSN@QoBwx8^@LI-N2qS)cwv?{nN4UcortI zx>~gs1%DX4Y}oZy$%c-A%s&jw+F$QX@)BsmvtB?hd4JA|=E|}*_b1-&Zr)G5eA@Xp z48QYiUtE`wWjp^T?z-q`9Hf%-xG>n;r}D!wX{$i3N&HCgxbV+=&*pwtK9ql=bfe+x zdo+n4G6ZRL;|qkdyB2b~yRwV_C(tjt+iI(dmBYJOonQRvjds)HZbrXv;dsf6y?52` zzE^PG)!w)TF=C&N{>Q4i*h%xlfbQk&Y*Yw>#|q<@q7I?oy@!e={*Foh0*|J) z;4(jj$oaC5xy|rblW&Ur49A2&_&c|Y?XEljEBX$q@yS*!gWEu8TUe;DjOP*;#h*QPQh z9BF5z*-#MgD#`jP%p}4uOI)}9faUIA-`HT=6y?JiPXEL>NK&&VW7MjwZv4>Z#%({g z@V?um>ngF|@ZFgEnZIx%F5Vg$8EhTSi#!x>`@!=5FL>EK$v5 zQA+iK3%tSD{Y(A++PFpV17{Yn{eNdISL~=;6Ov{0Ex4n?;`N4=TwP<1%-P|HCFdUv z#p}9S&-&fKO9kX9ZXfJDd?9bUW8bMK!5$|>kidvuRs9gi>#T}gp>xnR`;?`r3Kdb0 zI<^*@2VWU~bguO6u?DnQVh8f~x0gC^dY}#dg7>axLKOGMUb_pzardXZghxppzo|k$ z%P%dtd%gbQg-z__!&yE}TF^7TR+bILD?2;>6jk2oQg*uH%lrAQm$qTwG8q>(3zo#X zU9Q>_67Bre8ij)eHY?oar%zOo>cUY`)#lM1F4?ba{~z116}nep78g6zp_(pCP2B962pf`tk$*j^EO%XI18S)w z_b?)Gd7VYR`N4Rxo2Sc-_PPbNNVM-UNdg!k>KK=MF3X>NyG7DGRkakv*{Sg|2(CAX zJVF0SDoUb9x7{{6`WSk*rYYyUvWi=ReW_ZZ*S%6a0v=0qipl@Y+NP|#rGpvqxa6=% zq_uC{*nb%yx6KdXxf0g+lvl~kj#jG!RsO2{hsoq>VUj!>+Wp(nb;UH_c_$ zH=FypDm;B&<1Nw{Yu|0@DlS8RJx>WYGz;{+Znh5nfV=nNIL3FNabIr+@uomzMnF6B^g&}|~oC&%6)Nm%ag$0n6?a;L)^o@+NB^B~;EEu_$wr5>!Ick`^ zMa}U)dtgJsN78PV#go`XL1 zn3iZIK7bkib>(~=5hG^0^j@Wz8L^*)Yu&O!*G9lEazxkWGJ|>!Dyzl9^tS%x6K!%Zfx-`<1|NbHE`)nGH^dSzj|qg? znD{00geaM`ND76I4k7w&x%ac;Mt zWrUeVK*>SFlC{*R$>Y?pe?=(g)mJnatbm>$|5j)$ZqlCB+KESo`#rxDy6EsQ{@CRj z#sJ?)9X`sQlI6X?`orLzN%V`Rn~`Rjqjz6G+Qyz+3*i<-s3%B5ip4pfvv4pacrKS# z>MkM90#@0W$Mt_jWz1nREx=mvYt9A#J5_Ixlrd}1v8v9b^I{833E#r2K z|BY_z{`c0dG>*ih_hVb~>a0k?YWBw&xw&Yq4?6fvZPm|=jO$+t(518{?1H{aCHhI|3EKu{K9A|I5P5_F^XE;N;!T-^ZX?HR*aZH8a zrmJ7?h*r( z72+;)xZM<@$`ly8YqPa~;=Z?x>?C&Nr$Z_rKmu=BPcO0vZt+`l0|M>*HUH?A=hlRE z?SjPzI|rGhYI8UY!Xj7?7#1HwDobrZT`2559zVh;kgarJh0e=qR`L*-2Vzq_l!@DoY0H=8B2YD_FwcW?DAw28%%Y=8n)<D-uypZFdJra z7~hggXhaX^Q2^sRAsGOc=s`1>>7>JN4{!+0THN7)T=bvlAk(*`gfH&_Lbf5ut+%QJ zaAj&`{E4QdVm0H%LAvNvG_GqMd%}*n*P;^TkM}sDB_;A$MZ?(F>lt*Rhvj4~sWO`fFT70+nqWAPbgn7BFLq^1cLEeClp$J+CH*yfr3l4~@V7F3StvA;G4BqN9G*?~7 z1B}QA8efM#3LA(1Q~;25JOJg@(+5KdtX^6j&^;I`xPELgcwQgnLBE5jqY!Sb{0>S|7A?Wd1tpTJB zKy?bHtp76{F!gy2Xb)*1FhiIdBM|sBbVm;9I0dx1&#y=!`Fb$OCCI*+V+FjyR0P3& zMH0sZ|a-=;XQ_tg9Vg7L*As5U&JS43XKt8~5NCv4w z)cMj}&>6%&UF|c%a2+xNIa6}So<;9Ne3mO|#bIyNlU|H}0s7~u!$L(5*42`E3P|3t zenjv085328<7D^(TsV3?W_gjq1nbr}0P9vS($`OkP^yUD6A-5bWIU-?@EzdB$QThA z8v|$q%oRr312%o&UU&e~u`mw_K>ax`gSmAWg!r;P;Dr23XeNbWbMVLf5;sO+U;@*E z6s8NlI~u^6Bn|GhSg+KxCn5dPb6~FY$lTE_>yTwFQ3&{dd@7=EmO^g7Ko3zVxgix+ zSKiU;-FlV;&!*tZ^CuGMd4RQ|zwzg&Lgog=Z@3Ma{}N9%f8kAy!E zw15WGuY1I)?A(O4MUExRM?HnF$vlKhn*qM-dEx|-UXq)Xef*rVV<;MNG?O{a!G! zU-%RlDm~0}?1QVz{)rMc&C9JO-{IXq44RGoRW3Id;ORFKTzjxEetz+2=Twul z3DvsC=s~%lIlab{_5JNm4erBy}^REbqa{fSiIKUr=8 z7ktSSipAeyykf{#9{;-v)9}cF3CpFDi@V=7+Fw1n zAo!N#+Npp28|z#7#x{*qSc{Y(Cz+s?}FJUt%SwTG8#akwh{yVJ!#4C+i9GUV%@g>FV) zGAuH%i}T_otEHYAd!<%uS?zdpK*KhQvHtRapMS9OX12x0mUZRpI@j6%{o~$UxxLn( z-zys0{$sFT(7J@`@yrx$@9odmhbiOv-6L03Uq-}6UNhTlqf+Iiz1Y^Ypa;D^gd8RX zw=|zk+??$tIkEndIL#|~yRNHS?vQ?a%(?uM`7&AeaAZDbV1qF5npb~??MbISfx-K3 z27c>}C)G60_bJ`mB{Hilqo#s4Pzole)zn;k+!O!MW}Pyq)GG5s6!u$YxLYRf1{awD@&T&r^W^G$2l z(<2MdMjn{_JYj?TC5)JOx6bI!f6yk#X+5qJ>|&eMa&kGHtBS-Gk5+Bk7#&^^EkCfS zO)Hd&)*|9C;_S6SozcCaAA=q}d%gd#&g5y-t?KHL-VTqP-xD&^9L^+%>3)dSbPsvm zKOc76Fm^CnkT7dLHp#8Gj&Iwi6&}9T!oN8#$Y~6IX8eNN%5s<5HD|TUAuRCljBz#n z@fjE|mEe(GOIv4NeqJs$qU6q?z z!zEiEDJ43GP)mWz8NJWX?wk50D2#mS%H0Z|tjUnf@-P9zdX`vew%Ieuc$njVU$Zan z%DxkMYo^AkRonbrF$U~74Yl=IRkrRm(QAQ6)#h@+Y$-Rxhs%ld_pw`Pp4xwuKOqmSh%SXjSXScq&@RTLN7O0`{uS{L%FXElF*ou10G_GG+5MoVKDKpM z=?cI@#5!~|6@NGuDVH7fEz>MD>1Leao%6Xj0@;6E#yRna!QIi&vhLpE;kEVOcNQ(@ zpDJDioL<85D#2-pz1UP#JIwESTrde;13(})hS<=V@T~Ikjy<}{Y6P%-Z`mYi$QX|Q z)vKiaOvu!zV)Y;bF6D`7eY0+%8FGFNBr*lnm+h`~3-Wyi_yc7Vav$e(UXjyS8TuP> z-rU3KnD79U+XamQe8Xz^cZW$dAGT%6JxDDU-0<-kT6RmgC({=Oj)OV&?yz(pT|V)MdU` zG7Q$CI!yZ+yBSCxHUo(ABTSKwF?>N{%i<0;e#z}1iAB~>r*(eY);!!tD0APbT zoOM&&aMFpD&c79r!34jH@g@IiBL|fhu%$#EzGs?!>Sgk&bMc{sN8Fybyu;H?FGS~R zB3sjM3>cQ|)i~kmtd^3J1hd)7;2_&zPHvlw!#u|SAIjc5s;P608mD^ITdC6C3RqOq zxmKoHMJfVuxaf7(s}Mkj3bhDfPyv|?vE?cjs;RX=#UVH#R)!dvWUd3IS1JLNDF{?3 zGhs{uCM5ms_dU?wZ>?{wZ~gwTR)*x{ocFxv+56eg-t*z=;exjc^XY2~(s-$tptfFr z3kHe-sXeO+ZOngq`Yw+R0=JCwdybZxIiU5G zv3^i0PZZ*uLhx;ts@N?g)B!4&3{p>_#;ueGIEOb5@HWXDe$dMgC72cAi=LeI&tKC# zmX<&h@fex~pMDRn&qXN-t0x81dSXk;gl%k_JAO30H4h>Z({PW$zkzO=MPW@fP+%ZwFX$9#Xr4Z*X)WLw*zv^ zZ^uh%Woyjuy>yTnQ#B+K;hME?F%Ak6y4t85IEn;0sKt)tvbT{sAL-}7Z&C2#%f$@x z0_PLyCx?bs7AN$>>m=GA5Q;>mp^U;vIqpnpF)IL6_#8$xb%BM@QV@+f&N>k1&U``^B0y ztUD0RtVGXUajTKMBiu;KG#vKGked-2>c!cyRSz!_^J!@CgHmxvScY}$2uv3o_Sh)G zES3KAgZwe2?zDT^Io*k{V4rngRby#$igLsnrzz=)0K*^0b8-JWf@uKfKB-F)tBAFy zr($szG$6QEK1gv(gaMDer9-ZpCxim2wW0ipYe81X2QD_4kGq?9xe!{UR$jq(?ea&Z8(y4#fp=Y(?O52|zVjygKn9Xl zOhGo{c+Yvhzy;15;}2Bgxzuhg(b~iArpB26xn`%@Fe<^P^Y8}%piIfXdMhL#bVDPh zvEh_1)+vg1%0j;Ko{owDxB~&yy}A-VPQMolP>2;~mFEpXG%bc189V7{(OGGx_$Q?? z^M~}pNpb)g6#&IxF!^_;+;`+)+v{wl;{a%#GZ=j?vRlPU>KcG5UL;jet211hqA4$S z(ln!;b^HS4?hGSE@E+J9<}I;c{s{~N6*mQWly?BlL@njf%Y*|y{xLPjJqL|VM`FN% za-V{0Y8;7>JNO0?oajkt0xFeuM6-~CPA@y=2ikMjz^RV28qHFme~_MxiHC@cauX6^ zx|zI;_WP;`Mj!@#A#wEN<6IDB0JQYe)5;(m{&X<}P=mWZUXt|n2_%8Mo&5h|R~cz3 zlOsiQo{UA4%ur&yKzoYhek6#ytOhTJE-NF1PlQi^W&&@;%V71^N4?lpJ}ObOfxLM5;fV|8x)8*a~$Swke*%r&PiQoK1iCB$$Lp9s8#A!x}KdKnNb zLuFLB(pU@RLPBJLF2MLObxI&n zi-ey@q>}=K%$UVeJ20Kz^{Icm`czkb6VD%3eH{4|OcS{TD~$wBE|;kgya4qC0q^i* zAh6ProL34GnQM~PNl`HahOs+5IOb8x}?C^wZ?Iw{D2Q7wssMJ1)O~H zWShG0Nc)|fNus);(J0v=skZ#)I%ci{(;#?IP(p>QXtKB%B8fKKnV|$i&(Wm|W|^=a zWXyoBPd!vAk~Yi^;8aj^j#~T==(RubI+Ua7w&b=arw%tL)v#0m)g#UlJSDz>{yA8S zL~=_|>H!hU`q*+n94sK;`hWbv;SwT|t`J8T4Quk{Ts_fKUF%IK;d^*X@kvQEG82YL zr8)zB4^Rb^>dn#G$QcSOhSkPi}QQAwd0P*8sI6&e#fec+Sp;5uN{*MDA zt^ko8VSB_8KNJgGPqsRktd(&URDz*P%RmZgi+$tfv5#D4Sfc@0x*Q>bb8;vN_D?#g7`K4 zRSw^H|ND-eu;P49ZZ`kk962#v6500I1vv`Uyd%0dHTLhw74?tUZOIPC7jSf z#0%SR{*~UN@Z*9ab^u(TKYWVRR%%22kn4&eJ&)GrgYS;S4e9>-B*fvJ(wTcTM*La_ ziHGE#g=r;`N^Yw-T)reU0g{B!%T=|SY=NwFmU#5XV>=AL)IIoV69J3d_#VXB1^pax z`8T10#K&8e-{#()Kug^I-|g_g>c12e_NCkY`7iC(XwUwX&|>m1a^dfs-D#eDf7$QO zUf*O7W!c$2@n-YSEF1hGK&*PhGogA?mviKFQ`U`^rsjRp2&;%e?)SqzWp#OLP6v_L zI&Sp(5|-@4N14xM+H)73O>$1We?#-ltRe``FYTNxzW8OKz@({F=66(nAX1RS*?FV) z-A4_EzhaOX-8SRImg=7SlV2V^cdO0Qttu?R+zF$_xPIxLAWof!^Vq&|oxQHQmBW`* zd1>;T#ZScpA3GXlA+Beb)Q2D@H4(iaPm7NOOFqu@GeL3HSaAPU(grha8yn_cs~$OQ z;kC!Vfm3Gh9~jl>uR1AZm&x4)ag&>(qa0d}mHXbEGcX=@1=HB&?`c=1dB5gxYbYdj zSlP6ytv4z3^4s~a8`m62dtvCaF~1|lZbQ)Fl)h(OMe6#Kv+B|kI}6lHhK70t*Dg7C z=~rHPQu@!LhNfem+kIlb^F5tI#!VwGH}E2dz2~ZvE<1OHQ-J(f|B7DcV{C1L_8Zk3 zz7+-jIpJwmO=S(2xyMHipf`PMUt=s8hP_a-;*+iY9-AkfuD|)y%xf;+*3W7?Il#}e z7!n2WYt8s;OI9U+I4Uo%j%iRGc57Weoc8TGzK@J|Ojw*<56|C|z5ib1-Wkvo{FVzqJ~ZE1oW`$en=W zy<@~5dZ)o=A&k#MLZ;rs4!aZVZkaW(bF!54sOzJq&L6!cak)r?YY~)^eqp)ZHp-#S$t5&eJ0Ik0>brJ*%poC8MY&hI5owx{z341?o z|9!SEkT9r{=DNiHtj^4@5WKF98z?a8l#i?JkEkxchg$io;F}``%>!~`ZN`GZ0k)OA zyJ94*=T(H7);HaFqp&JXTun`QT;U4PYgUZ;Ua{?SdZvQwFSZ!y`pvnDnm9nk67U;g8X-0B=-qEVQMSCQb{(y4SD@*c`nkCR~uG_ zRA^ml&6Q7lbZjoWA_>zLFgyE){u*~L7wyuI-vmzoNuf5B+eo|0f&<;tTtip`~ap8mED-~0Y(FmKFB*G(RqQ(+Sa|i4^syoBdDfvI|1Xb=$V>5 zMLQ8g3M4pT5a0l=0nD!gIuoiS&l5vuTEPXGc@GJwAcZCL1bA4(p$uYv=~cix6n8k6 zY@NLszCvohHOXrA`8uw8*;`0CGVQCM>dg8HRE7WSB4I97T%_s#hkp z;FXhiQ944{+BX*#cZyf8z`o4a4Ly=>0|*nK*d&5qg#(TUo#UR*?qmL9J(BkzpCFQ7 z95_9*aF;+i43BS^r4#Z-rj3)=F$3VFnAL+FnPv&-g>cL@z5F-XG2FD}3vhj}D6?oI z{uQqW4axPGEWtxp5HCiyl?L|}Kz?6F9UzXh7X#c>nUTNpdviJ0Go@uXEQkLt2=wQaB`DQ6YsBFV9)m?Y-6TvN@u3%DcNVePQvk66<3&9p+Y{>dU>Q zpkI7N4Aadp%;w!0)4k8uMoDt(9VDqpLI8u-fUc|NKsuV!0e&_}#YAcZhDb8CA4(xU z%>KG6<$3oiM61SeRVGB{uMnwK*fiOzP&M23V&gJj; zLbCsu5#8VHgxtM{+!v%Pi(Ve#BZBz`I!p#6Z&{=Fc z!B2m=F8Z-vHTXl$j+k6JC;!7P5@AOWCg3KzcwPrrS5kP!{7@c2;J9L2h|eJ|D{)g3 zlUHKi8~-3*hW{C@t(VL`!GXP>ynJLt(9H<%soFaOqs?$A&JbiC-T~@-dLarz7<22`wVf%MqXhi41X>7fj%sV?#jYM^7%=ujNEYv4?tX&>4C%r420F zF{X7kG^GC~q5z8LAqFRS5mD)AlOEqq4e^QAo9NDwrvPC-zxFRS<|5yXpr=)Y6aWY_ z0*FG+0-zmu)JsUTtHEX$3V#EfDpsfy7?ZisT9Gj)W+;g=F4(vZ0^*Y0&8rS=Q`&<~ zcuXECQytlvSVE+Eik2#Mok`ePzzhLg&GgBWpXSiMEMpw7tXy{rsO-p^ zd*LIHECayKP9Q?UJwUS#b5(*Y=~wX|2EpQD(UV7m5Wun!mno}fROZg)9uUY!l>OVM zIst?vG5!G*N+b*b$(1H@H^QtlsS;Df5xE}3`ogljH_sEQiQ3pyd`z$gSD%9ErL0Hn zMK5uqrtS>C`%!&{6UJat4X)GEi@QXyh0%*3^`S@4iC})>4@$^Vl9vfm1)K5X1O(Yk z5AkCl=ZD?CSRybWT!cvsMs*JU#7H<{02Zhkv;-nDbV60p6q_(DYM>w7kYl`UCtCt@lvkxXPjZyZ+yV3u zG~AJj<%yUh85F||axzHYy`u__Z5XP@qql)9NIXka>@?(%yh5ZJgj9n#sFRLHV0qJ} zO-VeE5QK%o!{B2gUz&T4l7Emy;3}uL5H1RY4o*7i1#B8awXlC6`y&}mpoSQP{-D_& zk1_v4Ny%vqK6L~Ai?N($E~$!+vIp?zB`KAor(_MeYqUw zaY}R}gbX4!4uq3Q!?_VDWb+Rbev}C_fhdHJW5+H+4bw4~Tm%zcI2RF9ea3&yH=+QLxiKkBHVe zWa;v%ViJeBk@O6P}j&ZhWl&$|rN8>Xz`oKDIB? z@5Olq7o(?lQEZw`guB5^uwem;KdIXK^JDgK^!rJ7frmBNkcfNW_1C$kCaYjsGgz@k zxaJ!#r$;UC{Wz>mp4@6_o})Tb_w?8@Ze_WBp!rB}P(&u{Rhef3nxeu9m1yIzVSMk_ z{MyPHNyZ@;Ry-`@_}qB*{pgV6s^ejXw=Dt`S!?74$=AlUvW1bh@={j@K>D6!Yg!2FFU6o%@^ytLl$mnN^1pzzW9nC3} z%FbrEU0C7xBsC8rgfNRbVbiR8cUUJl72{1Y4^%^v1p+^>P_N<9uiJe7i=e1Yb;|9l zV2=LI;aTPD`rS{SjaDuH-l68ON5P4bO_^@JIc|%jZ=X3?G~DbT8JSXH;r*Rvt67_T z&7mkK&DJY^VJfTJcn+RX#6IMIaaZHfIMyxk&pFVMVOET~{6|Z(pIG*%BJ%Ssj)jf{ z#|6abd-T_5&)t_Q(;ITPx~Ey;>vx!+*(%KSV7-jOaLG3*_VS_sY(FI~D6~D@CbOFL zg&*hZ0qFtX=sl;x_b2U-JhapE#*?&50`(@;x*e^3IaztZCNP)27ZG5&JlO9SWd^&+ zxWH#sc%5~tC_HWAa@&TS83wts!`tVj-HwQPT49uu-fR^xHv3vL#;rUY({0GsvwQPT z9v)48!=HHNZfMQV3dqd<4~xs%($?2~;nc(4qrQHg@6LHxV6vp;H;@vBrJrg3IVr1S ziJN5oUO`63E-R9^X1e#SAFWmB9}DsD+>}!H`}vKHtxeoRE+}(nbn86(lvSmXA>k^+ zJD=fMl=-a4_mm=#<94?upm|)?79|Y#4@z$zzNff)@7If^onq;}3E9v#uKeBF``x?l z8uMm3-ohR#QyW=r8XAPkLnIRnWCb1J=p6W+eW^ut6&NY#ROW@`hi&|LOYh=1S+3Q* z=6w_Ixs_=v>e^%u4Q_2FlRo_Gy(!LlzxBToa`?z0hs0ke*K5Kj`L6Ix%kmmdKC>xd z(mLcC_$*09f!mnFoiIDIqKAt&owAN*8Bbh(4iC8muZhXb;8P}!(1Te2cPMwi>Xc=X z%(+VRqS!#bRbc4q#lP-`yEx7C?UJ%OOa~9$)(ZQ*K0d!+nsGl%U_-?As8O~XK0uQY zWwZajHea1R5#ggf9OTg4TGFOrX_s|q*9_FX?D8obKObb$BX4-fxok0VYGk80N^i-m zP__?}(t@m?_#=!A;H}ql=kT~Id-zP1h2C6Vyk>ydC~{3N zFMKhHG3`Z9_F!|Kh#N5C%UJvI*oQ9La?;wzH|$VAMlyU3)D|H^$NxNkp$zoVeRvU8 z14hC|XDC!rEJ5%NOFvHYO)v_=fn7Xr$An@K0mwcqe>w7mGcsSC4c~{leHtHGS8Q-b z%BtjKne~%WW%6WE-RXS*&2<&S8f!YjpWI)wF)P|95_{!dLvN|fE+Nm}R%&APIMUimJXI-KR^HU6E{C@^6=WD<8v%d$h#`_csR1Pwaj zDzxu2MDVbpDCEJQt#Q2MnT4i&+e4Mbq+FD?I&rmP4s8hlysR_C}FF zEZHvluvtY4PK#hF+>~%!YR3Du64QPT&xi2IRU70l5T*-ZGu)d3pLMU8iaHkrLmk!p*2odqp zInTkKy0ZJ2-ilf#Aw$zw!hc(l>BkQ$O&uFqc#fr=qp^}VWuiX@TL$>Qsplav6Gafj zA2lZ$#hWH$_!2n>S%ynyOVAmrR&GrgDsUq1is{<9C}1#|iaHKmf1qS9CZMtopoO6I zr?*N8nyyAq9tHzgBOSE_u`?D@O4T2F1^miOhzJ`ZGYj5ZFep#r&tY6~n_XQdrAWNz zljV2|l|alB{GnF|q62ixRY+DKh=rL^I0U`ML%`a%s-GH&$%eO9aM!9>EqptfMO^$Mcy z)adK?OdP)W7nn70rSKQQ%@ENl}346Ina9Lo~?Eg}fXvR+7T$jU}LYPDvoG@qY7 zw>l?tOcxh+7}vz1?g~+*l1>s;$xh{;&~^Tb*ObVW`~>a0;qYM3=n%gQlwt&MB%GUQ z4JAe&cpRl-0B*kbzq{Z9+;*L@|(fFF(ves`P3!N0WA zrS>qK{Fh<819_ss7!Kkq9#>y`q&n)bs3B`0KvdpY)$oVjjGM?bt+=9LspkbyrqxBg zJ=7?yA?-{^YHc*QyN2D@8>}#|@Qkd5(CiK0_Svl=*T07qAg;;cN);&s*1^PYU5hPF zQa5Mz`zk>!85lW{_d?S~mVI3wZx_2m)>^h{;yvtN9c>0}ykhOU^<}pa(}Gd92Iqw@ zf8EhJtyXYo;>kTWr4VKL9wka$hOUTDC2NngLSp#X#fClI-6sGdMms$S=`5ScrEeG=+**x?fQgz9mrLR;G3o?L z5`3wcfGRyqdN`5Y>bN=K-LT9+rZD|!{Q4+%3PRQpqY$D65p0TDM?HEkUXfEBN+GMuB0(cbtz zz8hl?QM)6Z+L#<13RrM6Zotu$GW<~{Mgyu}3HQa@J_CT~0l%JbgeV{la~IYig14h8F0Dvj0`#`JTLO5^g38t zR7o}szscX9<>V2#%zPMfwe@h@HcUp0G3>@%oH+7rXeMwPDFv}M7myluj$j=aeuSP$ z;B0y-E%?<_BM(plVJ$`B$kM4ojm*)!Tuy!mp)VwR7o-w1@DGlBJ-J**aH7Fp0zJp3 zT49_63yuoy=*X*WiQqNJTT-zGb`HE2c|H8W#tE|lDo6l@R>9mVR3L)YO}@h1 z-aSNqK%6ruh zhiyEKWU%1Ookn59k;$4D>Km?zS)TE9 zAms+{mV|ZSZmITTJ@)VqXV{%pg4bodoDFv|&BQJ}v}AK%HD5fSXQsqAgX(|lEZkxO)`S{C%%WE`*qZH(FELkB6yTbCHQXJkPj4P?vr9CF4 zgWs=eGst6Yq8v7KTjMOxz%2H)b|*#{wR<0OujwqQ9y-> z_BUPaLM)v9do73Npjq}W#qfCHWj^Lco{Dv6J;F~a*M$Yn8uI4OI&nJn=Pdp#PZt?l z-ri^GYEO@BwzJSI$;(q-suEhav^Ajvrm54;} z+*&<)Y>b?b`bggu=dxaZ)#C7r^4#;~-F<^6GM@E5c#&o7QY zR-ErA|NHOGv&+i;3k&LlBtt*rdb&MWd;dk6H1aLo+@kstSW6h^yL-4Qvo(+Yo#0`+ z=qyjgw^QCy)wDD>HAS2r2wAE3wf$+|3C%;JJu=>%F)gffcUN0i`0Vlebf=mJQLN(; zbwRVpvK(Out(ffJ7;IsoYuX>;TITS>MDzxmeD=u5;fnP7LV3v32i2RKzUyy))YlbV z8{%TZncS?@2*fV+k|ryDj>XVsJT`jU4&2ok3e=8~5wSI!-R=QfK;wX0$k<5&yr!$f zM?jNIl&`(jro3L12RZGyc1^>jZytCoaXEUn!_-DIpWh)FPyix8y!@C zbSlpbHVQe-m<<{c#Oq$yd{`V4kjbtY@p|?Af&6Ehl5=lXW5~U+sUO031DS=KD`Wd2 zp;Y*W0@E5*kGi!cWqi_w%&IuUt~Kl3*=Y69)-su2z$rBv_Hb1|J?sART|AS-JtH3s z7jCNmN1@CuBKZE*zK#uLVv!9rsl%)j%7lI?k?9GxD2Gm#Eh#T-u#U}qt+|hrY@^4Q z=0!?RW2#~@QhR0ickKnaiSMKqmCY004xi)?vyXZ1i!b|BVg(;n`I$smJlSh~%pDkxPf0$hHyGLNkKq zpeu;HAS&P>bw#)-=^{VD%0Uu_GQHCUNfOPr;2Su(HBT-ltNK|J6ITLwNehu10in_r z^ixS+@Tp5#NyorSbeZoIK7NL#A2F3e)$4`s9L?Gxnb@?d&GcIpKA!BgAMq2!G!XQV zpsxpR=LelRUO@=L=&UNN3fS0fm2q$!9o$}c=((4l!W8qZ=Ef$3mqyKrSAt3a=y;*z zRxUB^AFwI!wOKHGQ3#tuOj9ZZy6yFB0$b_4bsam$ta zDiKs6t$;?u2Wf>+CdSnBn*kAG*Y)fV-H0#^b6ulDi^>ETQ_?&~8f%`Qv$OW#$0piId5n*leZU)D01V_ft6tw+vPQOEKo6 zN|7`UmbIw*`Yg%P2}UOF1WF4@U53L`?Q7a1Rlb!O+ZmR1^kv6JRLi6q-)75<4fX?~ z3{!-`hKHf!fYW=vHV2h@|N0_5%wjVZmlGr^^N$+)$5j;?{Ha+?lk zqWPX#sCi85m0koT%vn7>jUgn`Q4c@{h~uR#h+AN1W~3X4t9-GsU(pJ!z0v&h^Z=sn z|Lp!?l^7ZYBI^in=5<0aLuMR+2x>gu1{;$|=MnsTrzR4X7Hj4mB<2#ZaBu_Y?`Iu= zSm)YLVENBVNv}PP8PWcxCNd3N#B*_=Y%SwLaNk&2SIuTO4#ZTd2Hb0;lZ7tN>raY& zeWJOpDn5)=l9!=A1ZC8)r3dN>B|fD{;oprpEwwWk-|~+7w%&EpZ)`8vir_eJ6y}4@ zp52i$3`synrn{vQ?$fIT{-V&~TS zmyakX;=PFfKQa}_joF*e_RG?@U7eF#&;?(a%Iw@%#Gntr6sg!R4?~~;^9Df{o*OQ? zIb|hvs4ky%uYpiil2xkW-W(pE(q`SxQvN z7*Zn;jQaW4kc9|B64`?wN5sQe(r?L8Em{P)r zLyHp5Ar$D5DQw0?2MZIJhUDbo%Fn1E>h`)Q+-lY8>RFVdz~kwqNtW&cu3v60ZAs&- z;#Ki0%qaFu=ne?;Ljn$PQ)bg|!i2#iTn{Q5O-U*83Q~QLSnh&5h+fB|kx!~jifGy*A#{8g10vn9@icSKq zr~m`YjGaXWR0IkGg#d;DL|{M3dj?UQ?!+FgfFSHGXo?R)G%!fW$_; z;u&>>L4rbowD_rXg|*Fzc-zD)NY9;q7}?Ss;@46A$|j7SFkS*szjSugCp@0w%_xio zQILI6i$#khQ5cQD6bEP)>jH~?>e%Vi+Ca+!%Ta@~Ow=oY+|=@cvOQw3y5x5-EP}T| z4VG*|4q$EWHL^-nQuQH-Qi$#Z3%>|A53V$MarWp44;lzuws2oZ$YZ9=Bv})xyV=L* z$Dnp~Ae>YbZgvs)4aD%Qoz-%Qjz@sT>3uL|zZq{uhy`2nUrO{$nM5$>8g>_CRm9-p zoW`UOxm5TD82S!icszXQtI?aBRB6@BYz#<%aI91CELfK{!~%l;&3b8-0{caJgMhRU z^HBNJ(Ww+dMg&c929&>|9K_DHG}aeljRpQemO5ST%ozmA$5GLZjWSULzGKLW;VQ7a ziJ>W-(=Y^GG4&IF@(ko20KfV_ulm<3gLom|{_71vih!ju3MGCmtWbPlyfH?ma7J-r zDNm)qO~`Dqx(j%CX1t0QptHCTge;iKyTlL|?%qHD&|f8UB9JJ7%=_yb|MNq;0?L}W zDpmpSg4}C#+|SV^=eo+|`G}@`UR#7gg~`dd=0ur>Q@sdmM{%Uf-Rf9U;w z)t{wQee4J;GIXE%N6U;qeXiYO{)gU?joK&a`-aq^nFg=7By6{Am(&&o?g>}9v%bjQ z)NYvGs91D}-#uEg|AohhZ}LP!=@GS4WAH+4z7)@?yLFkmCa$}Ei%{k8Q!sCx`b)R$ z(*~#5K{O7VH6jN9KE>~-Z`BxzGq}b;Rk-alH5qF#G^%J*Poyet7kyfG4!3W4^Vh{% zw_(qoU9NoT6QRs3X=3lI#>PbzLgB2Vw)yA0L)-FQ|IoXc+V!i)NYZNk(nT8YzUSx( zWJQgi2+kbL3Y1P3h~62#VjOiNpua4qv?RCM*Emk3{_M)Z$ik#W8=lIo=7}y{C_*$j zcf`I_c|JR;YE

@DxZSjg4hXQkZqeLUmyFg z|9-x=AosavD=XLC`DoU))&$p~fDHaa)A#NeolB@n{8bil&RwDoOnfg#UzwU)id|$; zze!1c+~$#IEFae0!ko+tIXrwU^ZKFVVF@j6eH|M?^Y>%9y7q7Px7*|A@twcgDKV{~ zIa_%xSedoR-n-IB#ktzmp}U;5ESS}kKhyEJXUog>u7UYx{1@X7GT8Gc4tex7j!7n? zIh;Cy)td?TC9j6}?C09}Kal5#7slp=nCuX%wq}?2D6b&x!#+%ZiHS!0ZXtiJp9x>~ z`7qLq<=h_>!_Uk27WT?^<>baDY%^Hre9Zo+m1U3>e{NQ)ALppg-=3|))jKHGgyn_y zMdZEo7zj6AV#Gx+HtSYhl*%O?x_*2PeN7@}Yk7 z@t>nv(8cd=YRC!zO@gLJA9-D1KGiSaiwlLd$DLZgh`4#1jWeYEw74^B+ zg+-ecUx;3`)%>2yw!6fcrCD&{Q@hE-&S7%T33u;bB6W4%wblOW1EsxpCe3~}x& zpt{<>&A2S@g2<(@PxgEc$IC%K+PyvhUnRBQsvYkFixVfTybU1|T1{t8o*l(Eg9-3>Z}<9i7S$4YlU zkm>hth{YN+s`DP=NYv1cC8xCI!n;z}3i)w|JWM(v8+<%idsvv$aN(N=rk4#{E|ymW zno0+fFcj0VxZ%5qgonbB5g(rCS-1?SD+g#+LV@iV?Bpg=o_l#VJZ^Vt5fuG*<)-A8 zrp8NA?P>+*`mx2sks2#iXR@|Xws-vU>%*cm%FpJ4*VJ@qKm7^|Sk3g`{PHj|XtWonZnGJaF6`tgfwPMuQla?~16#B`(* z*l6FAdyRDOBM_IpA9=X|+(>^Klho`#Bw=UZDI35hv;grz&`EG*)*Dv)qj_JbDOXhG zRq>1#f_`YWV0X$gw1R=~y>7gd8Lb#V0n(N{I%o@jMgqC3Yxz%Yo!N~;<()9fw~y2c z!c-=(rUqX?X7Mra?p!k@3ftlE6zLOnB{tgAe)Y!@)5qG}?;0cfLf$ZW!7 zq}ML}5P?ooR+9}x4^}9E$sff8MM+nobX@>P2auI)j^_<#EiB+H1SP3U4pY$+Q=-n! zp~;A*-HsTYvk3W0KvR3D7M z1{a)FNM|uSauY~4UaVrHU+#*!%nMDfMtD4B^p%hFPt3(Pg6lK-+Ilac9kyKQg`=Al zfG9p5W05CSq$kpy*k@{L({?~}D3b>b@;(elj&-iGV{8fq9@fX!4L-;=;T5d0c&I#pjW8q=yw|EPo3#pGY~LX-oDkTr%vP` zsXzT;>_w{I-PEXS3UjtBt5B6ItR}`&d(8SUx>tm%l{&(qz9wOIUj7xGXep1w!ve)n z;t(6GF$6l=eeikM;5=n}pZGMckxZlz$u};-wxV7)RI9GziS>sQfgyrnbpWFZH>ggB z68dn?wRh^t5K00R8^V#HO*#PrAm=xr!#MGVGa}o@QB31$JCQnLG^gKZbB^Y%yVA** zQ>q9Y1Olda&5Eif=nt+p^(QXP`3${PYU+shfi{Ozf*f_Hsg^922oU9Pct3?y)lg7l zWi4BZ|H$Safr_1o(m@YFC{4;yl=;W!O362?rOoVWE*oX)>L>cPVkc}Yo$aW;uE`4; zVh5}}hY0199h>~dH|D!HtJVzpfBp4VBcW0Y{WEPjlO#7NOUVFw3E>$~l@U$?o>qps zCZ8dglnhPAG}e&C0rZY>bB<=oH{|XtgO_5&ea)uYLGys}&b}%vkzR=sFeCskXmuNq z>}8l{PBR*l5bh&J-tH#6J;SOaqUCTD4)g7`wO5AH(c9JH4-$61oTZ~NoXgsX&zmPK zz)G~m%GBRh+HHtd+0ShUZg+r`h-=$DX@9Lh2`;W-q8#UnmM$oEKwONDsD^4!++|We zP&AtU;ECBB;23RDQT@*cLn8fxD@gQcTZgMEcHn4XS-EEWLEPs@EPif;jic8VoK(@n z`q}4nakWLK&CooZ=@I49y=rAj&&rhUTq=nTv*k=i>1G$sUvayZn+nAO#>oqV!9M(? zZrH=c4yvGR!WUi;L*9Q>F*I4ssWVO-1&kKH|8 zN&CT=P&eK0gMkARvgVzl%zI3pS6^k?{jomYGN`g4~q`w z_aKz0A(REKCd3HL^aw5YnXfFXAO-o9+aPLb_!44v8v-M!{y#GULvc%qc?SA>D6RoV zNqAKP}GjMP{p{^BO&Bve=gq;WWC?1Zom314CNOl$V2vP}actje;B&u^|C8 zfpDFm2r9k)v(9JAdBUd<%7Gy6f_^$5K1V&LiHSv+A|U`mnX`;|QlkIIIE3*J6cN$Q zN7Xj@cQx|%$fN-A)(8A=0K%^3cUH=+y~y=iiigiOECd;K03>7_Hp!t{0)9vY1{%qr zFg-H~m|3tJn8B+K0&Ir*A(yVh8Qp3^K9JddYEns#lRzQN60}e*U^4m10zD$tpcifx zw%<#PE{Sk}#H)b3DcaqhB$W>0`^g9b-!bCAOx?TI0R7hSsRBfT*_^JbzIbF%QNi0m z5RWuaK!A-H3~jITauVKyi-^5#e4>V+PBJi0^bBN{WvxD;Aa<@4;1>Uk@^j?(fd0Y+2mKWA{dt8vm2>ea=tPE73{3XO;7;lV zua_^G)Chc!6-thQX;>4piZlk^5eYxMmLfru^bp;P;pcqm>9f#rVE#)Xn$UH_djOUW zSfx?K?=TSy8jJ!N`f=kooW?3bZpd6!C`mJgfBP@ z|IyPyE*6d><-4!dy)MX~Aduf!cU>F3<~!3?u9VIg@0$9tJmHvzGy$ zw0i<3gv6=f&O~n?at=ufVI%NFvuCjN5K;3p*18l=(>E@`BS(OwMsiSOy#U2i(EtvD z45JXfj7%_$g0D!!@nLG9+xBnQ$V-x?Mq)J=@O#t@KF9Z`Tqfx6K(Yi z8O{T(1oBm~(5@cWXTOY~AC*}Cl2L`nQvw6>O4Db7ya(MN$R=UvJxcsPfV*+`Pu^ z;^%)|N@yrp$KJ8gu0cxZ{Nu#Rfu3r5v}D?bIkVIJ#Q*#6WfepYV(kgpA08i@2doRb z{?J>9ieWO6h*C>#DSbLHREfu|i6p1QCMB&LHQ_VP{qB9jTfYlOmu})6acFvRlRLBB z@W!#XA=mya`*M|3o?<>zcH3^Bs(JfALR41wOS}&WhoVdm1v)ew2@v>qU^E*0KwDM5 zMHuf;e*>{Z#-&B|aSycn z1~O(q8MupmyIT`DG-%099oC8(vvf@=+0cWaV=A+Gp@UJgliT0mh8^DT>(aV%Py3QB ztreMLb3(QTW(?ld1ai2kK{3sj>jo?Ymmj>aFl$_Cps?RI=CSHZXi&2D@0?lT?K{OS zpQrR5TW-;n9=vMPWc0;7oTf$I!+j28tFrX+SUm|GdG3`^cI_kXyL|(0`*?z9&ZpCx z%X84##66P?|_FK`x zQfJBemSZ+G4l|l}?8t3sUH`jQlvQ!W%fQav&b7U1a%X7%#NCB6ih^ZkKZgsAN`eEU z!VGc}_HE-OFNw?!e$Z+l9&SZ2C)@_N`JPoox|PkHI6V+;B;;~Kf_3NOI|A)&V9tRF zSiZFLXqi!-vD@44@3qD7?!(8r(XnqiX+@j96)(Q?<;mmovL2aOVx zt1Uo_-|gGW-17OE5lyjO4L^wk0*?RE?;;zqiHd8K@AQx3jJ%y|xJUkIXos$WeYZpB zS-e}B&~s?lgCoT`Grfx8XWwLwM~rMJrDphYrp3<2fPoC-?KM^B+U?h?yM=ikzjJn8 zgLcFHp7g9i)cs#o%jcO}#M|mDtj(e;#of|h7G^tW{#Ac$-N>h{x+)DBZZLf2=`X8K z%zADxF=3>r+#Vx&Xd-Afm$RR4%9`<@(D7i=60Z~Hn@*UE{mT!kZt30^*8~Upj@UN6 z9Md>$-y=!cd2(`Ta7t~2v1wvlJL?}yd|q^|Vfw? z1`p{zt2alPE!H)=$7DJ*-FhD45%@ePDp?;#=@TgOOB?C%Zn(;j06;8f+oEcwe)m zuHuw6uwwH}xmR$O!6|3{@Q)t*+FUtS?Tgq0{G0~M4=yL|J83o%_mCf;SyzDvV*tWV z`W;({Q$uN6V@#dy65MMe0+wI#BO#AC)TTE6buZ!*hwH6=3_WyFVbwJu?WMO|Qg*`B zmhoxdf(S3w7CZ{ly16p3YK{IqY|{~$#8v99+UfL1E-G}nILVT|x77I}tymX}oyrol zj!uX<8u1|5TV&2bgV1jae6_tKKa2FL5-}>i=!uo?@>1LRrbtKV*TtA^a2{?$@&q3E zR)*B?(8-I5TAUx741h`Uj;o28fI5t}mF9mvHFx0%WVb#QQ?QBD6w` zovKd+NeI#3UqQT{rBWzvL7?H|o|$Zza^}>r+rxZRDWZ1Pz=|-H+(bQt4;_3YY=RDF}YPC?cY02csd79k@uctxZ22^jR_3AJnFhSjtAC{9m*ByZzp z7$4;gfS(}70Q@xP6D`7K3$6nk7IuaF2u7sW$wzBD+IPlWeh?-Cu!k5Uojc5XGe&Z> ziW!Jb|@g%uE4^qbou#K}#o$J9uYcl?WL_dGETa8Dln>*l|9tnWJ(lSHo?^ zw_(fk4*jk7B31F2NmzoHN?FQw3q1uJ_Gmru? zN8n!XHmg6up=TRbp8SB+?uWOFO2j7fDtw#j;}Yy_e6I0?1DrNL4QH~non>{mPP;I@d8k`6BgY0&p7k?o=IEa920MB z==qX)ZYbYrQ}hqL@A^uqdk67b5QD+QDxWfm#BdBf3c>`$i}6+nx?rLUscSy%^cpR_ z3BWeAc^co7>Yl)iDR@H^449NA9e&VW)1GeBzMet}!btehy!t6DL4JG(H(|fgNh{xhky;)*^EuR6KLoOk&yvpGKz8S8~8OiH;#0I zs$^&t{5V>{g}br)hDK|PPdLeB1zeh)ZvE!_*z04+bw{6?y@A ze939DIAK9q%qxb`1|dy|@`OC~>Y|pW8r+~(hncQ*{Fgitf*1&aT0^c2dgeH@Q~&~% z|8$?9Jnk7Z3#%W-spFs?32z%o7R3b5*X0{w{1z*Afl!m}CM6z|C#lFhCC{~Kgp1WnrwOoN$qfMG z4(gmhhjwsg=t5=)D_KGs#x9O>AU8wXNqT@M|aExyhRsYO-lE zc2kFo?(LZa<=#vByXGHTb;oYQW{rU=tAe9($t}E>g*4PfYck@ymBB9~vZS@c#qa8} zGJLj48#9utt@lYxPUb%tS6)+XMLPQJF4O?#hpAD4WvFS=TOVB!+-mhQc#Bu`SfXn& zyE&}w3My-MS7k(cTVv);`sC6&-*$Ho1Z(ZvP*bw|x|d~gaC@N1L-#krYcnTA4^ECY zwjFF}8|~F4`*G|RweJpl@o|>LN`8S4u9BjhU-Pfpjc2c}wX(1cE6fUBcu$*nV?(51V&b#x zgf;HVR|u?&laeK?Y@D0!c*o@VZmzcyaZe6!?-VoznFsHL5%Hc>e-#V#!Fu+=@vPy- zb6=MkesFcAb)e~ERYQvW*vE(1FH8z@AF42+COnm?|N4gIpQQ{$ZXC@uVmgIlc4dE=vTpLnO=&3_BodHp_JZDxs0mooa2j~1EBBV{F( z6-oSFuOQK;5yRBF+`@qq-CkQD9NwAlAIa`DQa&jhJdnf@pRpWr?uKVqt7oEtY$(urO?30n#4sR4kUqALwq>P=uXKX!S?D?KP z<{y1Dy89*NKf_<3Xv7}reK>m(x3-a_9KLVn6J6|uW1MSorGjsNJ=AY_dR=g|Py3>T zgb{3;%F4c!?SqNg`x4>lmHRJ(2y8z#J^7a1^5V3<^G{>H%*`#{9lFB((OK-ztCS7B z;cLH|;5fh4+F*C=-<`$(Y_C<-cdYWx56p`;TeCC2b@AL(t6AOaTxF%{Ds&hGZI4Ue ztl;*qIp=<1yIZq7Dfxy$)b?QKoYyz_aijm6zSBQ2Fg2Ief3q;>jtx7YIA=_nZ8kgT z`OMc=_jdv4dZ!E!5xcIrbpg zcs!p|F0(PV5%wlK=doREliq4S-{$ck{QQvl6STc}BlUOQDJjmYKa=y~iIJ>5I>x~I zmzIbjt+Bk}>u#NJ8Q)-{OPPDuYR9R-FcE_Ol+<+eq8^D-r54z};;n@f4zv%^k zVtcOpWp>B2hULDQ+{^D3X@b8g?;e9v;AZX!t5H#zU}k3WQu58FX7gqDUl@L?wHy|_ zxsI0^R=C(XCL_`(QsfS0?ve4R3T0R6VcjJ+Q&V)5*-wW=Ww0`-- zxfbN1H&~3UaK2raEXUKw>D<*IuMns4q3Ujn}LELsnYQWH>F zYn;w$%X-iQZLk*ZGs#Pb0eGV4pj2zDv*V_|rbc2O z@)0ASxC|@dbAh!vZ}y{e?K=nf2%^`f>n!X}0jN_IeLW9?LK4eSzx2wIU=N7LYc_oN3Y=o-(084W&V%2fiC+pc zHqti+7t55q?qvgIpsP)U;?6fB}Ft0Gk#fkZof*)UVIcp{vQ+VmFKc zoh2?zM7eO2O7gNI%;L<7&%tafdvKkp{Wz}q&}1}*PqYFMRNM58FKUH|@{n(YK_NS* zQH$AcTDDNMvHCH|QosmEGoWQC(DxN(=8&T8r8fKc%HX8{HycRcdbfTKhByjLM3Kmf zD}|bxmbA3-iV0+XD(`#vjv>&~ouAIHyu|9U5sr<)4{)Xq#)9KI5C2k_egaR{!F(fH zm{S+k^T}Lm4^y~hz@^i3G)D=&10=IfedWY8!W(dx;eK+>9MJbmL_?9BqpR_=Qt`%+ zaeB;on~5bC>0)tySb3x|<%>Fg*DJI^=*<*K6g>FSf?E$QEhsav2Y_7wsIPL#DS>ne zyBQIrQr+x~3ghq}c1Nx??189&H1!bXg^>yvLgT-17K@SjD({L3UK@35t#)naZ9}&x)smj6~d0PA!V?K zsP=!cY<;A9gnj2iA#U7-lV(^}8T9v;tW&VoP+H&|OBl4Jhx6M$} zYu6iYWc>`5QuaaY-iA%7zdaQGGv*hJfxh$1T8e&*#u~{k_8?hC0j0a7I`h-zy`E=D z`JM9o<@1f5P{82-!D@93W^(@B__LU4tzvl~vGF_9xj-FQj->BBb3MApKv^48iKD>w zAew;iEPqA?+Sq&k6y9enSiGNk4V1~`?9+)_Xb@mjxqw$Nz}FRF8k&wZ6Icrniw<`? zWuGuS4e@gZwy&p&@InJ{Wv)9)d3Lt~?J>u6HA`E91{t~8TiWwz_oAI zIH4o+E{b&ujc{WVoC=x~iHi`z>r(W@zEdBKS@Fnggw5%w*Gk~lDYAegAEeB{Jzj3z|$nGkXh#cLW} z5eTgZPms}myoepn7Xx9{hSXAxj8Dgl2>@>-$`}H4u!LX&fGH?CzE9R8!2H~CFQhh3 z$=Eq?bn)t$W-I7Jh>`iA3I6{v_T~Xm)@%DX=Bu1Cq*BmanA55W$gxBem2o-fNmJ@A z_ANm}0VPFbbC`C~zw7=y*g5C> zzTe;P_lLQ}Jo7xC?YXb}y6)>Xo>1qp5#{%E%~Sn3kS5qYs9s-&Xmpcyy#dl|DR?N* z=LY2N08Ab3sAQGuxB*t_0rW^kNO%<%a%Xdmy)#-9ey|nU!#=f0$HQ2VhXEd}+y3yh zPf(n++%CthPWgbjE3*ERAhg>P+3?sRqmSc+-oTbt{Ku~**U0s)5S8HFNpIMzpV@s6 zCH-U}?(S;wbn-?#-^>;?`eDFbQuup0TOg#GNA&|W+L@xnX^u7+YD`G-5W}&@sAxhz z0Dwx?MSh%mmr<A@m+o^^O_D==(;? zXY}Vtx1a9*qeOa$N|AIo=L?_33r||CHm|jNvq_hAN2q?{<=w%DUEjA=vbxow zx?jfxujx15`R`V@XWn_QTG%BYzM11ap`J|s-K@;&^d(+bu%=q%%%E z!h1>b`zPW)4|URvwu>_jBOdCEErg1bcWQ$`SAhSD-42f-DuUcrTz*?kK_-?0OPl_K z_+R5IcR`3>8CbBVczm+RF%rHRDZ1BPn&5&C6unt_j_;2|)*lA%URLcCcj!|i_XVp> zi$+#77aG0i(ywVW-KzVmXUuOaPx~q&#vAcgtTnGm36!dTYtVkavGWs`-HC@Ic8WTZ z=LD!%@$!F)dSh>?(}d8ZSDj=x?cC5k)4S>iU3|0;I2t^B`uvHaSB#IR+!-CrclUNv ze}hPsZ+ox8WNd8iSV6bf^R-58xj}=H2Y6|>H>8N3pqT?Nt6{uUu;>qiJ+oRWjI(6# zhedDIn%z(jyiyq~aR0k3+2y;|GskO3bCiMmLdClu=v&o8s)!EN#fqU!U4BD3tkBw= z1~&gvwks_{nOb?Hsbfw5TOr%GTL^O1qozptjyo3p&Av6ousJ(rFe`X}bPMu@irAWvPpwW@zE)S^y=niJ;Z!iQ zYj<92sEXScslE~!?t#Lpl(C^F6M5P_p$*qO-9?$giekGPS7HkP(Nw!KcH#P+kps35 zhL7CXm$Wa?(!t%WbIkk2wx<5zMv7avn$_PjQ`a^d!&0oQ>~PA?b9Nye&bHtFwRZ2k zbKd*%U%pnb>ss?8Mf7^VK3mD;_IFYRm)dpjB);32TV0(~GJS`nLD>}M*LD|Pmv5UQ06dR&CvzMRDez{WG;pq~vDD1nola1NM#W@A)jBp9e)L#0mYPv)CfCpi<#bzz~ zsDKmM4HhA%XK4?O|NViaR@fy|hKLir0`e|6H|AVyS4^xoKR>Wkb*t%MBS;P?GLtve z{wg@vuCvV-yg9nL20r_H1nLWIj}aXC$DZMTy4U4myo2$cBWv&3`t_J~~#BFNYfQLeM4YH^PFqjL12mFM_sfdlsFju$Acq|TQX7vvXbzic*UQ<^@@ zYa-!&t$Dzf*-2)+Nsm*I(-yw1v3wJz0Bt$RF*Q7NoDUVhuyG=RVRK~K#J zy=cEjad>`o^7AgW=qi$l@CrQ)8-O8|=W@L26tC%KqBEmilC)8AI5J3Ol`a1<D6)!yBiOdnrD6)h%B6{w{&v*W*TT?7IQq4c25SS z2ov35V$rbv#wn|GewVuNwENbKG&|Jq_@@O+o=4d_m&*4;gu^?$nD4j}2NFyeKrAvo zuS*@fwqi(g1R-`ZI*4a9SPxD7)H};W(JyNrDyerKgwuc`O;VknBxx)`nAuNqWqlJk zyoT&$&>@q{8qDEeybVu*SuH5lQOhPqczy864X}mRUckbO7e%ODg1hR)AtU&YsRS1~ zqp$e97kS$jeG>Gsf5tLXWsPQ^ftP1QUMh^5anHij1`ID=B@ra)u0e|2N3snt zHiG?u6P~id0g7j&lG2J0<@3N-*93@hbK*q}Cs0rYr#%sf&H?w1&rn1^_}F!HkCVAw z@e8vM8(7J6;K>bT@mN=%>o_?iR*^t5kmnDKbJ~S;cF&YC`jVzBgkkV;-K+5I3H1YS zgd+Mkbn)n5zt^=PDiztJ5KuJ0;H~k9u^R`WebDSlcQgIXugx(6P6tv>#eexsFoR-b zZ;l=g_E4k=>q^Y|5Mr#MP;kfL%jS4pUJpqcgVj<)h!5v;@MhsT>BUp#=)aNZB-k5v zysEt7qslmxCGFM<-uW)w59~ghkmOb@jYI&@aJ=wV!ooj;Iu4*mxQNcZ+YBBZJa&mDy-76u~T z>&A15R4o<$AeNSYw5aj)-73WYhv&~g*EFi@L$~%d){y?XT;2#ruc>}{sGk&1N(9HV zl#mX=Q;`r3V-PT?_LQ=im(JTZ`!iDF>7K*0=rV!Ai31_+rCz)baQG^+Aa_|X#Kp-7 z$Yw;|udWT$H`SK2j6&cPm%PSjNcT>vZ-0G8va8^XTxXl1wBmQ}(|^gYL0$byOFk41 zTlEXt~Y?-IS{SitBCqa*7valLS9h?g<6{bV!VQXpF zsFK{ekbxSIi>PZ)D}7fR+9a$4TQMfq@tt>FYpM>uU@BB;J!ixpLIyD;N3U ze_)!NKR^YqG*zk#cV^ytVPSa5pClOkepHp|C$is=M8s<%q|u3A+QL@^eObtF5%yH& zTJ+k5zL@Vwvxpk+=VzI~Vjn@sSc3pL8Ne3ZVfcrwlXPn!8E~?+Yu)q^&_Rg!HKgxj z3+QAW`=y+b!!Ofl?!d|rC?@Plj(HeQpecsCEv?>4y~BZ0;9js^1WC<*3WTpetE>$T ze~;KCY!iYk2hTBtfI|~^Wx(ttNEK@K9bJ$l;DeBP5(H&@BuRc9!6ekh8|+0PH-no1 zq=-I{^dMm39=RaHb<(xD1}OWjDDcgWsRW3~%@46SCw51nGNF5N0K-OhkQ|`f!lK_; z&HkM(ioDS*UO+KUgk}`kB;*c_FGo&Y5`Hq&?2%w}p5(*vCCbIG55z1`PFb-0N+vu=(cI z&t^jkp8$hRJS#Y36|fXNJH1d2yJGTqnpb16ED0TI7mvjQz15v>!N-g#A&4gxO&Zd7 zaZ`^$#VyeQPN^CS?odAnYUqG<1FsQKt<2>OsZmmE(CxF2A}@{yAq&AGW;RC)N%Ukw zYeJQ6EU|+&6HGc!vU=ntiII+<2KN;-YNWOfSgreDZ0AhfFaMLrovLHn^zR6ka4iG4 z{~PGVzk!oxDL4FgY9j#gW=xscV|VvuUFXYNVXflYl8!nt9QF4~Wi=ZR5M)O&vN ztxxt_zmqm=C|i)M_cX7z3~{g6vC{f`vki*_K3f#fen4lQSDTg|s7zH1Yd^;#JpQF@ z;oHu(LC4yx7C4;zEmJyZ{mR!nK7ggva|+4y*(Ft?l^>@qL+GmpiKgcYf+M-~{}?Q^>Oi#mlzO_Z+6 zuUr}ZKJf3CR$TYnx53Tb!LQH7?Ah#++wvS^)sQMOpvoLYq=oC#g818Dkrl4q@!wt4 z_&Z#Ep=bS)$ah+bEc3K~6P!t%C7Zyy^$RoSH+^w`q;UUI*JgKA0=hicYd2?ZbI_)$ z^ZODygF{3SzpZorMA-3W{;JF2^I?6G*7>MTc|;tTHPBoT^kSb&LFI3iEB2%{-hSA* zAIC#&vT%E*me&kvux5wp(f)6n%yKexUsH&_&+O5XmwbhHSLayX`8%Nq@6)xwON8yzuynd^$mZYM8lNC}c;D;22lZrE}& zFm-vbS^)>Y+uIj<&v9=uesUx$$?zYUSufvSc_w8A;%IE00o%T8@e8Tl*XHF4j6Pkp z+YNoV)HzjW_k>?}P*=1SALzeTdEDH{G-TfSBhE*fZuj+hXC`q8Jw93AZsh>SUIdcRxXdv9KZCX{Spq+6Ed7isO=0!|JG#Rdk2@NyTMwmpofIiFFzmH}#aSScm15%Z0W95y9)3}E2jLJGFH+yol{hgG zvO0Fbc3+jfzBhJ3PLIaXC7a)yH?JMV%1Ao__|5i*VU%Ap%0O!^2c{*Qs86?&;r|vAJFpfd|I5hxT&!?f5AY-3X1vuX z15Rb~A00)gaxz`zgz{!MUF(mgAk}ft8~PAEe}sFy}Xi1S8jEAOF$s6D_ zYZ36vb6TI+9hxfINR1do+hn)1QenpqyGO^vKXy=Hnykx%8$4`W;jSf)o-|Jppscu| zLC#YF*8!SP7@dIH?ZW~bywE16Xbe!KFX@AJLeO6v&#r19)_RS$WiD5sy61B@ijtQV zAD-qNyVfmT-Hr*-z}3zoa4=QhLWvVl&U-^I+?aeJA*sPHBsFUGO z&>n#92O2>}`o=|%POu1522r-3#)!LaYj;TMnM1(PJU2@#fajrQ#qa!Bji1STK$v{!eU0??mrKNFVm=1 zhj&o&gQLeFf`vvF9>$RIT7DA}azw<>;=?9a)TLww4jz|80 z{vgQHuOFR6Itr}7va3ajiYq0+8C)VbN|4}-H6@1x;Mkeg|U2)U_O z9P*rw8NLN(S#Kw*B}Q6WUwp<~^jNG3+D5iY0D4W4=2gi?d#dBOupDj~W}pzv`zcPe=ivxt<5&G*E&C8-2i zm^CogRBH`j4z`*lY`8*JGyx`6G!}dv1<_S8v)`5?OsD7>A_3v3h68fcShPNZqwHod z#5aXYXCM;^NY9bN7Kiog93s_77G^TyZ=*E9Ji__@=sAq`ZG5c(~ z=+34PBL`Or3NlYd!g2(HK~Ll}k{JsAxsH+W$;jrYK7y$q5Ce)nWmBWX@(R@Ojuie+ z8!A*mj{safU9rliT`wVZ0o?Ee8=Wv1y^A_H3RUO{CfnNHXkqX?Ze zZh091wTKV9E6|-xg;Rwx0B)txfXpSwG^7dXfna|o`qTc5ocYl$fC;K86=A?}&IEhm zD0iRWE&>o}mzkU!-;DcRS_JkJU|K{;3c3Swk7FqoCN~Gr&#Cbv@3on*Q=s~2>M}>5 z9CGeQBmGEk-o=$KPbZ5xX2QmF=5xIOk}0)9(Ai)fL;_@1M(7+18GJFfSn2!Vw}wQ{ zGq^Bv5Ei}M7+{d_hvkZ1n4Jsq|j;ffZg>$-=9k9*QBO7?85!Y2M7D z_Xd2GtN&Z--LbvGykfPTT%NOcpKaBt&h2Z4viAomUvL*z)L%*S{dr!r)=*iW(Xrjq zxb)-3-dvtv9uAIe6CPW7YJ5H{5OoQ7$_t*zxXMwvbHlaOSu&9eTqu7z-Jo95>5wG6 z9uwfJ$UoTYHF4~kE~{fmUHLUerRag#tlF)yhsBxJ<3rl|fz}U#j?NKuWbPmQYr)LU zg7o9NT|ItBN39DHJp1FCo_belvrsB;NlDIsJl1_YJwR==*U2t6oGW!7n}cESlyl0P)$c=VOf(4_fR z{O(g}r&BG`q$Ub6OkXa}71`ac8A_Y9Rxhk*SADU@`dbv84=Eio7AVt?#a$g1y_OSx ztj??X=-=~ef&~2P0r8w__2YKsTbF7&+%vcGs&3C(n4uPD$c;O+Md%W9rXofWII2FY z-kG0zPFntG-&+ey<`w1Gywl`yDcIG{0&K=sX^v}B$*XuJo;ng_?C%$Qe235Ap*1C1 ztCnDA#pEWd1A3|7uVy*k+g@|dek2_?iIQz!up_r+#YSbt`J0QE*y}5``+w_d$g`_a z2MBYTFZ`-~e-&U((bWfDZtlT5)-KA7KKJNIv9CIQiy#2W-5zzsqXR>BW%}sY3j;aZ zuN+?T%t*mvx2m(f9j52re4PDk*XdP}Eg#0l27eat)8nBN_Sa85qdR#x-ELxYr}5mD zfIQ8cc@fL^{|vR_?VNO%d?X0&UwV?hg!jqf(}(+fFRlo_`)fzZTYGIT_zxDz?J9%X z3e06MwoM!wFb!?rhjl70RJBxwdF}f|-CxzVPg$tDUUZ@}C{~f8EGdh;+M>Jq=vJBg zmELZTiMV|)>&^5{^+$IcOyenEMi;per{cADqfAUMi(m3`|6x$r-CWdlxAa~^Zq~p8 z`=O@MDx;8D0mFZBoy_a(i~vI3yrOhqx8k(b3xe3K52TGL0hMF>{XB-3e3fG~yXyX< z<~BJ>ne*z4j5}j1k|VXPhnl+-t(FHp^dqm5vd`}6dW&JW4AD%#P)I3w}OP;JhLca z?TJK(u6W~8@7YJ+wB%>5h=kAn@3V7m-@R=%Po4H+r#>xjT?~NSoQvyswkyT`=Fk)m z52w8dcdely^LveB`vpc=U6-sLmxwxoJA_7pis%;>mF4j6U^9M%I z5B*B3;L8g=_bztak~nVN=w!L>UEhQYWTb5wz}NAWAw8mg>8_3;Ghv|dExFaO?Wpox z_CEfu8bq{_9;l+X^i}3&FPiYM(eCQj+$?TK$fwa_l#)uqB!21WPZikr=)+P~!(J zPySK$w{k(ZbE@&ei38+?oxD(agumxtG#jh^z+yt_PVYf#IEL5deI~#;-z=+7M@=pI<*^!B@j(CV zXudMnMm*dN-KL$iEY3MQ8&~;98C;;zjwYIAJ*tA^kDo;&I@TljQ%LsJrvo6FZ+xuJ z5QHbf<2+j2f1P70Wc838X*fVM^4V&1I?KT9xilO{A^;ZhPN-*@z`cD9i))~c063B9 zVo)f$E}2giy`nSbm#KEwK0nPJArHKy-kLHhQt0Xz-TI-=WBax-e7Q9LM7@h5a8S8IVmy@$IoR<7;dodMss>0)`;nO5PHVz7{NsxufYdUSx@K_l;>6*1|(u) z{-ayEr)V@vq%1`)$x_ndUu=*I^%}bxWQAZ|XGw06N*q&q1DP`+x~wrLGXpn-?<^kX zLh-`cYtJn!M%<6$3)Lj6)x_PAq+QQ1AdhH9eU79EFAJr|SjXs)>S@R_BO=1GbBOb) z9Rb@xaWsK+PFn~fbi<4Wy9)gYqKt)9-j__q(=`<2;z1|0$#V%Ld!*pDO)_G63SKUF zqcXn+L4#er{)dfKdpWr`kO5C2(S%`KNvaMO-_yY4iYpqDG=w^$>|kTn5-QVx0@F^? zRi34nDm4wq7_7M`z=?rQ0(rF9EBs+;e+9fWLBTd+1y*+jTk|-ruYCDUQR6IMN4@RFb76$-`3+;k2Wpji5LWull zfOX0)3=uYBX%v@E>9Z+n1g<1b<{4-Ync9_w#nf*tc}max)RyYczml3BvH(^mO{5N zWmn@wnD7-7fSZK$Bg7lY4Jmcj0vchaQr%1-0^G;wYC~juSf_JnV_XQ%A&kyK-G(g- zq=~%)i&v*uiNKfobu*OD)Y&swL`~=kBtBM9IY0j2@H9B^{-E)@PBfKemdgx+^-uM&y6)Bv10 zVIy0(Ul9tzNMZ+0f7%zEpcFA_3WAsuEm* zsKo!|3pbb|%Bzc?N{Ml^a@qJM2W|-CQKq618WJn~lW3a(H@r|{a9;*UgaTmTF=R0? zmYsypP)rP=3+&ZQ4fCiC*$7_ArS#Tbn#DD} zLsbCaAwk}0;r5(77?&W%ouD2v4Qv(Qhv^RxVH193#VN$T<~2rDqFfzc%7zX`rpnOQoPIEtY^Pp!a;3yvT7_~{Q#B~dse`xI;c|M-2DDZF{S_?>gaa|2AR6?i}X zuD3xX$uBe3gg#IVl>I78>5z;k@P3{r^v%t$9)d6U?bB~Ug_s(yOGJ`0`4*JQo!AV# z`p4g&mfgO0&4mgIw+)n{98SGHqB!B6%@Oa%Cq5akExOq0nBEj}r!9MzEZ~IMbtuLn z9(wx}?4QnSk@O;-b+-u>_vW0wJgdcF&K=_jzt#@D>E8Z^&{CbPT9cu=BKRa^I(&1E zrW&~+B!%(3{;Y=tU1`$&O;;{n?d!`AEY!|8KdE;bk6sQpt0Q{#h>X|!L&`$$C&n!e zCt9o|wM!dau3by1ipC2+)Cn@Cp%6FCj4QVbv?h(ZvX8@r3ftlad-Z;0@{Q`{l}WnX z;gXsmPxIiivcL$#(#pT~&3gyBZyj&3`A_vFWqtvbhxBvmrA<*So>no+(+wwsRz_}A ze|6BSWZp+n;W2eD{}kN`5B|x3e~#^U5#E*M7Z(E?Ib=1-uU(Yo3a*xqiv351PQE$* zk70hxkWp}_!R8bQaM?)7+dRp!Xv zKao|CwLkRi*>{ujT2|J6S-;qAR1x776jYGrSak44b4%=cGp%xdU-S;2kah3bEpUlH zZRs|oi12GnDXh8dFaMQaC+c!-C@hO_pNu-l*R6lRb*=YO5Me(9$p zrTX<2qaU_hl<{@*5nwE@|6H92_ODcVd|WXZnqvE7!+7A~1(&rhKf_0+c1CwYkolE2 z4k*J@Dns=f%k#_@H(d+1Gp$jmBBQf(Tm9woqi)Mncl;pHt~1r`_}FXVkfI{CE;19S-qfiMx>=b_yWL=kpKthX(l=HrrKx*k`+As9E0)OQErlpK6QXVFk{ zR&DWFDVBcQf}FZBSspqFMp%6oH9xGSGw77%*eqdIpC*jgdoFLg{@eHa#Iwi5i#*($ znh}d72-0p?%pE*l{bHKS7&nNhYA}> zuX;?i=FA&{_a=5OtXLt$h`1Jrsq5)--pG*ac&He$_}+}n)YD7uU6@}xh-NfF+0MMs za3LPu`;!vm7c;ao7jz_hL6(@odaPYU)gE89U^HXNRdU0<(c!}C&~mQ$p}^8N%>2h5 zbG$`ceJ8R?y#uQcSYdz~a!I5pC@x9TzXk@y>rp|~kIR8t<>Edk7EUGUUqSAoRwL(M z8<;20=7k(v8@n08vIP*l0@ma?TcH&=qRhRwhu;{BPuG|Pi~5_t*IR&;Yi|rpDo*Uy zi8UcSpAxNi%gy>MHRNBjV_o3Q{jMOz(}~ZOL?t*CY2Rq|)7=uJD^+G^=5tE1<_sEy ze!H@b#b-+2rFu_yIN$5zJc_SpsMjw~aSQuKniw~(u@Anq_HMl4$q*6wo=Nc!2T?kA z7EZbR0KU`kl9uI?zBV%1J-q8y8S0Ly*(p^6iWx`+{%sksiz?lqPdOcXAADo-ywKyL ziZStq7=}%WPoPxm^RRNybrh0`#TE7vc1G{tapA6{@u-CvMTOeXmb9oECw}$vflakC z#gXhqkAx?M)hdaq8MZ(!tiN8`3RIH=m}`s-H$He;Itc9`(^@+YVSX(9^U>lWsG2`Y597f zea>%VXsbFcE`O1(+pnQK)z1Uyjvp1*mJ0zN^PC3k%obX3bIu7DX0y2qiQat~&KT&0{W`a$<5I$!>;dSTbVu*oJ!v+WR~ z^Sn=ytY+Q5 zj~FeP^TwUNnbX3;%?pT@e`lU_$zWWH`6TqwezV!LOA%(x-mg$jfl!b+=7gjB`c}HnwA?&(`x_l6s6nfzs z;owd{-!2%5gETmKIUPj0dS=rhHaN4el zmX?_f6C5Pukt1omiS)C`JMRM+8rT{yA|IS&6Hk73=YN7m26vwFeJo^U>k}|lg@rPA znxFWIKIa-PmENCj->wU*hHM%kYli+9r+bENJn2P&7vc3zOMos&09JeqN5I639Nj`7 z1rk&i&rShna>Av!nK6){8(Fd=g9E}8={{BLGH+)%-m90<@^_4QE4AwFGTz82I+wF8 z1g!44{bLK)8b4BmM|{W$1%y|qZW`{+FkNH%X)K|ghlz7mV*?|u7i4PjC0L8YeE{qP zv*F4|Bt)q1iKw}mSUL2}*sR$WG zy(beUVFG}A#Pqa}j1jz5qW~p+GHe)G7ax2%IX*WlqUt?<AkU!7o`o`%~t6 zPMZjJ;C|p@A&2IuT9`=?>f}HemNJtSk?Awg9tPCmJ3KZLI#U{ECjT`>0L1^pz!L~* z2eu4S<%IQ$=qo6XWDlqC_orI3p%Vrb@zmkqWx8Nfq$lwQPA@>3XHM!xXqj81@ZLj4 zuEvjY2;p1l@W&IOMO{%RalnM0Nxs6`tp8y!4TDBdm(?y;qf{6o;}5XXs0!X0wb1le zh=G>EXdBPi3;0P}jS-=oj+w@RXlvzEl7jmPl;TJTaY&P3Fw!Hs=l%`NqJ!lB25;HF zEraR=a=bnQiJ1aBv?+k_bv7 zyc-QS&~1V!rrEMZFEcV+n?mFXpy^o};6E?VJppC_`B>V%Pct!3W!>1>q!&S^j%#^8 zGta_}GH~I1B~N5Dk>DUfdWfIxeA88aQ7=CN=Lf{?Pt<=HY)N0c;tzvhI}to$fA>E$ z9(s zi3&EUh*;&RLzN*s-;4$n{}rHz@p!`#$$?74o%8VutE?{`LzsT0)gxFh!kEMnrPXVl zajkmYmPchbV3Hfle-fT1zGGVIn5{1hf{)T2PkdWFs-|x}9uv1%JLCxp3ytTja5%sD z#yrX6GyTC$ZOEYSc6JEuyj`*0cGicx(M>D$&HL)MXAbs!{&VVBu6C7J>aY4#{@}@? z?Xfw(TE01|40WC-kGns)NjFL}-tTz%BI+bu%wdwJo)+wNbO+t}Z3_`@J~ z&?3hD=Z`Jz-e1~&=;o@pntPK=hZo0n!?n!ev0-dfMNgCWq89>s#9wNpXlQm2&INJG zHiHI>vHb^Nc6}^(SLpghL(dh2Z2fRgCtn$`J#ygwFOLUBS9-^t(KJ|YGqZUp(Ydf# z|IY`57Se+kiZB1@?%{=irm|)wZ|&Ibv%mWPVen#3!cQ_EizwN_7NukQ4sg7j5;Cjyci_t~B;$}av*&Tl|-=cb>2OFMVtmiWas)O4Rv zHZ+XS;}7?!3;S+7)*Z{!pma4Dq}!H!-O|{Kvz@*P`bUkzD-EI(`q$8TtngA<&>sd- zwlQ{@M-MD{zNNbVW&2w{r?f`=?c9^`#QJsK`~3FnPkmzdMF!&amt9XUL|oFf*#59C zVAH3KfggUef6>-^3tJZ|=R~LIwnR8XKM)w^eD6Y{v?VC*fMvh5+$xVmtCi!E+i}!Z z)n+Lz6oFK+>vdqs+$iz6UHhZg?tiB4XQx%|yE2MKmS{#jWU(txKQFvDdZQ@_k-(SV zNEMO;+E8n8a)7Db>(vKl#Erwf&8Fc%$gW$J?}&#br>u_Jgs6GXyXdZV7K)x7SkQe5 z*5AI4B?~4^GbduZ7Dj3%bK=+aor|9laq_X*32blaxdL@;On-0iCu`2V^oN17(UPs1 zlr;j#c@h!=7g8ZnoH%?R`uLcyHNO~(4FFVLuDS2IQ@Aa=1ggTHGfUFi zJJe^7TJBfu_&OiSn~F?-$qBm|6Y+yl^_|ggW`oFq|cU@kIez6l@2bkuWdLJlAH+!I8ntpPw*ITP5 z(h}~LSb8q2n~8=j?_k5YZLO03^NoOuW;Vm8drkm;T&9E>_%H~?wCH=FGfw89 zZ%eFb7x2-^!?4fR72YpW?0j84@~cgK=xTv`>5)}Vk^`A3n(F#=bJVRS_ZVm1jho2I zAzzQOZofYMfzG1QYmFR#Mh&CM9tNBn0v6Wu;d$*2U!9d=WmuMegEOyprO>yrsbAb8 zK9B!|loRIz1f8i9m_g zW-?5^rZ&X%!uqkE$*@M<_1I!FExK^!t~|Pac=4FHX(;8`VKa;2o@2ty!If)yDq)WE z)_T#}7f+R0xOVByv)~vdMAS`3*wZz&p{k0q)8QVsK{ zZh2#X@7+-GRBGkJtCu76#eBaIG8tLTg7skp9)C&WI2vB5+ zba6!aeQ5ebp2%dDC|GK?@Oc-!b>Nu#D0^iriZ^Z*XHds5eThGyy@28c!}>GP9Dz}k zq(1)xfW{8s@^dyn&hhA3N-qxtU`Y}(j=}p1rKa66sp{LjoR5gyLVEf$i?9gU<|6hn zvsa3$M^C?mMq_!d!%k$E_4tG?Q?6|9trvywNi)uBcRF@Gi;&XJ`YYGkTw1|vS06S} zf+CBkq0&A88u(+Q&0Q^9oMS$ljj=@808{r+$|zCFaJifY8umnV^W7ovc@(>Lp;E8@K9DjZ^_ zpaafTdM;{0wXbNhz)8?f2FaFH!MuC?U1?E;r`-y)_f-wjRj;Ep)Q78|gBFH)iRBFt zOC#FG82VvKyoiPzmFNbL;ST$1bPZqqE8Yk!mEt(aT*1m(<9%Uow!WJeAj!gmZAxaA zuQi^AGha^^=lY8ekn(P-7S#=c;pi&!Hq;I;tN`KbNQ9_-W0mP)XqxNvUKCcOhi`4< zdR$-bF(xyisd#+KF$Tb_sXk)%r)WlwNKZ+0h)Ff7ep#~};)K|`*aO5|Z70X|Sr>5^ z?R>2j*_=x&3H2hwl1JhMs~q(I=s`uscLt7;aR2=dF)mgTWnKEt6$kK@=WODxu!KvJ zYEg$D8iY-8U9^mwxhCnD9xcQa$S_=fPLm^Ulv#U?Wb->-N8`6fdq47NWpcYI&db0A z?r~U1AYDL!6P^HZ1BflgT407?DCh+dDr}-zXnzugKCBoF*^@MsLg3~w2Pz$Fp+1*0 z(4NsgIq@hagJmFYXO*S5+FnTrmM-ojE zND^L#xm*r(#Hg4xM0$yRfEF{+OQLNQ&ocrErfO4~P{v;s0p4=GsfoU0R2!>9{eHj?@zmV2z;`c2uU@PrKq%1~c++5%M~G=Z0H9%*v2SQNbJ$ zs;N^KR&dt^PI!$0pUOH=^%Cbi1~E?)AQAu;H^x-BoG{h?3!-LDluo!e2KH>sJ8TN> z;ycj|5U3yf8nxtTk%ZD=9@=Da((PuC>7eB z6sBRHK|l_tT;Bu-JVm4WQ;y>p4t5DZrm18z@!r@d+?!$$jnx@>ZVsIQ1N`-n;o375 z0axM;+)J62r~i+g!(dFG;wt>!!=ZXy6w{1Hfk@(Vukn)*+_@PLJmF@t4(HyJnBVl! z*cCYCtbK9HkUk>?0vP)5VN=xJLk)9+nHypRuYQ4L$)t@p*|5?;f?mMS--#eAF-i&b&bp65-h?9rzmat{zdD%|-8frF*Ng#V z+3UURho6bT+88J&G)yx>=?+fV@&RjKN4R+k^U?}u1H!tIsc>s0l-5G(m2y0^2WTyS zN137j@r!z9v6gyBXJLXdsZWd9$$*`u(4g3U`ZAfahhvE%ngLWOP$qwho@WURE_c8k zk!8Trknc6J3?2Wp>jnNc-hJ7l@emm zDn(Kq^wYZi*Mz(M&uLoEPlkDnm2JFs(M(u()An%LYn#iy&#FGKCM~+-JmAyT>*%Am zs=i|4xWdb=rOB&4D0TOq82cJ+es*$}=lJT|J9rA=-Vzv?i0`X}v)ruX@68Aj9&9c!HiiXC ze17)w*pJ^|%xmx!Dm6@DB>!^|}7ViGatI4PRfuq3T z-VwLlYb!^hWJ6AtsaukR^Bd+JmC6KeH4n-sPIa`%7NJ|$OD)a)g9WkccZY3y)c*Gd zixL?>(D|nS_W|2uH}Ja}olhS1ZQ$3Ilv3esu5ft#!;|a3%AH75*qUiO{xDeN={0ev ztxBfQ?=+u&k3V~GQZ%t+Wx$HSMpz3b9^x5>1l0XD@JU0V{Kj6p)!Vlp?5NnNJtdu) zQ8O5ppYr-Gk?x0(x3}K&FK3EW>Ib6P3B)vGaXNq1-}sR(yM z0jWz{*h`(7UmEx)gAQ+(8EzEpiPWD3PU0OYxcn&ROjl`}mFWe~pKGEj)v@A;-gPK^ zhrz9v{6y=-U5#vezP^mVFCQm;06)|DW|)IZNoIClMlrw7q;FTo644KPl7p*pLjC?0 z@)gL!`lJe$jc%^BZdaK*_3{*69}o|Z0bI9Ol~kw~;n=hC9@bUv3a-^8 zW%QaJmu@9HCztwkGHQA})&u$U=6g2ZnF;F^ReUGB=5|9!;|a@@aeCQ0pNp99`5Z&7 z%gP{6UY=Btg$hQ^UP6egLcMp<1L1t$ga&T1CkIo>Ttiq_=6w0*1DLXk)UwmTnjF|P zBgMDbskp3O+L)d!z76C1|eUz{-S+VJNi`t#csa)3|_kf<8 zM3sxs@=W7s@*tOl?%)Ryn%&`d59ppFSb8%nExPjFMGj$+o3vjucyz>$=aa z-!7z+OkR|OtgcK6`p+2snQ3;nG37b40fuxqE&BO)k#Ry&JySyvWHw%p&XBpKydO9m zpg)Gs04gzqX}2-Z-dGe^5Dz|JWH6_|Cn~Djne>J%=K?|+V9yv`O!CnRTW#|2VGpE-LWw=9D8fIlXZ1?yUTL`d?Q2fZ=yyeU!b<%BdXNN z(pnTjBmOyWn|07F-FY-yV`@K3evfwUz9otylF`wFl)IrB(7{g@{GhNxG+E^H86jT8 zjd1JX;wi@%c5AZ_NtnxTF?9(7>4;wmu@!Fg|C&*iNBRfA0%YemECgO29(Gg!qpP#w zZCi7|+>VQZpvMR5g*674+~j6js8?Z%JxAG2l7M->I&-BQt<8Mubnhr%#`@0lI_PBA zq+XkZLik|4UYe*3rxcudbiUplmU` zc=90uY5kIMlIowouie}chZv~zVkGZLSYi6N`Z=iYS>OMS`+ zFGowCyTz|AkvaCJeYz4|&Pg?l3?-N*kZ3g&no_JOA&H336v=}am}L0yfaMH&7?BCY zK&6b(W21I61S$_A;P!zSUz(&w#E#gNxGHQx4ixd1n#am@yjZ5qzU*~IoU zehTP*Dl3DutH)2NcOVAq(JCHz5p$i4Ey(1bO#V-m)*i{9(z0SSEVqSMKs(isLTX>O z2d3aOi9bGse&iBavnAZn6@P%vSWjEg0W6SVHH)|j{XiR_kDX#~*$SlsyiiYRkhJ;erfZT3g-VvC7!rNV%8pmG503iQ;l_!+Kj}sJrN?{OpZp=0C zY?~?wr3-=Tr9`bLM<~moFp{#s*vl}Aq?MtJR){|a7Ym`o5O5T?9hW9h(YJvwBGpiUj`P) z?m(Vi&D1ImcRvIBP@%WukSthh`eCLi;to4IJ)SsmNzCNJOq2E5hkV@WK2pe z(#@n7XWbBKV@@fjsl1txIDu!5=Y{$S+4HzIsg0pfkD)aLUkwZaEd>sYvTPL7959HAYXP^iX_-1USY8M_ z4)a528o^rRmAHJsRBbo)0%LCr8cd8hSa$C3^agqQi<*sr@h9B{C7rNNraa1s#%6vZ z5Z+KeWa>4?6zEo5p?^9G${4i{k2R*px9ByP+9AxRU^+De*It0f1p#nKelP&26Pt@g zO6h~AP@I0pfyrPvr8Eeceg z9SYT-NP>e@f$#;_cA>xX#^Ey+T3#7XHSDU#3OXM-R9j+WQlL%yXefG%09{+Dsw`=& zhqv|EJER1mwip?n?lFz^*}{o9)JF;YgED*1MZJ{QP;wWgwdq;%HN86jhBdqwh0=7` zr$^SqCi1!pjX1T7&1`BEIU>z?!d>C;>RNQl!O{7^?2Gzmb*a1lF!+nrvc{{R0pg+6c&qY6{0YOlt zLy1khO6h6%g_T7RGLMp)zIFRMj5GIrUz5JSccxrh&=_{J->~%cuusNyd@73`8_d2P zVD*ylm(7YFV@8dioshU^YuBIB+GuPv>qDn2P=>fRd%)9X-FrJpz+bRbDR~hkbMG9vFMDs* zs?dh^y=kxMU)nwdbaNdQW5XvjmWCH~7h~_mOHUs4j$huTaN_SvHy$=T#qU`Apr<&q zdT_sTaO-5T{J_yI@=GsBGG#+jw^7jxZQeLARHH#&NB*ih^w6r_wb#rrt0e1jx!8wl zFdN4Pgc;r*s7MbHcdVsIgq*g`VZJ&FnQ!m>WBLZa{^mOk%Y}6TJpJp!;Vv~VyCG-j z^V*(|#2&c;KrwxF*1z;QC*1St)c@Mqr5Bi2KIpsKQP;7CVHFiBRMbMTt~k?|8cv(6U%x;~!si5(xSmehxc z<{VvHZ&<4I-Mm299Mw1Td}-SodMT29BHojubQCt|F0XUCw5)dJ&>a7}jXQ0h`+Yx5 zUe9YoCyU-MJLWJ4^*VbW_8ax6%YV`psCik=8`0=fDlzF+pTE!@FW0S=f6$@yMSH9E z1L)J*aTZNS%i=DbN7QU0>u-BqFRj(Ldi-JVg|%qn-IbEO0KCuq@>4}8U9uhrX-rNZ zhFT%O=T|KVO%EgXqV~{#>qpkhU;;2 zq;d2&;dhsRrb$2h5Dm2Ap=3$7lLx5#NB%H)c7Uf=dd_H%RYX;&d3nxjdVJolgVZle zhS#E3j0}F#_iQ1)K;t3Z)z2-K*`I!MRO` zqmlxo9}b3W@cG-L6Pg)`8-cc^>A$^2iF!S%N!!wW!=g$(dY*aMG3ow^#bDao=I3FB z%@{sBviAW%WTUu)5L|!bqDxk+`iSRyt|Qxg;<#)c3Z*l^U26~KXhT94lw_#yqlcSJ zBVY{k%j0o`YZs?`XQOh}8Tw!OW0R@~!;N@KtCrR9^moyEruy~S(2H2q;i`Y7zSnYB z&Wb_Kq)QE@8S@~9>{yuSyit*Q8pvGb#*yP7i1ZX4TR5*awKSfT2&I(>4pP8aM(Xjk zdSS+fg`|1p)h46cFqNvgT!n5q5^=>8$ALK3wR&b*VmvPZr)M)1+A#HkczUh#nkp9- zWPwjX{DC?!z-%>NBh)Pgd)EgpJbUJGDR4=oEwV6Oa@{PvezUdvLOrS zJ*qgocv|N|bR7-4)h6eKL+2af2CCnx#jeU+G!K$k+6-CFQ461+f#s!dETnp0uOYnz zt-OxDi&M7kCw0GS0~#TlnITE8-LkiS42v0xB3EHT3%VdZQ`{=nIPTqNjStzCw7LO6 z^=41=cFF|k48!SDqnCObVu))ZhE-{i7znC~4x5Pm+RbaBT@YmVSz6E5KsQ0O(oxJI z>BE*)U#;rlhqFdm0V1Sd8>P)9rIDUnPZYg-Wg=*Meg+KC4w$C~2K(5oYgdG}k7(V% zD~%!8oJc+Geoq{Lg)4fFcy_Vk!v&2pa?l+?h;-ThR$Lk_!RUG* zCQvpx2oOutgI?3fTSG`rF4Z6m5W0grranq32(e9YGxFn#ARG#=-j1`j^Rdk;M9u!X z8!h?@#|hdr;tRd&viJy{z58-kQfVx+ipw7Q6@%&`u#bvT&Q=tlNLs2EHU+D=9J4UL*r)e+1a##^# zGoXjK0eFNdP{2h+_9#J?qaOmg&{8Oe!Xr@qx_&Fft2Iu7)*p%re<``#v&z!SA`q6|JCN15?g<}b z&3c@ZsQb*OXb90c#YSr?40W@VCRk*`3@5uryML-7hvjGVi+2@kt#b@FAoX}fmCV~; zQj|k6vc4iO<0>;4o*nFa_eiMx{73%ny%9GW{iP?cyE9n3b{@Ys#H+4F?{#y{UG}rp zwZUpUyew4&53m}|`-bR6wJO3igh=yZ69!)T!M!>=qsAgTpp3V zKVg%g%lar!I}f(64U1e`h4P%EL)!Hdfx^3cZXjKb!A%taRA@fc?Ok{DUC?$E#v+-M zr(0Vc-pwL@5>Hd)Pw)VaN;t2R(Tr}dutGLj{DCXC5HUZ^y=3pjc{clVt_2Xe+Wix< zSr{j|U`aT9M^rRLcz~c(g3B>Mj47nh6$pCGU4qiUo6Laf14b${6S9i9dH6K!ftXB| z7^f*IIkd8b-#G*XaE720fhYGxr2PakU<5iv#%oMF@W~L%FR)PfJ?(nCoBt1AUji28 zd3H^-hE}7bRWLziz9e6xp9&<6xB?^bBQ_c}CSetr2sI$6Xb@#_culC1KyaD`X;lP8 zgJBy-_GOqv#AzX{3d5qHP-Iw!odIT;$$#$W#iZ?j{r`2t5nT>FFasXt*F*VJFUAwv-HrouDcoSd%aG3&#)$Ba(2#0}yY10+Rw^(R1VV8XS?5Ab0& z$*7S?V5P1&8F$^nNAr<5Mu|NWzG@!=Wd_&i>XC0a&DI9^P=GcP7httu$AP5wDZJ(W zSr&*A766;RRqVcqqQkR53HnLJm>!RYF{H$`^P@{5TOOE@dY2_1_yr2=hL7F>ltN{< zLl71B6P`d~!bP2f7EUnS`q>9R7D^Ck-l4XIiB_+QBQx)>fxnHSgNN}2UH#b5ZCs!c z_o!zT07zM2btC=<Jaz;6 zK}>-k?758@#pV*R#?SnV)LI$@cz=a@6EeiOz+6Pw5L30|`qFQB?p9-QoZ5Cp@!$p< z+i^-CmDRHsE%BC2N9n~7k<%#wQUE|}t_D2&?nXxZYUUbG9fJRxA0voeBKO<`CPxFP z=oH2Qv#bk!4~Hy$T+N$9U)$ z@h-t+ct-#;OJ}%aPxVm9$qBpLjNSoIteDZxLf@y_R7ZTQ>BjeP<{(x2G8#^zjDqSC zm~@*i0M*^M8HDE@zh`T>Syq5r4PuV2B8@j(FlZM10*0T0V*HIINx41t=jl_QL%9O} z4@(VU5NIISAIt&h9G-4k6V#54brV=@^kK`3GHA5WSo#Csh*zU3fEmD{goW2RFnJD@ zYSTKUkHhp3W}#@?7kMxwad0-xM~0hHhiu(ay5pBWSie7C@zX1ZPa6D7|LtW(-_O46 znIq=D_Xeojlf)87>+ux_n@6B1Pb}vEJRbzr8e0sqlr}@YJXI^HH?}-#i^LF1&4w?_ zFf)W&P4%hEenxGS{I+{?)&Zy4^trs-@}_=8BcHeP*`%{rf`s18Dfd`%rw|CyYG%ko zXSlBY`n$OQ;_Bl~cPG9H?k7bNW;+*Tor`{3cqp$R zThX;NCV1JKpN`(G`m(h!`8PPjx)lxbh{3_zL*)YBrpT7D_hak_?oJ)@l>O6c@yLJa z-I{~$jXx6}$n~D72sed7b|6xEv)%A*31=2|ZqoStP(h1X?y?|kS?T42!JoT*p3}KZ z2o-IKrm^;6djL1~Y*`UHXQ56P!P}oDwO(`cmtR5lQom3Wg2S4j zKjYWh*d8=Y3cuzp{#2_gt<4l{KU=jYV0il%Yi&`h7IXt-^NMikZHbv!`oC5y^})Jb zh5g=c+S<0r`D9>#F)+m1Dp1$$Knpv+PdRp832C-8|v6E>@);EVCKe zuI(<1ZF0Xi7`8@X{x{-u!&cEB9usA9pO6QtY(YT=9hKxp$#w@OWIuoCgoW zUh!DpF*f3lU3hn{;>HHz#3{Hwa$msa>w>-H&xqH zXB_0id6yF%Cb~5Rt=1(?wec9qV~@GtoFqo6OAo!9DGkL+>8;^#%mfcCt?S3aQ=yAPiL8KbdY^T#exD z^J+<}uxB*7e$Td;#?o*!MgolpOruOJNLJeo7Bq=|^Q&j_4qG&VVH>#g;7gH%kH^I= z(&_1x4(mIHgSqu}2UmJE*=VXErY}QyinWMUDYj zWcVvlkVU?3g&>8_y)<_I#q~#jGO@6=%^ZZt!iNhAH%Q=gg}JjK z;r^*NM}DupNp+5e`i!b&KLm>pI8BBhS!Vc)v~z{1X|f+-DTKhlaiw4z(>sD>dTZ0X zv*ioGa3r`RMKM!oX%~cQX>5{;yG6Cz`v!#f${Mvp1ySa#@TH&GU%80g5cfUfbPtRq zfI3QIksCtpDSgs^fK#tqF%Po_M$regW_f$*D21C1l#=N9F=3II*f9p;&fGU=01na) zxC%&p&57pB10e`bCjxODuLUj54yo4t1AF!c#fY9s((HWVN-F0wrjiH!-%#Wx@17M$ zlFEqUoGKBB$|@LU>O+&wYmOy3r!`}@)Il|%56cv<)N~xc3{xQ{irdy@OI$fv z4N3I74?s*(smQrcIV#2xSfD}?2>;apQYoz2f7h}6#nDpxp!68`Xy>ZpHKs2D0l_>w;jJwEpHocJ^WN=9?^uN5 z>DbkU%u9mimerDe3?r%Eqj;TZosmZ$dhGl#i5$0xw525ZQ&H&2aPZgIA(L+!6MR1T zC>Z@W>b15wYIPY%=fgS&;&F9O%4p@}H{g2Hz!-^H{W)5Hvsi>6Ea)V)d?U#@$FnaP zXgtWn4uFEF2occ)@aoRMIcRI{13`!Q79x@g^n9(!MB|<|az?}NXW1KQa;J?f29o)| zt*sggjK=#LVSo$I9^N2%X#%ki2$eS7&SdnnGZ{dk<#{XtPu) zoQtk67CQxI>~g?kRT`KZ1t1xZUiM3bd2=- z`&^;M?&)LPr<)xnl1~rX{eaOE_S!vQ55cRa)_2$Z)%?7{yyF_8XO+Nco6TOa)q+hU zuy~Wy8J5@m!d9932RWj4l*!iJP;y=aPVTE|w$K^cC3uVewS=X~RSVeyMDP?hYOv$t zKk;2Z2Y;Y#P!UZ!QkxHmc1F+$>^#emIMtXVdsGi?E}=eF>W3@Y!9Fab zg?E$5?VthUTJh5l<`MO?` zy7InnY>>+8*&f3B`w*!NC}x4A5eBV~@G#`I)WQkqo?N*C34?>i6?PH(fX=^!(_;A- zz?itJ0fZkheRW8}uqNEh4hqOWbFaTkKkU{(<{IX+cfqHmZ6Y1#kL?mpty7yY1k7L& z(!Q8R%#w~%PB0;i(Yvf%2Vnw;NC|GQ_AfZ4@zy^f3kHwJt+?2W5mJ`&qVDh&N%SVl zkfBjDOx%C9*Liu%LyRwy=n40dvuvrVAs;vadls7O(mWv~h5V`che%GtV~bo14K!5= zG(vTcK>;6&BYuK*zgCnTkS2!R0s5a&(vzkrQp`O{I}uA_m~Qo9^dMUTW-{)Puz_D| zE{L3whQK^PqK7;VSKkIUM2;sChTlF3zC!~R{)Y9@mvmUe-R-ffD6M;|%M?rN|&NYRg!2C7%kkK-^233iYr3o-!=g5%?&j3`~GV`>BP z^d|rF;g@j$vOGN)WHx12K~2oxz$$p1(L$u^bKIq2J#832GtZc}utY>RxThI1GKx1` zrz)MbYE;N$^*XE~1wP}^k+{G`u{{IX1loKS?5rg0i5bXF0ovwhEO7a_(fhbts;>T5 z#W$^;e;c5G9V}<~84!GiA)K05$1wdjL)WB=!pSiY%q((2L6-=E&&NOE!!XX;wp~RO zM7s^|8lIbz5`^xBwA(>v(569NJ3B;vh(?>r6WB(4STJpSLFUqrWzAMIW3lN2W_fYG z-23V9{fku^lbQo*{|Ekl3aUxNj!}8T_)>t@>c59qRK%akc6^vRH8vm|8xwuV{b&o4 zb#zR$X68u!Q0a!QHIPT@?Fa>Z{Re+r zI9e8FIJvOMHv_^=!(#Md#j)--Qm7Z5fl$A-sOEZ+lB`x;z`Ovr7WnnL>zunM5*o?I z66UyC*JW=i<7eLX%k z)OL8d?CupmZ^dc5AjALc(mXaOXXF<;o(uWNJ?fg^`CKl)_r&CV5Wsh9Z%@vt%$spO z7{7agVESZHw&;fws^Y9_Wx3o+*V_JwgF5p1e1E%!+kq#n%p2D2LIc~K;s11f@`>No zusv^%T&YL3MRmouN4fr{BE_QK&8k0#mc%a+iN6gw>8V(fm-KS&|HkdfZz`+iZnUEK zyS4T5uH*YN^X%>S=s)-hiN!@nqketMsMI?aMT!rH7-O`7ee04xx3rR`i({+2^xIo9 zYBj>niw)2*n*)kUB5L&qw31NmW3Qh~+;bdi(+vj&TfL{gQtE97+|COF3Z&~t%rD1f zhAnfK-;x!L8CBlGGZS$;AHLeIck9>go``-7aee4HVHYHb=U?%wE#H6H!qZC76d5X?8>B%<93$BnQ0W`M)W_n>)PO7;rRUho?A^YCzA)&txNLN zjj9amxNC07GjAG!-~OCz*kE*Gbvu&c9*0bsr`sk{e~$U+LleMefZr_5D=fQ}vGfGz zmyVvmzB0eZjxvKJ^w^blm10IXvSRS-6eOCPI{YU}q$UlVrp;K^e=l>qr2J8sjWqZt zlVe9wTJU5nbMu{?zcy~yDq`&x1{)oW_reje{s8C-4ANu7h~^|R&fN~%5A{fLJrJP0qXs-ZmYTb1p} z)c(ZcNo}_pY3KaxXkjxba%46n=-jqo#k}i}O2OsrDueCMm|@Mpdbh?NxO_KH?39cT z<+e4!G1(XgBWE**$GapJaQV_e(_N+)8XQ!6hpWoDLygu$c^PP5KE_#i#QUl=T9>-8 zNacm@%Zb6-dtN>}RE>Uy|n zw69Y)X*|_mJ7h1&4lDoQ-i*st!-e+texuOZMs4AJR+s;Ti(AL#d!it`2x#i-Rzr*i z@h_fFLZ6j&J<7A42iG}R+%4shbW3YB%cwfJ&~sF2Je97PDG`@UTt$^JU*Bp8B-Su0Lp%G+*mgoLpLivN|0m*b9y66+5oj&E2Ymu_l}>%T6gIg3~UC;wh(07QN0V+9JN6=Cs zugsU8)M~8y39XMf{V;dOE?yKI)+5uDo$II)YdyO)wzl%s zm>1eg)TiqGW;%=XCCJ0bAaW;{sI9=&Iy3D7aC%rfp9P~2w9hAobB>K#+nbG&_@*fH zoVvp`v0MWTpw2ES?_ZhRDvln=1F1f0IKaqlAn8P&ypzv83VHYnl)d#jqv#th3LS_8 zDQqeybj89k87^&4u8Al4kX1hQr2%*R1_`4kt~QK8JaQlGng7FfUnqbZ3O1FyCYsDslPO)`p#OWCp3bl9@qV@CKp(m9Y z$a)^DcE^Uat=C1f7JVPH5t}D1*&Bo&ai`b8?!gB=3eZpeI&iS;T8nCb0Eq1)!k*5e zNTKdysA!B1HjcT=3*&#p{C$Snh0G>vV0MQs5C7$VQB?5md(G=EZs|%FmuR5mx;i+C zjlMMqKHEBfU1)D=#Ra9##q>qLx8Zux7@-11Z7|!}>%%>Y+kzeSo;)JvQ(RiU{oXV+ zw_or4>%wui|5G5 zGnNz}ly1td3z0PTLNaY6L=;8-*x(to6h+fMwHjW*80BIR!aQyUS*Ag_jXv-iT_Ztr z5LS!+Kwyj9e`{1mIj)vo#2C_F5(0d8WxS6HVz{X>@N?PU;AM7&OcssO4Ij7nnisIW zkuoKs0s^^!4}YEauxI8J)|5!w5hD5NU(^hY14qBM4A!BDHVf#X z8JgB(*Fua>72kSRE9!^mi3@%QVrRX#r1b{+IdN({0Sw4$0(AIDi1TBK^V@;PR|8un z8K#Dd53sQoCFU^u5A}iMnSLr#oPoIZ=QdAFeUC+2v=}im=Q?zK!=xz8^z}0YUfk*{c3C*vzNYvC5g~ki#vicSIl5~ zcqX#YHe(dN1cSs}*U(7Amf(nu@=unx*$XZRj)ZK#C_2@l1VNQ;2x_(%_~79&od zObR8+r{OzDOYF}a=mCD&M1Y|o5#Vqo$1A$qXDMpPxUpI>f^qjGS$2h{FuG*dcl|T< zamJ^UdOznHd_QptU5)(^6m&S@S9EqjZVdUG0M@+<#nPy}XU-k`&W+uhpfg+GNR$G( zs?CUZo^27g*B|*?Ow+0aG{cU*gw}BaSG{eS-(_m0az?%|gg zVorbXA3&Q*Ntk6Iwz(xJ)1D(|E*;L@^x!ZZ? zn9Y4H+{6Eou_Yx!T&9D7PCf z+~>s(lF#i;Z@71GwaKl4$Ap71_5XBFPVd?~CTt!7mmRA&CS@cyD(ZAFiAA-i*w!cy zEQpU!)c(ox;a{+-FHCvuI4O*}z2#b^E#%C6+>RX#_k)%2 z>bZYdhXruYrkK0tsak8lDdN-~t-f|VQh#LPeqYCt;SOD>=Gf$hljWV&6`G-$$OkRl zmAddkv1BW%Om)^MP~D488kzh2jJ_84IpyW(W8$|{0y|pP=XCfW>Gy5GM!T|@zlP;( zFB`G>?wHngYAP=?a!-Nbe6hsL$AdfAx3su+=FZT@%wvzM^t*OPE_?km>lodNdaS4m zX5ZaDFU+^iJ!~sDlKOb^v!UH*q7cw^b!ZZ*9=3<2x#iWw`DtO56`~jE>_qjNQ0Z@` zJh|rrwZ03`n&X6X_x|thP5kNMxow?G+oYj8DZ&U0-RiX-)zT>ogP;s-dcR!ooqb9F?F?xfaRsr~)OD|VVqc<;7C;$hAzl%O?- zrMl=B_JD-#l#O4k(pU|)>y3^94u2ls!aW~$qP$#z?f;O_U;p-#T)PAE(&6f|IBlEc z6%7|q?N&5mk=*s=5u3X<*6|*>^@>inV}9D9DQ(J#Y(v9i>|?_55wsaz7q@KBi(V@N zGtncs_JabVq~U9fE_`+m*$_8yt;kf|2P^e1)Bjp({l zvB%uU^FaXDmaLvAhu|4ygOddvy-PsNdk&ZUZ8GXL995RW$M$6qP(+m*zjSAw1?Dwk zQRA2WR#qdg#)MfL+dM7HB!1!Ge19j(aZAK-D6^`VR{t5;gkIGDT(zm}ChWMRm@;`F0}tgv4lN!Ec+nRL@yHUP z?pTJ!`Oq4Y{h^@{sWf{C?0y@HdULfn0G~L7{P`@*1NxGU;~5~JxgOesm}~A0cv>SG z@Z#yN=B3yf$HqXX<(8J!4`vNzN)-)j1aLSaD*4m}a%nf;u4Apz~ z15Q`a78_pLZ|3y-{jgcswod-0Qq$pI2)T$vnAeWVZwPZ>mY>(~A)_bv+=lI$vBi1G z@+ZvR8j1!F6!U)uIy+&)9o5;tWNTbgvh%rw5?!bfBsmc2NL%p>> zb^6WcQW{ZXiuaQz?U1oF0K>kSE<@OA%^-^WIGz#(r z{RQF$$|3f;Fn5?-k4CX#ya%z!-KW&dbiNK*}tqXc!n^=VU5x-UotmP6{i}q0o2!Y-$`&o2B}>G`C1%gJ z)4q(vx&^$|A(wga0ln-*amflhmLIFu_CZ; zei3)z{N5L5nT_;SknsXc=uwAV_sqE_*L4)hQ@uh8Fb?n?1?;QlRM5XEhQx!LZL5_& z?(#DNr^d9b5|;8Uj3okj{fjJ;w=f0@;{U-wIr~ULkxc&$P)=bsDXi7Vihx-|u_pwo zDgSJG{3aMgD8bjvo0T%u(Ij57{1ay;C3ZH$#xiaKLW~3#0SE-s7>;0ga%vQp-^UOq zi*fnw6%)HE9TyQ-NjEkQ@SOvAz%ms8;IFqW?8&L?n^d?1NQ+nLel`6q z-~SB-=`cZr=?K_6W=}uKo3wkP$Q&!1m}n&PNHYrxy=#zZ%7KIssgt2Qc5zZ8V7g<# zmWU-paD&VK7Z`5y0RTP2fA9vJFzm#MB2v5TGni^bV`I!A0@NU7T(Ib{t6)KNZM8sN z=5EFy?z56sF&wM(u{S}Fv@>USOn1I<_|J%cmPnkB0|#@_+YNQ zZ-q{H?AX?MNT0xSn&Bv>nIL&T48cIIw%OcZ4|2^VTz)qvGqxoQI|H%bnK zqIVUVJYai~x%A09jB}Z{J={5rq8ns8PB{|@$rl}E*MJIXdUWKIjh`3u@9gR%2@UIw z4nCI^fFqN*yhjBL7WuSVc!>!->XAU;Nnxh{V8GINhNK-p?@Ku4(2N4G#tAEpd(eZ(*|LIj2JyIY%rG|qv3`BL;SHss9gKGQz1eyO!{>Zt`d3AvUE}sM9tt zrh&c00ec190|5}$s`_JKW$<}uOo4b zLyMQ`6NOAu?B{F$H(mEHa2Y-9{u3_~R-TWY5kH5eU`V{rW0;#c-Or!T9COfN4la%J zR(^B}B)ES$*Q~o<5g&hS%Gf*{+#J*_4A>B_yG_~uBxX)WRe+f{@k3WtU*+-o@ zks0Fn609^WJ~-t#@#QcbFldq6Av${Lv^7$9YZQUpli!oDNlG6J+2cJB8 zCN3^{YNljG#&?Yzh|F6!F?3_uqB}x07a&U+KdQ71rxqq;zvEACyT_89&q6*g!5Qu? zyqzQ(Nc@$@a`9i$UDb77V^YH3KD}Ersad*oB`R^yv?yt^H5*NzG6}dSL>)qwKBhH06 zD}pzb*wut|9_<+T=*+`ptpj|pZ|WT?tgPyX%BqfuQ90cxF>etC8n+b61!L0{mT!ST zPWK8(hWtVjRJrJYu@8PuH}!kkG8H{F#S2iN+1eN>_XWckC{J#a3hzk+>}R&WS(fE* z_=2mOHL?=c9m?by*1+AW3{MC|WP1EcH+fDSC;RnGfWvb(cZrYf8Fz^XGV==BMvYid zUvH@#LC+>lz``FvPR@YS3q59hscCI!oD@FuncnRpu6Dw&F2eoLYkb{|?BGYjF7T%g z>dof>+y zD|e%^QFyar5{<&}b!~!Pbe}QCglfC+zHwt_%*5ec3lyI(Rh?ISh|+@89ZimHL0-lT z{8n)q?ooG=n5{=Kawa-3qHxw5_2p+ktO2owI(rd)FCK@k_Jtj|Z zh}IJO<}N(D)-#COGD2CNzcxyhT;L|8Lx?;P#?#H5_s8n;FzKKwZh|#e6W2Oa*63`y zW8G{H6+d=V;)*GnV+>Wx)Pzu{60iG~Fbo1sz} z8rnz?5jq3ItTw#jxD^>MKyfjg%`sv*03&n=B%@t$fe)AF((X1?ebO_a7?}pn>a1a7C!0XUszwmkF&@bS{bEuGP>&UeeUf3peRGUDhyNB&P=trg z41xMn7s~2x{=F_2&dG_%ewq56C;?|!@AR6f7(`7Wcx-;?hK*rx&gKy1y;Iz&D?a)? zBBpC~&zXnaC^$@P3KbbPPa1{65biS9FQ1yY$~B-P^R?)!HkknKq;=72rC4Oi zts6`8Qib?ZCuAHrKZweJj*j7ko-H({L(diNWMnv|UDJ<;jX_!SIk4 z0CbLZB+^cDb6*NVieQJPCHyL;@D_?`1l!=JNH{=FT$*oYfenS`M2;6BfHKF)fTC+C z%AsXYHbh_i-E38KEIGB`>dUD65NYw2#&c^m0VWX9zJr+rH*(B`C)$kGE{#YX3g32P zvtHPC2nE+HWWo7ugEm$3OgDmgQnh8=#QV0PR*Iv&aYt5MN!8%_51s*}jR`syNedG@ ziSl6pGrYl*o-)whptk?5KguJ(&Eo+$WtZP}75Nnx;IT(6dLwb_VK^qgvphZ_RPbEp zz;P}|W!AZ@RwMaLHAg=8kN7%ldIz>zTIO}5Sj1BI%AW}**I>FrJpU2CCKpLAdI zld>BPhXxM8Nd;NZk+;q)0VQ&os$ChTA;N+j?g_rpbRa7(F(Naz*?<$=YKfSUDhXXQ zw!#L$1D-(j2W-P6FqTx5WBnWz)y08coCxadW%VRczurw|(# z@hMe!E})njIT3m<7#}i5!&JmDk^M5z2(T4@4Y3TG%z`F_tk@jV_L4alumnDyasU_) zY|=qTiQn`Q#lwgfqU=>&OOWR7|Kv&B8| zUSSBk(<{0`1CL>H=Q17^a>jYF){f4TOLM|D}W?uqTjg zn3U<5uo{7y>Y(?s>;&n4We;(iEC&IB@9B&slqkcjhxfV{wo?P7H;;A|*!jT9fVvJK z4dsB*L!U#=T>#@(QI!nIIDK0ys9Uzzxig70^8+JIz%MFq0=Gz*uyo6}=-~hg z80#88#iK{PIVGr5>73nt1q&Ew47Jl@NRAn36J(}~U@Ch`sLzU)c=vL;v;Ws0+8wZ& zuwG@qph7k#|Aub)FCJQ}&qa+?-rlEb3QSIa zmT*OzXqu+%kcD#|+g(n>22#|F!KIxFhZoJA?9+>aZPF_cJ0!Lu`NBhL5boHNXW3-R z0scb&5*Yr^KW?Pe#x}0reEtX8C7naQz-C5Ysh*>_V{NH)uRCHZ-98iP_IGN~8XK&< zg!$hn{!;sTc#zg|I=QR%tWg-L|e#Bn&L-|zKcKv}YrTM!sR z^6r_`sSan{Bs86wHY@w?Qgs(aN-hKaz60DCT;2W!R6}#TJecsV}_^pgg z52^00M*=u2~6!X^N-`Zxc z`!RB7WgP{H&IO8F82Jb#{?B$gO6of0_IP|MCS{+x302~UqUv{sWc=ZdVXyb)5G2*B zq`H%^m8u~NWxuIU8}#LuR~F*^iE%-dvh=AT zAOce+xqwEzb`T514c&BAmN&xIsQ_KZQG=-eb#{nSS0z8^b5GM0k=7=)C&`n~7W7Wd)ZXR^XVJPP%Pv#kdYf70*e z(xI~|(JpISmOLIjtR+3YY}L+=?+g|J35jo5qDaN-u9>mLw~4&ihRZAR*%EJrG& zu#ZsC#7WCNZBobD{aQ=2Hx|p#L-ch6I*yJ;lB2Zrk0T$+5aRt>F4xAP6#T*%(Of8U z2C_GG^hBjAbqTGa>YTd4;wEvoX@0>Q&_*^M!<%d-SmzgCO6sx#3_38;K5sDS@_oFXw{C{AyD)R$0sQw8wBjW+0w){ zW69Pwnr;mVKNt+t%p<7-$z%WIH1MN=ybS&qykAoWyiFC@kSA15kFBkg?!eK;hv+Sh z4dh|SrKzMJBM{$~NInxT5I+9ZePe~TLnF{dX2W)Bsc3)76Zfx$10s6+-f&|Kt|B+E zS#JyXTNbfIUqjm-xO}VNmw{FwhFHjTDOR#ZT^ME&jY!6u_qHp@W(kKA6wXK{udc^+ z<8dWg)*>1)UO+lpWwZ}NM5abnh`|NO`j&S+i&xe`eY~zjB6Dv9JvyNe1@aWiG)G)|i(ZiX%v*o{(j5--c##GBHv?jWFRo=s0)8 zO>3s@)<{~d`(VGMbAxD9Fnn3$p(cI7PbM?rtq(-+hIga!@<1XikfuiU1fC?o{ERW7r_Nn?q)})sI6_c_|%gX zv*UU@$*jHYb(9n{+D!YqAfcO{(+Qh1wjow15__I0G{+q18X5t`6M=?J8=*X|X>vhB zD1~n1+wkT~TRl-uZqqf9iJm7PgN1}@o8oN1KzL!_YJfxpvPN*C2tz5APiM~ZZj|mF z^sk-a#20XlET@1d&gz@x`6f^%1kmn$i$SaOqtJ?FJdKQl4N|1C7$Iyv^CUa z)Wc+)=I3TizyYpXOM7E|5bAOY%8!Ik#aGdZieCp?-^de=c16NS?f9q7!mcW#SD*sb zx{RlVpw4t7q~cs{Zw9%{qKdh>1#Q^0#U`~_AJ_*m8_-~1jE`p{vyn!#8I<$(egLlYdQt$ga2~_E+;9+&E2fA-9GN~?9d81oHatje zrE+*^=9FZ5GCR5)9o*f(l1GvA;F1S!Dc%6AX^*`We;yc8<<6upYEX`fFhI{9xN!jkpX)3Z%2KCTj^9upkrBvxERK!xVt!K$APB*ry`y^cCdo~5ZbP6AfR#c8kv1T@K zCughV(e8H^FyCE(HhyqS(alPkfW(Q;ZFf|jh(^|02(ubMYq%J((aE81xq2p1l{#VL zL!g)(Xi|qSCVMJllD`&B6VQl<0S-!Z0=eUnvC`}B@gFd)yer|WK^XQ_klQoSYO-*i z*p;AuH3tsLmL^F5j}9@O3=ap$RI~J(r_FVlYZ~hDX$0BG(t49J2#>fNnLJ@LY9)Pf z$LgF3F7zW_^^^Tc{2V4p^YNt9B>J#ks0h@qa7o@3&^$_f2OAG@38JpOEa7fdw zQ0>6{Ma4NK6Y;9CkvQFId53{Av&;ePT@zcbt82qgCJ9TBn{PyTh-pHI4X7Ni!9Xyz zFk^*RV_b(u=n1kWU%=ck=#osett3N$6*cWNK)s@iRq4#Fn_T%wZw}1v3Nt6@7bwx- zR6|Yf8IYi^J+w~WJudKKIRS}GjHt7bM9Gxa-qBy%K-WNE0~?SdGH?+nKBf}xy-s+D z{q^`5v~o(;?x5L(#N7(%p#X-`N=mT|2#UhgiE>2Ddy22;C*)#6Rk>68E3|Xcr$4oi znuUOjzILNj9pgmvMVwO0HGr^$oZCAb^~Jiw;N169qb}Gk!{oaS2l{bfO>|Y_`yKIh z;Cz!Q)eILxa~`jORn%$xjr0s~DpH!1l1PH{GP^uHqpRrZ13P}gX}`gmz~9vMIDuC- zFi{V`M_@hg;0t2$IPf{hy*(oS6WRFtO%o`D{^RSe85jYZryO(`PHSlCQ zCLA5{)46SpL-9VMGQ9cpUUD%&^Dx8=FD&Z%mk&1m(tL5x^K*meWCU0WV^? z9c)TJ55v#M7CVu%2`*vp)~0TL!_vksx_%Sk#&bU-*ovfZg)(_zXTPy5qi&6`(?SAE zEo|-;;mt|4ey`jyy#KVyrgxcBidhVhL)y+%e(b z$-_fH4WSG*KmC_;eX9M}3?=&a2cZ85etSgp_` zb()D3?%3gzm)w5axx!aDP#n{Ci0gVXW>_TO_V!3v>2O8G`8>aB1g&xFB8T$Y;?~&W zqtUr%;ueWKy~VlisO0R) z3%N987(mMt^iaY%kt)7gj3RKsfMUuxv*sVLa+8D_ihN|l`U>s#_oMZHYWuZAfYr)( zUhwoVe9_j7{ww}?dKRz1|N6#=>Dlou)yG8vs*b{;gI@B7*vS4|*`di7s)2te4?1-| zmaR0)@PK9$&B=~1hh*v2kwxcr%a8e`iqrm{Ex?>?p^!Pc?`G{bfl-KmgEriZRD(pA z(`9%$Z|42sA&=pS#+aG^E=;UM1;;9tWK7D}rA00aE??!+Xr3A}{3ftA7eNw9CY)Q&j^m zD~%%Iz14TK!u{|pMdnvJt5N;jKd@OfbBe1Kl<(THU%unE&Q)aA$)E|+T zQrf2It<|6{Pn!6A=iR?^sVZBnKN&UDZmf%R_pX;1qa%Z~U8sIao>A;`1?k<4HrDRi z^C!@mX?((IK1n_ zgGPzb(Ahyn!Mc4zIS<6=WJ7RKhE3TGlaWQ(9IPfgi=b!H?-;aq(4y3Q z;1#0-+Dc=iJjgAYLlq;D%&B zdsp%GgbCaMbGGccle9g~{$zHLVAN8k-*t=;yzT_&oHLolQoyF0Q-0et=N-ohYoQT` zVX0KG18AY>?$Q${*2A%rFRlfGG^}P4LAdxl?$CVqt~25 z`7Omgn~p6zn+4wi&mJwU)k9$&D9=Mzs^FluzD1{A7Y&?nM0i%vbb<_sHHj`ha3@Zk zE$2NXn}mK70gmYY@+TE;oi&497nFMcX5?_FD6t^6XXEV}#u*{Tcf(c$P~(L{r9;1;oSQMNaxXEv)512w2p~>wfL7%I30`pG^87 z8K$bxeV-&}Kx61{$mu}us7k4K5a?G*0e~i7w*^mExE~hgh)0TRuZ|Y)+44LhhdsEB z`S7#`3ES%;!t(0rzy9nq+K!oD>*}4sVWD8v0?`(Nq4T0cjVQ;f-e=c z$)s7;423S}P#5O4QUM=S5PA8WQWM}4c0^aTv!G0lB-M7@==4an)k{sbd2##Dy+;cH zG4dbHgXwaVAK^R@t3~#vPXLN)c3CZOGP~HLI`DJR|R5}adr8@r|Qdh((j5uHSmqUZ;6={so zu7R4nQOcoE9a$t$AnK#m$teh!DD9xOI%ozdM zF%AxoM3;MsF5Gk*K|KS`RQx*&X;{EWZ$f!5iw=3w-&4mZmJr}|%p||1V}RfXnN2g; zf$y_Xs6SW{^dVy4jkkf57tA(i!l0R;@r*0MN3_s|pDXXE#x$F(^X#0bNSxWs@^}Tn z^p{YZK;ceY8*KZ(0nbCcC?CmUoSgKwZNQDtMIdlpbQ*ij7w!;vgO_|3eEWE7;3OH< zKGVlst}CcHu;dGFGbPyQ?n@l7UpmGOTLB#w{FSA?X9ggX#6Me6hQ-%yS2Hn!OGeqhVuEhJtbc73WzZk(B zAb+aX#@CT_6$&yiu~-(fb%&IyDV}h^Ps?GgrwcOZL*)iI^4WrEH-PILHhLXwD z&vn3WP+CFQ(`B*bT~ifpVGIgll!IYHh@Qxlb2(xNAi6Y%;cz^GH?hg$FX|p_vKTT| zEE!_`iR@)%yfAN~gm;Ej$9}z?2?1?9W@zfiFl$zWlf+TNO56#wCGh4;n66|9%@zYz zHin4J=C$h5?4xF^&H#cr?@j#Drh;B%MyPQz-?9QglT|<=(F=tq3-`-J&Camb3jf)c8jBy7 zMwV&RAnRX*xY;2o1pb&YN(Lfc%NqNv^Cn-Krp;ptgqu_zJ4yt~+L@f2b&If5~( zWQhbGNaEY5weaKs+xH$`?+ zP`NKXuh@@ehDPtyRMEU<-N0a&@UHEz#a{$kt^ob%_I2-i4l~aQ&os_c=hsyu(N;g( z;U!I+2rdZFO|~hzo|J{WUM|QRKJPb?`Jybt!*HvqP+ziTu{M6uB)V3050+3)C_y=O zwx-nU?a99#dFA~?kT-}P+4QDJ&C9guWg@6TRYu34H&xqZjT*=D2*+BBc(FP_Emis?Tnx-3U8db7utKYBf=g2r2%3!+fRT5kzBGSMM+1)PaMx&1L^{AP2~S}Uv4rRNJQt+0t&O@i@p2q{0IXpU%{Y_&!*rzSH{ zmE7C|Q=GABPeGf7qy4ZPMC;6{eeWL6su2vF>U#mB9G4ES+@DC(cEvSjX2z$zN&N6J z&V@uluIj}h7N7%6DMoEq=$5Jdo7pO>cji016r)<4qeB}L88%Hc8*JifLL@U%y_nIG? z<@`^{iiL;0Tb}f58mrK7tQRgRZ{2Afk)cK{vE$ceFLktreaQWN1CG&c&&s2>PpKa7 zSrHf6u~>c5RYxAjU4u>c43?!`cZ9tcChiu26cjAAL$@cxIhEnrPh~FzXQpn0%3YNx zIx<`lUto|oo&DM$Q&)j5CAp>0Bp{;&WUty(9rWVldGaVinKK&XN(B8x+Poo z#r3z=l~TpzH6-Lo-&9xy6fWyY5R&~p(U$$tT8$zq;8f#edf zINs*8eLEu>cM7K_3Up29viuQJH;F@%=D}-k95gmK?0D_6)wOzo&dmo+MQadzqy1{2 zg2#H;tkf0>K11*rQ>fiKMkYYsmgpDylZEmJ;(Q%hex^M4vnv#7RoB772ucYPbRBm5 zN4nd7u(=00wM&ooJi5Wa23)9~r~_Z-*XvaM8HCz%MpXmm$r&C^71u9=o^@`gQfuwd z%*B*PoA9{^ZIZ~uH|GTQEA3*SHYe1?qKO;=n5(b?Cez7N;kT%8c4&tZ3$3-T!A>GW zXDESsJt$@|kPjw%KW?OVGPEXqKi(`HA9A%B7!U`_zXchDY!9UKW&nNGv)TwFF&k zg?$7lW6{Wv%isX$eDnYUMr9w9*Zby!=X9ieNP|egW1=a4)5uGtnH8h4RWCp!693dr z7(zA_6IE$&a9r4bdAT2XV5V?zzIly;#kdGsYVVs9aM$yoZ0I1cYKeuFkLglG5q8r#-8;05p^1j|?x{?q}NsYOBYviJef4t$-tDXKJl6ZoLAITyPz zl;&@jmdT3?A?t+X$aL?uc6Y;9=+SoS0aG_y97h}h5*P?@Hr8U$8u$sGFIT)e%~6|_=RS~-!##ot!Ayz$h{45d4AwG%l-F0 z?yt@Fe1MXVtO(^m-ebjp*!EqOEoFnr`2hy>JB&E@gm&ZGKV8QSh)oC9Eq~!_UjXff z*dBZ8Haw0VDbJosYrD~gX%y_ou9{ycJJ;XRzY(bUUdcVe~tdbfEUK9i?Vd1*6)&3L}& z6^NJ6Nd=^H@;K1&DUruRm`OzNw!(RyN zMA2L4F|L#R%5fu@sJxz-dD$Rlqv>jb1(?lz z%;26*a0gQ9eW&5AACO9MxOap*y2pTJx>`ixw4Dj8(MpMO66@;WUpd}5NW2j|w9wYA zcM4x@1#0#~cRhL>UrPrclbW-Lpme3|uK9A|)Oh}6c@$;4*}_mTFT z_(HUHnQ(ld4$S6B_}by#Vr5$V<+wqZWP*&MUr7c?GY4?@n#A@--x_SyOdt;w!hZ+a ze*-=JyM6#TAsGj_CNJ+Mus#duC28#l7i?`DOM94~Pu$DonPDFXDY zTMkXHxm36bCL2%{vl>E$snR@33Nd_P4%sttmar{i=*q?gayJJUH!^G{>{7yVBSpsc zdy7yJsfl|Wzb9G~KZEv$9G?yA`D97TT+o~NRKaojKq~44CMScbr7)FfeM0yIEH2GZ zgmt(orr)I$%&O9-*GNQfJ_6NbPs%1B1pv)2x(U|~9QT~WPa`(w zz|2?tvWs5v%yykQz6_6;bPces;|0vKsFTGJ>}gZhhou7 zUPb*nDf0gV&*I!pz>;O3PR*uh?cybLIIfmr#^?Fb7BPREX4PIOJ7&{uu&jb=C9V?F z9Zd^URRKZ$gCi^RM6sDaUJa5t?Z8ne&nX!Mp#O45Q1E>Umt=@oZz=) z@EnW|&bWv?x!$cUQ<{A^z3gjvQ|+*^UoGsZ)vWWVM`OlB?GPO4Pux89=J3PwH-&XE zI_In;gcD}Y+H;``mLvC49RcbkzQ@AVPIB_1!}}V@tRJr0{91E#?aGCqbKOmU zU=nAp*8{h`omIBjE7Cx1Uv4kNSnIRjXeobVTe;Hc<$Q?q>ma9Kx5nhh!f8uuV~f}) zsWaaC)Jp%N2=Q0Z7hk_~tznZsKGJA+J z#Lz5AqY_YKw6ck80s?V?X|)PSScCvkimW5D4ahK)_j5mkNzb|d@Bg|=HHyq{d4A9D z{@&l^wz|}04QC$U?zC0kxT|F=n31u0$3OI*0h-%m<7;Q;>EF6ioIW&pyS4(ru5-=` zE#-zEZ6y;kY;eI_gPDXyo?dO;Z)7&%+|4)Joa4JkW~VWS)^J(3xs3Uc>jKRC!48o( zgKc~G;T1Rw5tSP({I4F(hiM-$u`{3Z$o5>BxLs>mBHUMPL0O2;{dAkrULh{c z7Qf)zg=@|x^g0(u!tUk2nq;^*=~ggvx~pcT-v7yJ_bB}4Zl(%cnAU*TY?mydnc0Ku z92YfXgzUEdc?wwVd=?{j4_pf#sJi&ui*uX4ci3~cq|>x_$xYXXuXkUv%?~k@omw!> zb@}f$ZvQP;GV@$oK5d9|DYtKQpKnImU{$Bv9x!QOZKH?Dr&xW{H7Ry##LnLugs9AHS>bYN<<2Mrmbx&8;ekxDL{WFxc z#~Z6bLx~Ao+vuB-mlRO-TZwfMVnHDv#_V289UQy$uj2=^qRr2k8zu3+C9_m|0^30B zC78AoTAt}>Ve6&ymgMbKX4?Y}iuXWeEec{}$=)KOLM_wSZp%N7M(F65H~G{JtE=0= zj=uYfb&dm`7bp0|TceV#Crz_Mo2jf}n+zxL?v88NaIF(%NYwk^HnT5E>=qz(^@MyA zq+scuVIe?b!p)5VbB7=UgC7Oly406SsGNYI5(>%`g8AfUBTU=DCGU$4 z<%*3x$_DCp1dLo4H|yN^{+RgH<6`3x2kTe$$9w~DeJVWFLv);{`)=&0{>ynj!Hx-E z53Jmi3YoIQ-Goo|m92*+WRZR?NsEo$?WSVkAgXE)a3@;|yBZft^Q6bJSP4WIJ8JhD z=#P6dpJ(^g$1robxoW%RsH8-?k=r9Pf6Rp)tq16;4LD`zr8n(gcV2`Nn|}9HP73jw4$)}dU4cTX8)4!T;zNNDH8Na zE7F7O#*V@@==lfscH0P4r$r98(><02hj1MF37!zb?Mqp!d4>~f``B1Hjp>@~6B z(?#J(-7mQxly*Y)lB$*{m6WsN;JBLtR!!47QF=5EW{qmDHW`}l0Aj_aOznNTX8A3Q z_QhGD!Ma{=r)ecp3YFgLBmO2)!wouhFD@QA?U%^3E{BA$m-~hO4Y0B5^nfWMKnPlU!A3C&{Dev9{oa91 zK*>9~6KP&j(F^_6-DzYS)lMuSuhY zt&yl5iS>$@i@KBQn+N}BuM|{<-l^00p~84Y2EpB|7;;oKG#Au7uXBU0+yqj<;uJVI z%5fntMik#3(;$P13zKjgn9~4t3XPg&2BpUMegpe~ds;nREim5L9Ki z#o76_hq#Kb4O6A+R2?4t9GA~jI58*qlA>@GSben`JUH6@9IF+uNe%me&5gC4FDPFs z5EBq7pF@!k;N!c+u%}=SWh1)BVfCZ^Rwzo#UR&cr5aID-&~3n@;ZdUdQ*G~rDf)a- z2`+z8feM)e=oPAZylGv4U5@T>?0PJ!oj&hF-%8|_&{K>b=C=`Z|LYx^8BNW+dK3as zFh{|DM@5fAcLj7d@(b2TwbFqTGXDud+Z3+(h3`*v#+2X`O-@rRz}kK)LKK zn$ym~<*1QGGl$q(29l}4GJL0SdfYBeJQfzt?gp2F-I7V~WjJdJ8;u<^x0qA!n-Ogi zJf<`O`z0T1m($Q+i`;4*t@CflQYBu26vFFJ<#v)#LT;q-`;tSN=$}CYLTSh1zDc9k zVf%m&Y&Zc{3YO-90aa{oWBYJ-lRx1n{#5pDd*94q6)*j!dYMix!i$At?dFoXsWfJE zV)&%9#560IaJljgYphe|j(Tjm_EHHtFFhmd#O*Au63*B@$?0V0{V;*W4;KS1c<=@u z$3AVinoRmH2-p3-#cjwUf_UNDNRdB60~t0?LaIcFuIZ+{*L+(R%!lKjQaFX85$|Wl-;i<3pTzYVa3d%&yKnLhrTm^Yc7S8$` zp@-qMm2#SqjzD4omUf{@lHI>c5R|Gc2D(%$gcbdRRKmf~JlIQPjX`BnbiDRp#u<;Q zyjj~(WsFgr_tsrBf?z}iPi}Ky`no_efxkSP0$j&96n6L74llL8my|k0=z7^Uq^+cF zAxZ-@ZKqD{@@2&)+IdaY7K02|7+8TlG|ztAAh!$-6*d=u(?3uasK`=#10~~$Tad?| zNZ%!C4nKz+g`(R}DNPYx`=%>7*VWwYCF2fesFM`3U-S~P;TPH|YdeY-ByInaZh?Bn zk`k%k52T)!QKl~Ks$oZe&r2bB<{enH&D38guA(&Y@Vk}`UtSr!E zUVBatEX;nq$F|vLmrXfu-`DN!8hHYHP5ZH)vzVl{P}r-@U}(E_zUOn!Qwt%Oh6fRz zg5#3*3MV2~V%fUr(1@lVqV{XT&}hhwphaE8TLc60RABJ%&iUi7?ky9(<=>CaJR`7u z+c3lj;SVWyyq?UqB=2GNF55t!=hGT&@&K;P%MDWk;&;WCA^!=SFyF1<{P}!>ABus%C^oP<)_RY!Fb?WjGpgDe%c47%G4QeeI7#3A9aXbw2loRVqvkI?+CJgr(9(^O*A}&| z`SdxGe$uMe2uZOuLKOal*aTcEc+z>B%f5=6Eza?4@NWNW4v(;|r{81d$0>fk4ED>s zwiSLBl@BceEGu=Jt?*&%H2kAIxx>{p9Xqb66E`T{uhcZeW&m>TS~8R@tgG zT~^!2K9R}XyaqJPElpqPRg;E+-0D5al?b3lUolhvk?kJc1jpm=e>-b~@qF?~(X!%l zj5L%NjadClBv5JT$)zsDK z_fFt*3O|%Q*=e)3;OroP0=;i-+59J7HUBfZ7Yq9_#(*+*wfW)VPl>8Aw~?$Cup`vm#+!REjA)B=W#zp0o22y% zc<@f=E7sK9HeL6x{~CA&UYE&Dcg$H*C{^n09B#6Tl4RiUmWiJemmKCfR#?tbKisi_ zzK$=BF9H8u?F3^Sok)FZ`-iXNO25P&nNjdWQxukWRelDNT5}1~OxM>*t!vw)^eO|NhODZ-+V?Wy>#f(lJwifqL~P2S5#Q*^j&CR2L_Bil;cr2V_&hy-M);LIug&=J>&#DT{OVRj%~wS&}xaP@7Vb8QqJV z?lirmzNXY63xq}e9hIsrZ^4Ek-fQK_o?q=~3a9ogpDPOSB&4?&dq}#}ZO9&Uswr%0 zpY=wkNCS%qktDGQ(E3%h-dvb1vmODbt?mI!yCzqgw3lq3iW-?LY87_mN1_(ph4Q#@ zZ-+uiiEG&Lm(b%Cen}f{TJR-P(L9DHp&q+jIiOXS>^O(Bj+Mi$Zq#`-ZlwFuZRT-KX`XPG00*ViuS+% z_7K@zoe_&M;u^~LYReb?vJaR>#`x@Te_dA>XJIogoal^owZs(TR}L}d6U=m@gi%gT zUezDD(ejnJXkThlRQ;)onqo}4@)^nPrQ}A5Kc&j;mPuoEH5KRo_SQk9SrHwRZ4Fa? zfBB0G_xhKl-K*an60lm5R79hZeivxaZj0*zUUo$R!OA#FOCUO6N-oM|sp9FiJ0ra8 z1(4A-;K4wY%a+K}67I^GI9Kr%4kt7T*F=Rf7N55m+bEUb3K&~lrbNfW*9zdGvXbO5 z#@Yam7$GUyPB#I%GXIB0-z$$O-yirw>6xKo2i;!QN`~wzdAwR1HgI2y_>6=036wx( z_qMO)-iF)7fh?7n0MyTow`DO&+!5zXMlwgvs}nmsCjqgH7hQb-0%aWEs)U?c)Ffmt zCyk+v0YBuw4i~ZRh=VDr>~MC3u&dDZ=hy0`; zmf^ntb)FQu2Cy#hy5f^`->+hhXB8__TAns8k8(HHDqbx?pR(tiPlljD1AGJoVYJqW%?SIY0@zSkCd3xYw7{R-$!UN39_z4X__G5v^$ z#WLihgc9W*qF{IZbamWAws*`g@!ZKkp|;huQ7Vi>JNpuy_t|_RFL<@KKrUb*0t7w}Ji3zL1^NFb^>2gdc5)#@ z1yQp(C^}a7NQHIG@Uq7{-oOzybC_dP&wy~H{4_arXw}vX8M+IS5mY6*dFHp@BGj0Q z{p8$cX6!ByeUE7y?*M^d<6;O|7St{s8!XK-vl%R^x;Sjw^4a!WH|f&IoFDZ;9KF1W z#xPZg^rZ*XC~*9LF^CFXOIbn~#Uo9E&YidMfAX6LU-Ac!O6LPm`2YRIDi(+MeC6HCp~;s`T&b+T`z}1So!* zuEwuDE5kfB_b2;p4%n-0ey?Zpo}%h|@TW)C3&S>)Q>8;f790ZFaH+!2LZK&v>V&2m z6^JIdxa2ji59fF2{vVfGB3FNq6OTpl^6kpokanxhCKRzZA%c?eU^a}ZTU~G6XLZ4s z`u4NJYx2;Ce{FVd)_vJ7Z~kb?xkO4o^K?RE&>f`>Ee^!483mNcP07BH6>FP*Z`St? zdqYC@TsAj!KbB{6zaIYbOHb#FekX9G|Kfyr^HaC@p_`Eh z>XbPXrNE2llAWTQ!)glN3DNH#ePjk=W@LrM+f#O5)^)X?MAs`c=@xS=ezF!9m-M+s zse_yPwtlg9@=;^!WEiwHme7sNYw){nSyD08h}!b1xr6k{ilmOvsMS%A``BW_ZpZ(m z&PV@AzfG%O94ZqB*`UrQPRQPi{7_rMbM=j5<<5_EOUo?vNEbXJr-dGPwXx&HJjT+b zW*B-kLx)os9x_+^w=DtTx*Ic6g`xcf;Fw46UNKixzf{9 zv$+5aZC9U^ZcBQ)dix40w)aA8Uu?6Gx11d18{kuRx3Dy|u*_opBqw6UnR>(JXj4OX z^J)n+yG8w40oxu@^&9?5H__IKUQr2RPEp9fAVdSJ!2T5YO*0D#!8Yhi3d0Cupo;EV! z^B*T#6=juu#c5&l-M;9w8pO~|eeB*92RyJt`nP6EYV28jU%QWP4giw`Tr|L+w`Tf# zAnaq4Iz%I0_BEG7)oHg!xmZj-24ntPXro09m*4o^v%7nBKi}^3N&&-_;evHV(fgd| z&au256|m^qzJD5O3R2#(P0C}&OS?ZUhP$Y)9@?FWACR2 zXt#!!YP?+QT5J`5XwOgLj|E|QeZ@-wd@!gc(*ttfH zL;9629iQWwn>{|w@p~_~DJLl6jB+X9(io*A6K-5|K`Fa$xRyzdl4dN9SV>8b7hu*t zxI3YQex9N^$QVN0v1|6s!-6+cOM3S=o8~Vy=TUMmGNW^(Ln7YrLr7y$Qrd31Gay$@ zFN)tV`o9Z^+GWg)52?EWJIwpjPPc0|<)%cELFb%%#_|xc)1RNFKv?hJ)jlwR2YNXl z8jzWC{0Pksn6NF90NgDpN_$Rp(weDTM~G46lfj3u;;%_wRqIn*<%S4NS7s;=nsc$8 zr|l*7&ke~UOJ2=1F4}I`!`eH3vr9D-g3hyYrvlr}b5ngC0;Rl&2@qgK9Z1b2a@#-V zw%YZE;2lvoGslrhv!i`pY>JvI$q}^Zyh?XmeJkHHPkfn``((0-(G*z!%J>8OiOHWb z=R>Nx+N&E<;Y!i%U@Pq~$Xm_bK&4fIXNgAL^Ta`_@pF`W0uE%*QcHiOh`5FDXjVWS z9je^zmN{2?eFHlRTVsAG%p6LnYqZ2NIEgAY>Q6j}&t{f_sYOo23AhtE4GlBgJ|V18 zfQ%%xdPszdXpL#)Q#(uE)7P1tx@G`&i*~jkQ^KiI?Ro^5*QyroqT6Po-O)#i-Xm_(y0ycPUMVT=1CKtFtftP&m? z+9UCU?m@ovHfop|HOIP2pYDF{jtrHejhK>obUlhwXG3q3H>Gi_*I6w2emA5kLH)z@ z*?e)$F$08HQ7({ZCE7Bkwb7W7G%AGNMnGcP;*-ogg+hIr`*wFXPaAwqq95mUAd?NZ z)jz~C$|1bu zrrGls*giLZSaoja6J<#od-KQuton4|CBy}Wx&8F0tkE*-Y?R}pPnM=9K5$&7r)<3r zT>rscDC%;V6Qp201&P!*f+Gcq$+}?u(AXdK7*ll7e+gqUQ?ZH=~~ zImb-^Z)CNu_0F!z$fovB*RcVZ?(7;lm&LGK)SX{N^OM;VVVj?(mqx3LA-eo+t<5)x zR9s|($f?Zj+8{yHhWq{3_K=E9nTA?kD!RFL38&EVc0?-I;2%3d5CZaQ-SSTXU9hw9 zrA*2~MxlLWJxi+r6j&w|s4m3pCWvK!g!OGy?%^-0w5wrMX1OuTPg7b zT|UO5h4WObS+7K86HnW$B$`4(#CbA#RB;!Q5+uB8+G0v^6I2$po8AtI?PaB4WCWf} zspLC{b5Vs;@;UWYHr2B0-!@7i{wpb_1y*yRWU!Bv%JMxbHOUUK;XG%Zb7X4EO5c~h z77(*v(Tg%Z6fX38hfk%_ppW*}bfLb58zg^fn}~ve5^p6F2QHke`W_GYRJe;Wo&oy1 zSCg;5Da{78tiH*bi%!5o4fBn(BCh;Z8Vp#d1$vrxcf`NWPoPGZ>x`u#tUi;*5M~rP z41ia3`{EPLR^2PxoW-rXQua{Eh~n4JxW(*-v8Q>yNw)(AtCVNEi(=UoE}+8hL#y<; zcr$B6*m~Tmvufmr01q8h*IQPdRtb$HRDs&Mj0pv~EI3Ibn@SF%t%Bo{3y0mG&*2t0 zaV>ztcB{K6P6bVj>sd8%`0 zpf=K-sN_g16lYYb#QrZ4X8BmbuAYlgSEQlkp!P&=f@W(iK zaa1vU!!URPQ%UbrGI^CPL9W^>+{R7?E0Jq*mny=UaD`_4=&SZiG3FtaKdsgq-3I>9 z@eyTV5>S(rr`HRkgp#GpkUp0(k=T z_!{wp>iWT-j$l~~BRDZrFvAB(3PAohR42ceJr+ebW?Bi^7U4ef66CwUNjk6M3{YFK zU0RKP-?^|gPP!-zEwavL)-wOR8_JSv)^LA|Tq+8e6!{n-^-(@^Xf|I1^X$YGo3s*j znIEm%k^H|B@Up?%_dj+E)#j2_L+|b1c+4LpyiNZLt4q@*-S)eI5ZljnBfs^#>C_vw zp)}F?q|GNJIIeBmnk_o_pn-JKqQwyt$;wp;vm=0umcCCx+ ze{cRub2}p4tdw^(ED`R$G>Hbs~Q}O%oTY`G!u2r?T7apEBMvhXsbHk zpU^s(S@MS)1yAbF7{0oxKUSZEeD%ETr&r(^C8xwy^LWcx*h^d2g`u`OaEc+Sgzwc$8feHTT03U8ZwYU&<~hGc?tlyY|nBJYAag z*hRC^R^RTz`!F`l0k{^l{XKs_soxujL)cNLKle)bb8GV(YLjH}X5txUuw>1g7{Sa~ zV8NC73#|KYujmzl_Hq9kAu|tue5~%K&*PiEeP!!yzmsm(G{mFk+BV`l8mbMMGqjgH zrfV-Hd0X>re&mbhTqE4*T|;?cb0u^A{ORkW;3*s$T-lX}@hTlUB8(E%k35v!RCkChI)v zwK|6_{cnf*FBQ?HJe0>RPx$%zINc9R(Y9v~4ZRY+YU?Rbq*<5YAC&#QLtqMWz{wN* z2RH2~+y@M<^VpkwdtQ6VKzBk<*Z57Fx7wa9!yDT7va8Eon4RE+uF+}Mv|yh?bJ3nA z$Mn6}!8%i64U+WfJb+LX2Y%OW?D&kL_N&l@b%4NRC#Ut!x|)OB?<(yzu>rpvDg#KZ zGYuD)nYB~q)XZ~orkix>?bvbD|Laqj?TqIaXpvVIcKi8Fn|0;*%G2)-E!(*~Kbl_a z);0I85AIVkD#FgbOHVP_{37cHNh6*&&$N~~QXabQX7iy@T|c=y9h5dNF*>W7F|Euo zbwjMptI6136Irr^<8S`n;i;-GcJ^0fx(?BG`eWv~f>ho46e6r+tKwU9o4N2Q5g;CT zc+W*0w{pi%$ZW12tcY4xG898;gtaoQt~7LP-x_j*L-wcg(^2|v&M&o42mapIb)hBxqnmJIYKk;`^ByRfU1B~m8@m*+Q=9^E&%NtIA3Zpij+?`zvp-`zE=>O;k-hrUgQ-K|wudo`8p>tNQt(>H zT)0{YVerMuFbq_E91Eb+K{9+@3cIY7;kFSK2CK%GxR$Oyb6d-F$68%^B3%vON^DKr z`~^F36}wC!NeW$ixOFd2RF^u?EYrg?;XBd06wT%boKpj6_QGRtONk~LaT?WN_Hf^MS+3i z6~C-eNzh)I|KbIO9TY-N6FMugbVq~tVmbcLI80K~^t2Jw^j6CFgb_#h*lH`)haT?w zibJXl5`%el8n7hF_tJL9DMD70KfX1&70fyZlxr$gpvs&pw51w5bS)Ktn6qR8M4$E7 z9~7|6QP;n+Wlvq4=XqlM8I>qptl02&kS+_wAr?&va4~U0DFk^Nj`qC{3jK zXOf-LtPIyo)UnC2_@iq|D^j(!#GONx3*|F7cH@RhLwDmk9`@kbZyzf*XUzO0WX7q7 zi%xuQyLv3Kt-r8mMB3Q3H?FJlx}nj%Z_`?%$-g>$vd9MOq;0Waii0!wn5mpvBb)NT zJvAbV%cAQxiO4^7#;FkjmnsP(vxE|>>=F4XWl6?#L-||?i;do1l?xK&N;2Cj!?Lcz z@t3WdpAKovwW-;FZ)1LD_`YXj-Ae|3yMz~R@S<$qyRlJ+S`busbv-p)lM-kn5jS3C z?lK+;e==3N41IhAcqu8M4R1YQZMCcx^Z zWgWu%;WmMtY84Tl;7)-NO5X@s%oz+;1_LKr7|F^kTV5^|VN8^d#AM;sEl!Xx&X z5JC1Y7FcG0^)b037}WU`goE(*oTW=FmvPL$$YxDzsa!<7qP{R!1FPdPoMDaU2Jy*(>f!ASv1vb5mC zg`$xvT;RmYqBwarAJd~(tG%!h)2meK6%>L{z3Cut8 znH(ZM_G9&LN!bgz)me|%U8SGi1RA83TzJ_d5iWnvqEsN;8LM#d=`S49r!zEZdw z_Wf#%jFIlp^HL`vLC%v`QIX@KwQTRbJm^j6CQxd~mSIn;$;+EwNMSaRJGNdj~$ZsqU1@8-G9==6o$4OJU*cYVkq+31pvw1W^?r zASNpx`=g%#O?BJVRaM)!on3u^m43=HfAISzmd^=CVMH-`VC)pjsxX=I=pkwuIakg| zHW$uH5T`=1@|)p@hT=eky>>J&dkzSkptZur2AzGW%W_X;tG+_B2+XNoqzdwy)P4C8 z>XMAiQwq(ibi@Lye?Y2fhAuBXr z(SZ&|+S^?iU!C91P;JT#=9ftqz&QPYsgO7?e^Y2beft6O@nZ6o z;|N?VvtNJML%Drov1jvu1ePs?CfF;p&WYbNH2GcoQ?7vd12~SMU82De$_U3EQmMKz zeh6d&hn~Mz-8C5ySc+PuAh1gqz1!<$#D`1&*uFigXZ0__dM00D0KsveGJWv1f4*6z zHSX4))kYK_4d#s>-`lsUVbrLYx)a8CLys7sO^qjS2B)zH^M-tx zUU9UjO*O?Ll&zTFnN(2DCcnV7Y0JF$tfuPtWY@8GlV@FB_IXFR8T7b@b%~1L=Vy=Qq)m)p{K+K8_xecA`@)vz^nSR`T)bNtWNqfZY`>_~ zoz<1P)@Xmx?(FjWOuUt#r9Yh5=d#9j#dctqCF4TGZ&?%Q;ddb786 z$$NcK6J1m9??AKqa}nbDCb!I~N#oi2NyUNYp?3_8x04LP4c|Mgqr46Fe?we#>-w5J zD`0c4QWRBN58J+NdEDF{xn5zTnnhit-A&(BL3@sU`FR_j@RlYW9)f>PlL|--| z3qUiFWILkI>uA`JvdeE3tEuGLHkT{aMk2Zc1-y}>1?b0oo4Y&kXuAe}x=Ls6_pv1( zZ=f(d>GjI^wZ$I?>XYhLSKrhO3^GEwW$;e4?VSj4{3@GE*RRS3ShxH8B0}~@KGs$F zuH)sziBZAPXAQ@EYvjWmwZG|~zk8S>R9sU~9usq$I~XMs=+m5##lTyU|$xzS(MMdGXFU z{bc-yHwRK0|8PDnKQm=((dF^|krOTJl4T6lpnJIE`);E?SE`BV2=7~r#KlXtg-xV) zDe)p)~ruzKz0l`2}2HWBA%zF z6m99pUj2sFcyhK2+ixIvL-&rdYfwa0F)Z)smw3rxxL-y8VA0!VS}%91I%fHs_L&T? zC(J~Oe5E9vTqySm*pI2mgS`FLRuKc~)a%4mg^10a#|; zbau@24U`$1{*qmiCcCqy_iOF&L>5f^<^_*7bkHX_$Gu@-iK)5PP#O~ayb6+H?C6Q6 zabcKU?M5Tdip(gdv$ua7ego~>icN4>c70bt)RfFK^H%Cb<@p!vv;JmsA z_Q8}C3=Or0vX)~#rb5aB(L0pVhEerm0gf+qng30mD>6u$`T}~H6 z!i7mhk3OoyU^@+~e|)LMsixpkgr0rj+O?ts%;q5fYomp|dx$YYWQD*7WerY>9g3jw zqf3LYmg8L;owe#_P2Xwa<6`|=o>#Kr*r`%Vj()4K8fLcFV^5r?igQ92>@ZDH34jHm zBb~pqb4X3R1kHq%yY~jOFip%$+dt0`QFD zfXVu=WB{)WQwnPrKq}WyUPJbW9jFQ-)M8;N5bI@V@-Ee?+b_V^bLzJ$TmWUYVv0eu zKDpmEkGw*L(yL(6~@~4B~cn4uvXyt4R6>_T{j|My9~SXv)N8`(S}XoKp_>*@4R!5yQJTgaY%D4s^s z!-XuC7#*Pz_h0?q;a|m5G+m|;7T^N-3;H9E{@qYcgW}kJ(WjO^vYU5%)fkm-E(@%; zMy`;$5a_HMZM(FkEQ8d6t~~uS+_#0+veo!hJ{!}Gn=4k5e5|L@l{Mo0V`Zd7*pA4J4# zB^_$dYdpV+&J_*mvIU&|7B11(2lCZ!hrHH)lTRg_2k5Pr%Ur}m(G`N#(LU# z`<+{9I`_a5;w}SnQ+8jNP=q&@Q0%V4y*9Q#7x(yE^T>z$Zgsr7A^E$K7Y9yf`TaU4 zyE3V;${M}D=WyQD_N)G9thsq3^BCbilHglE?M)wjUEBdIwnIa&tsOCaX;5Ijdo;$M zdS9Pa^$q&NCU@~u+2%(NVqvxBmh2A>xH)^ea79~Q*ENv|8jS4UmVY-`xa&dB+C8+T z^=FYzf4;BcW8*-B^%?6tx`Ov!Ok1-9T4zC!;dlj;Uk6Kw{LQiCJrfwlRcT!CGp+Fy zkJHhb+M91M^-+H-|4&L{TavZjunD~~1QXOxLh05C6S6OFxxFlC=JFu;7Vkz|^|Zbf z9rIkw+ni%#>tn8u|MZk)(?`+unJEP`+qJ(gA03sneqgqBMRohf>_yO*_4`Z4 z#a126W{}$Q%zVeVzCYhgJF)$e<+pyjP5Ue_UP2CYFh*D0(6M~&je5(4_^wW$J=K;; zse_c&Jn5Q)S!P{N`@+hJe~Jb&x+I2pPgpS&&LZi*+0wO+*bn!i$JOAU`l>}8$p=Q} z!NAw)fzG~(I*;=OmD)1BdBWZ4Gb1JiEVDRO2gh>W#WSY-Z1$74Z2#cgweD+0<(XL5 z*59zfGKw9PqI7rJw+K7gXJ#tV{9+9kGrnA2r%h&ifvS)U<~F8MJ*ScpR@yC8Brxy2 z!utjJ5nFNre5Y>w$a$$}>i0KwPPNzjqkUeU(9#!GIDS>Vw^h{H&Rz|EsDv$}whq3W zX7YU$`|XJy^Py&~p^e!!fljoE2)`l&smlXKy=tam;ESN_s@dOc-?zl_E0+GD>U&E* znD2VLWiAiLAx+I@9bW34FBa>Do;;|1k~p(`ZUv1c+>Xt)PP6>FyNwnPBFT(ajVVT? zt)Vw|UFuMDQYciPr7~S|cRUxF=S{40Gu1Qc8ztLw8fagGil( zf35y2?F+76C{qkf5#BDp>OEU9)UN9w?RCoBgT?#C9n6eo-RgRw{IQZ|>tkgUK^@id z2v_N2s5pt_$EFh#2Bn>{`$Hq*`p63N2UvQBLI$nK+$ld<8 z^&NjZ(gd!t3Q<&UzPQl6Y5njP^M*mXOg`P0qWz4;{Jf(2mf(Y-t%kAnRqO-j!{-CRo9gH z9e>z1-B*O_q$u&}&5DXX8vC_u4c<}_J>8eRL;OH?kK3fUF$!rc<>FuJqM_aZ^r@R=SIc$%(B)GM&b7;4T zS>1PY$b9&{%ab%Be|y0YlwEErY@#5&TCs8G^2(6&OT)I@TCM5bq^+_}Nbfy7kML4N zHZXCtf0($%?v(*+p!5i7bod~tr6Lq!1(3fY6Li98AWt zB_N<|dzkvqIn>_Y8bjR*ER`-3ydJFu7%Q`X?=(_q-!bR9*`WujfrlVxFtR{(ahXNv zjDWYhQ#7Ff+XvOA#1`AfoC+-)b=7-X3ovl5qBTL;fPpbjM7;^oNZapk8^?%FkUZ(% z%R}sDXE?U(0rYHY5m3vQ9VURa7LIKht|j4_)8%he$fsSwI9&YJN?Vc&Q>sk$H65(C zzx~G7RNHUa|4b@ZN@+W*$?-UELqK+Nt`LKOKs6>?rU?mxVFINgi}h2iqfy{4nZg^C zf7gDIMEodwi9qWr>m0-JGJEaLJ~lUcfITOAD)MOMptKIB7N@xzhX!s=aGR1rv3XLL zF6Pc!ng8bNna*Pk+gT#brYJz(XOfBIXK2OG$rYcPqY218S0HtBx4Km_qi85!UugQq zkYG);eKGyCQoYlnML{#X*($SN`)Xx}^z;06Da!}4T(P(Laf-PW-ctGB|0;bHxv$ZZ zsjCdP5VBK^gCR@`#JNP6*vQSI)go91Trc_0-Tg4E@OdWtL<)N?LP#X$ls%6 zN7fF20@Z+y6mWs$K=ZL=FO1i0B-BC!?D(KcxWrcgwR`)2IP@GuhRPp zS{CWW1640}>*0=&ZJ9m%m?1YZ&c;3jnx|(f50oT3iEVt&TA9bLU@i}lqR*S>3Z*Q% zhf!jmlD?8JOPic0!LnRVR6kZ(0mH;J4;=gGHni5vAW+a#^$lb*fd~H)y#9}d09!g4 zQ4gNI2ft2wXc^m;Sc=&uA2gxtzbfl4(bGqHTqh~~y9Ckx@VBX}fxK8X*@W-K$J8wy zDjrUc>7tYftTmGo0`hi4aX~C z9|A1xz7b0Az~5;qqYBU1_dWHoY9X)absMlIJj_INF1cKyDJpmB3R2|ZeFrjL8En~- z)V8HkuYbV4_VA@(Ha*vb_SN=#&e)tEGQwn=8~%d9U~3$90Ixnc-Hq{}&u-UsgrBTU zLKWGT|AM`-)fKq6W9N>9Z-YXweQ?Rg{NMll)5Y5*9}k5bZ_s3TYN*Hm&YSV6#A@xY zw`*-cGmM|pgRj~!^PC>|=haaG{TGq0;Y8!(6Tl$@;p96P> zE{m90+t3?6IzR_Qj3()mDigwW-rBA86RQd?2DcoTY+K~1*_1kY;4nO7sBI4q1I)bB{XJjgzoOS z^w@OUCDLxgOS;~OHqToffr=64Aq&kWBO@tF+eODI!E!HuBiqKacqA(d|fdrY9H$twKm<1w!-&1by?F5 z?fi%9NB`8-FDJoN@S8qZ_939)eS5~!|WAg2t1dWuzk~LZ8y03 zQefyi0lpYUs!im}@@NupQ*1Gz6D>&DTC;p?1Kssg_r3_3?ix!^FfTd!KY}&)KH3n! zIbXLvJ161D3v;9Y6K_^LkK*zaon`G{r!Bv|5;mAbBj;%Su?GD~ll#Azr?PWR!100N z#*2k(?(8;RS{r%xl80qLaa#KsQnTSIh=I1eOx6@1p@ zxD;%h88cyd(<)YlbM|;P!qsz2@`PzxI0=X2M2fYV+V>Ks~vuGIuGdGC|K zk`Bt^F0*&Z`L5sPSQEDdM6BDi&>2QU@9IL>&CxK?x<)UY$OuuSccCi5?iKlMNikevKPuF42o!f6#P1^w944aimYFT9t(<2g^2hZrEZHCBlEKpKz@J@3| zoG@W|+(uHv&gNK_2h?XVhvo)s`MU?At{9=-%)qLZi6f8;)iu{Xm!Gor!xGCIn`mYQ zxC^~2YbGVe4|~~{b=Gu_f77u}qMzrV0v^18-ID5cxCKBkRh_$?Vk%!Ek;S17o)za` zpT&&Nk#(bMP0kJ7Lg|MdO(gWoCsB*~O3+wrtA0 zC*8x_S5LB6;hOrfD7al@d-J9Wpx>HnZRWb%RrMLNtVAj_4OpS@g!64#qrXLpHfMR+N;$y-v5 ztrT&PU3iQ%)zdN#DX9BeDToq!w){>%F6O9to+hm?C&@y_Y|8Fy&oKOjSqSTG`$BI- z_m0H-bR}1LQ{y{^n;}SVeQ#2yH)&eUF@DH=5}HSO3wee;>Tg=3-n5Ji^B7pBatQ6Z zzNV_7?UnjfmNjK?Y@&TYKCACAyp-y0t1_^g9rh-=^3U?7fB=vInp*LY2{${3(tp^l zCN-llY;w!E=DqQl;`c0Lgxwn6Z&vKXSdD`GkRI7Xf)@T#m>&<(L1U6~jt{X+mnk`H z@5cC{K_TrVjnP}_xJWG`PfFF3NLg=&dehEi4=>$_HYmOQNHgq#R;g-1aEuC^MrSWQ zp6ya-J(wazE+A)RcupETt?0ENE7c$!U&o<8boAyKdNsZD0d#VV&R;xf4-}`l8NDHu z5lHj^j?!z~9%A!P4M`Td|AXBs&?WRMqhQIqq*(8?r+Dt4y2K41P~0m|Q@mFsPLBkLkO+~%)giOin) zHM7Y^Ub5}+mX28LRFZk#an^Q#*%H{cv6FX8wExp-MJm>1MaWrrWtryT4$sdGAv{ zS{VUtMoYkJ*`*!9)moQ5ox!Oe$0P~{!nP=6&K|jarF6|F4~454J;CrZYR06?t&9!W~d(MNfjXp zToj86D~_h*ke|dC<`i~TV64=X6m^QcoT_BU9(q;%aqJjE$Qx2855tL8Za7Cs?(Q%N%8`!1 z)A=BL(kpfjS^t61K>YO4%1v^ReZ?&sjL^(E!cx33;;>g@! zw5$4&Qppow_eVe9)s&3x`#6+v!H>+LQTS1AQeD#+u^w{IA?Tt9(-u1nBSA$s?^3iQ zTY%^Z#K~coxGDj4bS_Z03&RPLd~Tlu-}I8*FdyCPlqsrMeVG|pdU6A&p-_cNrJ}Vm zX5v46DW&PGxY>#AV@dAvB6?UQLzgi_XKa96!*EeA+GqPplfd0T(J5$+8wHz=LHCT% zWp;v#GveBzs#z<5TRJTJgWnuIx88b)OZk2ja!*I8y;!AssIYW^=*;t^BOg0a2YgWoA8g9Z;+lrc&$@ zYAS6cVGxoPC;KeJE)_~-Sr;|gg&HyfF373ZO{7zC!_W5j4)2Wd=7l#uh_z}2t~%(c zOUQy|vXEz~%XV4)c9pK%6kSW8hAm~u@s6;n&$yN3JxCCigz!p^t2fx&olIhh(I&|7sQ3;dxsQ|j|VyTIbT0ua&wW} zOYQ-SowgHAi%v55|80fK)2NW0z!-^p75g}V5~Zn(Q)sGJ_(q)Q3QgVfyUY5OK1g;$dK@ zzViVzROZq$^|zw<{ZRz0fFnO)+av1ws8y=ga|=okJQ!1p7AXMguA{e5P^#L*Q7bHO z)2{=|iZBEfgi^|~$l3lmVR`Jn-p}Z->$0ila{j|$Z-lsUmBLtfP)=OJsATU_U+~g0 zFwJg@Ck2kDhs`PYO?$V=b5UiM+JpWL@cr=L3Rki|xXL#eg+bSE?r6j9klh_Smj+wm zVBE&`=q|gBY6u!Up?_z6dB?MbEfu%FcbHf(kxHOwd(Oe}mKi_4ZcB_x>&ZXy-PsC~ zpm5Pa%w31O%a|21s`{q&SaD+Ov+*s3)nQOI!Tik7u4}8Tc_Tl_iN4qGWiwXj&U9`x zG`8hBJ>g|dUv{p5Br_j!PiIOoa=Dv|XL@kJrJAqL-!OIlZPIh^7QQ=wDakAKh_-w+ z|G85W5%kQu0I79)Ek2j8NpSSve6FtB7fZOACLvH{FVmCAnEeJF(^Jgz2kS9KH?`3`PPslWC*LEw- z-&NBgU&oin8}rUu##?|k(ff^-bssS=<*@lsDcZ88hN1Hn64auF5^YWpd}YSHn}!rh z4ZH~O3;A;3lVYJQi}Qpay;(!T&#DIfSM|Gpw)B~ut1EXtmDpA@tB>HIy~k~z{3LY_ zH5CIj+FhN}t5$Zcy$(C&Qx=jKE!+5tiwCaVOI!AOM%O#A8Kc6NTx$4vOXR#<+vR5~ z8$vpHUS|CZx`9e9VDX7~^Mb0{q$I-)*AH7S-^y4J;Bjx#N73e}t^jkqH8WM$l}~4W z+qlTzGkpf4E~M~=Yc!KDo7Fd20x=u6Zx4!P zj7iDOQwx3@aKYvGRWl<(&OrHoRi8NYeC4#AOHOPF*u?H+$vE8~KI>p~@Rs<2{?q2z zgL92F+M-2wo_b?Pp>vp__pWQgjg7YI?Wb+OX5wGv##0c%)qN|@RLwC>|L7tCI!q)_ zjl5T9miWo_I{ceVX)A}}}Buo=VH z)rned9U@08RakpZIDVNmbGo*{9Ky^|$Wp*9eVW@z`ZJ^cWSwcBU~~D4ap$V0W#|R~ zwnjo9t*3KI=dz^F|AJGmh+^bG>X}1`uIcEvC$BR9B6sHe$&I77UKot8%?R#H|CY31 zB6C1fqdRYpuCoV{6mQ+rP#sLrl||#77HaMwYk0MxJdQbt zWKOosoMF4L-B7;4xUZlt(90ZJD(SSPFg2C;*bozV3ckb}>Sg_eK+N`Y=TUYajBjb^ zYE%NpqwWVLL%T;BOm}&f8cS?CGC8FgHG2089Sq)hZFuWQyJdR9Z^J>Lom~FXUVZp? zLD|?@x0cMDihGR&ium#6>llsO5D)_-RWD`wQF$FukekD5zH3}_`;yOi(TbT_Fz79GF&L~+i#+lhz6jf}h5 zTu>LGS0=j4A6>8N@OZHq`r4Xm`&|sL6{r#XqvLGzgHLSUCT}I-8`cLdZVH{gYx+z> zmb<}O|87Y~-`X49X_@{5gF}m-Ir6ndGqf|%rgeQ`DGj`r$kH%u98T_OnRAVf9d}fh z#e5T_kkZYOTszPb-MZ97Wb%?Zn>a2;g1AJa<)r)hRZRe>h49za6S6-f>&%VB=h&3p zCDVwz=dC0)M5f@g#g1;3&tFgSy$Pv^5>hfd7M1jUs^tu(rr+&oUyGMH(It2EpS0bd zGD5K;Q2L15$amBlZP3v#lQ*k;ipfqB2R?m46PR{3EurUnWT$mBOBJlUqIaHOCh8Y^ zc8R#~X3A^`!VK1yUl^2_Ef}w9* zUiI}J3|3VHOGkwNKo^*WAc_&s)5xO3K5K1x$LyW2Y*oq8B3+NublpGgqb`~~UIx65R-BGQRbxj78q!X{! zw9M7UT5}?JGx~)_(~aN&mso$3w0*A%h&yR@c|c=^3@&FdoMmj4dHm@?PmI`)CK>)7 zXq^WMv^2fo{H$-hy$u01*4)H_mrH7`8f4|~&Y$(Y!%z32o;2K;-ZDo}=D>=34IPHA z`T5-DsMCE;cE&$Ft?Zfzm{TV-&)@jsuf*sn4mmqH03V6=OGLL|(07ft_vtgNIo+#@ zTacZdzEfBfz$&~~;!k;yg%blB2=^x)ZiSYFW0dB}=^{yE21hd7gwRmIpimR>t^6BN zqe@XJ(H+M-{8>DG{2hY1M?Q)-_emYav8__m=WG!8058K^P_Ejot0Moi%P*$S4zL$*11Gy0+HSdVSNW^IPOTvh~9oT~e|da6R_Ia8&f z$dlC{NPP|Qd6Vl7XoTZRLOuZMqnqyiSU z-{ybn;X+Ol+M(Sxzp$(YHlz@%Cwqo2d91x7=-^2!oos{nsb7d$4fmSE$}saptsUhV zS_U1a=z1E%fmQK_EK2k^GD-E(*+VnrSVPqV_X@<)acI zqksy#$5q%fN$G(=^*6;9_AmuUW<`pY4_;GOes|dE;E=}Pkv}m^R#`Pw$no@Da!FCb zT9#ApOAfG4;+sO#AiV}DrJP8+E(<{Q-@bZC1;arHD&8znbKmWdqNAj_!px;;6Y(y+ z`m=-dEM>$Loxk^J508=T-$$Yuoa1Sk)SpQY(cGo3K%fU5cIxSGgoQgj)OXgh{F=bs zVI8}+n)l*;3>|U+GKI^3;f`CVtG$VZdQ2p+zwC_ABb=qRwixBgu4#Nug`TOwc4(ir z@TbDZ>N3>2lNO8uOUhhezx%9sq~LP?I8HGYt4`$9Lfe0U!llv%6jCD7y1>dxSpQ_2l zOMtW~?LWu@7ey(LV6LXO&hH14xW52=t|1ue(6 z6{#OFytDk5K<~#s5fQ7>(A!T{wh9ShkS3AvtHU7Pj?4G}luL5v_CC;$T%8=2U4kV0 zm&8=?H2vFj8HtIrix&h*4ma0Wj*_aRe%Alv*Wtt^o;^7*nRW`K6bLG|72cxf&KJ>_ z5{&>O%>9&?`nWQM%h^oLZ>}zsqd-D&UfOT#%@d~PZ+!;?)iWn#E1Kb6x{cNan;eMS zwnzS-?T#zQ_X5vBwyZ+=zSkb$+yC*a7*>ZzXboQoj#bEXGrt6eswzsVO;wc6Dzc}>uk~P zbboKnxCFuYF_Xv{M=@mQ|w>SbB)Z$ zslHp-THlsP)@q9O+Lqu&bvdmruJ`H=+=&Ho`uSr#10_l4`TsKZ=5bNp=lVaI8hT6& zr>RP+hB-Y=n$(Tv#3gQwCdb%lqNWY2ELCbiL5TrG(P5HUV}+P$o2ZQ&E-@MwaRfwA zAQ6!ssj?UrAxM=X>xc*pIKxc7@B8y$PW$`)zOUEs4-$l7=6OEPXSuKYy6)?acm2Kd z8D&HjegyV-3MUMMA?sVb?&qmzCPe_9{^_V(i~5p_7N)=!B? zYL9GA{3ymPCA`TpmgUDz`FMJ+Db=44QH0QG5Dj^w0?InW_O&QSSBypjcdVM>b;BSE z!6Sa~315dXNM2Q?ZC)rv4Zr>t+*DFnKj~a0Bw>X6TR-|kZS$c28i?v9PR&uygNeP; zqQzGB77aw{n^S`Qvs)?~0g2uD_x>g|vo|6{)t_Q}Bi!(@?SAf+7yE*Amb2MqbrHHY zoOWxY!(#oMKfPz%9%)(E6`4_ReM4+mkW0=aK3MuR#(4a&dan1|#?ze{rxMDzw%skr zT+hVouU{H*IzIkj!<-5FOOy2Gj+(Lhl?cs=hU4jB*{OdrC@{nycdg9^ z$#zD(Q)+%TUP~y)PBHyz87)s7C#4L_DIFz5LY1}7Yr>|RJr}XxK0rS5SyJfz94mP* z!CbwYW{QLzoLG+2W89_@zkG7Uc10g99^YKP0h=(Tho*~bgOR=Z@>AtQTooh?} zuyv+0iP?7}ZOa?0><$n6Lh~`7?&kiwg0EQ{Ab_%lyk8Mr_o#o z-mPY4)K4sK_>96qH|OHzhsGGxd(Msmw}C^r7Dd1asPynCc5Fae6@mkZ#(MTpO?dsG#At;W5fK@VV}8 z^Se{lxe=oW>%LZMJe~85O*Ykn;`+UGX^fSf{PMSt+plCW!E`gebLr^{poK}WV6^ne z5wH%BnDB}&wyYa3P^GaHGnn!F|HVTL_8~o(>m=G*tmB34=m*x{QrkDRV^u{-k z*agaz%)m#(A+Tkc&E~@ZHg>I~Hm!B{oK=KZ+S;4X6jVcVS=+ixth)O<=*PmOt%kaRheP*U0p zk7y60FTx0;CCPq86=!HAw&ycBv!{3!EwYW_0$&=_rEj~UqBCWI!GVoEMIeP$|COdV z@hnA{Kp`VsJfeBy0tANA=p6nX{rB*3ky`3^DVo#d#LusIUt?z1%!m<_+_O@kYn_@< z78LQ@Nm#SG(To>!+oU<;%Pb>*3sI?F?RVZjO6n!S9#Z>TUK!b$Q0m-oux^RZkcpqa zYxlcTuDx5R3CxcwD=^m8L2@BxXq;tPhBJ0&NM!iYYq`)fPOW^|4}zVz2j_!Y$DW%d zq(xp92GfNJyJGADOw&eS=;<3AS@Ao*E*nLE=$W^SmQq~p^G+GR z_Mig1ramL0a*FLO9UE(py5V$F%BG zDYfKf;^(0}gwW{;e1ijrdlKlmN(nWkgf)4hi<0rBlBk}5hw?(A zR4OGBZU`D>QY%Jdh*Ks&VvC3PLH;{J!4hU|C|d?uZ9&e>quDY}kR5>Hd5(zvVSOI( z^a@x!1YVTO!(5k8>`_UENSj}Zp;a4E2nki>l!0fw!j(dXYJ3e(h!_kgG!#v84UHG& z*TQTfLm&`S3+TRuYDkgcj^<0k;kEgV_zMlgB1j9r#YTq6?fdxu4dke^WkrY}p73|1 zuv=P#Qw&W|M7fN{^!l89j=*Cj!jxRudn12Lm5>Cts&lUa7swNolCQgMOh( z7s)M(Sqd{l#pp5;uuVg?nRzs>DCv_3Sp8hCB$Id6XhP}O2vQ*6H7)iO&=FKnWP4K* zilbe~+X-8$CT$Sm2c4$^Q)s<*quw$mM8(Mv?I=T7CIU#6Jyfz|mRJ-Ii8UmSR-Z$# zRX!|ik#mIM5JR2no_xOxo=G)pL1PT|#)ZgOdM{NKAy$8-b!HKjB~_y>k6mE%8EKPQ zZ8AGD$^GIfOx~R5mCDs2SGVJYBwBqQ3mGZdngqJ;4klLwx6IWAhwRezbn~mXq(?13 zCAV}NGxi7IuBEtDA8$5UPYiCt;lbMCAcRRq{sOGC(a{s6f`u_0DM+h+kpX5o;EN`^NNFJ z;maJ7Lil$tBKlQ1B63Q33LDocq^Q9=wc^ywR|{_*qa-Lu`;yC)<*25tO#7jHj|RXL zv01J?6^y;Lj~~L)-{Vhhoy%JP*DnQ_WtV1=uz$RFOfzw=V2_o$1P#}zTnsZzYg|JT zuEkED>ECl%i9-v39;LFQG5pjTsTxsOR_3|#d6PXumeZ^3OWYCx)^&98YWtHF|GgTP zSv|D#P#7@`MeO_O7a0Jhe3i;swv2!Jg7D=BlDi}CCGO5@2Ulq-uEXO~6fiHc7{_(f zI-+nC3RcsD{@e`iD%E|@Mz3eCdld!N9++PC%H+Qta(`;8(W3n6!f$yBpA|s|CK@KX>2-``fR5=2$a<&mj6cJMFw7 znzKcB|AD{ljpkj4c5g5+yY#D1ZTl5{C2MMw|IQp&ctlpO*Un^_BIliF8+PvHMeD{vQXV@l?%My{tw(8uXrY$@3 zZFu}ulMRqwlRS|5>Ns44tP`L3XvQ;wH6pC`#s{&dh}O|X>elJn##YfK2Q))76ifdN zXO;J$@8=<#-o11C%6yMm@t}I%$RPXSr%p0}BPwYTq_YQ9LC2LQK;~+Hf=YY3s zHCazXInCSNA{9o*0)6H+?B2Dzu{PS8WU|Gii@*#;U9BV4QjOu`uwX*h(p?r%V}9n* zE=yzG0W?5o15kwyW8Xru5{hE-bLvZ}%I)v0N>25!`HA0_T50Cmn}RDSgI%ANk-F-B z>do+eoKPv`n&$DRa^uCn=XdY8(HXm6*P9ff&!m}of)rD-*L{q~e{exjyjR(VphbiK zvA6LC-A#JNwj?(=&I?>V5#K&7J@jJX4I^DxbL|U5+w1g$=Wb`BU>z7R7rVatGklYH zALJwYKN!nT6%_5OC2ek6A(bGd>AoV;#Gn81U-p|z8*Wr@PjBwOv9`G3U|3zS;p1&5 zqC@vzotW$JB?A#U<{NUmWxK--XG++i7F3^{+t}~3c^>U!XQx{^3j31BZhf+#qW8Sv zU7M}$_Dw?njvePK9ABoFEgJQg1No5wn?AWc!*pwl@sS%L#!H3FTP3{nyO+Gvlgqtt zpXtc2`Ss~l{>f|;B^iI8lzghlr`ofR_6&6`ed}~L|FZJ4=$s7UjFL;ozbL zR?mU{vcU%!g3v{`^p`!khlOr>e(w1LETh2(-bsQ>oO4r3tkfkp#Cmvi zKes)`;OcxXKxbQ6GnilQTfH<8aeAD0a2G{mdFN($*9P76yJcnKqD*wLy3{P&lc*^XiJv4ZF0aA5<+wTjE@nS%Z*f-O^O|DEMnF|5ei6NIYz!pL z#ll+s`ho)TQ#%H2@T*2?O3vHrrJL-6!!4IG^-`Dl`@xm;sYOP7q(0Ec;LDjs5OFpr zaJ*-h(#CpTeAwe*A&iFN6Hy@c&N-EZ2??8R@j>582)6~-bPDAFd`{oo;N7ad54t%5 z0x8!J2UAhTek5H7D>#-^^MaTOPyG>#ItP3CIHiPoe@c>jd@-#iD zVvPxKdyrr^NWd;(?i24`79*>wc$K9%PRq_ugT0f^&(7dFXtBn=jD2lrml*yjLO=!j zxED}W)I8c(vj*_`AA>c(92P)DDq+59Qb$DM6EAvj5`8Js-sMucb97?VS|xy1$s)wPSP)Mj(Ou5GlBp=GD%h1XIH z1yHA14SUMX5X}IzsgVD+ zs(1hvetf-k%`4tVI(v57K^=LTm1BahI}fC-W##nK&o=eT$mrdD>pU)(y_M3~R@h@l zR&3ZD;oKLo>iQz<7bCVTnbQ@}KKHp>=f{`zTGrYfI;U6EB(KV{e2Z_l+fL8yq*UX` z*0a+l+%f)P?rVmoQNMBi!qjW%-xzKzyOX+GgNl z+uI%dr@!ygb%t8{{1~^@4go*atFY-d!GV-}s!V>XIQvLJp`^ZtfC{p2D>({`cqoH}2#|Rx zzLtt~grqzH-!4WxGF`;nhnY0-`B|`SQe(q|Bb+6yn{)uerCH$F-ihYE!&lYMLSr@!`!Ov)`NVz5|HiM9vDI_}*eGo*<`rj}rf{#Vn zsF=Nxa#wXq2CI>dA`4U%X9DEpAB5V}3*JnV2w6ndvGI}Q(<5HZn+zxJmO-hF(;-qqbx?)Nyz5S_>YD;v;+OU)r512|(P1H*IFB3Gc~P`Mg_fR?%P5QJS* zloEx#RUdm*5$a6GD=8F;xMKbY;aD}yC%w01WZ|nAA5^p4dq_BYi`~;UHeCAKD_yJ_ zeF@z?_3Its#~ibJoPh!>dne)yCH--dMn85bDR@g&>5a(1Io_F2qXvdK zIaHh{!8;feb0ZN5dX=nNUx?+6u2-ow@hyh>0rL*+We>|hJ>>*Ld`Y{PmnsCOxENbO zOZD`aQEsscWXQ%YvR#ce^~BOBj+Ywg<|3^wUya0WVEjZ4;d2x$`!srUETy+Hn*r-+ zSmhFXx3GSR<+5RO1Y0<0;2^73zaFP7bgV-E zD6q*gS!NINJF;Hn_W38(6}sarKqfUtt1mTsTLl5gx_O=C4Wpbbb4P=(N*SsQrg&Bv zC_$Htt@=I-?UGcJoo4hhm=d|xOSX*~KP3mTVW!KS(P}kG|6mgv zSSVvp9(!qw0u|z-OD2@gy~QZt<1u&2WuZ0JdQ~kXQ-kSvh)f$P8L`q1cVY72cP^yC zE0D-yl!V*0s-?!X8S0xkO5YyQF(8Tnyk=~c(^9BK**lJLU?G_vUhKv&_YSo$B406o z#PMd8tdPknMg!m^l@9yqjMuV=C#)p96pSXb5rGVs zJ@R1*@)&^dkWrU*61QR=HBZc(h8`#_QPi#|Q-B#~%0tI1+u{=E`#ij9$rw#bD;7Df zRGF!)VqtmF$qM~C1ls^8fr2SmrVzIG{pas(op-n*Z|3q4e0pSNvc8 zFMH4kAfLovNX84Q;+fHspdGU5rsJ^eUsCHW3f_e}t-I3l7}(n}ArPO6_GN|HqoN+z zj^2OIU5uE&RH-wT^c$Qq%~xo{^G|5}pT2)+ICOgGHvZ`%xk+{_?dksO7%W1)j1;>t zX|C?bK+l46r}HT{oc+Lf$-}K+U_;aRq)A25)>U3tH?Iky^IBu?DPxoS#q&CkHF542 zOPoI+n=RF0UMF_Ecw&2evS)jqzV=i~evG?uojt_rL5TX($JUD1=+j&8KC3&ma)Yn$ z3xw{DA3H|oB5HfEVN5Gy27OgIschbb8PD`=$@j1qA@4b(6C?h;rHgB;vNoh`n$c7| z+tc<&evF5Qvj|U_&@Dhkf9pOb%(lCCCmZkRq61sbn4jBUr-M2==WY5!NcT&XmcF6@ z)daM%usqUO(O!^#ZLf6_y-g2WPoMjw41(ZT!-_`#nvj%Di9NSHx2-pxja*ky@kQP5 z9DhC|-RGJbU(PQ$%Y~fCo7xq1!8Ie$aJIxU{o0m^HH+L*4SCmUocR%YS`p?+pG}ey zRREIlUJ(bZ@6?Z-GQ!fF!HTY2*Ph>K!%=4)B3+K2@`%%wTW5yeUJ~-)iTV}w^~SP> zM=njsKQXwJ4k7R}U6CD7BoIQY*u?Y9ykS8?gEjHr7I|7rJ7&{zfy}Hfr?72|dKHY> z+nz6-k^DoWZFL5Myz5gEF6!y+eUo6Ggwy30U-8MbuSZkoci)TQ~ijn(6!L)&?_)&u3PdWXpw z9Z0XDty43%1pkRrxt-E{v@mJfmX3M9am=(#dpg8_mVH@{F$(Xwjl@;eQ#R|u(+oeS z?$I;NvVJm!X*aiByENvmv)F0ENe0<=Lw-0Xlj{5s{imqUyy91+Zw_o?Gr)Irp_&zlk^);q=0A$V_`y;27xA z@1`z?PNuQxA4k7@O&mDMx=?Ug+lFfn0_AJ}hLg)a_qz1X$W)y@)baDwhKAak0m;4X zE>O{}&N254)g~%sOCyqYEpr+s7JPE7HNLoJX4OBMiuOHQo0D(t-6#ge77+Gtxx zVb)q_Y`zZR@h+d6mf)ne7)Ji^O# zHbhDt%HQ3ty!_~`DGf%5I!VG_KogS(jKm@*6jGY|BxXylcaL*t8JbcMry^zC)C0q# zYVZi1Lks%!VYWAEOj?@qybC`-E`&{FR<*wJsG6X%k?Gtcx*p7mULw{ zrFPDFZ5Trq)|S;d8-qqjD-m?S51b@Jx}WXav)8#D#{a-WTad%#BpgT6{I9#vmnGNk zYg_Fne@yj0crF#6d`&rXAe0+{3XVD?*!6cu#Kl(dT z84{X30lX$DV_@(+hNyeu?+~f!vb8qbcd0gcvT@TVj^P-1dDJCQ(Eh4rUF-16c*Rz% zq;C6!Z8bV{T9J&6L6Gg9p<7H-P@HIQ)42;TSKGXN`n*xNQP?DAyS9W>YNlO(g0NPy zwdHGONMvVfLRo+WpR#ZFkd;?A{AJZB(~g31f>vWK@j_mw*p`|?!vaQo%=p=pV}#O7 zn%x3|QK8pE#j3G1XE6Uw)%AwwO)o7v)kp7SSmI-|}xoG7j}?H%mN+YE$9cj|;6GxR(qkyTaV1xg@qM65$}qWiG0>l^p#d zWm`3Wta?+-ng{F<8s5=E7TP4?t;o}T<#QfByHs~CNus`{nnD+RSe z3PT+;4g`^jW)u<%;lNHPrK1L=F~y;+p$m^p9Fb&|@kx)+sm2yg;wV^_{oC9WAknLp zp;7+P$DDKN_u9@-58)TNqs?NSLT=bO_eIfWa9Q67VJVMV671-$dIX8_p*`D>T zj=7bv(-Jx($C`Rl!d(hU|F;qtGz8in8xuo-nj-i4)=+{)O;#x8LYq^Qtxn9BQj%%> zIygbtQiiN><9yvMQ=`Gk8VP*DOVzIS4}Ve7`eCyPWE(IX67ho$caQ3`!>f6 zu_RW9hLEQZ-Ex$n7n^ix3HCzVhI(?cai|73ipr81CApS~B^>doyj0qmK>kV!f(jJ! zQ*`viG|5v+FaajRRuXL8RNaAKx)mj>vKe5L=K;L5?}4)h`;NIL`5Ry4to&C|jE_k2 z9Q3LZyDqAvHqCefN;qE^sf(~1l`cnLXvoXz;c3m=Y8w(Spy}yC z5@Ex&^-Nl!dCaP$E5Of0X{#pe2#;IRic-5GP;u+v7(V3 ztWkxppQt^R;gCFQmu*trjz*$B!b3Ucc*GLJRq$ybNU)QqHw?6exKj$@S&C9m+uV3gx#yH8Ti1j>W|M(^0@X9o4U6kxB zPaf$cG#hhyS7@+al$7`Wye3WumNF#;*ovI`DG9|})$f}^ZC{P^ z4d0}lMY`2RpS27YWfJHmb(CS5*_Zyp>K$khLpK?3y>yQZbc6AjuZO4GhkfFn z)Sq#1ZAtrI0+&@U-_-au5nzhb^cl{$AVTatnf{yV@pXjyy&Pu#0i_~ z__$qA@L6B_cY;eM7>`xYYmQES<-_<1#w3cx2tuwGN88d{>Zcf+@8l2fT4Z0HzQOs^ z3g_iDhCu7-+E2>5(4UWJh}^W}S7}`%*NoHsq3^|`!4VtljE;FhHN*OT@!6w2?jfz- zq0T2m%(TjQD+m#uVj~;(+NS%Wj|G2sQ@HCk@LE*Hl!#Y~3Z60BeMSlGYV*~{DFv`M z0k5@d!~BA@ID~BVy56oRKietOsc>uIdi#j0zLl}E0&0rmJbv80V|$uq>g2i0LZdgF zD3AWm<-A8o+RgO=`7lts53c|m?!{1fSusYv2HNi=_c z6AR4i#i2Jz^?$uOmc4QGgdOr+B=V!C&%TPbL6_6OQh`%qi@mdZ{!hMhaof7`6ot+uwe-y`xzw=AeraDn zQbD$H;YM7dT&!GrM7V^nv_?k5cN~VX2}>O^WNZUFp5Yu6kU)Ut{WB z+rmYZ+iF&>c{`!h+BLXeaf_CiVP)*^{&n1!pR=}5IO^)HK8D8njH!j?Cm(sm{@dUn zbhcPDJh-1m7Zk(0rAu(DbzZgW9SO$ID^%T4_222ln9$)2o>+VbktA_m9ZtQR{rkjx zPlA<9QCjZUk!2isZEAY&9M4SFWcy7w`r?kHn?s`lE&Fn0&QQxdrAN_YGjksE1gG!3+&Dq3-^(d>n!($FzC&>Lm()fWjgN2v)Ua$`wdsWr7*{l!k(Cl z$I4z`wVlL6BB}>bcsXn8;7p_QW3QrU=O=z>;@5OP7ijmiKBaH%{c+!i;aVS!3`eGw z+l_SOCc!c?^?_~iV z?kOF+ch8KtkhZha#GJKyYI@n=mKCU=(AA{*SE){&0n`#w;jx(*A4WQeee}|G$87Z0 zrrz!*fev}N#_ox0>}leSoqL64m|C?;oPFR3!sKTPfNR1~x`v=J2BIzNEzRhiIFVEwpL0D8 zG5D;DZ0b7OsId0B%cM1>3p6F{Y?f`<6>li8>+4bv-?M`BBIf_Z`NMlpag24KZsqxx z=9NR9>hHLQg$B6CeU?pBo@T&I7j&DE=z^E?>Rb7^M>q7bgGCx64M$5lr!(RItHV-L zxC^u{xh`5I#`sqyn)g4glZDT_rr?4}R#VT;RDAqkSQ|}#bC@x7QN}uwkp}mOK3GDw zY)U^wDlY{iu0L6IJ);>o4&By?%mdV&gg9&6L>U^cYy1KU^G`HXQPL?nwa5^K zL2tkqF&rWQ@yk^04g9DU(lpP?E5lp(P))&kLdwD^0kXsp5R8RU5T&m4>=X)!Hx%M- zxv|BD;mC;58eb|6`@~G_QwSFj44HO?VU-aPH#yO(1l{li<+cNWqXr9rIa>Z4IhKUQ zT)GXA1RxOVjg3cYtieQ5`MB&f{sfO|`Y@QTFH{U7Q+jwJe`?A@Q`ibtH9rj57KxMy zj$-)_c&LRv&CC5OeDs!*?_7pGKYKW}0%KFzyH8x9evG10k);(=U8ur7ric~>itXgE zRY|jl=uVfZWTrcu`jgtNs4z%;RSSFdnYU#dJWle#J#M#Vf2(WDTE_>>c5xltT3@Aj zSHO%gKqK|{L#7Ev*W7!QJaH8`$MY8nC81rfDS3hr67n&EmNXom)#MD3GRkSH22VUf zhL;1Z^?#>2SYU_9ZKi4COzw0I2>^f4=gKO(RB>kQ8~Jv}rxj z^b zCzc+@svJ)PC|n|xo@9iWji!!8gXax9_!H&!#7q;*IHYrOQM*^srwox}v$m5acC2uD zGosO%K!%iFqne)l*eeHKnnA%CFaIWh=y0n3?PJcM!nWJuU;7&&{C03*@h+N-I`AZe zUS{h@VtxK&9HcMUS3_j>%h-jaVE}DgCrVW{{w@Z>ZY8SyoL_~4xRv5~A~r_73RMl;yik<@`%LW;hEx`H#zH?M&Eek> zadG(&8pGd+RRoe{U*f~FLlxW;O>p4JRCW_MJzr^3fMVpx6R~lN)s{6HW-+%QZ$ZVm-1|_&%}mv! zm!fR7t7!WgvIypcTqkw>LMVgWbBD-4rIHZB6F&_WLIo>X8w9m~sWl9E>u=Mne%(lW z0AA&b6Mv|>^9ho63mB#SA`Rh-vQNePKU_vi8m;|Ge*J&{n%qi0Q904dIvWCg%UQy> za4P^PgD5I`*>%|0T#)10q@JgFzB~#Iq>HzbcuUcHnZNR{82Z!iF{IY*`JX`XBP+Ix z`*DEaPnB%^{{I}MC+hy`f+gvL`k1?)brEhFuPeDKzpAV)x+lZk$9ZJqaFgvFFJN>* z`N4w~J>R+fVXMw?FZHtTxTo2Qb{fhC6POc^F z)FPer!5W{r{#82L@uk&{w2kAy-4q> z#SEk4MLuqNC^k$3yvd0oU3VAGK2P}#6fNQ3D}bMW+@3MGnU!DVde;1$CnaNZ0y0v= ztuyx&wtaDS`Z~DU_>9`vjDh}Kp8uS@DYQ(SNc&k8kb#&OgoDSCW zuSte&>`QY?UE1u1v5Nx5NGl8aj^?@j-uVbuT14?=H3GU4U}xiJPvg0u^Sj4 zOoN$;jjg>uEU(&`2?ze1W2&^byyW#lLHD!ih4qm!4Rc`U$Qb`OYsr*Sd$0e)jL6>c zN$rn&oq5-mQ@a_sYtt>RJ)ddThu=0o$jm$ZZKu~y28u2gjIYw)&SDrOdgT@0V2pS}PJ75HZpK#n{_KbCB3?*47pOp?qEMq$&8o>U#P?3Fm zVf$F@LhAc^LDhz`I!rxpFG{>HGs3webRE3*XBF**G<}e60~v9f9cS>|%vP>E-1Od# ziE;*FVk#Dgw2OT(!jVU#A?I#Z>|Dq5v+k0|pE6?Cp)W~=6`aw9)_9egsx$i{=IF8w zA6m~R6?V?K>S<|Q&!a!3WYsYqan-M@1}Rx2Q6=;l^KO2#CvAQnyhX+e^MS)nk1Z*) zZAe8e-0R@{X}U|R$?;J+7^k>?G`*Czkhy7&6_@tbB)5Ho@^X$Y9>HyYJi+nxuE>O| zLk1?P?(zD1eUtRy`JEUF%@0%`#)gf@56x$Um6`Q7< zY-$_Bs(AnoHpM?-O#g|xwzc_@N*SBwTK#(bGIQe=@BO5!Ew-8kSNlE?z0D?i+>Lq~v^sY1Ng$(0HQzbv(cv=x5BcoeA5 z6ur)EUrcT27}qtp=|+g*P@YT1MoJ+M5};;fc5M;kP;}aKE8y~USzza~l88y;T7P+I zoX1afG|)f`(({jSddPZ}S+6hrL3$#6XKsEPbTeWv3c>a3y-GD0N+smk<^P3#jR*dE zn~z9MDpN{qV}^)LG^%sP&q8P>I`-sCt9DpY{u-q7W?9y!X(dh*{SBF}oGL5B;^}jf9$7mbXa>H3H0(EmgJx2BB z>XO>7f>LW|IVxSL;-uMR_2^6(4(d8y@trcUor`*=LiAc_+Kv(C%R|J1Sj z=2>w+>Zq@CDa+Kg7Pev8C$3A^BJl@3v?EOVz2X5A8&Z5|o|%w-r|v_hNz{UCOt4U< zcsm$qX_Zv=XBHonck1fh*9ui??maxp?cWBXEXntq7S%_x!<9$2(la=-(phA+m5Ean zn#%sT5{_(x_(-)VFjjOwy z8nu#-qLq&-yg^xQoq;>?nq$bgTb#j@UKx@R3T>M1<`&*xvk$``VRdQC5WnaHnk3=F z-cdiU=_UM#LSL(3QCa9jG*3eOf{Fz#BmAV;>pl**WJt!WO4N!hCE|itj5wg=$jT^G z^`S%gr{a5Mt>WU#xJ<;z2h)pHF!BV74ZzXeU_bEBP{REfzK_d?8=+;&c2)Tkma4FI zY;35=&MEGiSb_jD`x+*;>>vVLPWXFK>)`M* z_mDQxcz77Q!6*`v6H{VDj+*eYa!_ImSHYOV8A=Q~#0+5P!q^IBGFD*->f5x~55!7< z$G%gUE){Kc!q`uJOT=k15)2wwy;#wEQW_w!;Bg2DZVx~xatOjaalF<_%L!($K_H67 z&aCOucln<5{*@Yth0VjhRO&^sgn4uR2TD^81H63LW>~G!F49;KA0pvlVMW+wq@5m0 z3!?JLNOlz(7;X5-kzwgwPFAJvW||UYeHy%6tVNm`h6Po}-&dALJjBEQkl306L$Pu~ z?#T{wgb-b$RoKQ)T;WPpgv*_P$_RkFA6U281ki}q64)VL-LveTgLX*x;poW_+-j&} zW2CF?-^_~J)ZY)4zXcP@5kR&iV{lXXFs)&b@aVKhiJE$n2?Db#oR`2SHE73@v7Rf- zb{Q+yVm$pipf6MVMx)a!T&wxJ#+J{c!PKW=(M){F6rm4LxNtO43}O(WI9^N#@iw@~ z>`#<3LJ!Jo*02))uZE)+if_ep(NZm6j1Z4fRe&`Xx9AMDLU3|;f z%TAfh_Oui=YbrsxE2Z*sqajyDZ(A% z5K2$F@_8D_pW;fZr;E?~=3<1@EEer33&8WIj-MW(RWqvD_Z4L+=IG=skX1pIG9(6w zO9cv7=CKjIMQA5(dIi*{jsncnb4B^_(rH>~db>avrJvO%u`g}5E_>BjL7RR&d0n-J zRoxhxURK=1{0w3yGmTkT+0d+}Jme9Yo3*c)VyPN+S!JXPylDl?TPUe7fYAp>D1UOUJQeaadUuydoY_4@=+NCI z3R%&M;bPYO3ePQHDXwMzz)fBy7%Hruyw-M7@VrXUAyj6Xl+V_GY!=$bgNIY{$@R)z$+~suE;UdarT`mWMc0y_~BPRm?-6D;2f> zf+2;P90JLrL;3&n^*zEyV)EWdX`n#R;s(trR%ucpvY=eS)!dyVx|)=!RTH+^IrS@O}eLd#Wm=gVUYA50I?w^vh>)7=#O^XEn;d|9-) zu{v%p)u-{r{U4q>1v1{!mDScC(%oV$lrqrU?(b1}`kUEP`W>4aYF0KGT_x##!k^T% z$CKTjgU{?KyqB~oxwG0PfVSqNgqH5jqu3w^H|Kj77=BI{!DC{y>XV4ZCO`y2dL6_4ZRlUrI>l>*Ct%+3q(lsMfVLb~~)&M%+TLKG^Rc4@Z*M`_t5m@>I#p^7s@;D7T*4>2JCE|-E?<=Z4lp4d|DP< zgFdwX76DU1N?&j#WjSk$C5h#h^;<&PGzYBO6L>qp`eEMlpMow@=+^s^sFUxV{QKtEz`>L;fHRY6p z#=!Du$I9JkwM)D^R#pBjhU9^Xk{dEVi(CCd_jml)>;vSsY&^;q|x171({&VsXM; z=}1TU;OQ*2dU%xqCijFAZy)I+uP|UwcOxiVo8NHIRbTC|ZYQzazIxX|RkSg6nrrn> zxTDtmO;U8o#;DFl+^=iznpX$8&a8etsxzr9K17JJa_L0kaP6GVWm>O{%%u<9#9FNj% zG(=EKctAL#l)Rp?qNI~vwFP4fSDq<2+&rnF)matNxzVdH)Uvl`Z_rxUSx#%XF*FS( z$DB_6P)m97;%#W|Yp(LMyR!*82wP|(3nnrG!%TH9CGwbkyfmjC_uZt3I$?V$pB-Xt zxqQ&GtEhM<8%s!`G}4vsWgf(*6K<`swySzX6-OW4_wD1PlhhJN6Z?iT0FGQU4w1H0 zleXtWy&8`MyV*7g$$agdEIL@Sr%saq5J?Bhjxzwz%R6qB^>);cU+Xe0d05B3;+pzG z;%vIkjvR<`{Gicr^mb-n)RUW1FT01VXCm*6&G!X9KJ7ZId~HoKceK>*b|amg`^ja* z-qJT(udfV89%2eP%_1ta>292SMCa4}T>Uy@c?zDs1}jAR(K1NZZ2P~T`r!H^8YS0f zEVIpnNR17(BQok_z-e$BipF~+@?eugr+P#z8yLWfoXU%H_EFRqG#X3#EG!+WMF$QK z?{t`!*xwPmWcYuxePQ@xbL$uwoRX4j3!AK$TbvEgzw|ddIwlVieZX^KvC_NZ zeI!vnz*k)~?{4IjG_A<~=N&r)1|iM0L1-h?ZsCU>}aNkMpMuAFd_ zoMl*dOSq-{mITtmX>hCZMHe>i=PkZt>!T`Cw6Lli)vWzqW(-u6%h(;LXQzY67hG1k z265T(%o??x@tq2`g3H zF6fXt&CJfNhP*^`80b0wV7PZY1TAqiZfS<2(ym)Yg6^Bin3*QEzeH6KU3*De-tU4+ zhLcnB02$W?1vDw&(Na^-dE3x_|P(K*0>J-XP_w z#Ag7i@5yW0 z0#IvE05$@2H7h~5hY821W0P?hzNlnD(hu0_Oz{EoA1Wm_hO?hO=4++`M`SIPK0>!5sOW= zzzpfFZx<(LeQw6#x%XtqU~MUb*D9wm7|T~pfGVy#Z=U3P5@$*+as8gq-WjCcD@m!x zBheUM-0Z9J7LUOW7h>Y1G*3V-xXawHl(1V;Fl;4EL=+tl!JDeS1Oa2&Kze2(wiZmk zjsxi7Nhu487a;ycRXkY`(qF1UvgoBj@-i$4qU4Ay#Hm<`qd2Kq=m2)vFY2Uor=f~y zqU=wag}ZdbMN#pc%Q&-w)-57scxltJ>$k$FXW1 zX!|Mcsh;|NgdR$(<*5EAvUg&>>w;B@!lnwTj+ll4fNX}0kX(*do-7p%|HQZc%lE>y zbLSRHmdSAlGJWW?{XME>y-L$s7ur;Y-X6-+@N`H$C7PLF`gD3^U*qHg&{Bw@y65oZ z<%Uq}Xaz2E7cwpQ6LxP_8%A#O_p+6%D?6pxJ+-fpk-7LUSNP9Yj_{3VMWipe6s}8B z7s~!gJ_ujo5lKr*yA}fzKQg+0=W^?rY;Y7T?)`sf9+WP=i}yu7b#_sPL zRrhVU@qQP#rrTZC-ni1fY;aS}tkzrp|FGrkn3PPX9ogygKj>O?m2_o|XUc%y@*yC& z=xv|5$(OvIp=C>LLIWk1*1aDr7SB8=-bxr&W*Y;FQukw?2mn z&9-;@VoF;Sf5${T`{UGcWrs7;_K@bOSSxul+Mh58_~T_=IQ&Q15~!Yyq76-h54e(a z9H*As+P?Cjf%PSRl|4sm|J#J-R6VMC~Hmyl_>46Ycy{ zkn66lX?ohet{2{YgY8#lyKikmk>)!dq4&u=b+oHUM5nQ#H$`u(;>4ulw;3tWU#`&I2wj!I)N>`IXd9j+zor z`x4xELu_$}M>b-SYh0IdE!{f)N?BEwk#?K@$jlwNVcd3B!KpI+k8;ha^sbWzk{Rt~ zSr;r@e4RC+)_{#a_^ZBrYKMo|632`Qxiov;c1s35&SECie)Pu2M$Y8(nO&V{DQ0xx||Iu`Qbst=CSYq6TWltf9&;8NY-51nAumKpyj+o*A<(Q z7MtXuD5OL60D!jL96oJMY2b~BJ&jIdPCT|(N zKyihWp;;H=A0)*?1c_j#WsW(H@JDnSoc95D4HnWwd0zjG4GX_8S+E42-0`x!4=A;` zsst~8JMN~xS#hA*2y4Gv$y61Lr$#G!p9l*;SL^6V`IE}vCBI38#yfTQf@mqWXwtugz!(U_=tQbg`yh{x4hVj@G(oz zgS$9@n|t}_#|tLtM%7_cEfVn0eOCsCCGWhnXDHrXhz01IKH9NOB`&rkWFQE} zXVEoM#pB?bF{?f?J?PFmJVIqPB!r|ib*RAr8c3+2p|$}-0ScH6Ybinx^~4E}aF6-U zrKR^xL7{+1!k0AEDko^%4!@B|=`_TKvqY1EqtjElatdlyCLVsGd2EcJ6d;2D&Wbqa zUQw-s328lf)sO9e;`6$mrr(7tXz(JTo*p95v{q9`5>*n0Dwc7~DmX(Gmnm#c)B>x7 zhLEUrvaz(wQSj!Xde8imTp0|3KvMzmPrL(=NSYne+x4J3`|uB>_RhPrl>lsVD>_Gb z8N?@C-BA1|92{|sl9_5?lt7cULqiXusV(^F^HXM#n9$CHlT^kRUd7(6@`}+DRtk|N zq-=F_qQW?;-b+e66Q7um1k~7MCzOg{4sg#$uWd_@q{M)k(;Glctinz)S(Ty!Ft3i; zoqXMAY8iuHzBV758j{K#WkmMGz7OFiQLXcRF!{2ft3raL9@8*p6^q00XDD?b?FGkq zxnd?wVN;|2ih`^XJ|pdzSfXFc6>{D1#dLxq`yDmVCm|hb@1u$~Ijm|DjgBArOr=XM zKmI;`9io3Fcn(I0DAkwrspXIhS$`~vN9i%+8=-GR*)0U6U@K}LB~TPLM|l@(k6j+C z%uF&9R87dOFO~T8=zCAlW>yZ#af-)!{?4^_MIK8yD_vo;sopWt=-_#RSE)#K8~ouh z$k7TKnup}9(zsos;COEDZp175uX#qLQ{d-rY>f+9Puy!bCY>N*-uVb=&kO$3#Z38% z1~TYs!4s3PoR5d_CneC^m2KED{jV5_ku|+XL(!YxfL904zbVBOhYbbqO}dHKA}MB- zijXJlBJcW2$3jKV5+JK;=#qX$B5Wavv{i_7_FGm}H+6=*O}$O4AzW>2)bcBUqZqI> z22Z!=V8OMa`=g?RJI~wmm5k8BwvuY~i3$r|cdHUP8<`ZqXKZBcVuBOHk0tD4-jZYgd`y;cSpNL_W(x9+|OOO#=nIR8jcQ z($RFQf+A$t#1|Z?FPjd@16hy)1o>PnYmHx2XkcIXJ_W)agV^~Ux?s6e=2yIsv0_l> zqH(Sa1h7$4RTLl8>Sg6K6dU`=Q#dwB-8Ly1q|cb^QpgNy;@-3vk$cJ%M5gq_+}N`7 zsT45C0HM|O8pQ`Yx@r~6Ao4&lai?@c&$vrr2tz=H0Z5;6wXui^3|Xf z3x$jU-^)zb#&U>nRia!*6V->O_zf_F#2F8gUtrk3zE-{azw)#!aLC~2F;wn*((qCQFK#h@gS-AwLnC~YtV zB(-#){P=Q?n}~{8p?M_jgk%2xOX_(^P73wLZY&^G3C0^Q3UJp3L>oY*xMp+jQpUuo z<&V<3qV{fTnNyJAfC(dD%8CrD>dud(d4hHJ9h+HjUtdV4%Okm4@7mw#dv%Xv~%vwyVP8^o9}k^ z(Btp!^mRICV05&fE$2gLNo>+Yw=lPzFIe!a42y^gH5j6!qlhj(mB0#@)&BT-)?I8A zXQ%ns%&O@7930u2wPyC^+^~}R!j5Z{gxGS-S5ZG^9yaNbq;{UF)G=*65piYZlznKjr60bw0;Cwf0VMUu`rSOy9Xw z1)^7vF=8re8Ji#Kd(U9HK`T(}+FQ!6cvST>W#74M|MHY!Z|AtGU19ASgTnGiq!RB( zoMB^<1OFZd*qd!u1>kqDU8?&QZ<=D4aZ61O^nK$ejWfqFC?TR03@N(#3^|X8{S(X2 z>+8qnPdeQ5HxdwHuS)_uEd@wUDHuC@Oo5pHed zGi~%H0Z64bIP$WlcutO*c7YEMqxXpW30;8O9H`5NWXJic_;NYZ3NNP`KDO88{eUr$ zy?C1<(1LEjjd+#BIrlGUx&6ZP&(mo)a&6t^9Rw2p5%dI~(R4Qt{Iq}EGAok?ZH;}p zyRmj-$o(ekz7`9*|JW6}b>`|h#C>=BlEz>2D#Gfi^?YW3ES*57zuQ>I09lm-+!{TLx)w8BDyzql&Mmm^J#ZmA6<%BSolA-mbV*DEw_N9S zNhCakGtyCp1n*6zss~MUlQ7}b0IU!=l)iSjH^O){+{sPD6EIvIXm`7wA1=II*ztR7 z3(~C=Alu_?3&$RMrT_+Sl<&`N{Z4&kIMo~tYzilCEvoA*Y_Xmdqow&1aQ?DTjs`2= zM6D-D7dEBPJH1H8CO#4E(b*W`?lbr%kh%((xI$Myb~aAi*cv|>ve9gxH~rjCa30td zLh!IR9%O6AMfRC}EjPT1th!7eV{Jx;YklgMk&hZRLO1nMF??$~dUgtB|MI2@xyh$t z6n-HLx@(`WmM54$M1b}*9@X?UZab>M@93RKJre~dL<_ZfPc$ouOa5r=u`eoG%tDU<}l)El)m_s4I zNF;Q5W`B(EqRg&K&Vh1L^Ts7BbLIhL50V!(9`SR0(d0~MMcfirH;QhcnYXtc@G3EG zb|!Bz_|x&os)va^diTDq$MNVDaZg7oyJoXtb8LW^I*GP~g|fwZik6>Gi(=MmR>rRe zR^Myt^bM)sSLMsMY5qaR^$|Y$zLwxW1%^HP+>>xyt_Ma_f+~4s;ul(|z;laMH_KR7 zY>^i8L5e(+&=!pxCh{t+P6#B#NsvUSM3->BUfAN?A6z3Vb%~Vb@d^SZqK|cU2K5xd zM42S!2RvIl|=B}P@jFTQHe>3SToOE?nlj}E%g7+*dt0Q0&un|Y1%v)F1y zeiXASz5xKl7#Qgab$LWv5(uf>xvIWZHDTiSB<)UH?1Y^q2ic35ffdqx}F;tOZp=P)gaT z3Mb)t3Eo;=&<=pjlIsEl<10)_NKBE2W<1PJcf|i&0Wb=%NxUebWZ+5plHqa@GyoZM zCAHkxe7mGqOpe-uX4xtv1mloO?(3q74X+`%238_xWH1-HKE;3h>H%Rf!^E$%;*iz% z8}DREP>oni>E}Ea*=%DyBqHw-9Mavju(39L+_#cNs~|D~++x&-53NnlQrk?@e<_Fh zB8BaS6WBIMhk@Opjv1*H&iAd5%}lM3Zv^l0ENE94(yMs#HPaNMjut4`Fn zG+MHlA;;ZA?spHNTnZAUsizDt&_k6SYImscthz}t?Hwa$E>xSW0t8g=Cu3<~{6_;i zcCS{gu`8jl^!SqET+}_ZNLBCznoTR46@=<`<~J+e4;9)-5~JOYtD}8?T{~P;K3;bg zrv4H7u2r%T6lU}WXt6&MBR*svYfI2D`DFnTU5bp$2*de75*uSeEaZEZR;ky?`heaO z(Y$Rq0WCPUQcw|QdHkRs{gJs z0#cciDPrTHqfxR_g7v=2V3Nkl2Mf=DL(xF*VYjnN*UomqwRraKlNWmTx*p>BX%CK9 zE*3zvYg4I} zMG()W#K_QQX*Ki;yQ;(ySQI(!s?-|sz5c$!p85!l_(VBHm0YAK_*k;SdZP;hOwzG%ML=Q&uHu(LsL=7 zAwclSPDHaZts-DN@iaxK?J{P|k^$s(nrNHf^(eSf4v`s2MCWV)|K! zvH=eL2`fiAV1`S%E;oxY9{ap7ru1Ufr0DWrMf7S}dACa)MqV^b zuk!R#vt4!vQIX5;@ULJ0-{dJMa*wo|-6~8h7~eo~9Ia|P$_6+5 z-L88w0GO1GHm2s=ac50OHWj_@Eb5+Ref-*q zLFf2^2i#~*1k$>EDUeO~%R28Yao4rZoT=|^lN#Lf9beO-bwyWZAlAOVS=CV&7DZUc z*LOtGyJv7;#`yNLUUy$Qo>*H_bz$a_fg+3U-bWz@;t|7CuIrU!9s4Mm?X3BH?alGd zTBzT>bupGuXQ95SJ-3?>gtyj@V=;aku-gTB>g32tNy3JGQa=0n^nxd86^H zzMPS`o#*!N18#vte@oBkIZiSb)4DVxW7>IB@3)3=W@C;t1!oZ>#NA? zXdnWB(o=6~{KByh`xwW^iqpEj%k|iYqgE(y67)jEWjP(P`TH2HCAK$QUpt}#6C9na zn3~vikR%+*@8yO#W}Kb--0fvly?u}?((y_vAZebmW` zupWaQ8iWw+-eHJtjW5V-GyDeac%$)pP0NCczE2i`Q){Lrx)_8+?Ir znOIJrM`Kv*{34$UM@O=hBxfwA7`&r?#g?wq-?FFpJHE&@{%BLAf4OJc0i%`~T0w!N zZHkg_DjDa;CD=X`#FpxN$KxcHF8M#xoFDI;R2NK9;)hl^M9ZG+WJM=#+Hz@M%Qo@I z@?R0ieLePgwBvVxa?@h-{~_zmA2KhC#vj0 zatEXxJ-Yy;c+Z~Xe+}L86Ujbe4a!717cxCobi**ofdQ!N!2Ze~Q=e1^4UwU;lsajm zB{L?)?L5uge+kJ?IvRSd$+BJ}oVkBl-NvI5pvfiSThP3yz`G2uiM`mE(l1hGmfB7B z8KTY=7p0Mub$9)x%sAItw=2?|j>RZ@d{pCkAAlF%5NkYxC~*jTz|bkU*Q`cOFrSBc z<1in9!{^#?HEMV!aZL;7eZuA<(upOcZwYT zC?s$w?kvr{)zOT3(bNv~KbeAXwG8(*fb!H_YZK>(mze~Uu)%A^>e#_*yr%)^hchhR zx9E190(rKF4*rb?Q=bD0!{!983yuzhsRa9Uqh`f?8fjIM^?DWeQDAIcyJiZ0UzO=? zHwDJy_$s0(uzaBY56PbU0F9zRu)T#N{W;Aj(q0Z8nhZjnX8>%9M~2p2CXmUubg(Ss zZpzB~=iZ!Iu5L!BTgtE6=ZdyrwPGo^rvUDfF#T`=2?1~q??NGvyr)J;NV1$OS}A{0 z+UQ(>rV%g{4%!Rh6Qs|)GK@O4j-b=gZF!F89YlM$n(95}z31?{FI;mq%v^jF7hJ?_ zM{~Df#UUDjejQz{yGkx7*Q?BD?~N2Zc;Evhnt7A|Ay0?St*1wjn7)7S%Zl5PI>Dof zEE&Qv{~BrnFwu95`W0c|(VKPRv8PtLh1e8UC1X0Mrw?Q`-xIJ|Da^sqEQg`D*n_FahKFzBU`nHoFlabebOuoPG*MmyXLW<2loFB>N zW8a!!qsaVfNy2sqzHqh7y18ldVo5sksdcabE_r@wr=ocRl`ne?ZSNML(eH0UW7Zqz zC4espG0QXj)Zsz>YpM^fsIrfH1r-MscRLeA#@Ny8X`pEE+ZO32*-3ZL~w>`%&orVVF8RE=9|FEAKo}Ka9_w z2fW7POTSoyXNbuubzJ%2tP~GG(QiM0hAga8p_Nh#pP^O`&5=9g!Pst7LouBur|2mH zc3W0^Eq@NeXX^W=gDk%RLPmsFP7SeW0F6^pn4N1l!lto0*u2W@(7~7w&xGy2GmMX* zCmFKYvs|ICrlQ+uhhOR=$hch_4wLb#onq_e7HLsnlLy1qM7{&GIeY>YFvX{2+$rBY zo5@H;s4P1}(HfJtu|zs~QKPsvL5vH5*Ad~&jC;saTUxpbi;9I(RJ+DfwJ8)PGFdN^ z5D<<)YJ|GaK=xYfW4_Ww%|-C+SyLIQT!(SnBM;d;6Z9jX5L7A@L5B+0ML{1RJ99=W zuKolrE@h#m53n63@4?I$9a@55+F3k4_%?YwLs3g0Z5*tb0U%yLT9TY9o(qK?5=n&J ztel&NB>jl{70hQU(FZ3lK|%+rMpDHRbej*10{(gzL6GUlHL%)d73pP951>gmub5HW!^MQNXvFj*T0}2FA1;tPu7-a7Zc@NK*WK2;W4&3+*uF zi3!DK!bge4r~WrxE_{nH-5EmS5fp1LIJCIr5vDNE3XQ1;8`yU~xJJ|P#05$au?AyI ze7InR>s{1KK#~r`Ew1Mq@S0ft{C<)C1vgQ@?u(sKVIQP1QVRK5>;FMaPmTf@N3hU1 zVF6N2OhrN`bG8r$l!M#`R}LYb7o&rYI-71YoqEyIwJ^j>?Ncd5$E8Yyd=N7`#(}pj zOU0*ppxrQ1X`{_b?CCA+(3qFhmj_}}dMsa{5fIicrWktTV2lj=m@naHoFkjM-t05< zOP@SSTruW@rWB;>bhUI7$iG|VQqN)W=7JQn( z_tfAUFx{p?c`blThG+;WIWOy~3MdGkj+dFJH z+2XVV+NPWx*eXvARntf72FNmsCC3XOj)Cc*2G3BsO_n2g&sy(~Pw%_&i%uoyhD{1pxbxc36gx z?Z4PnOQHz;AOt1PRIdTUtox(~(L6(j^nW(JX^@7GBae@4E=w<48qG43U{HZ%V?fGW z?80CMktepHeO-_w6KH0z^8e-Jy3NX^Z9Ge{45 z8X?Tm0fpI^!FY(np-|(%(*Qkz$ij|DT2x)p!v=K=@_`5(P!Ht7fG^|lwg1Hs;K%qO z4ZH`KKx^iIu}PZi51Gi!T+uY38$u{WBv0WHb+K|mNNgW$P9Y_+vgi6Sb>|r7LuF@xG*RnpjpR`mr`O|slU%ZKOQ(WNja8K~)``4b zP^hV+xk&^(($Wr=|A^X4{ML89;q%W1{uC67%;f3-{hqr%MmJpopLhW4I_YYc?X7O} zT*63}w{nXr7<||3ki09+qcxtr>I%CCncVI;H(XL$5g?-q8J1CdD8Fn9A$+U%DA!@Wz85*qH7%{5$A3cHxwwH+skQ(>rSQ*28M_-Lk&GC$V{T0Qa8$EKfIF@_{1Gda8b-WX%-p} z>d#TQWBZ~{n|~W(r>m1P1ZBGi6}Z0icS?FJk}_{zd8;h-Uf+khP0@E_(D-&lw?8cc zJ6Y83`tQ*n%U^Q6w-|5XAMDMqqI2VJB`+?>&D(hZO+c=ki4BKy%ShfMuQz}!deo+^ z5f$}*3zCyb0-QKm;8Pd($`Oa^LF0?xj9%t##^Kpj;V;s7*Qds z_I_!ckzQ6u;7HT?MlK2zwb+h?ey^95Dh^;$kq zDKopmD#v7-#i?-2Yy!Zp^{6%;(Oqojo8^sY-IUp`nZ`bF+qmXIIed6aphQQy_jc%> z)=iTsA&qCwlzpfZ=R=bJ)ar47{TA{tT+iS7$&ib81cVvEaKK&ET9aQp6k%;Tvm!G! z$R|V$FQe;dXB>A|H+ZoIU7eNu$+wfz%!LCd7oAeK-K>k)K3tA(p(C;~Y8M%D@s~b+ zQyIC%s}AjVFlTxOheRJ{OSNa4iW-9X7xJ>$1XYJYbbi^K-i!uCx~x4NujwO^LQRsk z6xhJst=qjAfdPeL2wK;_18K9!5cN;Ms_zq{wVpD@)u7!8@{&t-4}g&g28INpY_fxd zxp1XPboHs#PkZ>|#j>z0sQQh%OEES1qwQzq*u8xFS~R7j4yQ1W9ejHV9$*0FwVKxq z_u+=#S-Qkw&^*__@F6z{qDQH#j=LK$yZy_uE+pWFMk*TWhaOrtdISQK_w*DQ?1Bf> z&m-hUz@7_zF|?;gQV#YOuGnZTF()A%O=ZI4BYH2L_{qN==kCDj3@&+)lo&I+J!Q(z z5uw{J)oo&^mru_9l80^i*^MGyf@tDW0yjNTnsfJVQ;F#N1i*E*Us#i!G`h2Vp2}?P zR$*GF68dGe{ugBKhnxA?v;WC^DLig%fl0F6ZOrP#2YXpGb(RL zb$7uVE}W@R!BGk`EX(^%TsLR*4B~%12vTzNMv-myOIULG6b@xe^;>e$t>xs%8IuC} z4NyMFpc>yJDxhgg*kiaQ`|t8Gr}J0%7I)L{V|xB&jC#%RqpF2z`o}uGR$vTZs62O} zMqH1q%XfHlQ>7}b$yJ*rA1QeyO6yrDd`P5$+ULm_eM8c1boxuIi$YCpdkkxxi6F-?$5h#Ct~h%IX2`oW_V<)nG%$@7Yh z&nLHJij51OT0sGhwSzH4gaogd&R531#iPD8RPkURcDQ$w9{Psr)^cM;iI=!*E4Dl@ ztmV|6Vr9PzRNGuV&flBAvTO-@qz9_e01TXLH8k8R_-gOL~ zD4TODe$?T{nXR}SC1$ZG0UW&-Cu@+~BJJ^k? zcVNy$Odrb63G${{^kH?vhvt-sfHM$2h{uh=@b(&7j?T1MXb(E;3^g?BBTA3e!VW+e zQ0PIZ%k}*kRiwAbMX&9iNmOZZ*tVOl?qs+3oF6nV{3hRA!&mW_AWE5xcqvq}x1xTt9k|+h7iuxk-t#H$ z!fDYS!QT{x|2QLH>@(5499<-5xelVkFGML*{4r^(6AMj!poWq8iSi%qjQp4G$|3+Y zV^lu^ZR)Z$l|2`pW?Xt8#6_?JzuknEDs#U?k0MD{yZ3?4r&iX@2qwpjzR&^%hj=-C z_W1kVTPS!$w&T)4)EBU3Ti8DFxE+zYna_lavl#F)xPr(t=VYdv$^gUxbqPe~V*N#) z;eUaKXGmaj|E_@CWs>=1!E}qdi;)2ggKF5x$k&3}3RNQ&N=5xj;smUOCM=$$ghqM~ zSr;Kj#zsXD0v6IP%P>wQkT!z|U?PEdE4&KGV_Myf*ot8g661+`0MHQvyMZXcP+*5;{!#kDQ8LeYa>24HiK@@U)$N~mp? zbKksbY)*qZ)$X z9>_|X2_I9uPT&?nYzgFWHR#~m&yX1@dZ^zKLR+Q-V1zV18j8p<5X?dW@ofmf7bwJU z;RrE1xgJP0mc2w51@%6Xu}q~&kOP!gcO9(jW9I!y34J;?RqO~9dJ}>Kb8TKdtBsE<>zDyO{Hn1#bvn^6= zJ>Re?2_?S~pZ0EST8*>5=42+QoMlmhKo^U{;uh~TKK9NOO}{S4 zZ%Z1#!7~DUac_pn7O8k4zSmhXd7Kf>l#J{eF5-e{{`zYgb2HzihK#;qn*@= zFf@gs)hq5d@&V>hEK%wX6@~eBBRBFgd;pfQPg2g?3(jAlNNynUOSlR$4CboO%Cu?S zfYFThwhN?MFH{$hY#EOlxzXhk)_EEH2lGA!O3j=fc@&L;Mf)~IaCLxF$k!X~`B`R5 zLo-yeBGk!#&8{n6NGDs2x`Y(`A&eGNq+34%#>9Y&WMjs_Z~Kxo1ZORk7Q`iV zD^c5mN^{U}m?5avV~7=a9NQUEndtWeLvczh!h3b;5tQxtcwr$Z@bFxCn6(rirP5HCjrBeFq;iZKPCB&2&I z#!Uj&LO+qAlWC#@o@H^V-Wf&75ctd_B7$T$o~Y-gYGfS2it;sB&Xk*`a{*z)L=4EB z7!8D6@W=-FqCkx^%i=z}Ml5Lp@N9d-Cd!1>WHeZhb|=^nac-LJEUirw696 zlpkBFKoBrwjd7-aUKWNzpaBFI863nJpNFO%Vr+I{aJ{pb_c0hQKyhkwN25azSD!QiNWUQ|N9w3ZJMW!9z_8CMVp#SGT~c zNwI%aX>fy2vbPGqZlC)58l*0L9yO(W`kl^IwW6B8{Yl?lu6ZhGzcIQcXIbgGMs6Ij z-=A~-_T(M}m5Q|ens$x8XRMQEI>c8G9~HA+^rFHw$ca!J$6Dvio2B}Q z9MZ*m`d>o2*DTKoUo!VkPu9aYCRPjhvo>J|=!LJm2`Jo0?QdL_6xNVj>#K3+b&Fh@ zOGnkZd&!j{t&-C)6Li{LkA6Nik-HBp6?bYy{-cs6!1$bfgLm@v>F8J`Idkvgk^+@^ z6P$+boykH{9_eu}6g;ninEIAC+=Wi>J@(e9=+xecOiARo(QpLwHpOu_IAu0={^lv5 z>iySHW|>!V$7pX@*BTDIxqPpd>q?hn&nVh9YjAlF-oWGamlnbl*7$LhsQIatjS1nn zgR5h4eU6iuf1z|yRnUZ!anEvR?JE3J6@zWSM(y9Yvf|rLsa}uG`2Nsp*%|v*C0^9^ z*_n4nr-sxz=e)sUB!G*|GsDb2PsY|KNkt81YsMCRi%`y`@l51_2!B5NC^!=S)YV6A zP0b4ndJff9k-EJqc6PMnN zn^RU-@hD0~I*seG?g_5HGA+TvceeA#R#bnzJtb1dbt6&%EBHhKeM6{Ybng?G7U7xld-^4%C@+)!pLwpnREYL|~i@t1rZep@@SL7b;Xs(W%;w`~|7-_1!eN%~8cz}?z) zMCYxq1Q+n9wTLnIf3aA-Yy1svD0bwjRl>CqcSY{zeEp`e?^JwPrOI61La=a0*Nr^T zP|j!8t($M%xI;uq@uuc5aXr^}zg)?`tms-JGo7wj?BB-Kw=XrViYrA3=%v=b3^+Q9 z>Oi|dF0}Rt+Q*jWWKehVO;d&sR*4#YZiaevc&iT}_typ-07ot?{Xk&NV zZN&5gFD}AT+YrQW%P{;P@p~zfcNptyTbVr+l|8SwwX2Wc$T#^x&t5aKg6jrAe+2}9 zQrdC>y~0|Z@22eU?FQVh@)eC@7U#)YbHrT@gIs9Z;BU>BK7+*Mi1(0UKUz)|-N6mH zr{EzRj;irJcGP@%L|mijJQ6C?O{JMH zmh3;vWd?iP#j;u#9E*K7r{TnC?iNh-cWWaQqbtj2!JX?k=4sE^-Tw!^XOBEAP46`d z8BV1yV>`u%FIFyGth?uc=WL@M;+|-IXLTw}v3N}spp9IjIS=clS%E#^8^mIxnUz7~ zJ;Ay?t6$GQ3o6HH(T1FS?z>Fm2gYb!Pl3W*;q4LqSiBgMNU2SH09!m!d_|S+g~zd% z;#b>!6*<2#S;d9d zG|Q_&*f`IJFn@n*mhUmqSU{g4x-C1$=~ij&q^dwtYP9Kv$XpU>{=Iol@0VGyM}Lo* zGVVvwoDES1-<2ewap&#sp>^VrG-EJ-dF8St|H9+`&N`U5w6|dYunq)53Y_Pf(L(Kx z{966DQ)rxE#GTIDuu2By#h?fh582o>e=hJGMylI1j5cG3(FUnLbnYq@X`g@#9ObA+ z?`U`w(4xWBYrNH=cQOn`pRL}s7}Fy2Yk#c6>$41*LkDzM=FxVPEXr>@m41?_p_OZD ze2;v7Z@eDe35L*?8Sa4GP1}sFpJHKLCM_q&an=;!i=85`Rp@BC+?=7*xycHcnmQ^F z4aLG=BQQYvWhiDdwYVIYzPYbqQIY->MI9Feh%L>+LhR7|_XH`g43DI<{|yROtnUfy*L+8;H9$_+c?`wYah*s`_skn_zYZ6dVuqb>_E_RvrDTBMNVUkHZ7Wb~_A7KEb zZQyRxj9#Z?=JlS;)_slymH`O<`D8p8F?sUo)*eGq zbxIuXC8Z5sc||hLLgyE`N(uEqVzc;2x4qkl8kk8>xTDx&+y(XTPRU5Zc&5#5hD0z)?4t zWa>bv9*kQcQbJ~mtECJZu>k~T1wjI{eZ_F278E}Z0lt3}gz%Qco9Gmdh?ijVY=M!1 zsM{?C&obGu&jrj3$V7M~qL~8wn59RMpj3sOxobO3;~>pLq^jBXWLH+qih1X-kd<^er7&hr&5Ta+8_IHE8uV-{b z(ryN{sq|f3uce@xA*C!2j5#|MATo>WDSgVC`~Y3!?*!k_vWP!t`QcST3}Osv#!d*> z+Q8JR2+kRbr_frUMJ%q@dHBs6RWPcrB`>u~mXr&Gmy%G2OoWfPyNVCGNf;HN5#MD^ zcj5DZ|6^o@d_9i+cwl%rj^b~9g(4ESq5uwpa>KesHoYlu=Ns6>G zzQpQpFfdu^S|DaMR9szV&4E)rFr(Vw>#A&ndAD$AVq6HLP!#}4OQ@jLnQkflImt!V zW)}noJcNDrGaZ%hQgG-HTR1w`WI+Cp=5wuL<9ng z$`ssQBU^BHD22u+_lx^lrBGmi6vHWrje`4tRT*34^Cjo`%QvmA4XetviHt78TZk|* z=mXMkXxi}JjDj)%tLGaM896FICRd>wz0E`bUTNu_Ue1oV=PHKKu@*337y(Bj{4sil zgXjrJ5cyYFRv?p@#t*B%|NdFH4O~0!`k0Onq9n;1wl&Z?2m21tIO^O9qnATT1L7b_nW z6@2yy5HI(XfL)?9VPMKByZ@f~bEDQ1s|95VG^&9l=Aa7#cSoMsjQA_q4Ydj+S70)& zNzp*0G81ajj)vCi{*SR(KpkT_IhI5Ifjz>~ysr->Ph1OR?t+@1-gp6E)Udb3c&~@G zH`N$Wu@Qswv>HKKQuPvvwhU;7ZXtl2kQPRZaP||NY1=ZkQGmH%pF|Ho%U@VwJSGwo zK;W!2s{bW|m`%U12=z%F!N`XHt|Y+D&UE(RCQxF7p%dy3ur;vXEW`FjQZ_6NxEV09 z7pT_|h#=rVF33p0Ff-ET+03B^d_+vJJR%)Y5Q7PF41oX-1rP?V#t4C&Jz{^cITGedg%GHe!Un>Gy@X z590-5X29aX48#-z`k1Rj7soKzO7G&AkJ_J#dnGJ-!8#AM#h45 zu?YT%x=ALhd~e+Q&zKahV=e^Pp&acGHRikpUWMg|&JYGTc>_-fJ6><~`!AOb&{ujeCIiKvTM5%>y)7>X}Sw zsGLkb3wLakj->cjA2-yJ;kzh4cfX=O>cKiJ@*;7bf0L-o`LvGY=I6ZME6fdTsJNST zz-}rPy zB7%D5f{VCST(ckg1HBjprV%c(UrXi|1j;rVwDm5}xh{Y3X|#*2e%Gw;F97Yf zc{i=zQ?U`o$JlKKI!#-;VX`JUb9ccSrCzO+HS0jugTT$t^rtzxPf z#~e6;hs?Lg@*8BP6LF4vg7h!zE_M%VoeL5&69!wUjLUU>s%F#7CY{-wA?~T_S_8sM zqS1kAcKD;wk7oac;&krulA~hiWCz7&n+s>$8>E}ZY_&fc{leG5i@fFfh*ToLZw3g1{}Ncmug?8 zl9b;BeHOR^lj8FM`BNYHLsi`}}6{3zdamM6F(F=EI*Bjpt7iq+!WgE9*C?b&bj;j;phnS9V z-;Fj7gJl}W9RHQ3`)$;+Cn3~{LUI)u2J|MH7m1~K>f5pSCkT+^WnDKw$(%^4>Nxhk z{%e{mpSHFu%&><2G)k&X$dc-(vJjFTOkIw**5A)7?Jo!FyHUW4^`CwpdFNxEAt;Q_ z+2f5ag8{QsQYL$v(6Ly5dRz&+Ydpa}8&_Drz34eyV}ENUoX>REv7BPdGLR$|Cd?w) z#2!pq#2a93$*KM z$mgAg#QU>FIGIX-JF|VjKh+_#o+;i6$qb|%`3Oy~)enh2PSwa+8AAqvqiOhrUx+SB z3f^jbp1P$4`*(iuV_89wyj&cHojh2cS5gFyA`Sh8qk>VX0y+m}AqjTVc_r5dstx09 zLu);E$FKa+AXg2vipzcKmP4A7FzB?f85i51A8@9@gW0)XM`teMQ>&$pvex&G^}cR1r` z-p;V*gJa<8OL-wt{6$w)gj=)3^RSClze{At6^Q?xapXMm>2~t4vF`3fnZYjk`+eJb zJ1`aGCo;IJ;9_feWwdUvb>&^-x;ZhY&yVGGt%D}wm}lgHfVa<&<#xS}du%ga95e1f zH@5JbqdGbE3+Edp^Y*^&cXAUpjP$d=1io%sk+TBsf-gRZq&{WaD+-ITSY0|NWVbrV z9nYL{$YEWB$D=Tp$|+NEP;I8UWW4Fp;KlBVOI0Jj1FL1f;i?jQtYwq$hjd7a;J`0V zz}$2i7+T4HdxWbG7kQDx$o^#y)dM$ItdUNek&avy;WKqOx^)w5KFv2WTf;@cy;It? z6SMz`Vz`|_w(wqz>At*zQoAXMz0l0NYk(fOSH;$Jf>iCtjiEplB%KJCb ztx`Pk1p5HyUoeb`{QC@F%gu8Byx{eKTS7tY=8BBOxusPj(|lHJRaIM6?)1W(eW+}t zEaA$v4<&j2{pK?eGOc;-_Nw6#SA;UJtF#Nj2>=NDl$Alb_h-x@^dw8h;k4TW`!=0J z^71*DSGS@_V4n}@ms~{kndaN~R4Can4mD0bEZkVd_zqa4eVo{5It4le64ZGtR^Z2UAR-Ux8JaV|Km z71?a+2jukGlZt!@0X&ESSdS#`!AwewdSH^EBg+H9Y8#=pCqpKXhb@KYe11r|b`sFr zi&{*U{XcrAQ~7284_>PL8-<$f!E9lML&=AV9+;KljbQzkNML-RBckjrTNO{-g%$2QX zHs!e8+pJrl6bdECNeX^aQfKI*P+`zOvSt=&Q!exhXQCa zk-P1|L?{TRg^_)=$c90u_<7fFiKdUpQ6;drBftYCMVKsvkp$r~9rL)2GXw9IETkPC z`m3))goNf`WN^p49$@nAnzWk)5;i)3R$Re25|cv7B9>~bxpe3u0)AV)u838vJQg>? z84%FO`gf0P(Hab?X9Y`>&buz(%|?*W;Ld{}wTy0RQw;i_k*~Y8fioR>HhFAbd%R$5M~?0?zp>@uZgECxyz|H~77{X^b9;@{95l5o$_q;hJ@OPv z3AiVQW#9l}fRWvN#N%UlI(Gh?vY`CHiC~fvStX2DxZzK&}Mf+6;CJxbd;S%z`qZl&OoPjySb+8-uLq zs|WwR5qyLH!gp6KeDGQ;+M$jT8mA-oq`^n4a?SavvqPrtrmz6TSjmiz6 zcOtq4_2Ez@G)PU>w6FncNzY*^l>q<`MdNQZ@=vlDw51E?&k>j7i)?1qq!p+PXkePm ztFB{c$x?zj$Y;-sSea2lbx!r>Roa*F1mxjrUZN3_=@RsOD5rW3a961Kfiwfn&zno*NPt75hhpPlUZ-+|f1IQM1Fu=eOgG)bi$4*uoGsbN7$QKg+6(EEx8VWiU1G}IJAZe>U zJ`3z+lTZK#tUVr2n=>&OLcc^dfHRpg1`-YIIU2(AU}&%=nQDWSCk#!)*dVIf(_Kt}V55_bP3jghHHf8Lhd4eW#&9}D>N#Duf&br>z9D|OrVJ7Z660XK z6LB?cAqX;b7oa0yBm<)bh#FveEnLzwTI0W14P50j`2!XXS%ENV1)dg`8UAD&54o>1 zz5`+!=3;|?f_M>B6VU3=Yvvh0U}gCLdtS>xeM+$np#L3)7Q5no%QYpquyLe+(QBxd zN>%{2=yZ(lzyf%7;SyZl22ANzUIjE-DI3=Y zobnNz#1Ad%l<%-PCWRqG&{1W5r+k3?-EILSXh zVsNK_+K$cD1&@k96W8baiH7_>Yg*SJ4S7CA_M$!ybU;>sq|{AbT)SV~Ux%3P*Fa*P zouz(~c~wKLiRA^7-;T|FEdD+MGVp%&_zT#g`-bbvDn`CfR*#uuul5%qE$icBSiJU+ zyLX5P15xsB-|d=7Ck~a&s_Rh>B7F-m{7?O6AgPEb^bb;4AKQwXd-3YmVNIfm{#{dX zwxij7F3#VFhf|$NWc}P7FCAN720RfTth8UNp6K^mZ~<0>O@8(JxxwM;J4VH6-8!%j zx(Fj+I7d?TJ&+nW$1C!3#QnqLCofOTRh){$ZY3R=*n>#)@2*r;@qCiGpuGGbPB(@P z`Pe6j$#)b2xt`|V2nBMljr+aih8p^;&{=m%q3wv+$8|-SFCD-x$_0bD!3fw4X}Nz} z`zuVZieAbZR@Co3)OHae(XYElKl*Vco!d*5wU8SGmWqw}cwQqT*PQ!Ow@ z7+ZRZ>s`~886FGPAulIz4T%@}1HhcM6w`JWxWSe=Jy+9He=x zw>Pt0|4+y$l25J3&XkoJqru5XR6M{7Z!F{=YaWD{t;H=M9mc2e&T?L(Y{%PVFN~<0 z_hTZAc$FEqA3`uZ7_7T=QQ43y?$@16g|I?Sah~c`r8e5WDc_=HRW;-;nVUIy{C`p8|BBgi@tT;wnA-E>_>r31a01M2;~rP5!U&tRC*idhV)*Z6R8 ztO|r<)zp*Jd2#6LV?VSOEtz}_+hr!+MuTiV8vfmZ4FHZ^i(Y^(D-U9i4rrOWo%6Sm zyAnfEg9%jGwu z5?xtfv-uk81k{pbyXG+0#V|44N3Iy&2D=%P=cW0$d1n-}Hmi4Q1Ay#4WOW8d;l^?D z47zwD?MAKM$g(Gu`|ej9D4?0hx6d;iH+-wo$57L^s}#Wm^%o)~eQaF?-qWLJ)VnT8 z<=EOE2Z%G5h<3S)qtlEqT#y=$=A0bzZsf*{?#(0o1Khy;EHR2>LOOP-B^f0r(*~0- zOHQM<2av0G;(radPziw=ou~vq>BNZf5FI-awH^G%@L;0@f*k+cj7oJI{0DURiuo+j zuJt7CLIa2il=h6almR9p5{wx^T|^};{1konN?fy?vtQO-@YZpmmD1Mmx6tsV;;!Gq z9mglDI+CD#I+6he1+wDYbU;kdUWVJ2za4pMZtH$EdV;)Z`I1 z+F?W{IY8{SaqrvFf8VIsiZiA!x)iZz2nArcl;WK3ZF(@WVzdbBEM2H!qNp?}$+vuR z?t=b9xTQ==673#|4fd~!lo|Iet~jqlR}6S_2E}?wpTV!D_uR$0e*xtF3DS4@q(sCY zJGGahECv*uGZ&%9;$;a?IgIq|SNjGAh0IZE*EEy+hs6>m((EJ7{|S7Tsvq8%Pk3?o zTO-h#iT0y?8d?l&7kQy{EVy02|K1bXEiW93DIH)SqIc!td!Y0$zE*!XrY8lu#e)0q zd}63oC8#EXrVkfom|&o0)Y?771mo%{qWrs9dqFW~4tNLs3zs^R2||Q>^N_PGdWqoO z{Y!hZ*tZt{|HBy*&wo*k_6Yp>k2`~=Tt*w18CQ}U_@o)0QowG)B z!$tY9bxBLkAM25#6ARvo+gw{DNMwSF+Z46n%EB6uYOtYb#Y{&TM-A5fE_Y|@Gp^E3 zhQc>fSA|zzRrw*B(tWFG@TdhMJY~_4M2O4_|5Fs7+&V8J+m5G@gJSk9U#=;U`Y4~g zO!;H~P^?h9-N*%Z&n&k%g;72PKSEh4Qn0{5lzEGvcTi!D!y~Sw44+*pwWUT_EySqA zb~xvwv8&uI#dpV_m#3_@0_erFGv2{#Vm<29t)!Z2^A3JjK#qM!mFrCVZf zU-sKKtI><;9Tybk3jHMtcH>fP;hXnx!tyC#e9BWHY{43Yp(>oO>X01=_!i<_%0N-1 zIwVna_nruWE~6Et>#(B+8}lwYS+_kPS1?O3!`!)9X1Hc{Typfwa^Z?8rehtlccNF) z6F`MP5bavp zqTm){l_FNL0SIe_ZHrhDVi<5mOw}!9*x3@MAaM<))8m)@8W_9C5Mi3$Ngm}%Y>Le( zazZb7B&Aaa{OpD_j9>{({kPK?+I3A6vTf`Fk;JLAOBsYgJLC*A?KJ6}*XEL%Ub z8gY3kurr0N6d?=D&;&3LK}5#?&>sS@MSSg+0;a;kkg$e-!Z6e50ohTD1E>XPJwUHO zMGlnNBr6qvP}P6JKp~rs=tt=A2jqo1p@o!T#ZW=m!s23xv}Xx*O>pQRM!5Nin#vK- zTZ0bb#ALz36s1!|WLA%h!C7FHfAlYa|Frs8Mv`y<&1#m&x2ik~|6NF50>*$Rtq{j6 z)!hOoMrhCKBq|bWzP7L?c)`_92=z(U1c)e%ud!nKO#%b_C9?HEDM4JjTYJycb*uU8 zd~Et6hyDx9HEO+zytwUzBWi(w$t^uw-*F+3g&gA4bvsrAVY~6Zc==agwno}hS@+BnAJ*DOs0G{&muBrGyYMVu# z>BQ%GL`ktWuhw9t8#C>>{a}!Qre^3)L9GOVM7YLwBQbg;L9ld^ggX^G6U)WFVU6Ap zXOMVD*%+vfPi$~WwIoDfiDM`Z*q2B!#LTWK-%OeZA_faEfiT#RE#M%<_&v#`@J&|x zgpS-e1VE8z^^Btyk$(<-8`KxTBL0S32mjm4aJb+}@2bJzY-7K;B{_Y3IVO{Lo&^=N zXNP%#92K@DD2Uc#W1yyVNPMFPVLA4dgi>^Yz~`C6!_0Ep`R9wN16?2O1E9VzYm!?v zK)l=0)(GS&c^Jap@DrwOLYigp-hUhknMQ+fD9sMCg&_ouv+W^nlu#*N4|KJb2}vJB zyTjTCJq;)~vJkm!rq9^TK0WU^IMcYh1`mN>BY@AgNvB%jrZC)KS|$Y1mB7r9HiBS2 zu_HqLjDYP0LkCP=jHW;hu#7)_A5IVKW(e$4zd3r%criZ+qC4>~)*KR+GU!M0HzJnc zk=O@rq7M7Zvo;0jVPHA5Kx{^y0&R#HWWAX!B3L1UXr)`1Q#(D>pID4ZK!h<|j|vnQ zdWNlYiU8UjE3MyarVBfP`l5vUC{zJ4!HN^9)`xo5;iUlHV}#!%wjnkG{3310U?h(S z8%`?5iDXr9%VeH^!ca5e0zOtSf^|3H;YnQAkl3*G&1#Ak5~y4225ced!&8cD4T6Cc z(Qb@!^e&_4sKlB|P~3w0CHE*(aN)&kX{J#3WGE^S`2kp^HHvGoEBSLM^iXtqSH7tm zxkdJa@KXp2Ao_Llg+Hbjk1HyrEwhvki;QlQNlAqT=}0hCsZC`~(sq+m4c~BJcU=9x z1K@|mQHI~@s^+%&2R1nasLQ^~jhBrR^Y>@Gqf5}RVB(FvBgNAw+gPWleQIU=yjbF= zZ55jol61aeP;2{X5I!4Uwl4etJo7Gj0N(f4m5C~^EJp0Nf1pe0D>tq)OTZYkO!(}r zvX0*vSU0ZpE>W3&SJ)()>_4j8gIbHMX?)tNw~bdj-5Lw|PGK{+c$ef7>QDO7h+tG5 zkcZA(OL4jv>%2ae%7I<@S8lLR6D;NI5X+?@4-nF>7gq-Lj>{(_tfBw^A$+Le-XV!k zb3^W-4Nj=$qa<2x>HVHz)7Bi|+N8t+Dpf#OS?5@Jm1RrMWwf>)H+&s9qHRh>k+rAR z^)ApIpBUk_CR}WTX9rcqiX1105%-r^Wx~GA5}z8#{*{J(laUbXx>h27@q#d;YtaU1 z;BKhBBq4Te?Byi>8M0x(iW(BR=v` zHAD&fa#o5bE+qOw%Ilv9w+YmL=Xe&^L{3Ft?1i7dcE2vZng0R1y{Iukfui3=ywZZ( zJE%#Z%$!KTYz^wPp+nZZn!jRoEX23Bntodw$pBuEsf-X1-_lWp@Ub-Gi4 zPFNF)Tl$qst;?w3foQ%+) zN9#e~6{liOk*mB|@z*Tl_T5U-^#8;86qnES4-ZQ-;TiG9Q>$MbbE;?*cOC8GCib~X z7lbbmw{WBJdS_m}axk?a%XFYyl`knoYLAGpuw4un!sGXThSI(9++A}51be~lEtjfg z{c8pnT(KSVl85VF2-bziXf|#djYAc>_|S$+3GuSbqt?3519ph&j#*34swc1+N-uW` z%^f_4MWY$~?_stv_J>h-gV@!f3e}VS0dDC-)}wuVux~GTu>CU4!-}kG*dY8eF1B3*Nzqzg zQLB>-&m%q;EuFTIK->IA9rvNC<)?P;h9=r2QEA@P;qCDiA?;Zc-1JPnLn}rY@0bs# zrJWFkmzYbBF{-$DY^htpE^pkj1X&(9|n#$TdCJ&1S zUtO-h#J?=Phg|O~dpN!I`*=m7yueP9EVCI1(p%$8EyhPE{e)6z@YUD!VxB5lAP$Nz zNrwI>+W5M@3wZaKQzVGvs3|wW<3dO8ZIO3~yP+zpbsEJq|7J5=f=P68t2a&|1hX{(hZ8p=%hIe9zz0!|5!A*fp^hw9%HbkcJgrK z?o~g1*lXDFt=LARo*46mSBe}PX?J6v*AHpO~4ryXnp75G(@rS?k8n zAWndOm(WG_+)|fZ0s#<&FGQso(9lFhgfi?7VSM`{dp`99Zx387zb{K&Cz)eYe zd|4DhV9mdCGEGEpFZ-mc!Uyp8S%&&mLSl`b%$DM@XU;OL(XXo#kZ7eRPB?9zPoGJ3 z%?~z{td@?6qhodamYsUm;M?5KNiyifRR_zxSr@^4royFG!*q7LA8$gR$TDP=?J3gi z6`KWv+fd}_j-7&QBHdaR+zHqJG$L2W!Y!Tri>ljEGV2q$og!st^s)B z3ZFB`*+J+Z?0t=VqMaDM;iBH5Y!=>k0Gae1N>tG&oe^={@BjA#tI% zJ;h)Ljj+X9@et;_a{SS(|iH?10ZA~<9Wh+xIjeY z1S{FSE1OF@n(Ln64Yf93h6otZTE3%#mr~jaIXxBtQp_crj$t8vKJ==j^QEi{Fpt@> zg9g>%3qVd>D6Ybq&JEG+$@6|^SW5@xbrfS{@3q*rA^4Otsj@I|!er?Yo&!o_s7?d0 z6vT^=DE4GI@d=n2Hirt&8=88I1Cxcb-qUO*z=97kBlQ1dCbt@ya}=Djpfh!~lliki zUy2okXT;9*Jb0jVRF`tTz`--FvYrbIDBid7W+(9h2HR57@3riC8yz}PQmpETKWZoD zoWS!@e`v-(NRCcHG_peXj@u$j2s>~@w{LtGPdP53jnL-VGLr;7*TFc|TP`1oCjv3Z z8$sVp&De?MLC%L!T0xBzS3UzW!72FS2pr8U8F$jthkkw^qm5b2|&8fh@9gty+zz6k#8 zkJ;6qh{m)Lmcx4vk&K_|y8#PmULRaCL-#tE4l))T0vFYqh=sC!{|u6PE1E~jGn8gA zS1$tqLT}Uen)~LGZp%e9V$DCzrAUWa|`V>4Xs0S-Ge^=8R1n8+Kh^_$}3{_Yy zID~PT|NsAwQ4Cb}Ljn~B7g>gurLz&O?)ZYS0#uFF;|6^^Q3Pz?v!T9@M7XkUj38eL zDVF1gW_U5Tk~ZQ#aXKi{g_9nISs9SH4JZy*`pe8}H18uyFy|A{21%eA@uwBhQS>&b zL4S854&X*X#{_}}kwAr8^KmrtD`8iG1#h9YaJYB`JHo1DT`5_n82NN&Bug~`mU05j zzJFH`mh@nD6wg5PGH`XnD)R#3>u3TqTLPrKSrVJl{Zy2NDye7A4W#cE=sKqFCnEz& zkfTe(Ou%FCrMP0cZahX0+8*#(Rx^N4TW$wG#?yJTMnJ!qtQbM;V<2fQ#A76{bK zb>qjwX$;sd&PWYTV_>Wejz(Qmkzyu$mTKrXNMl*k(jmuWB_iQltb4hyZgzZ5o8}_22{Qnu0C5PZDB80 zsOm_jfhGD-qzQUr-2BVsSZExT3qk&phq)TRKCdKo7EKm+p)JF;l6kMOri19tfQqsC z9#lH9?AnP_G{o|+IwWTYCM_kg{89-5%^)Q$WS-`0S-U#*-xZ(^Vq1<-M)}P6^SUx%Y|_R)=dv z{70M?bpvI&>wuGN(W9x|)~`MXwQMT$LWS?fU8A{=M?PDDJoRJAI>OuNIroJ7H4o=O z>nN#yyrLM01-^&9hEmrv&wrny`pKx6q4y?HYO>7q9BMvw5%MyugmQUtac)hbAOEZL z$FWGcISup;Bja~zsxgFrrKm})f7!jEMZ;x%x#|f&0<%k$S}b{hK;-VG!PA3ly~J4U zlhZ+|_S`M5a=VD%tZqef^=W^zdPq5jPbCNSSJ;ssc#(^1j?!H8_u*sC(W4@Ndx*%# z#@y83aEo`DuiZn&r>o*I)u8LhW7+o+(-M8;*M1PwBhLRIFRv?B z@i*>;l8)v!uej_{uF{KpGRx==r1Qw0`-LguGL!5-(&Xuvjx9t!^$Y^1{*fWrP*h8` zKgi}n;vgIIL{aRcMv3~k$ulJC-kG3d06c2Z^z#zD=&f)+r(;#79}MllPoWbPIn|st z|Cnt!E^aZE&W9duxcZgJTAxVX5Y6T*9TlVb=1ZmnNAD$!=DvsfR7~utspCFU#3!Ss zV6<-tTlM6$dqXp~FReRvN6TlF8~3e%Bl;m&blZ+%uLARLoY&h0$ELx-gdNS|t~EVP zZHmtMPp!T!=sE1*U_Neof5n09BN?q7kXoS!tD$tg{N>JZ&g7`D5lsxPuq6OJIqfBu zXT|LDkzDJl_#q!(vIU+I6{$mW(zcpEH`%7Onos}WhoNk`eh3Z`NKUqln$u5Cs%}ph z#pW24aBt8$ATwzkg19vZDA38Y9<}UHrXK(zO1cVb0Mn8@!2EOGsK}pkZ#WyFO~YZB zVMXr5peWzp+oN;jxY};xhtWFGY}5K;qHWNSc#qkt?$|o26Sj66&AEh@*4I-)0|rGJ z1%xXiul2YTX@Y4J_yV>5>#P8C>AfqC^@j#wE0~}A=TVvI#oW#ZFf}MpS$0h+o4jH8m#*;eT9j5? zZ4+ja#`EWxK0Gyg6(+p4m-QS71*zlmD}(A^LFNs7c}BU~5aKI@OS8}l(L39Qz1@rN zuYyRuU1spy>m^Y)yH;)XS7l6I^@RQUF!kRCV2g+OJ(C^njLznAwx(@p~9E z4yI-p5!#%*(I;jRDuJ7;)RzA8b2z;iYpgmP=`unwD}Q3iG=_&tfC7Q4Q=jAT3EKBO z`Fn9qIzuyyu_8}Mkywb_#n4D{>j1@4Ffl<7ORLDK!`xmWj1NyO&7=N@EMof41Ch)XhTcnZ1MT`_yOx(Jb|x|uhIZLGPEYN zA-4J%Cq6@UnB))y(pk1psL7)CSq^wvqH&)EJ_8G~(Ktib%`SJ2#PW?GY-N)k(AW+k z9g=OsPW=81DPDk{yX#ciB%#fUl`21v4RR<$Q$iQO|2pT`bd;|9oy|ZwbX6#@{V%|M z9If9GFA|!g>x20qgF|hTmIuPFdOrFzv#SNB&fFrjdhih1`=PN!UA3~2K}}SB3*uvW z7)(yl>gng<1{n$sXnjQ$)~Z7bfnT6PJUHlqeJ)v z_B*RwfW(1HyAT)!AG8YK|~P=*UIRiG0h8&}pg z#d$Q21istE|m1#+_yBh)onD|ETnBCPQxF zo^$;(KXIzXWI?fer4Ix8;a~4P7=ExEDq0i{yRG_qFGSe_%ojZVDT^x=>!IbrfjKw3 zABuq8aeAmzuc37$6=ehXIt#%31AG`dSYLaRf!5LzxdZ3AXHTb=F3efDMNjlaV_=v` zxW9R(fnXrnY8RH9u!c}bJ`2qo6ePM$$dV+R=biivNgXzg$h-{A!qdc0^y2Y@*oIHj zK&k#|k)>jUB@G-B8MJ_EVRxBo)6H+nreJb5cCB0?lTO$8A3Zt)>Fv~#3?Vd18wsz& z&5@EmXg5X&gBhLoOLn-pP{F?PMv)bpZ`5oKKxnKFgs&IEXLL*Sz|9uiFS5E;{TT)< zP3>`17^`JBpib|Z@4fI5OKHas)_sGl9_W^A!1=%5)OOAI6g#nTY>@l%>qg&#=$> z%^0Mbyaxp(nL`PB^@*6vtUy%f@8}h}NLS2GZpd|6mK93WyPe_2ZxvV9Pg`IVwn4jlXpZ5CbL1 zZD?kr-E%^z%5Z=xm|6Wj))-Qw)PbQqMbtla2xgmVRC*&H&!-#>zvNma6ZZA20v85_v;3@~8{yCq<~pYztlh5TpD;TEAMr$jabg;X|zCwGlt z7%mZHjgUzt-4H=QX$kVG5C*U%=GG&)*ug=P*e87B*?+VES(=>cF1T8E3?EB@dhv*OuRO-x=n7d85Ib#W2CollC&C%?E;vcUE+7W1b#oI;mK9b1L(&$L7#C}Dx z5)`@kKW4J)G(G?c2PEs~LK%QZvJo(lkR&!8Ks^Ak2}i2Dw_}^ovK%}ibW(-O;kxAa zQFf1_3iyw@rEz$&rwbA!Mvs zod}-BHspUX<`(P?2UU8wq+DSldYcKP@GXKWvS18^ZuTZ5?syh>;q6XPOjPGHlud>8 z7%aA8De=$nGW|iFXaJq*^E;92-b>pzYc#k;F&}p!;N}_Cu%XYdZfsQxVZ$NLep0M%fyOpuL_{bz8f zzGLx+=p6&rhTtkHS6IdzbV(U9>U}~<0NDl8OclUyBY|Hg1;EquaGQ||8ADlWH+T*_ z24=Xzw;1hXguraJDcWl{L=+U6H(=yci zo8u=Wx0MLlE$hl=5*iE3qhyO0udAt~AS4X_?ex|-VsOn&BqnTf!G@HFVhD1Pjp18- z?E;_hGBls1aQ-OE$<#!x5E3y3fmtAuLQ(Kh zJa*sa(!7z60SO$y(D}Q0XG2lKzIuGs@X!FO(yOsVfR~?RnI0kKvX>UQ@QY9vf&;mJ z_J^Zz;h6n~o#=f{p*Ll!M0S7-Xd`9?4|RM4A^t?SpB;;vB2S#ty^alGwd6Cd{y{iA z)(_+0Tz}A*S=D&_b)Ew77#7j9I6QxbG&;nckiyR=M|CDdMat4z^G%wKexlu?Cq0pr zu|n*NxJ@_@`?wGu;vy_z5>SVdT_g(y*vVMrQb?~B^iWrJ6n}G%1swD#00u)4RYKBj z=LpxvKK`FXM)5anYO$wSgO8(!X%bqi{ol_vu#Iqr03_tXOPFP*pqWbNkaw}iI~wXh z%LfQ@(G}wDGsJ~<{nydZiBEVwmTezCjlw;${)6nVNK;wh$H6^-5+C6Vj%eJ*PJs8T zQ?598;w#}<(Oo|akhO+-zZ2N75mhCdVjE9gPoYLsM3#p$6osrcS{&p3FBlO2pHo@k zp`+9k-zMzmy8xmCv8v zb*iUUp3ma--9{Wf1H3g0p&{^uZpiDD=pz^_6PgGM5#l?x6C&wCTMI<;gEb=DnSD}RzBv3yLqH4eNe|8N zK=ck)bQ|(nAE6;mQ+~vNvtkdR`=Kbyi>nd$_a1k``xlqX=JL%zNZ93LggN^phALjo&GlY>5%5g)mljD8CHgUXYq11Jw`bl=O05JDJ z3dAMD9~mA}bP9U$;leS_G%oatVxa^LG`2sm(fB_pdr78@P=($Au-=C63VYMC0#t12 zc=$3jc%r_!sQ4W>yDe^M$44QkMMekl1IHQ(odPvn-qgUZgV31cfcAYGp|bshLMaIeKERDYF;NB68Y|kcR zc-wxr9!y?J4H*#6@)W0&HkaTEE)An|6rUl>Pf+hy5s`Epx3M3rVa@RRk0j;ns__s9N{}c z@x4sSPR-%a#9KMF26PJE{_g^@&qfntqX`mLmNSJZ*=H1^Sg!{=uA$&FRYN7IdAOQQ z%kt<4cnBhZC?RA-CuyO(8$I_v(-Gt%X`B#;F9)@$lTIwELJpN?2h$Li&|lT)Kgz<< za0(>5mTC_*rmLI|Fx3<4hU4%#hN*kT*2!9~1_FmMkngs~72 zOJY^Z5HU10;5G=5p`SW1%1wN$b~2C<3N|k=Mactr>!z@^2;tT>p^cM6C_csnnyiIj zyp{2HlnPE8`2Prd`|zskGu?Lq7tIuB$~AivXdN}z%)Y$J0Lsj+wn{YtyHn24X@zdX zTP2x+>4u`MwLl%vxM)4L=YrOpYf8$lg@%SvC_!loNJTJ?J(w1^kbsE50If_UyhJL5 z2zgna&wW4NwGunC&-o)HBx|kT@ArLqp8L6<`+2^*s|THaoJHeo3J9U~ z41v}SHKNLkFxOM!c!mb@BABuY+KgrDFS=*ifRTtGOpU+T3WJrA)t%}~?dCeuxMKVS zBN!QLMYxQ{+ zKC82(W0^9rBsdQ4W~j@>LvAp-`oH9r`N zF($pDD$s>{NL+&0Q5|!WYTl(CQ2+c(K3+r5?RJ1ZY;5Y1XTFWMh!W;xER3qz9PI(%DkJCVMDD4RalEKNn)RVyCedwRkTU&$(iFOplnkII9FMP>4+(*5LlY8*|=l4rgL zLhgQ)rwUHk+^Y!pOmqD<_yj&;(YMTW1E_70@z{TzN6V&`1CpnQ3Sca{#K6|O$WLi-+W7FVw*0kA>SqrJf31omPoaR^#c=1Q>MXYhcc(~wfdy;NA;RgrEP zz5odox&hAD;p4H>Z{Y$Hpj{NyYsoydhUG&f@)}aQIZ>Fr%KnwcCa{J#)*M;DK5re0x=mevc)w}s&s$z`~&uX z!F3j}A1y-_*$N1?h8_t*a!}R#ng5nYi3>P(`n~Nz%Za{hgl|9_WwHkM!gqCDV4HmA z+@xQdG4_!Isu_VFq1=c@A2JqWBvw+F5iB-oLP~vVGdFT_=6nelsG|ht=`QeqLZX?E ztA6NX`evd2v|%5zJP=^auTw7-?-039`sp-i^U-S z&!M7)92xKud1!+txj#h?SghjTJo6H(?ydlM@Jxb^(>?>Z+5K0t7v@f9ZhEguw8hB9 zsVik;7e@Wkxk;`DTBwpR(eA1d0X%$Tq2TY1!F#aO73W$0dpE-#bEE{phq2U(zg+V> zYY?U>z^J;#hI1Yas~TCX7r^=gEb;y9?=f=R)rX>{AoF2Bhb}PV?Tcpxwue|s@Ri(; zmb^i)GU?ieQPBlgd~7eIXvfKm7gQ0|^JZ=plO?yqWD5#69L+v`98h?qVX`;M>~)Dy ztrLRy#EHG#d=UNz#Rr!GG*z#=Mt4=$T0Y~ouF~TV|93(+g2s-_OX6oKlt%|mcCy?u z)jzGcFuMN5<(FQj{=@gDeFnmG~e1AhDn8 zPL>b*IniM`lrM<`+&g7JT1xl<7X+QBp#q@e`F+d1v!&3I_B1MVD7S(bC5=H^Q9uam zf?@>U%nqSy3 ziD4Cnj7g~3sWZ2rT%&nwxkHyg+Ma2_Z+{_$RI3%7LB<_l@>N5OE8>}8W=D=O1T*rS z$vIs@f~7|Q9#5Nb6_CWD^#}*=Xezb#qGV4|3DrDr=_L#$@y0_~sqFWIR@I=A zSl}&fj%Pm!m|UxffESg$2^M;Jg}UdVTViB}&|->kQ^TI^E75R3Kt>?{*+)5n(rD6d z{BRnV`DouFfI3s4{C ze5vS(>xX7JE*oQlxojwPVU_F+5~**P+5m2<=8Yqp%E^Tp_A#hqS~$h%L)J^smv8^o z2cJdJyn6wrR6}sznm_I)r}@n#mGCGAgO*5CETbh=z`r*Tg*&%B>D6aoT!|pgehFNP zk}-#m?5EROK2efKdy)O0A$f!$sEEYsKU)Y#Hz|{xOd)4ZXpo zTn<7EhyRB(;phqaSgl!n5=ViaO~hhFB^A36CHf*;YOQF3j(L7g;jYLQXwp@Y za&93yqcYnRo1jo+oB8xJj0O$(C(y*ho=tIaKN1w*h&1*%yg;poKS1P8ZT1TXRg486yT^bExN!!P{QJiUj+q4VYGoSJBQ=3H6_dw`k_sY zO6qv`K=mtx**rrx03)P;#>ULh#z3y+8#ie=QUB z)_A7;9F+$;t0&J)g;L`VAnlgB@21Y!p5Jxx8qPHOzLu~JBjbrw#_S_vmD-fo(gZ=henh3 zxd^!pw$%~vEy(bv7uDLTQU1-jb?#%*)(yi;a|J22>vUk-rjzu=-`e$R2}zvXGig0PLmjuQ zQel@&6D%U5ce1xScZ&?_bQnpBep+xgH)K06JxTBUYboetYGmYcsE6jcuUyP}h?hb*=Vh zHtE5Q!DcMKW&$CQ+3L0M3Q@0*IihU;t6Ht=hIG^sujs5w#*zW-`*g?js}MEPfHx|z zavG$9wF!bnS~awY^QS*w^3~HFC(qoX&_*ruww!IG96wdfNSQ#h$t{~Z7B+*g5CKGK z@f0DyJDsOw%{|52Hk{o0#mWG=*>$rOU$o%r>4bO3m%eBc?I|>Qs3v@f13QJoS2G{G z@a1Y8*1iax*7x@P({~xuW#ih1oY!_<(Kx?qqBH@jfKU*5fVn1>t`>DlbWa?~QUP_) z4d{4ZQ%6~zg>iI|Ob58wW+Qq@^7k}_2D9pUAcMpVQ zev%pTE!sYTD?>P5+DvNHM~Y5Mgud_tNJ#J8B~1jj=2&jd0^kiL5JVV_5R2;k2J+{u zN1bxO(|WI~>x05n_{T-xidt)hWEl?DY%8vOe&4snsBa3}V75<5dP+9Jg-BK&Y1u6b z^RTk(hKR2Nie01Z0qTC|h!3sYt8n9*yTya+(@ZC|r+5|x_5U%368&(O?a+E4z zjR6mR;M|N~KPLjZ!r#F}UfSECb0hrYT@i^(yWOQ}Sj^0PuuMfPja_=~$_3X(jXh386 zkT_w4^`i7Kzm_^^Y*RR#3R!qq4g6*ihCpaev6%qS$D{w!o~0H35?px&Qg3Uk52E z9f7zQpzdXV4nQ!nO`2kYhxw75Ov%v&71n|N@%{kvpjt4X*`d?7k9<3eB@dj3N|#9# zj7%@QtD)t;8sgv*i0A@GA3yR2BUON@!kl0|J&iYnLPw!2C_4QP1#tnO{dB1>s&v0z zotVR*b3cHBQCuZt79?i0;Hl%3m_&L8@G|t1YQpkgFK;sZ)w%=HdLhSMxaVYd$?SyG zgb?0%oN0Dc65!1UwsGReF8g+6G!8*iEA}h;Smnh4mKaH2tB^V^Jdy=sEDDO|ESb`i zvl;X>f=XCg0ZC6Ui^NxC{V6t7zCGB%gdgRGbd22=YgLeFRU9 zWHHY9)-cnSl0&-3K9;Q%!Km~<4JGfM=A*a8@k*re9wb$|K?V{qGESdDRQbKC?Ec3O zLqD{!TPF9ukUh9(#C5;(Y(?$g!CV$UtOe|0R>J?=yB$ph`N`f&Y$}_H2AO^Z^?;C`;9kW2IaaiWb+e=(8q{&Wtbg6hiM@sIU7xJx+}5Tznlt^xP}cM7PA1 zAKp8fyLTC&wG z;{=d7bW5AxR3Yqnww!@Q83V6YcIxdDYU07o-B3GM9b)9hO54jW_2DiAq_<4}Ye3RB z%dU(f5U$)zuNaU1iVW37yjD4Ex}#uIuld}BaU54Z4CDI3s3SXGY#6zEmE2D>m(Env zvUqdEZCvaJpS(O*iX`0_v;Dx89Q{2Sbc7QY3aq>1^{72_Pwo9z`egQQ{}%=NJXo;7 zFu_VQPVO#_T)_0vZ|!^IhZpE)^x8xSPJC|j+6|w2T_S<;WtUFf_cyeinF%jc`kO9# zx?jb-z*c0qBuPDd&KN`YMUUHvGw$kg1Ge`_$NA12d;aE&M<`fc4~Z z-AmSP=)0pJu$fr=UcN;+Y^{#r-l-nLhjm5Lae=v5JYH}WEs*KZf17b)g6u~d%jIKU zS^>&jK~VKwez&YK4#rlV8Q|DYQmVO_a6_=P>Q5u0rVh<4$h(vEIWv3!@SdLU;ztg~_6qCj~7|lI0;z z5d5+jQw2L2A&dTkiM@BzZuT;KkK$n7dxBqj(#8b@1ea zLuCyi5%{rX(U0C+B_|9Ab$iQv+#Y@H`$<6}*r)*rJgMTPu>lqhp5U*e!bczA3y9mA zAhZ^jE0M#U*~5_&&JCAmrmB4ErUwWiBO{`d2A1FJPPS$nw1bhypXFjCOwk6Io4lqtKI9To}8C_Jw9J z{(GdPYHw;Q`tz#us_E}%4nq*Bo@Cl!b{MJP>MlS*U~54;WStGKnzvrC@Ku4@HF`DI!|q7KQcnOuS4 z5NYo2l}8&T-p8bACvVRFww!E_J{mx1#&|Ss#U=N`2(250gLw>heAvj`=TM|&Li}WL zjw^crZ0;Div|$8Ikezsvsak*j`}jM+075&KJH!vhHZY&uraW)gwU&JUEtfl6c4~j_ zj*V8!$f&*HJZi7i@{-QV=)Mr!)Uwa3$#8CYKZi&uXCrIiQsQk%`$_7#i>05p8m z^l|6b{1H83RF7g}2zNRsq5^i%g#*5Q3)$2qWu<6ACiP=F0n-Xd*$* zVy(JiKKt0Z7(QzdRW_0su2P{Xz&?Ap`8!;1!}uLj2hGQD_LLXtY!p!Kah2T^buT&H z+a$q0)pOH`hxn+WLu#KgDQ6u1}M-UfA--MCxKlD z7eM#Ye&tG!?Wk_*H;){W-*g0d=;6;zpqap09qaM*0hu+Jy`PYOS5!e)%iOnE4)(@3 z)RJ*r^c|(j&sSX;?t&TAihgnN+#m=aJO9nHOSuJu#+_{@X3niGpRrZBP9{)8p`#h3 zNa$$pE6J_ZucSGFhiGw;gO)44>A72GSDJHEP7CJ5kcVRtIi#hod;XD2fY4gdLkRld z#3%RmRU>rE5e;#7DNTNn(^deN=M|gM&S(uOPq=)PsvCa>1gD3ta6nDp zZa$4eDm^WUGNJd$vaxHfv1xJV(&i3Zu`L7MggBInlyhq984#%w_reN0o^$y+LLr-EC~*1~fs^H9k-FZ-aZ zKe@808@j!7en(yVtwcIo)T?5HSm>mT$Z&Ed5lVnT5Mk?QKRs(ErcLuanj_Q>ZZF!(oK?ZZ*#a=c>;dE^yDw5UuH3)zxJfo`2q|H*R%^qB)4$apntR3gj*7IPlS zN{;5;#D1q>Ap7VlY~52BQk9PRj*<6)&GVlFt$+|YU_nw|KH&EF2DD(l0kUj4at(DqM!n5+(C`DV{d!u^g z`9g&`J4MVk;UG!fh=o!rqwd2E|KPP@_O21)(O z&Voi9!up6iD4}{0IKtUDYzmE<1%Zmd0I$*$?h{f@U$#L5X0tG}1KCc2xq)B(>?fsv zunvQ*3>HgLOnER5!;PS<1k4t0N>m1!5dn773ECj~SwMcgs0{v6tNkp1BF%@k;xL#q z=sY+%e5Ye+fL=x=z2r8gN2Z=DhKpsv;TlwcMzf>*6S05#SFxQwPfn!eUeS;O4fL8> zYhP;cxPV8YWOXpqI5B_qv%Cy6h~97>6A(BBJRBfhGowC7`LT32+X!K`*;zh3nbJh0 z*t!;G#+F`$Q*3k1=0@_~IvRGTQvXDWI$C>Re1LNfRx`8ZwY8)zw9Gf-O@?nb`qd_K zX({TL+h1;AG@ep8$Q8-)LD2IXGO{G!zioO-Fo+SH_%RZY3K<|TJLz^a7BcjZstp#c z{SXH-NZ6TJztoBiILyc#RxcH$Q_0Lv<1?lUk_AT@m?(Ok*>jcr12v!%af?s>gnQ5y z!QdeZ)YcwSN(vdoNw5M$jo#ZZwZ=R`UCIo{{n^Vv!4A~z`5BVwTm}iM;&hN)zScuz znrYK~wn#uOZn!JULDn35bgLcesM>uAU_+p+Xq*GtVJcDr#!U$$deD$>&K^;z%=Crb z8xvN?f)IfY!fP-c^7e5}L=>!KXTVpbmUOdcom>`=77vqm|E@}+IzZ%>N~VLg#Q*C2 z0-fk4hkG)T%Mlm5hk%yciV)llceJ`3rF`aAQS8%4 zcl8WEx?r%xahz)2b0zAdQ#F)wNe7)y8lK%z{Um#7eRSta%qJmxwq>Ipwsx){*me{H zjERa68#(UL4(IjG4low#K-tr{<9l%<6)#|{FL+=i%m?a5ExlCe(o42RI%<7gFeKi>TC9__6;M?9TY zIXo4PToyOGpfnh^Euq+ek>MQcc z5i-!5n_Qy$dMSfTH>T}ad!0SM6=KmVVhgW)`JMfrs`^^}g}&j>*dOykG@H!??Zu$p zJIjW-uTmng=5;jP4-tL{Tl5Q>o^!0LDO5e{EH=BCo)^It=Z4uRyz z>-JyK(R#5x(SgY4P{Xsh!nt|+3Gz0ZbJE~9)IDwu>BON<(@vW>1as!n0;konCLG|t zZdh;SY+`F?2I0P&AIE?)j;b~XL>-$TQ%F4X9#RzVEkNG=2$|&GBW7 zokc%!Rza0n38#;cXszmX<_s^4o0Chgn5P{5D~LHs9)5Vhpu z)z*z1Q_cbQ9Uf2^G{rFm z;*UA#y7nVq*8?%uq$7q12uhwBkWdmp#vqS6^5TpDXksGM@YFxi)J3xZwRO1ghEEd( z_&B|v$3}nfJ)m?Xy{$*~7z2ACzHx8Ze8xnV(SRmfD`!+hSP{!tU>Ef3FkM#Qu5>R1 znP7R!?7=*GE*7Wzr~y_N3%+3+^Vv?7-GJCi5bJ3F)l5-j5oT!!1cp8^nk2IPv@@b% z-3ph*GbX%4ofMRiEJ8=7g)k}EZ7sC;2gVSiR&3{>jcdmjIx{TY11OEsz z0G4s6f%Ddawo)GGE^?30Ula#yf?#qYutWxE%Qm0xF0M6QvT*k-lYYQ_>IgZD!fbQ# zKKjbbg#ao@oAYQCVgtvWHcg~d_P2Kyqqqd^Jv01DA9Han^qy-og9}?!>q`bGxB5@}W0UPTQCz3LJzYrF29VAe=>z%@HG( z9@6e)AKa^#8#%|oQYHV;*i%u+xBAMAmfsoXYhYG3p1`awoO$z$S*mF(33qBos6ay! z9$`$*Xkqr3jXNsu@-;o3C!^%|1xRw2kh*@1Wh|!*$P9 z(&+uH{r0`WKit*onyXh5u{rFm{WM|UHf?(8n~(mctDB%ESHlhy)&Ci@OEaGR`tZg` zzfXR1%Zsl9Ax4$WM-&v)@IaGPjriKy4z|wQ)OTHF8QhD(zCb01lhxDPDC4r$#)-T&I99IKx+ zQ6kUh!+nu!Zs4SX4r54GEc%;ka#h(=Z#956T}dL|_w!giY=#!3>uShkMrj{CO1=lb zGXXm!a|;SwJ^pGhujXt-X!4^)z1+_JJwhK2#wQNlpaw;ixMAP`UPxL~312G2)5V+1_QC@q<`D-S~>y3T$4%ZV~^Q<^r z-a9jmdR{yMeadP9%aD#yvf*$svj`P{vPPsYIggtM% zwS=ndz6LtZ)Dwp1Vyr($X(?xvYFQi7umeDv3z-3)p+LFRudsMRZC+Zsqe)PBAjJW~ zDSQGHhQ#=Gdc-+3S-2TRvx z;EzOwAoaoa(cFx_+Zyp4ro83yWLE}`-v8In1p*2a8RUpUHSJZ4zhcNl;$zQ{QUoa^ zEk;6SL!@b>6dU-48D%%$RVY*$kzLof>C`O@0TM(xN=2&75L2ku#WD06ehrA`!TuUa zqx%0kW+xsIwPo5_Noin#d2k9U2B^y2$phCvPYQHY_RQ)zj6}l;umzBZZE&=)d@&g;5lZ%;#ZG0) z(E(;C8-2tP5&16?V=)@?uAmQt1Syi=j6|qCV$q~JkH>(L)##N8E}19mVsAa96!+&b z!U={ytzZ6$E~p06BzQ!yh}LY_u`~IUrMq8kp9?NQ>RUGg3JW`>ASi4e%d{U|jAtmj zH>$&tHFn!Y%E4B))gB;7mJoI;nLecAZJmmlZ|oXM00M?aFr1U>(27+ z3}t*Z{yIeO@Wc<%e|@dlMUm2%*su4@K1Cxr5)B@rR;6x8Gued-i5T7r(@2LH{G8na zqqvzvQN>AEw~|*}swp44(firB8~>lT7ue;JCGcDtRv8z@v>pWeUWuBN1Q^S(h&G(u z0o6rk3+7_So7pMQcPuwb=pMHyGG%$5Fi??#db z905IY@uH<)dGNJ-#91^EEQ$$3zsD`^z~MwhkQsD!ExUkfVrQXxg9C(q|JS2kU_+_PlnRs}g? zQGg%$Kh-?f$Ne6L^rD|6u`F-j78E;4z^)5Bi0Z9)Zc}VO6Bf#uj3i}K+qlr)b>oEp z4>x{mh89_hl(sqdw_4NXZB}$;B-a85T=`Q>r+x$!(@BMq!r%yRP(5ehbEa#R4fSAk zbov32ENhWt;@eN<9{JkFo>93WP&qc#;_pV+%S_(;Tjm5OCj2#KbOIeC5JP5)?>+MN zIW}S9Jk>^%_tg#+@FBgZOi#(&^ zO(YyHU1ddeq9k`V2n&oCCRZ5n1!EN>G(nfG;T6*&%pxy#E6K zVu(o@+#NUjzwv>-dI4>u_YE}IuKLvr%t8|R;pw3_GjnqTeto4t(7m9wKezMpu`jB;ni=hOgW8^)3g87pM)zZlQe^9V9 z_z$2gL6zq^wc5uGi!IPE8%>xvk~FB%`2Yb$cyy6bjwtkZo{)teNX5edVh9COGM?H_4gj zdsE^R0l0ouG%BHvDtl(_0^r_o9m|ttu7g3s2OPHn8Ex&x5=`MzR8;{p_}6#l*|pWM z0Bj&)9NQ{pl0b=&nYJRhppn1q8Q=Y7duWBR(p-T!Y^sc9ut(7S=4a7ffF35%4fp1I zEE|0&_uV(1-$#XN^ukzw{-F~@Jv47oG~bUIf(b}4#i33ljVfp_=M4~PASjhz%+mRp z+B8Bc@(cODN7L-Q6PFiCop5#2c_xBYt|{i>m{dz0<>dH{B;qJI65=5!*CsBM6;f@W zu+5kwJeh@p+{VJ;gw+>+*1m4=x}G`j9o-ZntPwZpxyF?Z?2wt)T z7=S+IvLJcU%V=~QaSOb04kQJH+F7i!!`ow$0@5&}Me^W9PSP z@dk6{n_am#IX2qM=9>4vxd3Yq!>{c(gdQ}I!^E4($%bs zu@>cr`)<#!Z)Go3?vqWSgy6Lcy$>I7i;<2+%j>jSocPq;tLWEVAfpcfDV2s1vzQDR zS4ll#>rnRq^}{O`Y#PX`d{t zBp(%yiznj=eA;Y;%n~}W8yP7GS&DfbqTo~)5~;n4)8vl%1hZ z1{1R@1#RY;dL$9a^iVcBV)&J=9k{OW!sCOn@y3&a6iYj)`RKEy$Iy>YE0?K~GDBnr zN^IODpdmGIla^>E}sm>8)Zj8O5&+ppbnB9~vR zL?8^T;?tf7dLKT0wY9F3!!<5!MK3bsygg(6?#K4a1P+&J#ufy>YJm5<=kepPToSpyl z)LOVJh28UcjtWi}c4krgxSc1aJahr%#a=cBzZ%2nP-aZy8ohUE;zX3I35V;a77@p+ z*F2Zqz#ZG<@g{^vHT0sZUlBt1%wLur%%1LAt;*Fcz|#7aK6Wa&g5{sMaL3=|Y9E|8 zZ{DJ=nk!cuMrHw}1jk?b=b1;2-%AnBYZj@#dj^}#PhCn$N^Qpq6e#k7VWi#q3n+Q& zWf{u$KknPw_aT8*eU?&Pk@5A6yLXAv$bb=yv$YMKlIQIA^5QIpU(Nj@7}+6M4CqtI z$k}{Wne#8Tg4h(N4=yYDl^M{YLwtc#9U}p+&T^3;t@!J67MI5+wHV+r} zAuEc-BLJWeGd&Peg?_2bvp&&r38@X*_?##{2jN(V>wae-Iw0y;fA*yj6)VR6`0RP#tual*&k@3mc*(3$ z!Fmi_d+F!EQK_31GZcB#HgAXk6}-^#2;JX%B8c&VurDPeckZqRoaTk!v4tJ#G2rqp zsC0$x1SX5;(mgN)EsYM9af%YMpJt;oo`gde6qKnsf`(U6r=7EyRiI8f(YTpUq@gwa z_Cb9LJ18f)*FupqqAdkl6y%e|V!ga}_!!iX%nnDiCnb9w4R?yqL-Oaj0H(pX<>2`; zj`yvhj42-6)D3qL6SM0n#CnsM6B?N%f>kujD$iL z{0_iF8j8$~*Y&N=J-Pr|F7E`2sMjWEI5}ixWD;c8!C&iDGOp!^AQQ!3U+FvSU`B{D zXr3j_SmALY&$v9hj^_qq5IAXHt+XLfeLZUuKpaI~t;j z{HS@SF3yOT+tNLM3UdLhc|UV)@VCkA@?p|Q$b^dhw!C2Kq90;T|6Tx9sWdM&HF+Y7 zNaOskN@i2~;_*T^AaO+V1caV{sr$Daz1M*NgysXkBQ%V?pn05HcKVfHw-AX%#$o^# zz-{#ZZ!5qhd_}mB0e?z=QJg=}_l{f)>cxKR>=eb!95V+110Ls*03$BA>2a)PW#Tt@ zQVq337gj-I`KVfnKyecUc)ZWuto)fJiW(;*}r>-|6zw-HBfpA#u8kG3% zcYJGn#2-0C9GARd@`ON$R^CICh0t?hUKkr>(1%1_)zY7aVRFeQYrH_L2#ZYD>8E-C z_w+({BQhYK4^x%KQuVb=vaOYTd-wP+(4+~4B+NiE`E0)8kt$L7 zHENZb1L@(%C9j+AT^SK;F*xH=jo5tMu$z4B=Mg7R!OA+^iPdA(L-@)QG*VKMLR&&H zp>$>G1K5vNNwZvJxI(fQ@=13EL6PKp`;V2iSm+RziU>HMWkbuN@9OiJ(_PHBL)c*Z z+CR~*yU@ArU#L36vUpga*Qo&bB_SXzaO7qf7C40Df7mAE;p0|(qoNl=ook%QrR@>L zaL0o4_W}3bp8=?v@plYQg^szgQi(Xru*{TS9#$bkto z%3=r-xN!rComx5su;x!dyAsGbrO$t*#_W7{m-PDa*lhdNe>O(8OqDQTMobb=m4O(4 zY+QBwAPEz6f&)NIt6^cYMt1C)b~q3pNYMBYCa3OkRY38>Ej&tNVw#i%^JpsKiJfQx zYR#>piYBoxQvI1U`!J$nJ!G#2?>?#ei)oZO81M)kP$hn(?95(Dd&9@dc?yh#w0$E9 zR6#v_q?)EPZ75{e`sB36_Znt`tQ*_;6qYizH`A9QJgr!G3b|aRrH+V%!)HrXK;zMYnq9#2OS>%ejEg2lA6K- z)@jxmB%e-Hu(ADU$#&NfK~3MEvZZ;4$O)FsKV5Zf-+Z}{a&e?KSY}g5p0if>h?0|E zV<&*azpFW*>w62u3e^bNse1R+YkZoRJ!Tt<%0|#BYkX{4ObI^Hh*o#TwCQ?e;_0?9$V3S%L#czf7Vb61 z)1EkNKwD@b!$E6{l)c;W;G`Cyg>3@uX~KID3sn%+NxVxJXK3}dWfVgZcV{b@Sg^N+A}3x)*jpAx;0&OY-oGQ*(^NSb}UvEhHgxi(f@n zNu&^+&ifA}vZu%RM~)Kcp?x30xek}`+w#l7f>mrmv24;n+x_oa{i=@Ptc8g5 zAke5+P^ z>A`|YkE0OPZYI_2+8q_7khmx86{p}4m#o1OXl=QByL_?&KRR)OImCJwuH|ULi$hWh zE}`|MTYC({IQDDT^2*V&{00Qc%IL)M~2GgjQF{L zxC*P20Z(gy1*cmspXv2b(EP%s>MyFZNWTmF$PP5+FgG#~`PFCJ8h%O(SQVM9OS?5a z&hl%TuD<)4HJd6+b`41s*uS6o{SAelJdUDa0G=k80_Dno4L0#RlXwvvMQMn4u-mic z-6e-kvD^%A*nvN#UNwfpcFZ732M@w|1l&?Rp!7u8%i^=Y606JyuYTa_GSHr|DJ>*} zw-B=DUS5SP$5Q5M`tZrQW+WQM6T>@)cQ5^!<#6jib@IzSe?sK#DJ?m}|Gv3E_!mvC z97Ne1ELs5jKEado;yNCBZBGO$7o;V`Zu`v;;`3$zYMhL$F*mLGP)$&XD(XufQAITF4%CPlx zJj?z!AAQZ48DbCcD|?ia53|jE+Kk+ZdK{2Vz600q4m&@k`9Wq*WmWYLZ2-Ka4Z<_K z1fmrwlyY(x51{^hVG>HHE2;Z0aw>{{rb*Vy%Qzb2cJ-{p2u-*~q+ueJGoY6L#)drE z`J_}Cj7`~rrc=#zPf0-1yZW`6&i!H=dgk}tM(YCG7qBT-pIRcwN*UIi;&Q-rL%JXkS+BYLUi0AG~j^`B)J}s9$np4RNk;=u*;aRT*DOT zOWh+{iJC;=Z#Ki_xhp4NgcK^0TTqT(ivuQ!DjDZu7i&?fUbPnEfJQ_)A)Upy7WXe8 z(e>Cmx%3#!sA%EFt*p{SM@7s)&lf0)DBa%750OU&Y&ndYA~+CQL{bTQ=F{y1eXC#K z1)HY-Km3`n9FK164Kml75L&BleEb~LmQDM2JV%J!)3^v#x(Iqm(5v2)keV!f5 zI}-z5Ws-ys5kDyjv`)DdJ2R(<-Vewn@L#S(Y=FTRw_mf9&i3r&S{N^54G6y=?;BZr z%KFoZODWvR-47jYYH0*Pys#Ax@WxAGTsUsadF82#z6&>5TbbB?;`2xp%6WDkZJf{k zZ`o+c-)2^d{v)TrdP~cNxHZ}o_h`NuVVPInJVvf!$vj~i&KJeSh$qUFo$mfMX;o+{ z=3E39LZz4=ud{s25rG?_s~NjLK{cms?E7WeuxP!$1}FGEL?^g5uPFQdUohs%v@5Sj zX`?KX{D7<*uvedutKn+8BDRqv37=q05?+F_Hy=<;$?g~mM^^*^#_6w$DB75rrwJ@t z2B|^mtizPGV@1R519-v)2?y!2OCi%aHCVMEyulvdTft>ih(5=)>y@7o9;2}e(U##q z#`!}(03&VFfx};=WAOc(NgXWlrnWvz%OSjjKQbEM)NJRJHaQsW2Tm?iPiN|vy5FKf zpN-gVm$_`G+2@(bN3jI?6l4ri5uaJ!Mgd=>pfu!~zJG6TgNWqbI%L4g&osZ0hSEdI zQ6N;zc9lJ+xbUP+d%_0f-3RyvJ7o=|c8qdL>^K(Z|FZH&w;}X%UXJNq)N@mla$=l*Ml6Z~0rQ9UW_2U1N z-Egf-1M+KwwH!5hiAu-%ngM=6MVR5HhREwfo0{Pk@uEscg7iuZlpp5iCOR6e5n8Cb zj!pG%~yDN z$+lMZB+5tuyhPD6gM#%(S*q?AgB-qn+_XJwQ|CW+9PybxE8itN77SzLN1boFbJ0QFu7xExLR^Cf7 z=|&#(nDd+P{&ewkG^gS&oZ0{BxDAyt%B4Py{KMa4_=Z_^!tMSwi@~_cQd~vh|i>Kth5^W;+I6nEYnb@Ba&pDD`*6 zngA-X0u=a^wsWhfi=BTQMi1o2``34zuz1l%6te()Pdyq@WeWE2jx&=hf zK2Uoa$*9^?RIDU6=nuEc3lvVpTudU0bwgeb_vA-41`6_wtC1CPxTvLp-D7)j@{XFN z)v{a4PQ99|+9+u{viPVeW*=#}W9iTkK2?;{%dH*t)(RAg`* zH{7FrgN3qFCmvT?zWP}{cPxC&oD26aph{tN9It=_y77{%jzWmM8k}CP$n9jOyb<;9 zQExq77^DYukaBQaI4AG5)1$UmZjOTtR!x%uy7``2${mU98|L;RFi2u4IB4(uEjVC! zdHJG26&l^3jTc2<7?$PH6U`teCc#vIcI+R6y-)yQN2RK zdjC44WrP9Y@%pX5XPx&m=hZK$4V8mcwpZ8>=9AMYHFD`o`_#lT)RL|qe<`>$;PB@8 zP+GTeALMP!OBvg?Q{SfdWr(`W8!d&bAoG0bY|Ovs9} zhFIBh)*Bg2I)1ce7xoFpLEI7lnbkU}g*coWR8|Y~mhd6iUm}SV!((w*P6$^PB?~N3 zLp9+Zx`_%WUN`G#Yf?Z(B)xeC3lEcdxOZ2wSu{Dj+pPdCy2!k+OQ9XsLn*!*mX@XF zGCb}tAZygnw>9_3_oW@vx&fu*^bU5sj)gtPAw%-q`8h2IXix6l)%RT$Y$^_A$Y@2P z>@*hbfA=MSZ(xF(V{`Q*0NFmZ>Ef4RT9XAG;NbcEDUtlke|{bke?TBth>xK>iYvnb z>%e8~vtdTkq)Oumb1RYxf=oAj8o$fZnn3=_>t}Lzu><-@5v}F)s{}Tl->2SH{fdNS zzAXq0`gY(rg(e@D1A#I+^Y*^8m^6)&{H^`6HdW!gkuiJd&gM{3-LeSdIZ{lJEXF}* zO}E;flnm3RW3M~YlIPgRlFmzigW1=*cTTw(0KRk%#_|GC^(<%1RaftXe{4Rk_;rds z=AmVcKwh0qad*#Ba%sqrHr$U#kjq$TJ*W@}uLrj$+sIJrmN3BQ0Ac9y;M_$tiu{i7 zi(RC6fBRgXl1)~3>D(I)hyG7KgAou4&~gFjGH}wem)=Db5h&v!!YNA0t8>S^apa!x z*1-bkEP!F1@_Y7z{^<4XoGG>cxjF1+coYP{$Id2y!yQ96d6c;^oX&>5`MLp&koM8d z_oyf2UaY_mJ}OVpyt?<7DOVEJ9J*A4x+EE?sqazOwW?9mn{Qy_a&`|D==QCCDnTA_ zGt;5We&_}DSfU4T=K2F)TA!>BL*~Njj-@Bd7C0ae7J>l6|1<5v1Ac)S2FIO(3< zzUJ3L@5fCrYd=@mZgZuE|L9M3SHYG+j1Z-v%%*Nl_RsjzxFC5p5#oA$uPCgg>OO%( zo_NGlMH4BTYV7DL*_v`j5AXfRxAzbpNsRIhQLp$UN}MsHtT0Xo^LBXcCtaQ>8;%m6{&Ap=@>{!xJ+la-3|Hc%B6upNUga-%c!m ziG5WOiav8jBI+~(!*Fl24iN@%3Nv>j53W;=P+52k=%W;?d*xnb@gt2wwIvhx0!Iee z3&!%^cGLJVK7vyTEMVG}a|=gr+e(dsPU47J6_BI%GGdOZjW!C6>TlWV>^?sc0P4D)BFVp3s$BhaulJ&uXNY|p|H`i5fEAZ1p zaYK0crTv5xPT3+oJnl_on#=Jk?1b;x? z==$@tgr+%NkArB(FaCY~1+=qmT~G1=DnuwO292L{bsrLmiJkBsMLUtgqLP0_&V*jn zm4uVCSKcMOBK~ty8q=>mre5CwQ+$KE!DulvYdR!|2@&sRz-7)jKSz9LZQJY(nCK7w zpDAMWm~92=3Cmdql)SwBW*!o<$z|c#KOVvl+Dhr(1nObuF}LA%3($~zBYY0iAbclp zUrpz>Pvpw(U*JwE)F6;6`^eWF_`60LTfKJR2k7UtW-whbiSFEKU=iSeEwqzaRW!}R zsPk*+Q=A09OZhN%ihGpwjgTsm%4Z$MV{c30jWJks>tUktlHu-^l)^k~k>rC#rp;-; ze8ke`{bIRj(QV-*$AN3)?Jx8;O`FUyNiYr;Lv7)J-TMw~o+bFmRuPxfaO5y&6D+YL zT*N*pQs7cPh)w_lt0WYq&^>Mj`p8gaJZ+kVLgEF&1`Q{sKR#kmUB$V%X!aNVC}ovR z1Il7^zM}eVw0JaS-o(IMl}M!dN^+CxZ_}rTy};8+v!C zRZ{^e`;tl@uKG$(u`F1fh7|@B)8-=PsNYEcbe(oG!a2B;JeBUxqY@2?$5C;f^H6124E4I z-A)=kPWECW%Qg^=+Eg+HWmBBpGik1a1ktImLGtM+_6A%V?1UJjX!F*(lr+yhv0(*K z>FX()pzX?GNkmK9Oa4JI5oU&*43Xm9wN-GTh_;K_g`ZE69Zx$d`}g93yfeWZEkx&fg*hcC|;v7O2v(eIRZt zatB`&1;uG~-kP7sjM69cVd+HdT)>3E!DW2h)uv7*CjkOgKYn#nc004No#Qh9INvzF9A^xv4<&;jh7 zz^=!}$b(R=_0;0i2y@)svaf4l5c+gJs6iMwkp)rrX9p*DXOEEc=(xqXgYKbE5!NTY z?fQ;7Y|iQn+oo0i%Rbv^{x^m>OQza$BHh!=S-1kkDd0<;IgW zPQJvG8>=VnG*+zgZ{nb^%xq18`KZqm1mKVOzhXX!We&uE#j%%5!b0yg_3|&8DE*xW zN6Ul9EO9c!fG}g94>gFni}DNsJS}?P7C;Kb$pc|JLZwOcAmWD~R{ySPCbC|GQ-r7924>BPHXS zF0Re&B4#^z9{zL$v~qfo8|@r8p{z}&>>#(3!whFMktCj&NmzGmA4(a)%@1RJN6hK_ z4y~Qd`hr@ptEr(Prr&KX0lvzN5XQ(8CAb(?hpo@?KsMLygOdOQ5XyQdDWKYlU42ky z4V-noW2*Da3*_3=?*TutP6bpqI;l+mf4Ef3x)k8hQ;v_-B&%vXALuKh6*Y`k8y$II zVj?v!w8N{gFu0n)k1+dm^R1N!_IK3IyMD|43-{=ZmG^$uQ+D8=YzEnNhy^r;X~Wv3 zhn$QayrrGB9x9Nk0t?Scq**)~0*TjjeY#}X6x{6Puo~*(jS(2!B+(BcNu1=7VHh~G zqp3HWT^}`W5L=jUXF79_(>H-0eV_C#NqexwN4BI&-FCH3-*#Q~3mqrl?I=KoqT8!h zemV*YoZjYK9$M&-6Wka_FeZ!S+p-2g$F6lkCI7usJtj1@kaumUR*w zFHzg>ZL1uMF}=)EEc*JwcV!@z4MO0<-bdM^P1$wwxoo%9+mD|-)^uR}MkdkR&|;3J z{Um3G{G$3h>Hy~D`TR>|oz<%0BwQ`Ye&0#Ni1Qo-sb9Y(I`T+BNp;#irJ^!f?FGUG zd?Oh=;|q`C_@L%_c&ULgdoGw}&y;| zfT}n}8g^o8sbi#*W8Tw0MpSe-&*DrUjRKCtc^L-96E>8pB_J$BB1jmuzMXh+!_ohY zV9SJK5!S^5a7Dm440&z@qrB2LCz0O}21aC}04*OFHA;|XWQ3)YQXXCbEMh&|cb$1J zTuUhcCu%#;S7y6N;`Nf=@A3owo34iIJDT3TPWKen!=nO7lgz zUHiJh?}!B%>@T|>Z5<4RjD)|U33+M)8*MG3<&0zT0J;v=Q{2bkMU<@Bl|e>E&CaC% zCzS zzFX4<&FbSG02A7vmh&~K;~u5{N9JAl8`1pla$E%L#tDga&JyS(^;NO0Q3$6de+%6i%vZ7QJ7!{bco~k)%{C(z*oFarX&K80&O?)W>Yc>qp3tc&w6)@H{Xn?M3!|SI z-g(Uw;95!3MT7^0vX|bS_i%N0L1WpLn&N2X=eLPu1Ic#AiN93l$+w9QO02$^BujXeT0kQwTOfNWc#UKF!abb%{EL}=o1TAZOL@L= zM9C7ot25R@EW(h)wPa2h=!a0uo81;i?qtfZmymFeD5E|qlT?d8HQV-9bEju_tBsEQ zEH}(l*y;J|Gi1G-d@7@|1+a`Zx?jVbunqlxXtQdlToIP*ReGGHk&)RR4dG$i z)l}Bnj!C_=U6bw)CgxtLryCQ=^#AeGbmM%OVa3~)-VdBkN1#$9p ztIUgoK`eC7+b=Iq$6wjh;_257>Fs;g_M=ztr~}af7@!XZ1aLyt|dQwEEj+bOUGjdJiW)vZ=i~ zoHESjGYKuK`iG4} z5Ja&^n<9DY7DnfG^!8$M@aK56vs#_X=jdWSaA;GHej3Nk-LOh-pHaYgqZiZtG9c1a zu+_vpLWs)eLv_(^T1kPzry}JMd@v$7+pI^KRsCf06~@yIh?TKwmTTgBrFXl!oDFWf|hNC|C|@DKnSNeN3Vx(l}r(fcy>?G zB#u8n6OEyG8l{5HJur^}0n`9(kiUjz6&bdpOSvp%=ru_jR1C{0#3CGYWS)3D&6^OK zOC4ayDhUqK{9@AJdJSmC1fbD^)ERBuchC{e8jaAIq_RQ4ygq=e@{)frxz4Mne0rE5 zPEo<+eTD~LWSC}-ayAg#c8!4 zTuMQg;q~Dxy~8-7DbNn%h>rkRlNIk#K)g@2CV@_(zq&qBH)UE@IX6#Se~@obrVt48 zV1`H&hfS42%Vkz#fRYMGnC)}>d#@ZlAenUP^$`utdAl-4l5jG%V@&#bBs>Kw?Xz54 z*>NsFf9oi|Y#*+jP1{?M!yp)b#srbI$Yf6)eM=%^N~Veo*rcv+)zcF@dQ*E?v6s|% z;ihE+r=&$Eet}z+ann5{>%5Ju+ixUmXRD4(o=C}gme9~;9Ds+lF68e8vF%o@+G-CP zQ=-F2gKLS--5GWLZBDP)JVRxy9>ZppMd6SLYPyJTXmCMWVLa^qT-gGt8n8#1FfVO#Z-GQYS*W-Io{0;lKK{60G#5&Fre7sYF$q}`YQ5) zL-RsCMhiN&v~`V)lwyE5I}u|}ApR6L{8IS+?{e!128gsOZ&4G8pPfw2_apCj&#czN zx#J7i`gb`)RK*E-b)CQKZqdz>pRT3+LmN{>y^*(^7k?yqjY>|B)U1qP=mBl}3&|*j zEsCW%zqAvJruuE5-C&{8H7uzH#P!P{$%xSXKJ3HOtx`{636VDSrxbl+4>QHo3wyHy z`N;*a9$PLQ2u+^N-EGJnFUBHLQ!GmV;tHP^KqR3BOkl8g`6$sZt9ZHQwPEEo68w4% zmMoTLNx#*ZlJ=LpigFhLldu@9iq09g70K$F$&fJOr?X_exAUZ(^^0{PL zVYCMCVQQ7k2)Mf4+GJv3rKE`ih>EWKo^>DU4+T(jbtEbD14o?5KW8W*XfHX_G+*`r z=g`az<$lcP>IW8P)4Qkkh%@iHz{Ws~fM}{sp|-q=L~aTvF{4^A^<+*6FToFz)Y44= z=)8?!h<&Ef5ktX^=olYOM1~L3eiLqVH05}RHiRQC?}Ji;CS1}=X+*!f0;W9g`B`;O<#%PRvRa?E~2(T#aiBi zTo*&$y(Q?s*FWI1zO)0PH`p)ai3)2$0FaQ=b?96}Z6< zSe9_pUZwFDNuDt7p*ANNtfpNvM!HX09Nb@d@S(K##~}DFp$u$|r6ot5gNTgMnWcJN zYG_w8B@<`XaL+`kg=Q~d_5+84iugY5vrS=-T%ab3=PvuJK!o*C{oFhS4VQS2kV|uY_4Tj z_H?jPoqvX)k75XFF|i7`WN0tN6#RH@h4A?Qr%d1o<9EkBRv&oSHs?xMol5s(& zAcT?#PK64gvQO|f@dB|h>SIf3rQoyWuTku@WrV_+qU4(~YTB2STH-LB=$86Gu&~hL z&&8Pn+*g?9K>h|p6KqO7&l8!1k%z(p8Wg1(YpxO!!&6%sRzk))$gWa890G)XA8JR4XzFx!lYs-52lvPjRs@Q`LJY_Jy|X`8wJ997 zl?$wny52_*eQRm178Cj@L6owVO?%Xh1M()FRq3|KlvxS)szYA%R~-S{UV8FfB-tms z=g+olb<(6J4vf;>AJ)AJ?@Ke1I5EPZr&H3rf_qX#YA83r?9Rfxhrx4r$9u(F=}GO7 zDTV*U|G2yNk$+{2hRfF)4hfPEvCz9U*ZPv#99?p$?;p592zPu%`|U5=0H{frPi08J zU<@l(-Ni^A`xHM68X*O#=g*w8@~?Y%Z&S8;J6)mjerC~faY8@-o5Q&mZ(d+x7H*D3 zZ)@79G`>zXcS}#AC9CZ6iWUjLI=_VJL~>aQ^M2db?S#Zmga&cy@?-FS?}wj%TO6fL zElCO!+u|a-h+Rp$9VmN-oh0{1ty@#?Bih$|$w(+!ct*%Je^c^6GVQi;%Ct|-5Ec2D zkyeJVf?*LH&bRZ?XUyF6Ue<>r(m?lM?!-pe$#DiNzmUSf2N{$e>3VgO@&z97tr+Ax z%ajThk0D%#${XRg6Z@?VI4sE2e+x7J#$60$N~K`YP^U3RYL3KIQ0IX$i9|fV4rj~{`9cDR zW;2!Su%8#_$X^B5wId_A0lgKInQ1$rd=8;~uzQg7`+IjDa#5E~!Ah!bM_l2A30+hM zKDKp%PT81D^MRd|gI;sL0JT3fAL8)nsVLykDHR5nOTyYQaVT=wT2HWZxaHce1=f9_$PR^zU$&c7Yoj=La*dOt?Zh0nW(K2#hP^ z;I%^IbzVUAT~LiF5s!#7`*d@<~i7+Ktr#9#PcnN<$a6d-G|@ zMNAxiz(3<}`D5LctG~3d(G07*=bp7J1JCs}VNnR5Q&eYMp_}S=?OUctfKHOORlrfL zcxZ2diB%QJquIk!{G}8{*a1i4#^M}(=9X6rcZ(E_W6?h#cMluMY-&3cz7%R<9o0h7 zzh&l9B7$uqB6;z3zZ0JY2NCkTeIYKEZIcOU+QCw9W4$Y$fJDw4Ov5A5t$Jh9924M6 zr`zZ-j>;vjuLmlIt({e8 zaxt?^w`Rv&D^U|uEzzmqQq(}bx?eduS$%Rx1uNgtD|VQAD|HcJBb%rpgbhQqscQ+J zE{_Fz@cJ})&Gc@eTF4epJI&e#+e&g$2(Ouj6}R_{@9i~=O98zl3wIlIs+MAZ?v9P0 zT&hFE0N9;JOMfHkdR|A}nb+BKnNdH?UcyqVVEd>${sUDsDH*$@ifmuW^s#D_hjkJ3 z)IG}54=I_~GdVW|F70&pmk+%%extYL`i_U1F7-h1_N^ejeX!|t585cPzmU-yPF%S> zN(EV$FoLQ}b&r#%?e%VpH5dv(tHg-%$&1A~n^NeS#9))U zp}MVJk_|Mfu2Sa^BPm(gcY|`DbOhnNY2I)jL3fmsspU`Sxv^VzTnm-0`QW3uYsJE~ zEFwgH>~&}Bl0DEg-c;6-Ykkt0Y|-tcLb+?4+h2m#&b)g5^+B^S`pl;_mVX|OUQIe8 zxU=9y^aU~|B73A$06I)$)tOuPT(b_X4w}}o)a|09kl{gyPgbZ`dv=e6h9)dAO5wKU_AA=;!2CkgKqF>QBNog7 zi_@<=a8a@Xiw_o)c(QyNI$M3u+In)8f9+Ev=?YMe8f2%13J!8*W0G8hr82vcnWSth z$YEW?MJtR}@YafFG$C}Bd~oSQ54@(GuJsgF7cyt;y3^wrZ@;EO6*U0n#Y~>AWh?1v zZy%<^%#?Q5>bp<0pMREEg)YMmp0FnY^7CLn+_PL3ON&S_oFXHw;$KZ-`o+?Mwn#KG zh)^+h-anPMh1194dvP+XrqI)Od3h`)vdcC{hbfn?#DP8?(41e%9s_=B9$L3#2dHJL zr)>#hL7UGmLDoqo^o3nL*K-4;y`uvi-BYd*9Jn=+B;EM1eabzZ?EA;oi6h;Xti9yY zsei!4sr?d^i$bdGzr8ZL=c7FzKL#n@M(m-2V~7TgyEeY4DK>)wt1&8+%iZdh1Q-y8 zS3#mZ`rDp;<Tr+!EMi!}y;Qg1>_&oWPLXd`~E9u<&TBYcZi+WsOH$ z@*1ac?ID8nEHRoa3DsvIcvEUqCz}*s-MW)l+I!JhD7MNqdxN~eTGHr^wFUtZkJ$C< zg@xp>WGU!q^8r1FZs+=;)#_BE!=uIQkkMl0NNcVhFOiAX_-@x0BLAcX*;a_t*nm(T zl)&G+a1RcX90{Yq%WtmW-0Mb}0(R6v!d1?y;jB9kEvX&$pbaJf3kC=xdNNMN@~lX! z2&VaH7fVHDGU3_>^fxDiiOW$ifNfweQW!&Rm*g2n{zUmAxG&Pt@aHEURDH^dvWr!> zNajdjo4ZB45WP5lLcIoF80QI%=;p8E+NHTSN^EqnRcS@}tV0wgs+z`$^E?;kMl^G< zDEH;Qdw^ZL2PXjOp@IY1hGAp&tDV82h3)>!>Z4=!Z0No6HaHJi1UvL4HoNoFDhh`Y z&R$*0xhRTiNWxPR|Kyg3Jnu%gYE%&ZS4Uv#?X>&TH~ch%1;UKSfwfAg8T~-u;buY z@m)D;`%Tu3YDu+giAWtu3Nk{A>cfl#2A~h(Zp)dh%zB|`iLZS_ZPoNQ#@*lC{ng70 zaEJ4cw|9MNNA?U@Vm4iR7E4VgE}orH(eAO1oAaU>V>8=m(tuw`0*#yQ_bQAFn`33u z&(ZdPSa`n%5(YroG8QEP^Lo^{87E}?ELf&6=pJEvL54fg$Y2MLJ%9&tvAa0vW$DYK zNJBQ?xJU$pT$Q6La=uxh2dPqw6~ce96WWS0Bh*h+FN6@KWN0#{+zWu-ucgk0N#q+1d*$RCJ1@XeU#(u{)}DPeX!*67pcn=V~aVCE^%d0?z6GCP!n zIsuSZ*2YjzI`%uZ_%Q7s3-`*MQAazF-h@?(Sb zForfVBo6vny$h*C8bMIdO56{YS3Ms4*eSjsmRQ$1_L^^Y^>$q9r4zk!xWnN}cNuGh z9Ek_6$k#-H7|0|09Gyc?JaDs)?buSTe#tBtM~tH$s=J>}c2nu`$T;j;cMbJofHnoU zrt+a9H4oBSyMj;<%@{|U-36uEngEoRvQ|8aEJ&kQKRxQ`*yiWIhN7>$VR0V<-2M0~ z@9n>sn#x0XZ@IP+fz6EiOS!s*y5c(*pBR60$5lZoTkWz_`wn-!V(j#c02OQsxSo6k zk6ZP%u(V@Sb?;BQX1t!U6h=0IauA++DLX-DTh(8ra&)QqDW^ft!!W55@hkHQ&Tg&B zm1Wyp*k8J*tL*(uMe6dzL*Vr>i^EvvLVo0fDS&Y`ub!*t<9)YFRt`P^9b%*gXkAO@ z^1kn7$7n636F;;t^`vVyX<3jw3!lRF;`|tu8Fi1ZEM3J~6BS|c4qEW>rY z*yn%Np6cw71Dx;UtS@km<=yB1nxW>$u*||rTFk;Ig@IqLK81!zYz0svITDbUb+aH@$3q||7-wWF` z*l!ueghiERQr?WiM!J)QVs)QF`=@uZJh+F{&WI5&KVEmtzfbq3&AA72s>{#Vs)N6C zSuHjeSqS{v_2}ABEQf=+M;2krl(T1*zy){q@~6+&8@KITQVrx7@k2iU6HuUBSk zT)SbAOt%3@zK`Z;NIG#L>N*MRf$Ly$Rz;Af(tX4)RvY#0R@5>dEiVaJZ@3C*;G%3nXkQvYNYM zv_ql5pk_MiFRB)>b03PVWy49r<6tdRg#4u>kpJCW(K4ufINv+da6xKI-XTCNJ~t)O zSih=^H>VZ|#apj`Ghj^tH%vi6cgPAb%F^~jW-30`&?CT`3@0L$3*J!vkCOfngVD8+ z>#r3!{AkC+^PGd^XDkMt4fiUj5?zTA-uaQQQ(DSSU7OODwDu9}v%SByjeK<|=tr&D0ft{B+9e!j! ztu5PqCu0ZVLx;%#V8Mc>R<>EuVobMgnGQpn=bUiC>MH3*4Ru(bV2uT`=nJ872w;{F z1))PcGv{`+h8$aD-b9Y{@NFNiXh*C@6HqM92htkiAwNm8&v6Fo;$7k#W`oa|LgQ*n|ezH1DUOnu$YbG7@10!!&& zHcw2W#<2CWm~w?K8TBP1E}&oWV59+}MSXRr_C$ZQN8c@p%LPBfA1mU03e!X(0Vv}# zKMX=sPshNy*k(-IlDRq-SD{;2d|VN3`QC;#!+7nHR(w2b5jstZtf0+I4zqpY^gAKX z%(U0q4jFJL(lIN@(aX6fV_2ixkXIRli>u(6mau>JfVf4TeGJVpj4pI^=)M1MV`m>- zWqGFijf~9{R^_yXFsENRYtDE?pt3r(j-n>CHHTHLMXLd{1x{hLP}D=Mpal$@)}>CN zHES%GQKUhTfdbV=LQxdb=`5kb2niHX7+_i&6FwpZk`NNIIlt?=pPfXlXU!j*gknT)T!#gniH-p_>{YFmX}R=58Y>>>N36ZTFSh$&6-DrSH=mUIVm z_tX(pW}EKctkuZZn0pruPSjRq4@C(lPly3ks{lw$c|f-#1gO$qdX~U6{KSZoqcSxA ziM!U25dM97=w^9Gg&=+-5|PRZEtIln_XPT4eKJB_*gGFUoev2y4o^ECzq z=}Cb%58&c$@HfOpjGOxZk>K^^%Gc`<0nT{(%^W_$}DoB+?3q%C5 zoP$u2Ir%f{tf$>xL02B2pf!1atr?H$d)Ip86Fq5m77Y#x@O29u(Gt`p>l?>LzcWod zzJyY`QT=7qgw8$3X1!A(xKAWl>03s3MpG^bP>hE#l|C@gwVj6=B{eL{g?GxOTQGST z<(+t3yt=#R`eD|u*G@+u^9+vfkO*cXi->oy1wm1UJsE&ra}D=Zm3%&4qx7r%&R0h$EZpcU%9`gO$pV{HMjv|119 z&N>FA#$j9qlV{*++~XW-A+N6Xy`auK4&M7LnUvguTXYb%2?BX19T>wGr6@#-Le*Kj z@-YNeH;C^BT(9t?)^5kH1Dw!EzK$Mf3k{D1v8&zB52g%hjM#a26X+wr9+)Jcb|%}@ z!Pa3{S&b1}wE&rj-(^xVoErkGFl(73)vC@@hO;hl3(i4 z0XtC|eAgQbY?qvP$C5iLwd-5!Zh*+(SC1;X1*D{+0sHMJQ52qlnY2r^b7S?pjgj}% zOC?@6K%2Ahwm<573v+H?K0xPAfR(vv+8d(!Yk)wa%6;rs!=)xf6`N9Bh1M;z_Z0qC zg8i5{7Nb>0BCVmFuSMcZw3mF%#)sRU=9VV4hhH}g{Ng_ zDbRs`I9emys0E5~1HwQAbe@PL4oxyB)Vrpwspybn=B2`~)Rr+H3ydA97nLRRY2akAuQDuje*9K5YFMG=j z+t)E@Y{b^r>|kx=9%-|Af#sOr($k_EpXaCmlVj6E9;#PyK=719_$*FK8S8Q}4}=Au z&`+j52`Y{uyb3UR(4sO`(bbbLjAm7?c2}ojK9l>TnKjJFP3P5^L%C{sL=M0T^$ffI zp6}g;QNV3@h_9RigV@+Lt{OD7SBlaWe1qYq?{94TUAo@Ur&xaRTP zl5+f=2lP_>hc&V;E|OqHIDk_1UARMYj#Bx_o!Ms&PB~F>>NxE!*m_Csa0ouWeeEZp z$BwDz=e=1kmo9KBbfTWOf-#+QdSgM$kXRw2xXAdKBr~0J3&W|%L%uo?o=lrT_F^+* zH`QV0<6<%5(oeAlxB_7o1IHASf>~t9FpXL<6M@_ek)VZ9MOK0#%+-m=TL5JsvQM0D zw2L0dW#WI&{y8sxa-_1M%JeoTK&?+kQ@DA;B2a}A%L?QudxY%}Qp7U|vGX5(9jUOk ze`~A#6&jkEh@WemH>|%5uKjmeY>^M%cCDMvQkHi+Ra)r2$G#N_JqQU|S6;6)6$aEkO<-;*fkX)bNzr!Z^r&5GZnSSIe-3rJj6G z+Hhc$$Bkw(2vw#FoC%6RiS(viNkAAbVB;*qxgQXz`EwC7a<5>XYI%6GHr1tk-~hEo zaKy;rtFDx@jhuslw~HQ_^afrNPeD!| zQ%Mn-WTt72x>)8$;VsEc21(jh(MNH3F8WAS3auND74LV; zyniJdr-Lo7n{6P@Z7WQm`9bBJPBpmLR-b3)T8yh2wH|0YCRTwt55iGi-?2mblVA6^&WImzbI#8hxO^^onej$V1!q)AC8scW$oq|le3o6j3B~|9B z*XskKd$5w=in9P-7*~*e3)123(W)ZZK>1ECqyQ0TYFcRU3 zfQPP%b_mC!h>F&9*h(siPyRg}+9ytAK-6UGT%NoL;bCmQ!oAA#0TxMDaU8n(r>MCi z)?m8*i6$j#gd6+=k00ZyiwH4iqg8Qy6^Nm!Ls8(6iw5*0n?J3_VD;Wg#&=s!SM$pl8a81I(_ApzQbo?uAvPJF_|#UJ7B}3dkVP#evJ9 zNmGyeARY^!C_gz(o|bU5*k|PqQtO9+VDYgnE*`}tvDd`JC3VI7SUja8X$25#q9-f1V zYh+XbO0nEx-ct{hy${Q5PQ&I&4iog!bbI+iXxcRxh$y2F%COHm9r0ZTayAAM^u$B) zkWRotePN)ru(G`JfhCPwf9JZW;3gQ015=AAYk5ZI~j)_dtcfP7omY-?IL^`!o4 ze(}g$=UlaJ+MF-Qh7DEukW?k;W5s=P=2nZf-eycP4}Ees?gY-02N!!He@xdxZkxj7 zsr+9^1@@_0S}v*7>b#K2H>gDDirpAvBi1GFD0Dguq#(Q*~2Sdq__c zoq9Kms4cE|@Iq!JH{eIWtjSI;q!6HS+m%9$sm6xED$EkO>876?CS-7wgGdeK&lqp5 zHK&i$g{6eq`JtKej4BOb+>bn*j`h{a(S4g|s%U@_(lxg_v*5F)ifRboR6d}>TLDI` zYfRhL7MV=EIZ7X-Xk{mutVGVv(MM=Kn;lDwQ8ZH+m07am|MizY)i4w_wbbj@-~D>G zgVvUibE_BEt8Pct;PpPj)_*KW9+PCaOCaqf-=pn?W|Kg>dB7*vyBw{RHYo|HPIt92 z$RE}!IfMWlQXep-*I3V(%PVylM2(2{N4YC_E+rxCkbUQ}k$EX|Aj$o%kqzf^r&Udt zaJ~GeY&A3`9|3M~LL>Fh=o1ma4O4_B6o#A6(!PYD>OMq9-ZpG=@2~6ipY^63crKdU zJV7Ng6*ji&Iiy+%G;!62^H)CcPP*U(F zKPRNkd%6t$(EH6e`K<_8H3w$l7b=I+$H=5oj7xc)*)WtsAMn!7AJs|_t4kZ~b{ON5 zQyfOBSe*bfUF0H~Np47lgeJ+@rj2);2NIt4_0z0d~DL=u2I$mXDWN7$&2zz4PH0GscsPbh+R21mI zN=?sA0|aEdRP}{0hHZ6M0xHi?%B{hhQ9(6n-oXSRO~8-PW^6rhX4zjYUm(iDT3oT| ziqs@-4js%lku%?)vJ&Nuyq{3TaYhW>z7s}dgA88bD=B@I0wDOf$Kc3lRUr`CnVKI} zz{m(I;&?PewRAp3(pC_bOd);#nsm%k-1F<~uFCz>Wa}M;tEa|vWbk%K9Y6&(=3v(& zyPjeqj0D8Y1W8=Badg);<&RLQzh``P2;P_fx?4crjSqB(8g@42h#R)2WGu7uur=RdDL#QPfF5jV)gtO7yYel{ zYmDf8kV1PlpRJbAU=BPj4|9(BbsF?I!5GMj&(akA$+jzi4K)VvCa(q0Y6+UGgnK&P zF~}40cZw}#Y9e!!1`z{i51%gp&oA{bus*7ISDGqB#p~aHyTWpp3egcGR!jotI&tR= zxY^bz6+U{|<>NnW+XR%8a*MGzhj`ykYQ2DpJyx3d_l}oRA#Zyi~sw+rlBl;=x}4h zd}zsd2A|0<5t)K#T>5WYAY7B5lF3bJf!L?qKlZw?95j#>UQpmpaFrL7uJYIM2=P4% zmGmfiYv5%Rual6LQW)vDD+4ntJE!MnEm;*r6%;4H_LJ;Ln-3L&MQv!mPAC*S|IlqT zvyTI6k!hyb+8kp9`Zhu_bprzVMww#64WNkQ>^R*m`E|jSs6O{VCtR*MG09l>LleP; zbrFabX&voD>8J5FHWc{G)r=EH6|iQEnCq8Bv#JHpge= zhz`#cos^$YZE07R1&HZ1&!9)eirE$Dcwif%$q}DCugr={SXEx>i4z{z5lg7ia{Bkz-ouh~!n0LJ=;OiMWM5QRNFv5b6pD#VF$EG*6x1sF5{n3Q0|t zp{GBd<^a>nfV72UepL_0K-|FIMrmkUNqhSipqK`oXJis*OCG zPJj~G?2e6UKvL}sw=tvR2NFMU6^+SUyl7le)1W;kvk~k z({{z#U$diHNyhsd_6T}M{2%r}AfM_cj^}Gc@+VCr=>GEY+&`6_Knk{RQej_F<%8of z0WZ<#BlY(*R~7p=P>y$|tL+n7wBlu;+m}UmtAVWLlJDtKuu&>}pc;n8uG0fwEID=N zq>Ueir|SLY{yq3x1eG@4Wgn6{J|!^)_$h=}4{AaedeBT%zRZ02@uF{Yzb=!jE^971 zvWNNfP%JAOkW1oPHR!MKDLCT&#Z6xdD9Cq?9n7l6#Uhus?@icPBy<0dpFRGMsn z)=se39fHY%i%VVEFfjA=t{1i)dR*G;NfE^5&D^(Q8NEKFdUNys1AsA}3?}2aQ*~|i zpUpBkOdn{otfSn6SQXvon*OL>WNIy#g=b+^^-B1N%^{GEHci>;yk{}AfRIoar%0Gb zG0QHStavJ;Jhv~Xk47QLTdNNC<}EU#46Z=cq$Ws`pprxcXtTDkvg+vV;8U+qW(eJ) zsy&DA+`;Auqv4E{g*Q3M3c6kl&rJAr9lF^j`!E(cftZ+nCqlP=`J;S(T%XXKeOjc7 zXcZg*iv)?hhU)klI5E3tvF^HJg9l9JH^Ku#FqIj~t@kt!IN84>lwHs(%ubS{0t{t3bBAgy?`fQ@Tdlyrk}F|F zljrxlT-5*eC-mR^nUq=()s)3p)OS4frZdmtt4)&~HjiT;J4hF7OI ziOOvuNJEJ{cK%4A4qR7XR~2Bt%N9IDOjh@cA;gag!53js3z;*gh8swnFkrUe~3!?3`??79k z`{YxHCZO0P;-neLc`tRjM`7*JrPx*MWTV7o#;I^afM z1DDHDrxIQqhbGFedUG1rg6l|$cBRI~U0qnbHG8C^ZQF@f4uO7i$@3e_A3d>misldv zSUnspHEvnHtwhp8s3Bkv6~N zOj!GFFDEOX${DR3QV?EpPg(?h2YG@6i8}9ikdSv;4|Q0Tn|pMb1}>_wO;4$j%BAM# zJr|MRdQgGEVGN&yWtDCP>yOi%@D>q7@()hY+Y1+hnWXFdqUVRnv&FK7gmBJM?#Z^F zAQIjC7KYnGl{6wf9L$b7t`M8qodc9* zA@X(^Q_siQr&1Fm8XcyA_w)vUJ&XSrl!v}33cR`(8H$`GU(cFTz{S{HVt-#sWY2ys z?5M{^mI7o-ykcA}XWFW4cvaVPP9k*cOslJB9EJLp~U z`FC|Q*=G~V*K7xS*!781m`z=|D@_)#H063%eg-ZilQmb@sjTXFfEBWHCic+OMb3Yz z=*aGRYc&~5%7=j8fwaJ-^1xh1T3YlU^&w#Pqeyaoy)|I!NGH`wYLpqqSu(8im>Tv; z*(*EhZZqUTyKY=w#B&NXSA2$AcW2*B!o>5s {A6#N|(fy|O2GFE9EBVnQ)mG5;v zG5KSL>3!|>!J~86pBORCB*(#qf>D72%J8Ega34$C&!goSu|l5|MI!m09x)dJJR?Qh zcZ-3lDH=QOVemRlQ7>}j9x^V+79>h0CW)|wAb2Mq7#yoaL@@narVs|l=5T{TS+zdvj~x@WtCXhWB=Ouk*bdSr!ChZX;jeOz%8)Mh(>?WxX7qw2!&H1U#Plq8BEF} zCGCnt#uJyV8qqRc6ST3tWm%3oPwg3nvm8eEQ}G}LaYfzQfwfpi25;_2c1igFjEa7*sY7YwN>x_eV6 z1K|5U?YMJlbA=S$NUW?!%*R!r45O2(xRhj(yDqK9I*}17xi{!m1noBEBPXKXbd2kpsGub zHqDRz2tdWnis?XV957;ChBu{_8S@5t4?)zOtrLkPOJ|zVa7O#9F}a#_s{-m9aQp1a zu8toNmy1ai8d%}5ka{>va3;nbDF84L73?j+^uF^yU_6Kn8CJVH7Y@1MQv&!~m9qp$ zRAr1jst~ydH58q4nFH5OzdT!4+t~Ja^rHIF)IQhC&s4iXQD`#^jRKT9a1HiANCf<1 zps3(QQMD|D#_bxvR1eg<$bT9hXc&HCS@Ff0wE)w9$_OzCF}kFd6~|o$xET4{M^oH zP{9iqU_cc+k`(dhHt^vOpr;Jrz(uE*%snQ9>Q?0~Ipxh}!L*eSk!IvskbqN?RjG)W z2|4#z2x3Pg4z06(f+cu-pD2Awyc06e_R0vl21B%~qh3 z=I^)n*8*UhK4Ugh7kj``d>V}e`P=0pg5up2cK_{3n5@sR(q|WgZiABm7CO2}WmYQx zc1O|JA<(`$b{r4)M)8i^qhG9xgg|YZDr10k#16kZnb;U=FFO;*8KC^1#x7-;g z7AhiM=(8g__y(cDO?g!#PyPH|hja5}k1Gyy=z^S?#A*d9kSaacSEXG-r&GCpy}g%r zFO`$wQ6Rfo%FXx2u}w&d&WkYlJ#G2vg|!6Aq_>Hfs%KSg$Ud{13iINtHtf9b#~dK4 zgC{g3jiMYALHKrAs-l_Bjw*DPkwYBt5-X8S3Q@3(U#vs$@S-_oPmvROx=2FV!lWuk z);?s;Nz4#&g>z8#7GFIFf>14Rx{j*y{hC__GWIe?UWB`w-`_`|Ums?Rp=QO~a*Ov> ztyHdj6)pB&Ia(^7oHb*~$(A{(Zdj>JjvR$o3=byxL z!O;cz1OqW*Qd8MC;9XP}DKqR_;{N5uP%`+~+!8>;w)3;O=FVwyimX}rAK+o8V}A)y z1*zaj*G-smqP9uSmvpxC`8OWlwY#Eaz=qwTTIYXk7ejPvu?7Ak5HRa{kl<-2lPBIj zwC>g%nLSY9LS|*O9OIGe2li>9Rfr-2u-2;N+&PZp5L5(W>T5KYfuB!7pSa{iR+EOfuFY!2p&}v)rm=!+X%5>x%#B z2T8Ervm{n@761E%%&yl>E@0=mny{nns+{1bOPbxRl(O3I@-w=sjWnNtmf&;an}`R< znU$PszLy3ae3kM-=$B->68><{w2>cn4_L|xy=#r~zK5AiJqwrr;Rk#j1Zf>kOo0F& z#%HR?+mn-YTb2NpK>OPb9`ZNruia~etU4l|{}B~67S42e84aQD$$nAvN($!&7wXRZntUQih{Ona?ZHnKweeM zJuO> zs)*5_4`<>XOE$6O^(zD2!236dGyWmQFc>vD~r zr?8#bt}$apr5reZa6D}fx7jc3+3;CDy6SL88^0vh-FPPf_d~fQZ6to;*jgSOTq56N zJdP#CXv^Re#laJU5i~+1-5^xcud~Zrj>r_mF*A^d7cVU7C?_nWg3`6;CQ0&eU#AY~ z$T#Q1*rh&dGVb@d%fh0=lKt02B9-jF+&KVFUpi<%2nuY(3D=My<4IS}YCNhQt%W-1 z{+FEe{2bXTEnn?c55$Z)OqYBwt!Q=Gai2TDC<&nhT!2-Br0`)69wEj0x)L?*U@!o2ckm;CLn-E;Re z)l^2pMnEe_nic9qWf66LnfgZ1=XH0MKeB5<%;FDv2~0KOYsEEoo-6)oR@*1{m(2W7C9VdGqNLD87o;1ilgZT->j?etVl}gmU zSmy#u94Z9H7hhl@=eTITWHl4Ux(4LFIZYZCHVUa=UMB345SRv#T|f6f>Mb}@bKVxg z&Vcjnhn|W86}sf2iejqrvTAuvRLhE=@kB$1^gWs%Zc}us6Z8Ljk>8)XV?S4QU`-!a z*di3HAeUuIvp)z+f}{zrhmVD=2PnH2B0Mq>?5G-%VJWD@G83Wkl+B55RWw3C`N@*@ z)63T$geX!f%N(4H*(RX%AZF|ce(7!dc{?VBV(mfD%cuX`RoT2xhOt2P+KgdsDXvKY zZ5**YFyl}$xIhfED+O-pqizDlp+RgR-L&|3U0Xk@pzwY&BK11=lPW6mOR3O2?zNX; zH!ctsitVHGtU0nPJX!o;(B z3z_uazm|y(SO8T_^jc$o?|Y0TOiSS9?Ec=^%fUuUVyfRsKUA7ao`|YTnV0;t@O3!) z&i?iMMEl5DD3MwlksQCD390L;-c?ryT$Na??l@OBGq|b+T;6LNTo@$xDSs(@g-wdI zp{i0H-ldA*AOZQ?xa7gcj?eO}a^jH#%3rlxSfY5$vY=o*Rm^GeczMo+b3FPCFN!w@ z8+YlMD4M9CPAX1j*3&2nN&Fy0YS>TlAG@z|305xF$2JU+8&TtmT1Q1O%19;ny^xMf zss#5hyhs3Wrs-dB`X7=Z^2V2V5+nH|ctxd+@}SS0bHUZoL#&5ne!Od}JR}<)S7%PD zak9E}oJzFlxFY)xRT+1}6ua4XsTV*^OgdV<^5f>izqH~62c=xdbd+h9>#hSOb<H@wEQ&<57rg*{erTDhMI^?gX)vE&ZrucwLhSbb))8ZIu`7I(@5e8rvfDU+}*gZ(dEtV zZ0wsWUyW0IxwGJ{X|C~zLD;S)1M~$va2E<=|L^8ZQmW(8MGzKMv(V6-D?d;I!*ylD zTx98qqH-iInV@8{LYdIz^NaskH-{K~$s9TVQT>ya#n^^BL*ZRcf_`u&ULL>)z&WTY z4gt>6Okq`1)zr>e3VcNf$`nM+gv=iG8ACb;&pq&or0p@g!RzboNSSVI$76TDvnJ3E zi5yDz`tv11B;0|%T|3C5cbQYCK*i_8*)wsh69`r{?%^Y=_9%~%EP}W^;eIE`Z5$EM zZtviVU|D3=2a{WmnUwsxG|wl9cfXULtC0O&R$Kpfznh|USy)TG-^2Lj7@M8v6{7&n zMi-%=(=)zZ9;fHAG$S+I3vD~G z9pphhzr^}K8TD3$1XxhS9lg&+jr#A^xtg8_cCrzfkSNF~Mtv@;X9d$PgQikK+GHe8 zN+b(D33%sQO1G!AP7=ieT3oe|ZX9F|4l4ol@OwcW8tldjh;4Z2=}4CROnDVd)SxMv z?T4|Ij~>n~P|3}Af*h>4kO_P)I?&L;n)c=-dN50G{IY-*RQe+A1Q0nEhH@woC7geprFAgp_(kz}#1Az;o~Gt`Gb!~Vpjy<}VK&(a41LHwyxs@ zl$_3-E^S~%wB|_}lQ2t#6d;}{oOXozE~aI^?vRRLKH};Z@3z~V7}2x`wkjrD^8v$ zInyQimk2)-)q~nSD#O}83KwaD>Mh*PNAhE!B@II_iBng68`UmV5;JaUu>Tkit4Jv; z0?k!Oj?*~}gBL+{kTF2Ebbu3cWUBRKFf!XXPG#ZC#cj7s{9|~Uj*YFJD$vJ~uyxo4 zNdm>fE4n^JVV-=YinL9&unREyr-8y}kIIZ#d)t8rm}^IH|9Ujv6(E^7msH zIHOtSk^rJ2kstr2D!Z;t#Dl||B<&C>*~_2bu` zINq4dfYP@dvPJbju?A+88MmAfc5kXoJkmVr%$cz&A3U(#vF!Mow|DK#4OO=PLgr5+ zsVG=Fe)*Y4epK`a@K_hA9U8b*^~HibaCSOa#Y-n&Obk;^zTnta)jR8^m8)5C=9A5L z{Nm$htLE_6!ubmL3^2c1^W)Et`e@mLihrN6;XBV_EB;5rBYmNJ{G>L0EZS%O`6{+kM!j%`&zybs7h7(e zTi4gfQp-qNem^nOaZ`R!8Hvj)@BLrYSvi5AU9)MQ@%YbMPRazeQmQ0dBvRXGulpxN zn6mLCdh1hKf#K8oW2N0~O(bA63>%WaE8c4G?;`|~2@+W8E2=I&saNFeSXyZ9<<1Qj zoFVH02MD@Ldug=--BGa_1x{jckS(mA-ICyjwyY;DE2AhiK}f%xJNZr>YYg!*brg$p z*BrT0yyBk*w{yLO&<=0U72P-jC$(@pkB_I?(F}UjQh1!;nzM`axKrH>A;9NJEv{yZ znCECa?GmMfE_RbmhsX zGLP%?9?OmUa|d9#nsV|6{cFktf7Z0NAb0(9nD-Vf_Y_q|kNd?vSH4_Qb8gEWyWW`p zw+Hb4SfoQ7r4<&Ua;O>_*^oQx?dc9*dVN!$y-{qm+@~9PAdkB8CS{v^pD%dl z)C=y~)Mw9?Gi?Hp7B|Dk%^yZ9(5jJ~4CkuNoiF_cH2*``2L!3rGWgBauW09_KNB)h zUokv|(zq=NyAu>=7fH2Qe${s9dshb4ZdFg*hj~{~>p9MU-Jt~DO03bXdN++^*u3$p zpIv;Uu;J#<%j#?sB<8AD92z%Wpg>f!&CxcwP3{fGaXc0j7f|u@=`oUB{DUqRvg?h} zIf|$rxJEK_^Xc0T=dNq&^ZTTsq-_LL^nM}OC9jQ{lX+y9PDLr4$)63Hv~kJ#i8Y}R z(2}B}ht-ccQ?>ueU~Kuz8;V}ux;=E+FpPZGdSQteDW)@TpV<9(uV2T(;+yY#`VYDZ zxNDt!pj_>jg*4Q^%~8Y-u=i=k>yzL8Sosr&7i|1y;}<6`C^$d&8`fXa{iCzP+RN3u z0DGZ|IeSSt1}9l=Kfj@KEEzMa`S(|y+tvJ4v}MMy4HUwu!s+Het=jx$8MaQA;n35H zpU^IdMjK^QW9s1xnc+E|PdPZ-)~w9JOQ|T722?D#%OM7cTR-JS)Ev>mCM{CU2^LtD zrsv0f^nS1ki$aZvs^fa*)pO_P%2#{UE{a}?t8CK0aA^ZW9L}xT>4(Z!b^RayVAqY6xo_3Botu!6+z-6% zd}G_c;K6A$pr@<-qot*dB^yROQ7fy#qE@vulR?({$VkYgWe;wAy!QSlw*U6PEiaw< zzqeg>eAI`dsF*h>(zo5f_+Pev(=C5hhME5%mw^p!J%?Ves9k)%81<8MYda@qku)t0 zcwgLhX5ejC{PC-MPRM97jHsJu!4Hqvx^U#>XTQH?$jA@f`t@^P+4%al*LN>p*4Q8F zyz11+F)Pd)bvmNE1|*i!SC| zQQPpx=l0)F`{u|Qo!L3_|CUm^Q>|^ss%6EOa5UED@hdm)X})_v$KT)cEE--ZXK3K_ zue{+d0D80m!;dT+sau!PCy-`9qE0y5+)(_^xL0qo;a5QyJqVcnW#x}Q`|?lc?cI}` zw>pgSV-QY5$9%4K!@W=Ov7eifTY69FSEl~YPcC_E`n8f21$nl-mpyeMv;TpDFI*-V z$3$8BAGVJedc#k@z5AViSn$}6uHpk-m%WYmgPVZ`aDhNYH1h9xk}% H!dw3j4N9GW literal 0 HcmV?d00001 diff --git a/tests/ut/data/dataset/testAlbum/imagefolder/apple_expect_decoded.jpg b/tests/ut/data/dataset/testAlbum/imagefolder/apple_expect_decoded.jpg new file mode 100644 index 0000000000000000000000000000000000000000..d7a0624a6f95f4e702c17ce26301388474cd6db0 GIT binary patch literal 432358 zcmbrn4}8`2eeeHGXiN*MnOy?At(v=Ty~_<~?pCczD+hEbcXq9$GyJQ-Eo=j^Rx3dZ z8V=4nw{W}MwNTb71w#>+ATa~9A{eK|Y)~Nq5e1P}788Vk#6y7mIl0f*`~CTz6V&a| z-~F|ZV?xe3-|y$o`_JqBdcWVF@8Ii$zlwhQrZ4~Lm!q7VD9Yjg(cq!z#wa&O|I>eS z^}mk|(|_|m_OW3fyD0CXi|m#EiH~2D|B3vIF8cV0kAGsg{>N{>HDdU0=^wg?e>FF6 z*s#3e`4{EeNB%$kXYhsSl8=A<&4L$(<$N~Ey(DMYB{_q?h>96!o(+$|`u}orhkY#X zqI`xif)8x@ZH7N=7=zAZ#C*D%_oI(plJ}|K`=bdL{Z7R<^FRBY!asiCA0GSo@88h$ zFGXK(d+!fEKl|>#{>1Q+qki|leEM_$?GH=-d+AkIf8m;IuN!;gpOk-j;{W{9oBsTk z%E?ovPMbdGFaGkc=FYoq{(`E7_uTv4MT;N&-a~)$@ZbJj%@2OKe8rDe{^O5-Qn&VJ z>mFbK#6Q*l^QO&Po_^-%&u)G3rC;oLx#eH~ZRg&7ukL^C!0T@u{Po*M|GoX4j?QE6 zp6-4>b*AU+x%0i6S57p{r^SCia$c7(uUzdxzUGyadoQnHmwYVm_x|XjPfe)E|K@jo z=d*wOz{d-3_=m@u{^b+D|M{=KS2X+Xw&5fH;HqDL?zE<6GyBaZ_SgSUGs{dYJFkP$ zZw<>~^M+j#C8B}5`sOwto7n&S{+h;~J;}XGKiQn>C@((Key*)&Pip3))$^j~`aW6Q zc}w-CBdN9N>r0}wCzc#}viMBjxr*_5W&PWekA0nA)*ou_=o|QK&A2yu5~&mXq4-;a z(aN%?T9X~UDZSsncuC=mH`)fHvdxon%0H7lw!0=OS)0D~iNR?7sl7{wm#pnto%(fe zYfgFLq-ZeeJ~eS*(UPkw#$R-#cL5)(-#)c(Zts+@E!lkOulYn(UVG8Z+SI{|?`kOi z*4crQzPYcT>svTw&4}XeuAbt(4f|^n?LA9JGpF*Xuwrteedp4Gr|(Ozn7QE&?l-n^ zFuL-VoY@UbF}Zicq?|>%+10bs;*~q@0QU{na%!B~e{XqV)-eRC`lN z9pmIx+T35gzvkk*cxT_8S3EIx7}HprUNY{`|Cl8z{Ot!4ELI3Www(`QLf>*NTaA8IIkqNt})~9&4@UoopkyR`c zqhKpoX)Rk+S$qEOx`x)A$>ZKQ)myo^B)6siS-rDmCQ~S%(!jO!xr*^c>6WhE{WT?P za%)PS8;t(!`1rg+_F&X<{!?ThE}qjfFi&s$tM*)9UD%E_nvDJtzt*N_#h+}PY^!m6 z*Xj7HgV7EAI*!}5`8^6N7e`IynqpKaiK(mY9q8v<-CD?H&+ROU>d(w(4w{+`R{LYG z{PEM;CNiP%7oF<8zs}ZOcPnA#S00R4KJ#;J*G~r{?&06Hp9O4cDtX}4uPfQbYgtxS zQ46Tol!l_+3mz)0m=P5wcXdsvd-jE<%Fep27ytE+qFL!#^LIp3beGZ{Q#RgZJKR)x zqBEZ3nhyT-;F`>WY?IWoOta=49*k~Yu4`6IL=RfdUd3h>?arC_`tvP=(YtjfAthsN zm6wezvFn^lj~k3W9hGjMiX0)9HI3u*rt01Z>eRZLMqR8tD(>zpHxbE=eS*~Zio`{v0A zlotlw(p8I3pDi!aK25pS?lPrr+0ICd<|W`T%+x}p26`*? z#?mnrNEYL(?9AO!+}%F$(Dj4SO^MbkJ0q9I&om!nwOIiS8Wt#OEH6G+P<}*~4_l)} zXJ@cWvQ5iweyY_JAbDWrQ7pjBic0Qza>I|$MsZB*Jebo!jS8JK; zx|a^uh8KRDTkVVz$6C(R)neV2OVG71O^mwpmWuKDuMb8YsmjG{TkejcnM0IXS8d_l zw#BDY^(nq*24Au?7!_AB8W~6pVQzWh!old7>1Gd*+P>mvj#v6xqt1JJzf-p_eSKnh zl)xy|)t&}#ar-CKJinHr3GD<!+{`5xqp7A4f zwSi~4R;Ra)=}dH=p3^%iC;dRr){?~gM;FgC8!4~Y-8b>|f#&H^qI=d=Gn2cTUr$bF zIK{^{Cm*X^_LZIHW}=GQdXA5qck}u0)}LLTGiM-oFZMFM;yY+Ll2y_3Wj1ksqP3oB ztQ|;nRmnSG@+am0yt0N1lx>;4JsRDn{YdU=?EmiR@(5KjVd`Ew;^wD#r);yn)7>{d zf8g5m=4tlXA)-)Np%pZ>9fQ|k@NAJ8Qj=8at*pzidk4l<+?#&*wHln5v~|J5^1!Q) zHNW3C`A(+w3GO8Mxbl`M*XH*>-&Wp^ehx-6XMMl0y7tf+iRh+=!W^@wQ*g^D$FV)! zx3Ro^)bgleQF$S={{jdBvCvg6!^1w`mVV$~W|WWpeEikHsPh*5uH3J=kqf=lFhc=B z6XeY0uUkwBM?acdV`|X7H3k(cOI;?}Zh5faFYvF3Bv!zdO3q_uMY(K_JhzO49PqBL zsF1ZEpWnZ@5|Nfm?il#u?4@I7x+>vz`K4T1im!cl2QFnrgnE_a?$EePcLa2UuHK#X zj;@(}>(F5IwS`zH*(Pji#;?dYNQ+!RbMX*E6*lw9-3_gIH6W11l{JyAWjU}%P6N+w zcQ2Z`bT}HnW`zG?$n`H?%EhmHQNF0tCBRqnqkl&_ieC#u*d~MFKFfv}n}fOsAH~Yr zRop}JuliKP>XplYPhB%S#mJ_b70%<2CNan>me(zEe(xPx_QK|GHJ$xg#@h``kLxt& z>Lxm%B$;0vJAxD?_thRZ)dRqEH|$^5oZ7u5Ogp#cchWQEo=P!O-xrvTU5dNa(;XNZ z#7tgA0+@XpTiY}~4;(cqZ?VrQ_`15K1#%1}QO3;jUAaQ@2n!yVWnjciXwb%}2qRR# zePoqDX1Oq>kPWu5J?m6srKwvwS~yUau|ep}y%0C6xgv3N-+Qi{!raKJM#5F+f*gPReGT9(ctz8mQ-sSiSa~C=c^m=Vu(lG z27y~?b7?T5z*BC}`<)D;7T~hAU8jzGrPN{aW01NS?;~8c&{At zYpIIc7NwWrh+mZ?4@SaDcKhOU=YL+;Fc>|)zcz)b1<97I8GiPjg>$$~)6+7!OzXzu z}~-zoecGKF-zOoRs(dBdPP1oidJ9%(wqJJg$Kd z!qOxC`2>dY6jQ-IqX`q9N6x2gLzoXD5@nmev*>n&Mj>2RGk)jH{$tr?c! zpA+iO%o#|Wz+XI?oF1Je*kO3{cT8ySZ|mdPN){IK3q}w>G`H z`6g{Jz-u=H7Xm_?Ge{-*c-!3asO00S-wl9$Z_P&9OkFPV!ojljKCU-++vZ-&$<|)_hmIIn$hO_44zurDo>(p0{@k8D(U;WUi zX zFdBEwGUvNJ~q9&=O&?GoK;gCjxsQSrWv3qAf;{* z+q2sM3>1+R`;8BVAjRn!of;2%qMar`1wgJd;Lp5eV*w1ixqNcXjMl0=SE|t# zM(cwD92r!km>>qOIUbpnY}BXlPd;&lB$%kMq2`~qh555Xit*(|*&j)SgTYvpgocG) zVtKN-)bNXk>#@}NPZ=Z>P0Gg0#1Po+8?tdJ5S)#PEWj(lRlz19U3dk8P4n5K+qB)> z5);RN4|w<;55;0kWZn>m-9q3~1jO^c5!_G0n%I~AH)t<7ur{$@80wb8QSlvp#e>nt z8U~QNV~T}p1iPJY?*{Wn>pIfOCvS^)H`bYv&-Rt|UN&W$jkE-K56oSYyKH9&?lQx| zm6$WAiEdVGcMtyStERoF zg&0>>B8GCAsq&)Ppyc)*3wP~Lt@SU%5b~wQMR-PA^SotydaJRRa9s0uC@zw-J-I>t zu>Q<92c!14SoH0_nZW@VgI%Ym%d0S)aC+2YqoheEs|ks2a#a?*tRKszuWhR zhdOg#hVWvz$F?vxg^WkOIUx6#I5)6%*%$E@gHZxYI&jIC`CLQMEP=MDSTMGDi$Q)| zpv-so`}{QI(pTI&W#*FQIkR`yG%j5I&p7Eyyhw9z-eDlj&;bGXwdp64sKc-z_(Pb6m zc&6~{tn7i_qR$Cz{oDN@+~(dQ#rhZv{JYGv-(@(4Qs`uI$#pN`*tbIu3n)#-XEhui zcsQMW;<}dxzJR|e87n>G&Z~SC*;v^%aimbS4R4_2$Yw$6AbRw1H$j3quM{|4hmFbcVG#UkHOwa znG~7JTIUAQ&Oekm_R<*-VA=r!DLyAfZWBV=UXLKcuU zPF9RKDF`Q_Grt$=$}5(LPD=XyepB;#b3dfDZ1c zLZ-`OYzw5ZObVnGaF0ae+#Wad0iudrWhaNT2pSLTV_U0OrQv{vQZ)P77xX59PEF$?mP!!^GPJ+8 zx4VD7ye;vosWkQjE*^@qu;SKV*?2W}M|9Y6c|_5t#dn_T;f69z=TiFR_}TY@_A4X? zleV90UGO6$)nEs5xNK+BqVYv%;qgLLcyId0uhld;z%%j}3P2i`p~K4CW$1hFK>5$c z?j`;yH-l|9*CODwv_P0UjKF>x;U(c%>%N@#GFuX^f{&Fv`eGXc6a3T0y9!vBdF}*s zZ8&|qw)36HQ|wA(F{d*#XJ@8G(@(_($JXO1SNAWn|}WonuNNrL;W5oJ_j978N?7vvpkWx0kD z1ZFsjI;9SqoGW9*CI+P~B+nMx5X(xTS(g@IkoRHF%SR=Tt(qrO5P^jsz>L3k!_rR@ z<`B#E`A~%VMZ8tQ2U`?M{Jf|f@|K)bytQ)!fNz4&4T*&iq6DN;NgRn?9Y`!$ZiKZW zLIZ^|OF{n#oY85MjMzE_4B_|z&m^Hz`UFIPtgIz4$6S{3u+6nw zulP<+V`{8MB}NwOxAI_A0^`|Gd~6E{XG|xX#b3Wh#8!JS(SAJrr9~OrIX>^mz<1vy zhBBoWn~0ehwPGeI8jW8g;;qVLs+22WfE_U9x~PjvUs}-rk8<;3nUUROXRlc=v`6DG zm%^V}8PVY)yf^f=B7MYdW-E%%U1H%J0nyB>XP(?MFcROq=k?@GLd9!xcZ-ENb$Ts~ z)Y#!?lV^^~{T+W<4ydP$5%xZD-HQjB`}^7!9!gigD^C6Bk!$vg&wOlB)hEiuaNpig zcxk&b2rR$kC$|~sEc5*Ar2N*x%B06g%w&567;Fdp4_v!M=|0&aK2-tCUhdj*Lvq&+ zo93wHmN>vipFVr8t^DvR(E~;A?g%8*+D`F@d(x}tnfM84dyo%qcMN9E{-xfEXVQ;$ z%7`h@g%<|=<2Kjum&R1-PZ_YREA$${vD6Y*gFPRK8JmE*H=jjXZ(Ag9gOth~ht5p%B@(*u=LSTWb;+%QmkB6+%9vKHK$ z{OWoD9rKONkKo^$VMd$9cJe>`>7c-wu*?+ehPc-3?@9vVrNZD9h0c}dirtMIgs?4t zvjmJ-D6F@oh01KcHg9KOkt0I!lb(F!zT%Y4oPID)&D;$%8?T#rdPi26ZCeNauK1R9psVEKoL zKrNYr1Q|d5%VHv#ArLq?8cS=G?yCH25i7+@&aiMhx#ELLm`F-62NPlMEO-beFnBgn zktmRM8HN)nWHj&&7K0(FS=>Wt8ULd!{GzZM0MVL>?HD93^8x`NJZDu{fQriFc$i=& zJ=`k`#xx^+lI!HYvy3JnXVw}B3vs*(_XZU)RV(hi@X`@u3BMRMs56tm$!uH3Ln;i8 z34o7soEb?*fEuCGEau&K7m;scd6nf&ho^8EKppI(scM`Vv3Y{iFw;)Gx?>HX7u@P} zjdZxON10skRE>5_(95_+i~8Dk8p92>6IQjBcyUW6G_(lJK%I%3@3t~6+I zPxQS}ac|G3eQ6V|SLiz0BALs1jzfkm+ch1T9qs1+Zv4%1pmRV9{S0-hE0VRED@LG@>Th63o(pBS9nZa%?J(` zBoES^_Q|S?v*?>LsU7i9!o8*2N8P*J(hET}p4-HRA56A19;)8~Cq{;qDj1^oHPf{U zm9t4Nm;C#T>a9Crb|D{4m+}ubAAMP>N77ccBRKQlPj7#M$m!3HBPI(LEu7fDxFY?~ zgx32WLitk{-@V;dQ2x#bjfrNP;2OY3V5_-Zt2?h!o*CEXv{!k4dfc~JDzW_Q7Iqw) zsBGq>oIP(OI}Qp{$oG#JYv%2}&j#bg_T7X(vecBxur#`h=sHBbA9=jn8MoQ zY~MJr;kZtDNNIx7o=19%r#D$m6+(|fziF$=isXNJDRhx0VAg$LR`?U%3{E=@@}gBP*fU@%lo zUjVF|s@WeIMA1yR^(Njibcr3bsLsqex0W4jo7i97%3cM-hz22d7rxf_SEm{N&;fn; z_e@og>VYP-`XxXcRlZE|a@o&SR#X+Pfx{q*X%K^BgTFofbDq3m`$C2Paaeyue7QIM1sV zMhoUH^!2kaFZu7!&l6_g@HNG=+zGdiDkJDceQoX37w{Yf}a11|wsS1MAcY=!CB9o1vOBf|z4elje)nDcR(p zra-eqv;KNJ;USE?wkom;RgnooN!UcJHYQEb(FT8HU|X9qqQ}#Ze5L1KxcpViLGE_bimS=ToA3-&lQg&$({p7i!PFzqRiZC?hWE{CBq3Ik2Si%#=mGjm7jVc_@HY zxFHuv9A~lUJfB+8EL=BOsbw!-gsdW8U0FK%(Gi-gIm&G9Nt2{mU`ho|Oo8Zr3chZG z6xr*P$cNrqk77D|6QWVL)7$6Vt<5-9t|?u@gPDs_N2q z=-NL`KSF-`mgmr`rTGZJVr80RUq$9V>zM4XmeQ>hxoy54as1GfXR^|AiyB-LSTQ3^ zv1iW?S-gth0FZ*Uv+^6!&Z64n4dv$-Q34=>Nr43mk75+hp&ah`p1oEtMY>{+a|lCr zB=5eRO#qAh>y%*|MxlK?2z+Oloy{^6>Y1o!KEDh*eRBR<(VS5Y ztcuuH3)^LGJ_x3W@qE455tlDdO3P`%b`mh-L^`)$OM*_YMA?;SEllPlXrNP=OyUZ1 zMRpBi_AmXlqV)%Z8Fu;P(t$$9POU73Sc7S!KFA_DX||UX=r{BZ;7XB$l}}Sxp>m1Z z^9!+WmphFsvev`6Zx8FjEqbYP;`IC9F>Z6R=sajED<>5iS7P3xO=a_JW%qV_o!6L7 z-{d%LH@C7gC)s{{;0x=mScoA@r1=V?r)J|bU&32FMnV%RQc^hLxx~{m+Ek?mqC0); z)?cBitb^83y2&0_AshRvA%c~mRfd;5FAg1+?p<*uif3f9Z2gfZJfm@TiKFkQSDE?& z4i$vis9goP5J_1tE8%;utlICiH?;0~T*qccFy*id=;Z$Ed zxqI1jiN1R&et7Ur+o8tsp$6P&Vm5Ibf=w-^WS=$(R!fOHjH|*p6X(Wz38|V>r+-qq zqxgK;Poz)VEI6_i5%G8i!`<*G;lSk4fkK|rphV?7@QS$Z{Qe~~&i0KbPfD}}5>`b- zN$!jArl7z{dDVolN`b(_T6U$g_%9lA-RRO%x7!4eA9wF9FY6E_JjgJ5i4|J~&02l_`h|1pU+a~R4)*gpRt zZyAM77*TtZ!DL&zf4g5rMrOYxOTbAYNXfg01PG!OC}X8dgU3 z_`@%y%<3?BTBJ-Z{g4k5Y=QRcVmXr>uXRBC2WVz z$bJu<FP4)u#G^Hv)k6w}yOPH;;3O$gN7EQNUn;k#4A9H)HM!Kl=A6YX_r0!G&pm zy;nkY2c2~Q7QzQACT89KXs#u*wCS#X?s($g4ZWHjw7A7)fj7|B;k*``^|aVbh)$h5 zjN0loG6Jv`F@(dh;GkUIph*A}1&fO>F5I-Ci8lZUCLar5#F}TZjzCBz>KzhJ$;o>a zZjs0aOWq>pZ5OOfK0;=+fI~Fv5YaWNR46-h(t;J8=HI1eLHvdkF-O!5OHz!Hj77~_ zYz)6*5#m1%`6)+@AZ47i4-uPaf3Zv~%UbDSX4Z90k?yTv`Uvj0PRwxgxxRj)VHHps z4AV+u7Nq)Ac4irLv*Pih0=VxEiLuB{9mzEX^=ReF^vvdcw`qFyr(`oJXQy7@2n*S9 z?I;$*5*r@MGN2>tGFpoSDtRj<#uQcF*L?o;c6`{=Ghq)OK2DO^sc}|cl|Wcrnq1b! z30dj+QWpyF7kg`UtQNNzAW5_zNPlU`D`p2Mx1-&|y4koB3!=s!3V-SwTIHNQeG}xs z5`I8KzkbPDh*PT2$z8i|Jt^{jlFB09gXXG!u5179fKybIaRYcS&P9M+-TUnX8Ypds zCA%S$sU!p@x7;JKucj_yj&dV)gFF{m#=OrrO0^Z~yo zEERSzB$HK%%h-?ff4nzHZO6d4{r5FLI#Z?MAxbNy@ z6Cj-}6XDXDzYRwrE1FVcA5v?ee_3;HTDmP`7zOw-?v3=aeYH}%Dm+gdy>PFc*=yN%_0k|&|Q9P&bFZW;i#^FhM z_l;F{am@(J^rHMn29lwkl)Y3FrgGC645=FeDv7N?30@cjd#L+Z$)`8q&_lyYIU35? zLruZ8kl#r6?u4Wi`zuRO5^bR@nAG@k&8)gjI76d^nlY-KMYS-Il9`VFMGUfNHjs9a z@qr5E<&WbV3F*qtmHt zandp_s#X6A@9EEqyFQ9ruG%v4l(|)MjCC1;cS%3 zBnomPHpX%(m1htn(~xNXDD!6vdR@pAv4_OyGOVDA;F@HyL^}q;F)JRK(zv)@FyyS= zs)&@SJOw9Zq+j?J6MT*u`r5@&!EhH92_tHS%} zzN%%+lNNVY&R#ltVAH_p^$8MBU8~>k#lo4Aj_EWr6zds33wo>j7P3hgk)2qZ&_$iq zE#1~dh!{#{!r@~`!)H;bysitlu(|EBwJ5JXe$Y^HsX|}l*k(P2S(G^>!84iaSt`k& zd#6`af#K)cn

;X|HE63+9Z8lvU6jvOC>fm(F0o}-$A9&$bEAd)-wUPlv1 zB`XYhIU}>y=r(2F{1ijI2toANYa2uXQm*1e0x*H6vGlTOCww z3eF)5UchPhT~GDRz7tnLV@&ok#iA~i-&@AaV8IQ_!Qi9l+msW7)f=^(77X)VH`J%5 zwcYwFseQ~f`0GuV=TkU}9+Y0}#xeG5om$ODj0Q{1P%2IOcJ+yRT)XvB0W z!E&J76=8@Iq`7lb{aKKnBnCM!PdJoEG(n6)2hB}LBDDjJ693#C?S<3YkOXp2>Y}k_ z59KRn^9_RNH}V)+OQE?n8yTm9&0s22Lb@XX-Rfsf zB-2})^qbXui(Ro9$A>E1OZHaHw_0lJxVf}l7)FlSV^vGi*g+6ZlBF)onfv7>Z=@fz z9(ki{!-8R`Lz)RZA*POSLZwzVDU&o`*h*?l%$2Z_Y_E)93@oz2zx9m_*;B9P5XRv% z&%U67jjh@|T#zX~Vjy7Z4lOXBR=G%Hi1XukOZk1ktSY-gBfwN@wXMo&_}S%JVVThY zljPSV*5MokDzrmn{_j_YW*}`0{=z)(W(4@Qr3y0IrTvd@hh3% zV=U+LtmXmY&>*$KcMKEX!Mmr@%};ocO$6?4%El)>P~cnpbh0jWG?T%+Zqrs9TQ`}V z#Q7z1Ii|u;1Hv&B>0!XGyhVL9WW1rR8_2*sz8};#F8 zQ&y6Tc3Q$4yJ2ZPt#9$d*?3Bk;CL|^<{Q*g778p?Zb8uJ{VE4Im7*Y3@XxTRt#@9* zG#bm!oyGZk6-PPNf7b153g!jdesijHbnA65)(#XiA@TYsv_OiTo&u1RmLDas6cuc# zG4X`P`FgrRw%PbVo0lBq@KlDiW<^w`MzJp*2N|-`-Z5l0bOyuS8}Fj`_XNc`>g4E> zQG}owwSs$DcL!d^@sN^q93AjbCDkD~F7=Sc)lqYxDU#>!Rj|misE6C)JvigD)}O=V zk5D;{yE4+)cE~)FJnA}uN{h5*BZ;7hRU~6t1?F$Bnfzi)E~&9XBn66*D}WUY2&>6E zy4C?#jRqLaZk*MTKy7v|&QV2pLr)7)_sbZyBeMeZ!brfG3_%_4pV*;!SHK5t?O8R=v<9Rf<^)%Ua7% zDZCMp4oDNdNV`o@OlPr(`@Ql)qRtz0(zJCdRG3C|0^%3q(6R*wEkogE++l4d>MRI; z0{7ZMwh9OT!GxA9k%F^~7sy1Q?p6L14=F%$fg7$=QV86latsQhK>_$9;!#yESQS{P z;A6OezB;Y20uSX0O$N-xYr8CU9B(r*y}^b}nZ3_d$%0{$cGguAEwa-bi=2z&_$nj+ zFp|~o!Yr^UwbjVXOFm4+DIv5ydP$eVxcv};H4c_l|&l`9=1w3 z39G0aBb`I){MzWOw9#TkWUbT>Ly^t8YZImmsya@|30TNH%qGNk_><8S-C9$QjkKA< zTa0Lo-J<_A+rsMWlpu6(qk%mueley;UGvs(7<)tg6j97r@oy}-rTERRz^g5W@-U_c zG$T)D0jX!t;*nfB82t<|^C|+X;>U>t*H!-wt$o@(Lx9=ZUc=*#%{Ul>@c+7RlcoirFZdxynDZ6ZW!AAR*8{dfT~FwRWl$(o-koRPa@mTKgey``Rx{Q& zvqRy6iS(b%rpwGM5ztX9#V~zjM0Pn1D_vAP=lY_oUYZW1cqo(_U7&ZMBOVP6yd%n#L*?$?U`56p2F%2MMTDQRt9~GY=gLLlK7g zGA;CGcmo45EM@(6v#}^q{Ta-rWk`FfR6DX5_y|Bo2S&=_13YbmZvB<+({3n~UAL=S z+YHT2f>9oL6NM+OT`2T3TSC19BjyD)O)3-^S)~}qnY9fSa{*hKk|kudPFv<%Tiy<6 z$JMojz)4!|ls*j`zgX(MCx^%w}6B{fsjh?K}Ve zTPL506E0r!m71iIE0riU#nu1_Z&tQl;ypAds3>pu!Wu*t%k8vKCZ3T&cbMWqohVt= zcA<}|)@4s1S7kHIPg;2^H^9MJAD{ed$c31Cg{m!Kc@87gpQ`%dC8Y-+?4VMGOvq^PeA1E@i4tZGFsz#*6=mRt5nW2HW2i%;8 z5CE+Uy{y2Dv|mYk0|rtb@+_K}SLi9`OPO&sMS@8XY{<+^ZpE(cJijpJ%LtVY`mqYl zurOL2mDafd@dLMv<CY$ihay!MF<6I|yg|R3iR04Y$`>+9B#)fq%VV_CTluQ8mN_#x0K&|8 zaaHxE6ZbTq9vFW)7`fsgBA}5`jF(0CO1W>%IGvawXxo zKpBft0|u6*7t9r3v?ypKR{HWI-|YLUR7aLUNL@z`c$SucjJd56tEfHH%RBsBu_ZW$ zFRcvGw%OtvFa=$ks#735F(d_K8L(`Y1%)tR3PQ8#mEjy^!`y@B^HX^^W!asH38p2` zI|KuEk_4A7rBh><+13ySnFS=MWn{5wfk#kXmnz&deF3N$QL;2&h(~~j!SF%utdGls z3+Eiv7kQhBV-j7zlphu$CS%>if*1p)LP6hQZAWL;8R(jV2(qL41~0CL&(sJ!iyg$^ z0y4^nGIT*c{5uI}-vrMsmeu@G8o=pRY_!FcB1deGvQ?D)0Rw&c3UeC{ zN|-zhP;9Tv;o>`(90rlIB-BKi*N$qK5yN>8^(OYa3lCFrry>rN{pU^yj&Lcz@o$yp z5Al)`XCf#HVrazLH(x}iOFDxx-l&TNiAkjepta|^S+`GHH9-|gO{uE!r0^cuoFt`b zgoRwz=og#|y+VohLnjjJ1mPs5`4cFf=!A@=KtZJ>iXI;3%1Avzt<5&j;7->)x#)J5 z1EO5MGwf9HUl1CU8bi%IG?(B7qG=fL6*O%OMh{>q(vMi5gx78c_~DlcY=mi;U4V<| zNHanW$*IT@Eu9+jL*xW{SM{+=B{!8gj9Q(7v!~#|cA{P46ug+IflN3CWU+aLu|mrz zGlielQcwI{NZF>o>#IUkZl_t8fHbNWlB?6Dr;5t6;(vzBoTg`=C#QjU&8JYGk&!R z8Pr^IMeN0_3(Vevhp3@<9&$qKRL=5b7-YVfmkRjHwm*ST7?D&RRPfzV@ z&9&vh;2=zcQP*2QMKOU{)l;nMhY8rI8Id?W^K zqz9mcBAST2u9c@@$j!pTOA%5j%Nysl+(@=hX(&LBQwF*ioaQgA)MUU3J88yCmc`f+ zd>9%VY$czA7PJCcq9RbMv4OI_l>%r)3BzNP%=e0B+xi;l7goxei5k=ca>TR6;gexE zm@Jz?)38@WaTjxn?4C`_otg1zQhD=tApF=rsjvs{hrw34yOP-%63kGuG_~B$NFT4zwiv(%X(uFPmZS^#w_ zry@ryQcrIwZuW!phpR4|Op$En9MvOW3cbVQZqAHqKHB#aBAjhF4EDuIPDA@H!rEOO zBxk7bcaf-tn3r;)7oU)Pffd%L9}$yfOD9=Ze!gt8Eu_T%vK$Z+V-Sfhy-uzw$@nrQ9_1BvM2d|uoTKeBy4DKyso3yS7gP^mU`94RC(N}0HGFbDP%x*ZPStaSje?-r#j$S2E}~nZHs2GUS{wzD1zH^W{;B7F@>kz zv$nYXKxLtx?IMYVx9n59VUsqAbABj(mt%{X?oB_0f3BRlfv8`)D_d`-%&kNo@fQPt z`IL5;HRXFWe6>jU1;Rt`s)iPdX54d_`$|6)I-iI-EIo!l6ZwJ=*coW;q|e$Bh^wra zHte0z*FotGP$QBWHl&(Xmm(flSE1*?S<@q6ZFiVjYk_i0k8bo3mhcZ9p61XLL`Yzu zgHR2Up#c_ca5_LN*NCKQ;Y*0Lp}H}7KWX%jESHM zJ`lg=^n?_MfpM+^Z^#BXZ(&e$xUZojH`Rdz3P=~?SC$tO4WyQWdwLMpZAHwy$K~aP zh zj28Y(zFobce`0>c16-m?E0~hv#s>#W!=mKTgKwLw^3@287XQdN9b-OGGzY1KWl~4L z3n!jzd^B#pG9)n$iY@VY+>Is^obj_zGaU*HN`qFK4x=MZ_BT#8s!0|%_RIVXM%UoR zW$xYNB(O2G2J?&<;@5+6v~hUFu3Z+u5~KgYdDVa8!%M~vM^;IyEPpP>_e;lcy^Vv5uwks>iUU!CoeN|*F8_O#euHKS-tSy|v zWGB7=bIeX(^!}(>DpSR#*+E*sW*W9)O{qeJ6vsUKLJVE(u&)`6M(1SlBOFd+5us5d z3x32ttVKj7RTZ5G?gwizMq{dga^oii5OWO2z|W?z{P1s$Vo( zkYq`-;vD>wyrJa``IB_=LgCT}EeIgi%tJyNQwMLR{D>fdvLY&X?EJa%^*Yi*!8}}K zj-4v@bw#xncA3`6N^xkx?&Y?Hxiay7K8G3b_&6?o$9#p^cif0h*R|C$%YXs(&Po6# zdZWq(S@<)vEiWQsL(z)ND`A2_sC~j^SFBxjI4q`xhRAuSsgN(gOxYipGNGAEJBCz& z8TL1JUW)1PE_4LKADHZhN)m(P=IhK=x(|~d(@P&j8|a=0pLVHZvE{SbMQ(~sH=wdb zej-qn86#mXLo+jkikDWL#@ddtPk^_>1+qFCbCj%v4Iv(iay|iOcny3w7bFG-`YF=L ztv-^-Z*s|=0h1vNCogAbm;)=HnY5iRt{uQqgPP}C_~kV{v0F0(Ev`#$4zd_*y-BVL zryv29%E}(&ILw`UH*5)INS0)A=n~undH-wxW9QcNBzlu}D3Zn}of|qoYdzraSpl># z)u4bk!$Ns@jg~5$@M$NFkg+Fxq0)Nz71d?%RmempJ?Y|!jghi5^bfw+0g+DTxtybx zJN0B7d=yTMHD_Zxv6t}(8NG0hSx{qb)e|ObXOip>bMS|=#DImxlku;$lln>t2-}Uy zkh8d))A>2UOLgRGSvR8A*4Ggw>A6thVmi%um1BFQ$kFpIh{CZT z6S7*nPB(~0)=B?7Q%GkXAqjT;2a&60VrQ7KdF93LpI$b+B=>oh@J@XF)b@Idm%RDy zNcv-521r(o{&URaNeeYph&OX-(Fw>AJtP5y;g$u?=->V~4WUFpEmUic!;L5hgb@^L z95symQIhQ8%6Fv*RvLoi<`f{dvE#&Y4rIBV>G*0{2t!K>aTQXF5;lxns1p`gdn}d0 ze3pK!d!yM__-m~L`RTu1`^XpTl^1bgMXe7 zqw39`%fM;J5+!H?B)q+lgxTsPF9nk2ip0J|`^yS2d&splk$%EN%YE^(B5-a-{9!PO z>VANmKxiJTGM>ZBcvywbFHvo6&MXC-ia1=9)q8*taW)5qyO=NH#gzvQx_x!c$JEeo zc2P`i4+On`m_PAL&{cgND9*RNt@M2o$4U-RVqxdgyGGZ6KIM`8oNViO@<45f)*FXv zGMnKFx2c(fA~3KK5s)`{vw`6#ZLliBjWxleTOo@3Gp8vO!|~`0i^AF*&tmm>FF<)N z>4so6=fqHzPd`8gvK#?gXC1qzphWQ6KSHdQ^G(Ss)U(6Z?Z(w>*x+BDeDn#&f zo7q1=eWnY=uSa{5+^%LSW1^sr-3|Uz;6Rb3GhF(VmDpo)aL|UKH8=z(YmUvo%MA1S zaQ%N4vwzrrn>`;|{vi7jdr`TzcvER`m93Jy^yQE=z?xW;u4Pfu@_7VVCIHzF4k&Gf*ebHa>31YF`wA^y_ zrh!rGi>Hr%R8hkPAJ^B!Y+(LWFs`HAiA{+jtBTlB+Nt$lHP`v?Q}mRB3(%FVjpptT zLa;MGiA*IhI2#)wdu#I7g?@)n4g{~VcQ#(ZB~VtxiBv1MXbV4 z80i~4B%ry$p33Pby+U+M9`o*YC^^kdSBAxwy^v29+OqTc@ndRuGz=Q|aVBW=y>-6V zpeb5OiJh{cD1b6CfS|K|vAt1+j?o?Mnjc^Zmuv*6?Ndw|P%q|Uwy0e0)+T~UVfuTT zUwFrz3K>9s{(MK(i9syv?xlzPQ$iPHFSp-D2R;n9bS575U&TmyeE;lRt$tus2A;qpTdD1V%(MYGy}yXHml@@p4FgpL3$TLbRbNyRvoIx##*8Vht+KZQ5dp%!rQN zdSD9}5j1rZwZ`f`Qb`h?L=TuzY8|sjO3adSZYyP?cWa4dr)bSwe^5IK3IaOkty@(% z&D?>P1KH+_6;w&h{p1bp%uIos%a`YHq?SF-%L^3<4D}FM+oISO5^DruaN}xpQ~W38u+;BlVH$sOj?pN&f@FAC`96gFtNNh`N4=+k{trPZeVgt7d+c>9LmFQK5w-KK#RGV8fEca`U|YQu!54&u5MN^J6fetv_*2FDP5v(K+LKr&aYKepyasm#n8C`% zQ<3`NXDqs=@kA86TSXk?R?Qf@zl;M>cRsjA4|%9haXOY;SQ9FZ*fPKxxBBhF1VW3l zvc=AIvLJQ-`cO~#@RYA{2Y`)-BYQbcyhmDCSwF5kvWowNx)R2WFAH3OonWE@NKXaw z$X!2Nf%}=Y%}mH~kP9A3AWXQU0_~F3lY|tfuZS7X!RQ5F!A8>Tcs`FV^Sw}50F&tm zJ#tG{*SHI;yFjUzTZm_2=PzCgA%vA0ss09fW!F@1)PhdkBt-z`2i;}y}xMo?oB(_5aQ>??` zRfU}27?+5N{gpoug=(2dhUJeCweB1;&@%6=i1CMV#JYo(Yk2Xdxo4ez0cNrXUChIQ zS>u-bgc8Vujg<79V$w(8ovz*u$|r=FD&4aq-E8qDUztAAGhMu*_Tm5sq|co=t8d{N z-1;j!yhPF$!a@X$v7I^zQ0cq#^}%Rhp&d*@8VU=(pZpL)+8GlG5Io~$pWSfy z9%?W}jqXctn-_G0hi&NT0nKetJ0Lk6KO))i7J(0E1~UM9x>Y(sOD(-$r;mvWEN`b$ zlc5k%O`)QZf-8=KZ>9_Hq4A$6zkD$IcHh;OEpOqAD)&I2zQ;!C*nfJSy{AKs9u&~k zHa>+xZx1XA8qKkYJoQhH{@yW1LN9C%J=G;=&2D%xd5VUg)2jR=rP(SeQsGN zpKBhdXV23!cy<7089UyVDH1XM!>i|o0zSaU#fC~hxtZrPjJd_^G*u!)0Mw>H50obY2q&x(97IXdFIm|BxdYt+-DHn6877J9E)be zb`BkB_ILU6z-C%T$&^{LV`bu?TVWVXt#VmrWwe~me#LjtPZrp2FqEFm23grpS|ZA~ z7ssiaTUen`?*cx~P1c2_Mp`b(CfX>r1EN6>jN^+|QI_GWtf@*dMrK|-%hVQ|r3*O9 z+c3;KFl^&IT{9KAyuqLB&j_^#{0AR136oNp7jpS#>z1#jJcqXEg z@d6}5!68gAdJoFf+mT-2@R41LsAD&&G5Rxj%}xs8sa~h3HGV3<1Uov#B_z0Uui=1; zG*&ETlH)D1y~kMvCBhEt@Y{gRaD|hN3<_>In0tNJAaGD3U#4Mf!*=II6b6iuS_?v2 zPnCj9@6mxt2@573j(_Onn7~Syn1-ZnrU&22I!T)rTf*aUh4C`RrlOzbd@S;^4}>rt z%Sr%Gm5L67^$fRBCl;Anmd(PhjA$0gM7!Et$- zPaAFyIy9_V76~As0SV5@bE+MU{&=sqWos?yGW7}a!YG+j14jwl(>8<&cCFxn6FNU> z73Gbx0@v;GdPd9BkbOJRN&Iz{juE)KzGpQ@&1%u{1D9Ip#=1CpLyix3v)OVIo-E-gQcSYo>)+uil+T6AJ zbevf>eGvE7ZIyaO$U6ac*ng9 zUKywCC~OSA&+;=Q2Di!?D9W^F?*#wAViYeF9wL4O%{2v4)W>8v%M!LUg-uC2zsC@VcC)c0|o0WoRg(sZKxSR7)S8< zIngX$EhpDHe}^bHiIM0w_YJbf4Ki%_6X)ZmY!b!GI$23pu*_N6llbxqUeTRI>^Mbh znwPNzYDM`3d^)Fd=3Fn5#VZZLQf}k2t}Qc`cueeL3_asdf5gveZ6EPR$#Uimj|;+p zO>yk^Y#V}?9?OsW`TbG|E8c`Sv@ke$So_38G)0Rxa6dfd!xHgfk2P$soiD-<*7u&pQ3WYH31@&*g};djtFW-b5Q3kiq6Pav z^42VEfpX8&+O*tS_~hfQI`s*{DVrqaSFeM`M$5$#g39G8wvz-%%M?FLBw!$pzs#`| z=TEO6Yn)*pJJfwfsdSV(DTs$YjvoVfR`SC!%^i>osydj07*r)Y}8vHys zwzY3mdYI*IuSgsxyAz(Ex~THxsj}@>?XzL#EqLahpF}qgTv9%jWE56TwGDPrPF{+5 zKD6km6|B8dI%w5H>_X6N; z+8(DQ$k`atR=z4SHliiYOASkpy90~i#$&`-k8@)CXj0iExFV1fpUEh%W3uC48jgp< z?GQ9sO~3gg(~Y1SER@9~!lABzazUOyG+N1Apm4p2FKiEf3p=TR*bI5*>t>W>n117z zQT=`qOXm#zqAdgEJr0J3z7zk>2pEUFs7~awa_Kx;D>6@WN_BtK+s(h;pPSSSQFxpE8fF?qVT_*$oz-NnUL+Yv?nwWcz9a0Cc4)T@r%E_!iNvQPV(&z{~Vh>p56z}0g z^vWp4hY>q=8nSSqpokfmYxLrdLOaCOjE`I}FWF%`%!`Mnm)IM4F1f-?6$`?sUTRKm zDR_F;eZ&J2JV)X&Q1g7E;E2<1lDAOcVPa~59{Zy1Q27ra=~Os) z%TYQ@R63=og8mP42ul;EqbF`B$9IKVo-n#o{UKvVj(Sq-IXx+ijnY66KC2Bdp8-GQ zbau>RX*YRi^rHpy;*|HX!RRj<$6Lf~Wz6c^VGb&5IPz_t{^~IUO%TJINYIF@O|eBP zqdO0YxE=`B(p!C4Ch>RZfaXLGW^#>;f|Z%7Ig|hgLQ`M?vofUM-O>(TqCcdAM%CXX z>RD)2WLc@)s+Ev)&f-RGnnct-9xR(#tvuE8AnUCj`!>%hPQCx<`(<8Grk%Fcp@uiV z|GsbR7cpkBgE1(|O(7fs>o-`m47eir*%_c|e1w5XESQi}Bv306Eq~K}yz5md2~g3F z%8zsgiSY1%N(n7jYZj9Ct#2c_hTlR~SiO=$v4&*GzgT6&8P@O3Hsf(*LNNV~OlbfK zE!SlbraZWvaWcZhT#3@(_gJ4Ztf;^Y*4RqJZnns^5BPY^Lz9Ua%!$>X%4 z50=EFxC?_L$D3JA;&>1eReH;(p@vAe_vY}631X8`A$i8f>)FegA+jWlBw#ytN)uYm z{ql9pYcxH{6C&AA6(A$Kz(jFSnF+OnJkeQRvPeKP)r6rZEO#J)_38Dm&oh$3kc2`} z=nXz7qtb<$8*4EQkW1r}oh9hBe1SN37PJ(0yLx}{OgMCMKo}LkN^;kM{^SDwf@z8s zEB=WI8eAhbpTw633S}hrFiFMx;;dDspuX2KEwnKbx0rWcZBqQBNrp7;L>6IANon^ zGW#Ye@1Uoy=^=*lDR{$kP&J40&_gmqs8&dr@lTD)fH-dqHMZF&&wg+WA+Z!Fe?_fM z2IA!8=-Txk**^ANOpg`h>Do|eD#p^{EpnD#1v%y_f~QbbGePV&Yk4`4PW+sBrAo^U zYr+ZXG1iI=t?r>4BI$IC74pb3Yz9;rRa~OzMxgI zS3~Q9AMqQ{Ewov=i?I_jG+9eXo6i#9io;P-k`g9%&{xX2=P{jJ&=fXg33OkkE7zq$ z-uYQW9G8;`X<15I7c(kSo|S7@M#JSv2Lpx_0N$m}uI9yo7$qy*#Yv5!pwAlY&_>6H z)7ntfESH8UcW_eKxlQ%Ylp+v*WwhcxtGeU>0cC1=N93*ggXpR?)M!loJlwOD(ODij z_SNR>qiKCJ&K~?Hnh&R%2$;|spvfFADI+KciCyHYmaW(N?3>?AE9{op@oIX!U>d1A zuNk3}=Tv`uE1a4MxmBgHIke(plKCF>kG8i~o^P>tcp3s5mpKAOiC{UL$6GnLNmX2B zn=r-yY2bJ({>1}JwQw$ z2<-=3`#uhqv4Z!3VO`V^9A5B17sOl*J;k{cadC;CZ+qoH?;RGn^OS7sfHv!>8B24i zf3wH^c(J%W8MTut*-j!)OR8Cy1SRhGw|Ztw53ckYb)Agp%E!>1XhrTkE&$S(ZBg%cR3HY1XWvvn-q?Z~E!kXZFvu3>r(PgE88yFyJwIUf2e0t))A4 zSgW3iP6yO%Z?Zd*VybMpo)#w(Tq#A$+&sx_Xb)Fe3M=%N2`gANk9R68MyI9M86+62f`@{( z4n?zL_A~}q{4_Ke+o<3CXIu<{?Zj-x>3W$V@QEyd7ieS^3s?{I#SF-0m}NmAVkhGc z*3~*&?RFs`%!Z1YnZ`5zO@C%lCPawW#j(nv;>qfsA?4$AnbVIMj}t4q6h3>Xo-}>v z%WbGF0R?)QOuvOwMz_YM+J8FK4x(bTt3amHIdRdLZpRDNGl9ejc)5&6FJ7t7y@Vyf z9KM4VjD<)DLoXsu2sP_JE*!Qc3Pgj)t~JPxe2LO3V{=z_{b*cjg;-^*3h8M0S*YI5 zx=g~-=F@3gN=QI_Fl6TyW*W`Nf2Ulls!oQc?F`BBA^YhGOjm|*@^6r8+CBUt`XWY) z97W}QoX2+EF8*M=>v(mKqSK@t|SIA>j=fyS$sR|qD2#p#qq;8hqKp`_ zu9AK#Tu}rl@&j!rg4Y9H{(YWn{?Wb5ct$0-=RS?BZSiTHlEkO$y0=GTOdCP#ob9L0 z42Q~^0vHjiHIyB`r>}rRIK(mR20WQ}ky$Abxn~SKj7$g5ll0bumczjRuMO6v+ca0qsChL#0#?(CUvrI9mOyH6hG?f7 zG+7yzGi#B(wpJqjn;2d(L&VP3Z4(q$Mp+|t9f+66)X<%yp(7<0pb*5tTZ|&3sj+7J4<^Z zll2I&No5Gt+&W6#czIlhg&6|;vvcbhQa`|7Wy&T@;KSDWkJzcxJ-cbNo)MpiYOl+z^2?k%{+7p%iU4W{2#NBs#Ok z0*(r!^FPwdJjjEtsLOe9P1qPRDA%K|o&^`)s=djcu0{EC7^h)DC(mMU72KS>TakJa z{q#^9tnZQ{D&-%Xu{Pk&VBI7s8nW^o;vJcy4511gh8;H&3%MhQXf&s_WQUqNb8AM0 z6QJt38?*t|Elc#mVuT1+R$QOT-M*TK00#Lr4{0_N`9?1@cH1M{IPS8#0F12*3-9S- zQUJRnUWoou0Y_Df`DtyOtmCCbEL-B-0L9l94qE0+Zjz>5$R~{^Y>N^zGUV||j}n)6 zcj#awjGUi@FoPnkBS(WoW5%$57MB>JojWiheSiHHmqH}b0uxt6yS{=s)!sGbGhL+`@*GN2kB&+(EK&AW}!av%&+A`^oIC4F&^8jk(u z&7!^(Ou6@Bh>28ns2615tD9K-J?HCRr39WH0ZZyrUSeT@_8Tsi|LXL6Jw)0b4t-Mo zhyw{FaCJN=WMcnw$z4yXq=d(=vjse$zH+vOd&bL#BGzH)M6#WSjj6F1LG!bTn2&;G zSuA4$w3+xgeA)_&ZbF)I_*1G*k;8|L%sxkfLuk!;FgGxqD(+-!>2F)BY6Crw+R_o$ zppStx=1VO(p)^07-eh*&ivEL?81rfsfeOd> ziOmc71O?rpjh0`BW^goz?CXP_qFJXVI3W_ss5eC(Fr-+}tS22(({0`?xF$<&d7lz^ zMpoA8PL&s63bwK@#>JXG!-a(-of@f9_L8a?7bpb+!+y64J`77QDlg(35;!r1Bbj51 zE?5M!w8(#}dunqT&AenW0M{=Urw5^2q!W?nu1y* z=<7C2PaaVBsD~Q@xBAR}<#OQ7qWpp1I(j0}DbVn>S>NVewp$N9(b9$|<(Y+J?d(X; zV;xsX_Ao$-WN`p7&uMO;Gn9)@Q{d!jT$z0?s~hvO3P8T43*c}vWM4T?e9oE3qdjWI zbc(Cu5l~F*t1P)ct&cI_J9Tv^7nv0pFgqyDN`Y)f+KgNlc`Dz-Nvhci@T0hjjbX?h zwQwB4c&XVVi%U3QA_Np>5>%aU!#7h%0HdO>;w^Yl&T3G_mzpo3iQQ&5mS0-^NP7D` z72Rm{wLCN?Xss->uM8Y&NtFC zofn}Cg@bT?Wtmt!~1Oou1b6Gh;t z+C0FP3FJ%$&K$#ni#8P%8yK{@@MK0=bX1S$*?2N$6J_V$78CoJ$|O1LmL@csd-QY^ zRZHlh@{#p0HNUKtFdDV5xIfKbS?8_toBZMpp(5QaAcqU{;(M6_c~|C_IJWD=Rb(UQ zmUB9V9?|?PLV5WLah7&gQm`O#HBwfnV&Sr%+3E;(UXu2ozYGov| zXg$%)MySf6D5#_wMUv;{Y+Rf~Pzk%G#Ldd}mb>>5^gweQc?I*m%Bn5?gNX!5fJ zd#ASNi?dQ)cOPsx{ML6b-;++#MvKUjV7 zor!nmwT;Wsgn!$5$sm z+;jC{)PIiD<&6k#$BX}X;+Km@&rM&_aO(VfS0!J(CfS)N&Hd%ebx&Wp`(fZ-@z(Zd z(icrBd%O7itMl(Ke&y8PC-x?e?cM$C$q}DeFh{i>#dDtT*;xFGSx@Kx%i-c>R}_5X z)#P#B?`uBv!yC7Cyi}b3?c~WT7WK@ZHT{34?mGWhfArI)fqz(Z$}IV5J~Zdnmi`|; za{HZq_ixZwWOM${<$rg_t%K3QYtg}P&RKoqn+a5M_xu*BcVD@GPs5V8ii^fTv`_}K zbYbmaRPbNVeP1Je=CuRY6sL|&?D_lOPL2A+J1wti-qj5!68~!}ZHM1O`&*Y(c9->k zY17+fECBUId4J;H+7mvpo}guJ7TTl|eNK?!NtK-_7SrAFgTcc==HJaOs}@ z>u+RWt4Gu)HsAF?`k&tEI4~H!G3uvhW}&6;Jg?JE|N7Bc$z3O(1^6#r_!#XgdDk}e zmG!@reDC7gu1o)Gk~^eYddfSt7r%b?wdCo(SzkPsc<<|bh7UBHxMu6+kI?D3;;H1X z>V9(B>i#ADm%X}i$t!5=Ju6q=wzhBL#}~}L_V#ODOkKZLb;VPDsBacL(A(0# z>db@bhu(hj3mxh0C0oCAs{ZKje7^I^5!;Iof|chCMvw0MZ@=7Aw_(N`Z@ppF6-tp{ zd_L+WOe;i{H4as%Y4Q28pO~M=2^t)_Zl_pn6*6$FLKfM2(|`IA0%^xZ&Wm$|iIvJ4 z#@bz@`bcB%V(gIp37k{pDx(2KA8eEbE`gG@!MWceT;pRgB%PrTFL?CV&h*3)wBg2p zdn`LIco-jOAe$!ZBT}5eP<0a+T!o_21g_0Vz=B#-G+L0rWA91rcV@%+s6rKV>nrrs z7z;+d!k&L;u!s(|C^bq1Y%`{l0Gj8dU>J{WzOcs83;$zu7-)K}1ISc?R1IexxCeu+ zlETKkW=(0x^=gmuN+_x9?jgWo{MO#iYI*ijB#wD4c?Q)TI{7` z?i+Zn>uTf0^z;wV7gVRa01#gj3rM-ZlGQzDU_L#GXUVl^_4H&zYU9zzT}Vf$&I=oi zn>HyDDWhL3Rhp`{6Sw2jbHH{Ap?Nsia8(Y2K!AE0JaCU16gk?E<2G2*!D$q{&5b<=02T2VJW2vt$-My5)NV6*nB0i@~M%*q6WvJ7h(MY@RU&@=ZY54Kp(}hpL{` z_3i1pstqYE0eaaTt;EKNS`+W-NzoI+NRo5)eyJuN*y2ss0*kgt5@YE?fA!Oh-sBhd zcV>l==w)w|J-=%r595A(ujnAehEUaGU!henQcQZe)(Y!#wgn2#b3hf5h@79pY2Qa% zWHfZ*e+Iyqxe$6m4^B~%UtRa?3wR1 z_0W+Ezw`h2@2pMxFrLcbvC3s%8QXN*TB>_*o%gXvFyysOuXpw|fBUt@C!cIsdAgvb z_JMQ(8*l_spZ8{R&$@LB25!4->bJ%;{kN9a168(fVozh@^r)@}-g{?Hdi=rSblzK2 ziW`<3D(-CVoO0x^2BT-jj@$Im#HPRb*Z03Q<;Ko}b6;+Jepk(&{r7+6SIJKfM*n*I zj_G$iGw}SBe@%}W_*vrVac$N24Xj%*7~Q#|qxYGX?_d1>ErZd7re_>dEl{eMZ^XzD-SOGOijm`h)xt)6vs?l12A-Q+Lt zC5`dI&hx9&BiGk}##%RQs5*FM`hVR1qgmZY4O`5u`}rUK>)B=7XZNM&%q;EuYRka& zZ%t12-&}TO(_dEKF|%Od+cz#+*wZz?Z{V{VruQ@U?;+8)v0I^_9=;Nqy$fsmg6Ave(=WEuI;OSWnS{d ze;tgT*bQuXU`6-9`dR^?e;@IUk*hyd)<{Floy~ho()&jB*46&(igBgYwdq+K?>Iht z-_e(!<=&UiEPeIKE|#PFc@D+-yOT`|dybbCRsZDe%~yOW|MZL92Uh>>!8^*HytMe; z9~CsETKdasi;}xm{^E{1fsdnaIMDMw!$emv`+8quV9|kJ4o2@iaISv$Ye&+bO4j`7 zwKKE#{o7t+F0~)&tqru66=aSCtkasrpi;)>5}|kW;420)kQt2!bJ9 zt)e2$r*q1x#dxf(oQ#|VgsdW@m1;PQvk-wHhY*e-goHePulxN3Yxn#7e&5&c4-|zY z&+~rXhwHlU>%Q*#L!B@5r3F?wH11W7Ey@x58ov#_fP-Yxv~ItqOK@y{kFLoSeKa$@ zq5M_6t9r6EQf_|Nw6N$Czd!Y-i4C6)???(N&o-}tIq}7Nes^)VXEY%J`m7`+Y$vK^ zZJBquy0C`M>t4SoaU6{w)l~X@0XVZ$bz}5F@jfoYQXJy_(B_#G^NMp3SF|L?4+Y z+FjPPG+7No8CN}^c}duU(USxpyj(R$kQd~y^%eC^;Dwo|0WX%}yWMXpmu6lSL_?hZ5gD4Qp$<Mcmrld^4$ls>lzENO~(@m(McdSc2()pXa-B@;pN(}t;er%G@9GtYQlD1vi_t3^5 zG9U)swzKk5Ppl+-Tzaj$S)aDJ%j;;CPFizg&o_a7+d79^*Q(}wHYnTTsx#Ji+z{6L z+wVnyG$7&cW}Q#{MBd$7|M7SBUOQ;}B-bHag46RqbGp+uws9FtwT&K&fw*5ZeiIhS zoen)8DVlPL5ubTzgH7z;5yDrSmM7&F^_BFtWL}v2g}~)V=fYo`_H>E>@V^}{u%vWJ z4@wm|PIwb@K=KdeuGojx7{B?d&2pLhq+v?jIV=;j*(f087~wLc|H$qH!GgqQRbs4a;hTjD#Fipkt+=`$UXXKb+|d%u-_ zHvdGC#XTUz?`py;h7pUxLl9|L6uGu=)J|#fY+Ca}?fCUGFH~*wE)-pJ^&DVPni1*G zrQdE^zDlDP9MQ~L;bPy-CF`7;aj|Rm|IGL~cc4>xxHY;})LSr^?;bSMQRSU+B{l`p;&L0K-q&H>UfaNYAOy6W!MvLu44U1C##A9zJ z9mZngRI(h4HBM1JRD)PY0HZ+I6TSt6By!QLf!BAiY9&Gpd<>98iqJ`j!;w-UzgjA5 z0Ls9v^Pq3m+fYJKohBKjvJm7GtmF&fJko99y75JP37-IXPPc&1V{$>f5$j`%YA|pr zz9|r8lMp`*g>JwU&PI@h!4T^LCZ>UFFg&(l5-w^3i+dtFV$z86eO-O57@EzN5NU5!<8eG*t6l~%U#oL9 z`PTCR^e378Kwx~1fD!>I9=Q`0Vg3vFlT62cMomg7Yf6D(`n5+%`93}@Of#1c6Jv57 zILjh7OlV5}q_4YEYLRyhg*gN%-C*OuUljvSK*OY=pePg{)CKr_M8sqz!WKhEj3Lpv z4-sPzWseEZnU5Sk*^Eq)w9llc+$!s7fi|5K@uZXhF(1|dgF$TDDefBMytF`G6zDbi zkeUpm6ICe=Ic1eq2?{pm*vM08bgE*`te7Y?zbT51(I?za1H~r|5kRCizgkR^dozt| z_*^I}(7@$Vbu6n-;C;Z^TN0Ef^#Qppg&|kE9PmGp)KAj09l??V%dr5N43s{W#U3#c zTM3ifQreV~Yy{%rV?cQaGF8HZ#RMeStw1gdjho~!tfe!{-cL~OI ztmOay6(~OnQ2w~n@bC2453Fzf2wLM~5VoYMB`h!O97rC`q%iS4zs+Dd{(mr!p9C6d zYvB7q&Mfx-{`;pvexBIOygUDD>n`!%FNB5Fk6IKO7Ju`ii5~y_+F>Og;d%-FIav_C zvoTR{NSEB+m89L}uHnuMJ5`teifa@6eE&1?2G?gT7jA)R>iaPr8`KjQ6W@v|dif~# zv;LK57MQ~lJ=SLJe|uz6&i;t5(~*f&;wrEL*DBYSq-TvwtJ>BTfzfRoOHw{B4hfd- zPB>lZ=jLE{=kO2SSE{c&IRFDE2(OPS!7y?w(7NlO@88xfe<@g68IdFX zPfyr@sp?=DcVLGkCMHPP<}B?oC?I3!;Q(=z$LDlQC&1}VD}u&HM9u2F}VJ(65qhjFGj&fz_hnW^3dk^iICwccZB8k z`3d5?P?=%LGM8;PqV~kUDZ_F(W?-aczSow_gbIrjC(0BKt4;}~mC}nreu8~?e^G%v zJ|PN`h^p4~MsF|k+*huGQH*mzQ8%0nJ8Sau$`P!2uPjGeq1kPAZOaYkn#-~H1p1xA zYj>rC`Ed@(9`q|RSN+cJ>9t*(WBg0Zl74JD4F$5ofP2zgV~9^WrI>e2PD>XI=Wm%f z8N2P;+OznOjYcFE?7=|cEP+|#f5_7W!n}? zt~Kdee>>T_OPhjEQ%{-tw@&VjN2Ibx;??-F+HzR-e*S3h^ecOn`cW@n#6oGctR?gN zp445;G-g7(A4hc=b=d*d}bvvwzne7sg$ z`0YL_AVlmKm5C%1YJDR1t*aRLy18`px1$)<&FAaJu?OB%a4izqRm*j3**G5@-sY}E zNpE`T5a2&48tvSHxgTX-l0lPPeL0t3!=@oQ%P0pP;o7DOM z1#QTAgMH__ObUhXa{1K`scP8dSr%$RN!LX@^8)WiZ_z-<`a5t0l&U8D%)!IcT%$zO zrtF!tyoR);>T+kW$Un@jAr=01I5iwsY|m_!l#fWy`5q*j`TpYG?HiGs0ty&VM1%zh zFwVUzGPu*&od4{&4~)uDJZ4$fp57+Xc4W4OYz^^SeX(hE_2GgaB#XV8JlsH)Ur;en z>zk5N{_6fek2IDUhK(C@c4cc>mH;>WsZix#u)IDj?@DO-O8h9 zo-8!SR}3EuNXH}XuNy0S@}_j;ec7=PTZ9@$FQpacMPu}(eGQ|c6lLMGTO7!gAyqo1 z&rTH`6KprHOB*)1StLz={Nao4?Fo^yYAi1q!OWhTWn9&5s~5}*meLUj4#X6^K1tmYek1a~Iz_^VUd1Q`?PYInoRF@cb?%&Uw=- z_3-^1mG)qV?6lZ83bKVhUw*x!3Qb#Rcgn{IOGMrHQw*;0y%WO%ThaZm<$a(R8TXSS z!L0IgPW-de_ROu3jyLBh^p$qi)1Hrb>40 zVQY(1Y`Bb zHPZJwuBnf3Fcb;^ALvRI3yF=jWXGzS=QX&5aw$fKFFrs|;F3zRt-c!nb$G%c(;ByD z{$T5RP=(0fIjv(-RWY>g+{gd((d8VqVS;jpe*{32O4=%%7vjE&2FMZ5YU7YIHm*AD*FD3w5pk0lekBh5K8) z6>ZVG19(G7!s7CfQ+ekG zR$a$OJCUYgdHyStd!_nrQaOPJA_!~EPm}E`GS8W?4+ACYt|R!2TsLDuASVPyS&D!u zaB{JKAZK!vpb$f)C;*#9TC_?cb|?lyCj$85;@24Gp_{?w^yg#BlCT0oWXcWEyJGUM zg<~ibnEfawS=bQ_RSbNYFeGFf+Lw2p#L;{wM>0@Na!0zEP^z;};-l&#yt~yOT!AN9 z06%VbI=6&ttn(S-1RN@Ye6)dx!|>L8O>KR|i?Xu9$#92_G1aj8SvAc7)O=ZuwU03t z6$Q1&h|$rMYQy@DrD!}=w@6r^-3A#5C>#Kn$>?^bKNlWU30};rdFUSkF1VkWZh!m& zOSCc>E(jXN10#bTQcuuz*iln>JTqW<;&iOL2NWLI==zR)rffsu*1@5V_#Y}D6qj$s)GJd-3S#lCDB+PyxwK@O{DI+`} z!l5aBe{GlgrTuRkqazK&^Aj>R*DJL{W2w#k{=Nn$wqDgb#ERYUjN3H&NU13)*A@W# zYV^|biV@eFjZJSzYXW_9-A2PIy$^T1;o#7*c+}^P8j&Zz48Zd9)d|rKslvl}$=f4q zJB!V>Z8>p^)^d{Nr`<;%+8h=aExr|5Exu}WT~k;KL_YSx4Uz9?PnvvMiPS(s1-u7) z45!8wZ=XzQ@44~9!GWm#2$*%t9Cylhj|z)pz6Qu_Zw~|1ksO2hhMy?D%Nq`{T!u{F zHZWSAFX?dj&9|NI^92B>U<~JAg^Ml7RoU-mCo^ai4{b~XsItew&T(2s8}kz zNAS|(V3`dxTodgg?KWi&XvS==U(hbzc(YkGYJ1E%O6xd$D;@JR{q#=7ML>f~LDE~F z_2TRvpWMnh=NV(}Tat8*eyt}j?1{rfPM41ALRKz*KFL4G<+iC%A;GGeGN2mj6)1AdG(L!gXQ6pr!#>ScVdk8Z|1=f!On?`N8zmizR7}@QhxC(B ztcKJNigsev&F>MW&OLNhl7sN&_Ih2@smsk#S5(b|(My%}S&^X|Y9HEkF2KI_KY|vn z{+)jYw^$I*&yd|CxKbd=P$)Q*G7uqSga*!UwoJWJDn8dZuf~G$tP-^J)*b} zDy07dO%yYYNw-e*9+()`sg3Jm+jnXPvBze@I$?p@IA-3=RwJlN$o&Z%&_9 zzAx}y{?wUW()vizV0;ti(6#^pp8GnjsxEJ6kfdUcewvnlyHpJP%Q-7bZQ3?sBv9 z(5~U!vC_th$mLu@5h=x@!9NZdiMZ<+D0WKm=HF4q(CdPHZ!*%2AF^1m44ztgdp^ zifCg;Xd`6#&(4LBTzcJQY7)1wCsjnx>IC5M<}CB3YS$@Ad3Ne1bzy*|{1vKdS8kHoHDhGQ;$?ET1WrqAl-JO1h2fUh%jjJWE45FtTUp9fH|85 zXjtHVNvdki=@5c9nHZB)70SOm~3}wA#TMN`J(wc#?4jH(HUX!uxzZjxKH!^*_iLwnz+081hxgLc@U40|>u3o!h>4;QP~ps8OGcrndaR z;pv~}zBA&b``hTBI)fXOhNZ>d1hzFDw_IZ>x=5Nrvetf6ZSVGvez1SlytXra+U_IM zxaG;JlPg=(l2U-WkKebIxg=L488Ym<9w=RhCQ=tvqa0{{&B){tlm_dyPObVkT0S3Z zNq=VBGSN`);QHeaZGKU7YKpm7Y#IY6`vI zJN@D{Q~f5c!gwGwrprk-WyTfl)PLCWcj24MI&XaNqUit~<9<2fPLVERB)Rqc)T6K1Fd*V|aGQ=6gW0N6ABjl@&RCxSOZgq+H-C;l@Qe?1Wlh>_$%xaKv_K-{3 ziwk>T?S&KE+SJ9EVBgN}(cAd4X%85TPP7~@efv==Db zVnC{i3|_3qFkyHVqSwSx0JjKT%z!8+3Id3D9JdpqB`nq;)!P~|lyj`~f+A@%DpVQ( zxIW|fr!p|4y;xLgQ4}sUQH?geLQ3X-~oiQ%%(;f zKbDYCCIn$r(pC^v_J>AiofD#E2Os5}EZ0MUsv zARVk7{aVaCoZ`3Ybv0C8)0XB(doX3x)zAD9#drXwJWgVJ#m zfTAs2aXyP0`WoG82%z-=n^H|8Tl9FeR@0+*o=)n};2EJh!Y5>$3-05EW>hrt5%4=) z%x5wug?K({S!+0mG0lToTs+f~;Wlf8-!m#o1dW3I42Tyqe*>byw$uCX5qE@-4FOR~ zet=EjciHP{B*!^9Q5iVN*`ezjuZ_n=xfT#Tvo%f?@wze8`yz2bvx+iZ@O+=Z`*~mn z)2)G#!c7-m-Ykkv)Ce?{YW$z#%~okOuqi#~8N4%7Cs82jF1(=uRaqVc!%87;tSMH9 z>7)IU&2rMql5{15S{qVLDP}kc&)6(twG1<}{?W|0`uFxc`hiujMx=G7JdId7=)|Lc z(`;bOodwG2Q)oIs7tfYG|6C}$$CnbYS%%?aJQ8&T7#gUe$X60qitAuPB({}+E?VWt z#Gi44{Qu4X>Yx1d5ySo|cK(rhv;T(+0Wu#}TiM5GIUmHIag)ND!EetjTD#CeX;eve z>gM?>FTk+IB12^2)5oLN3hTYfZ**ntY*qb@ThQk=zc($enqub3+V8n#T$5qt_U@ZX zocH;0wnhU7K$IRCFp^ZdYWVr!ph)=(`+Z`v1_b&^OoWEs?R!=VGo3qs2yr#`e!YJU zjJVa$%Z=-lVePhDS8G8=LYd~!a6xKjT1;2((VV#KwcMG3s@!f{g%roQs-Wyc?$@Ke z-ui)21i(!p8!9%xmVML2pdOAg%_ z&VEH7RP?I<`@&bt8=tD$sWM5Y5n#Rm#mnD@$}6}uYv8lOFz}#wFst~rIQ#UI!hg^z z$dbIKMhMvwV0TyGv;JUsaJWIVlhXY$$gA7Cd*+&=TQX&%ww0R}tjclyCU}v5P(hdM z52d5M2eXE^Ok&q*uK_d6^7&JlJnnQBvo=d z{2y1_=HzxuMs$cY6G#Rk55v(K^A6c`d2^#Mn@fo_XxrwG?^)pZS>Qp_H}d!-2&;=i z8a>+)2L^5gcxp{869hFby^j6K3*v+{P57j^&sPNfQf|ou;itI(ao+g>7?%18zMFXP z-wZ)gwR+lwFq4O87bo(M`)-urL7&3g zdn>o)d@}s)eW)Y~-9!`pKESRj?vwC2|0IkaQ8^JdfMCAgwA8nf%J1xCnbM|-S?>u# zMPv)p6Hb{-KrB(gtdK*(>@ct9a@@f{_MB4pc`^yfWU6R|g{qHq_G=cstvadRjt`&xfE>62q(0>y)5M?0{OZR=tsM)JI z^Ew9}^)_TJx;u_q%j^TEwXb)Bc-Sfc_vG??2%JdAoEH&>T!)}zan^VjL`qXnoIW-8 z2XmQ#^Q^Q#xqVDK_=@GVt}MwhOQvO|2}I1iP+DD>@y<)8GI=xRkq>={<3pQW*|v)2 z4qH7k5eu5hvh4P5uo`#Xp!GTFPsEU6vtQqA|`w zrB!~Oc}GK30Cx4}NojvUNS&G!UjfL+dpW+Mb6dW!Gv2TexZtw%0{qfa)kaHsF+w*} z__4ST5OaJmc3~&}0lZeklrDw-=6?f@mK!e7XI~cRrKa(-_AjYQZ`2G9Df>HMJvNnW zj*9{V%Y%?Ezn1PsP^9^wXo7WD#%pXF;Q_kEuu70vgX`=M3*BW%ocEEEFRMf=7yLdV zg8}xZ@KoOpsRvfRVfKSb*9bn)>r2goQuD|O7MJybkwqI*B;E=IC;cA9vTR*eP}nt zJ=4+BE-AIl2U@==u*d=hQRNV^3j0hoFX3UD1rKd<@VmB*j^ z!0RP`9;1Fg{Kok*yCXJr(ZZ(q070LR%uua$R~V{+v8VPGnq6YEHr&OmFPcswp`G)P zvOf7v03KN69)lJSV@jW8_P*M&&~oVGG-PN2)umbu}g6*_Cdr zsf&JE5&TxvpM*)j|Nf1H3lpK8>AEc=@m-fqwr$G10^2El?mMNmS;=>xnA0DfmW{}d zc=%jk@@nOvT;~j9a|oO z=NS;q!v13i&dx}zDL>DGSY%wxm-9Wc#$s>F-tMUxE8%ri5Qj@*fbVr$=L~TBwPC@@wMkvnXIl_qw*J!__90h(>HS{C-{;YQZj)Oz%H*uU5Hs~RzF;Qg%ZHlM#$#EA8iMs#X{~h zUkJ>i7a6l;@#)LQ`zxz}kQwr~mKI8y-m)ilwS_{|DsuXyHA`-BW7B9lL)?n+JqRCt zkyr+)WlSv`a{u7db&nN+vbl&Xiyh12Flde)gDKI#k!onB5|d{nfhRpI#K*4(s{@An z6BMzhwBV0L=A@ygN6m8rnj())d-|SD{h5KF!Juv`0RL7lN|^;~RGfv#p{c`C;;HF> zFX;cvzd@V$he$Jqeepz!KAeap3S0mC7w8u;h)k=58^NSlQh7<bZLZZg%$)ygSQFFF;gW86o+qOp&ntLf=lI2OE_5&m)+%U7b5-6UZt~V;rYqL zZ*RB>#zrx`8#ITicl#lrAG=DepCi&O>79rap%Wg|@hbWBbqjh#e+J#k`m4qIx1yHZ zys)-&_z(-FbN9-!Q%kE6w8zO^1W?^ORE%6gvc*X_GzNjlwwyO@GlK?QJOq+`SEQ=R zR*|+YZ+a^-fFkr63~lc-xO?WvB|9YC=NK6IUlDD^=U$eS>Q+wN75J`382vUMPdyi5 z7@qn_K|2f}tZevd&F-UylTvIG@8wifbS%*BcM^2!deY96?McD`&N4Q#O(jDv%cTm- zdDCt4Ahhk+jW+f^yS`GU7hD^7!eDB%EKv**ihse|_?5HM+6}l30?PB7^YLA}?NmZe z%y*{hWpuAiD?W>s)@F))^8jJXD|T?7W9;8Mq3F8v8zUmhlR>zjR!NFZ{FjF|>(6SQ z-D4_hQ(hZ#RrU@*3WLTWPFF|EdYgxo-71S?v1PI4_u4_xwwwbk6@&ICrK3*fxw_B+ zgy@GYX`yU1PSgP65U;ezEl-#HY&V6%Wfu2IPPnA}@6y5*!+MvG9&~iHW=TVvwGO@y zO41g4)eP#D#!*20oDXkB#m@IWq!&FOF6s-=sSPO>Pk@1jaLM1Rx&4`<3eyTKH`x-! z=>^LS0*g~lOz#0-+A1-6SCKvn3x)O6lIBt{gB<6AJ8nR%7TMZLm7$~95P9^;xy4vr zl_@QIg09!B0n|%KXRfL2+M`eS^57nnii6N!=yKcZ zZf~2t=W3)E#asQ`Ah>|njWN^yWPH;}<%BL^zPI7QRcXDnZmHARXOl|Rk*9i|7 zH7`X8=4wBI)T*f^J5#`UP?X)K)Eqx;yw{Mn*zzVIM=y4DzXnC8XXx6-rP+f8nF77c z6nnP!&4IE@J(HKB60Y67eOHS({pfH|nZ~rJDZr*VztK4gCF;pil`f2$GdIZi;_`R6 ztW-osg03jp5n59h`Q-y!qXi=|AkvOoev|LAJZZsgkpiVa%9^jA6qy(3zO)?5ch*bw zfl@sf8rCiDqq{5B`)W&jXh6AnMT{kUT(Rn;?IFdLKueKjKKQ``#f?nIUyNv`U$aM! zV>?y^##&yYh&O0gWu`6|&Xb<6ow!hSYx}zoZ3dItM8>fdato3W6oKz!D2R}oEPLTY zo2_qK0@tIacAwPfJbvP#w)xKp7X;=X`56_7-8@ z5C_qisNo#l0tidf#k^OJA*+E^?*nyq6ejWJoaCXqW=@KS;F_haso1`I))qP|vm)Hi z*flQI={?eVvNM8HX!gIXfc1l@{YwavdDEZuc&2pRA?A z@O6{NEJM;Fhe|&YjtluFrT)}ZNWBhaH@nSq!y`o78Yy+v+Euo27-am&Xa@MWM#@4vX#C4Y*q;v&}i^5T>jSXMlFFtPamU;Nx=hB({7D)qsA*{IJL}?%WO(9dK-RdS}x7A6dKyL6y?5;q|mC7 zcvZTu>E$Mud>vwENPA}1`X9e_tLL9T{H_LQ$?lv5$rT|M3t|*HQ`2Vks$E1R5F()H zHjf8L$w0}F+edi-iQ-e15rDqI=`vdAB9i=H?W76R@f-`zQ55k6WkJ>g-~MkNk|csA zEcInBQ{W>rvU;Y~G>gSQvSeYk3|32OVmFk2c?!B7Mlu+@ex!Gz5FOYWBm@9fZP@%{ zw=iiJ?t*>RS*EukTL|mB^W_@|e!hJZbnH=uGewel=LG;uzTpvXFaoNNG$(IR;>s#Q zx6oZ9<-ouwc|im8k!>$#-$vwsAn~?l7Od}%>=CB2tmLuJ0;heNnkw;a$DR4;_Y>%w zW#-#wv$taD2`1L31PAL;MA2@pg*#C%&nC$=Oxej!nrIv^x+ktojD=@L3UY$Q|MbGj zmpvB%PnPBQS;+99UZ!Z|KLUvXu!U?9>+cFUG>REW@5Iwlz2tnFL0A+1XMcf4&6e4JCo?YG)NOif^z8*^ zTg)g1%>-*D5cy!iewZ;Ej4|)O!JZ;rK5s`ubdrb%wKT`YF)L2=@_gzNvM=-_29KrK}fOilNis@_#D4X-m%k>i!enIK*(mpJ^Has=WI%QGP(|z z6&6_lznG+GROZ5#c0@g#mGzmpEB%*tr(Qa^0s+b0G z!bPI47f(b~R}%BM6(o6%Ss4evr14swZL%c9V|j-kD+^|FD=dUEix2iAb=HkI{dr zi%rtr?&V(F{}auTbFypEc9l*%KDx(ud4@nI?|X$4I^DgseW}E8fy+(1>9t`Ns;_MWIf=fClIuS4 z=xA`t;b@UUUovJ9`9;YV>Zsc2E3gL8~FTAPsX1bZrN8ev@3JLna>2zI{td}khCUB*+Xcc&^1@oqvATWYX=hrN8ZA99CdM~ z*tSp;o`c$BJH33CZqwj+5-P;ltmIG*5Zr<#lpU0uV<9}RRckisq4g*(`luWKQ zyEPs_vqO`h)p>JTfyz=2ITxOfW*b8O8f>Qhuxv4}D7)h~GE6vsr+n0%;lRr!2zWby zN|-}q5ZTS+4q}6wxR;Pr;VT-qG-f5Fe9PI@>V)k{p@!2(dj$}*6K29Lc5F*LSk`DJ z98)wqi_E)Z(-L2}+{90wx>TM5%(eU%x%bmR76jXfVpNbTU=`>Go_cFrL+YkgTG7;= z;=)|f^J>v^N8X%>T(zKDtaqA2+AvG`T~fLrIdoZ6&aZ~m+Ijp$*wD5S?xxPv9?lPPfM{62#sWM`L6TnlVB0hqm0Zt1dPvvHY zGd~%_&UrCw>Miq{pl@rC-&Gir2Gm@0lXFgvb8-IlI2SC4_YI+Uv+9E}N^y-wx_e2PZzgtj**@|yA%aL8uq&qawH0dm*ZrJ!g}9uSe8-ud3^uvtnsS6qsYr*QqT_3Mn(t zE^pR;IsU|u+Tt*{LLOmQcn>})r}F1M*d?vWbQ~TOjhRDNe{v6X+`42@cWWLtBc$kl zVL4T}bl53<^r@c1C1>@GRR=GFC!z{?AEu_DZ+XbpX6awrG?YQC&LxKKRhV<&7@mP^ z0j0zXuszXcC3D0@a-O2uy>Bi1ZIU~F4k?m-tl!&RT^mhGL(g!_aMlgL#N8*h-@R)V z>|VKCKBnbPZ;IGwKtk>8ediD;vN1NRC|uj#QuCBdCmV;ksCvq>LcXrOx6SGN{qTTD z!T5iqbu6yzuSaDxh|8O6#uSM8=mU^M(z@ryXZ6L)M@>t2W!6?~f2Vh7aG>gVlH>zK z`0@0$o`15;HnsNBfdR^iD0fI*{1Yn1cah=43ZD;9pS4w92%R4n^1F`%5-J9zVdZ6_ zu$&7O9CBMZ&JbOSFv(Y6F{GzA#`}~ncz)YN!EzUzXn&U0ol&``YW4xA5a59!}kJU3DEVHMtZlNE9ikR8YB!bDWL> z-MbQ~2`^H9kb6<+$la4inLTC426yE^1k^iJ_Uy{Pq7)}(Swd(B!df+^+O6CR#&1S? zRVT04##Kk>!(${WxNmZz-}#SWYL7Q>o)|#6mqRm-bf1utd4D<2RGSQMcGngE3;2k}7D|@Dt4RNFT?^9>tpp{h=>0_#st846yHNZF;a9zH(+q0^tyeF>_7iBOV{G7o$)L~KycFxbaeyt2LuFAZP5UPOC@($0Fe=1(cwE6V8@^9l$WF^(giLbl?Xw zU~4T6D~%r~H{io4%LDX@U@t4@v0+Im2&b&93QQCgPGbFC%9nr+;U^TZ`PgH|P)i#u7#VaX8fzyRc3RUU0@gJ#u1wQm4dLGy#TAGuM}rR3xvA$F0i` zQtbgf9IK8EwwY%i&%cs@&OVl@^{DQHckpJ_|3p98Vws^Gd`k!F%FLzK%fQQ?V);nW zWfG^c=fY>1$j|#MrGxm~4}xn{3C$u1#P6>d;_lkOQoFciyo@Q{tZFw33;g7Zcr?W|;A??vN4+<0iX;$h4N zUScJ3KlvNH4KHFkT|BYJEQCqsN98xx-Hg<7G9*()EP5W7vkYaH#$JsQ8Y ztem@Bi;Y3OVIoJ=6@5!)eAfeMMoG-q0|L_~e^Wv4^z~GP#YgwN{kcgqcO<}@PYa;gKhgI$7`{98^Z zy^s_W(_G*R5ZpLPAVbksn=;*<)htRx17|*^|}~C ztM~JxDSk)Q5Ht$hoE8Pq6)Jc3Qxg*}tUAlsD`=m|PtqZS=lY{h`u#opj1x zUz(~#Q9y0{2P@K+YJzIBcJIo83rd#syD-bF6*UQqJC#UN)kNo(V?EXt?@nAN9ZVX& zh7hC94Z_dnmhq%OheQ$Smv)5ubHC;)1fv!SbUiUh7gddO9$CAxL0zX$q2N0-h>~@i zFu>1~z9i>>Z|yJ)nQ97dtMf-BI=RIItNPNcb+syl-%*L1-cqhMsPk(L%M6*aLXB@_ zhb$Ba`*$bgREsh1lW)y?WmwT-qXX4d6}m`!dpQjnA(dD@mFGkAB?+rcH?6PDYUJV{oh&3fup=s6f>k zK5Bc!!}h-Aj~e*)HsG_6Ep19~t@;LhXIP&k*PohqY(=%>ufYvUijZFT)R*Y3Vj5)V z*XklT47G5`Z;MiQw`6Xu zY%A6-Q3GG$ucaWY(Sd!*bDD&;TnhJv4=j4-atBVKEnP2~BP4BFCAfCaznc?Ysnl2! zPzpp{BHf?G|1T0;N<(s~=L~ABuR_L&%yn33Kb&H;oT&Khysr9Nht!*jXk0hsh*W}Y zfR$}XphQAbmUDsfiATiEn?sSh%hNq!K<4j&dfD`IVTM&%r%(w)n$hYszq(i{23K_= zP7r0@kzFQhmez@oNagZly7nw~?b6IeL~=orRy{Qi+qh>)Mz~smXCe!juzhZ;Msnpk zWP@wLS>J2S3OCjE^;|#GJ1^QfxmsQ%`3fUk7=V>hxfk|PnE0y6L$29MZ9NwfDIB|N zx~tVS)J`0)HGMC`66jxt1ie$JdBI@t6$BnYV4iVm-D>$P3q@Vk7NLR53ZRlHk+i!b z)AHFZ)Y3QCJY9LYS$d6YHw>t8NmM3-tlVBV)EY?)JCQ}gl=>|UMgn^;Q0UY`z&>dGsEjK|fL^X+jDys!!;g z%^eM?rS2=hx6-*YOUm%E-6}5m>$iSLDnm>HqP30?sagk3A(f$haQ&f;y>xeo^z8d{ zK2oN-3RQ8HmN4QIREpC!f9W8ns?<$jxx@BCKO8*k;?=<#bC^NZq8Fje2E|I+QzB^9 zvdh(5x!;)Q_T;u@DQo+N{Z|DWI9x z6=e#ur~jSLgov^(owysb{llt5$CE6dM;hkWlp-&7DTezs$kImh9wf~Ukb=;by6*&c zh!cG!F=yf%zF>3+USUuMqj)6KKK&o~Ouro@Gde+%t*4p1qy3@Hea~1_rKn3u#8Z=H zzCB#AdkVcIPpR5OHPLw3c^m|dSM*o!G|!z=s-B*8(Htymy^_+FjPcU)$RW|` z`KJ$dU@nTNIc(t`L=^<64^-x9PgM^tPzbbx?cOIZ>5zV}k_a@7mCukW?k;mu@S|os~D5 z(ma$s9{iMC=Bn&5AVjwUOz^|)A6~AVzq$8doLs*slJmr*80I7hczr6W3|RpGv-IhA zV@+dH+l294Q`}z17e;$@R20~BUg3reB15@GR!P?&ve69GUEX zy7<1uZ_ur%X@!1Pg==d}P#%;g-opBjy9<6VbX(S`&J?`6B}n;+&?7-`OWrphf!iTb z|4`|iO{*5vLa1q+%-ym5xG?^Og9(2u!};0KQv2?M*0uP>dA@ zvhS@iX|N@hC?bqFqc-FL{Ue6N!(tNf!s>@25+Mi9A_c@!b&RlCL+4L`#qcE`g#^c41NHc@7Q)Y5Af8kS(`~{qo{Gn<(R9{`3ZZj` zsjc9Y?j9(a+w=f^CFu!UFZ9Q3*JB`-I>&&B`CuF69|ckZD8E2^Z2G5cqrzI2#Cj2v z^lYG#x+5qaF;x0AD*NYoDjWHCUY`7iwFxCTLxkaCVWoyYsRUmOMr>7e1yer;s6bg9 z3{k_&w<&euNrVwVnpjyLXc;wKV!)e_wv}3;Ip>cZg9%zcnlDI#AfW?4;8brYRnTGx zTevVEyoX8bFydk=4F{v*d;3u9Kjhn3yH_?H!O`;0XThb5G%u?#IL<0!dH6v^hr zU=*wp1+!+SSWtj31jEJZTNTWm9$qWcN`~k`s&OZevM*pq{5XSZq78tJu+o6j#)sYz z(%BG3&WI$iq#v6SI3>OUzoM-S|4(ynOuJ& z_E7+Vl4+RUc#AdX?;`5p>pZ}kGu0vqjI<`?f_!HOI0*D%2=0558?ksNc3sM3oiF}Z zE-k6;);L8h5;+ZQNbqdw2V$h!sMsHfq&}Q%0a#;ql5Mi&vvr=+Q~{l09PFpnK{MCM zFJDZtANd%FBjz3IhiD3blfCm_E1q7;ii>7qKkS`}zOgTs;*LY$fP|MwjDso7Fe?6O z(exrA>`XM_r+@#*9jyVN?bNu$I^iGto2Ap~Gm+5ZGpsEC&fN6qLQp!yOKJF=Ey)+} zLT<lD}yL|*Mu9295lw4w@xM_{-~KwXiQ06Aya?~sVsX8 z%Ki+e-aT)fmS5O*^($2?0J>o?FzBA2$e>;*RT&C$0s~oTb)k=}P%KHud7%lvp0zaT zt-l`So*C&ll0^nzXAHfT1&xU2BBq+UPLU=3B|wLGd^o_-5w#NtA4*CGDTNUgYo9J} zh97ZRuAA}TK1tTjbccpMN4G}T{7CUtON!3Hu9sr#)rS)u(zg{XBZUS;WIXK8Y2 zva)l*s?ERe4y?Tq87-PN1xWtq{aDG4g@`M+lPYs^QM~;r&oq2kbgQ)PiZUgCe><{f zWs*;BS6#hzCVBDm7+3?QW)tdLP!Z3qIcb$_WzOH6k=8q{WPvdhjtSgH1BUh($BH78Sa=IS8UM9TKaG_r-y`3O*=8$bI9Wi>SBiHh}$<<%0HCd9S3?)+MUJj0L)R ztj_CYkon7L6Ivd#a?Op zt~@_HVr9H$R9&8p+yz6KdPpa69tb&}TOUbg5c_`_LudTvC1(kJZQG)A zRB*qQHpdq=YQnN4fmH&CJmX~3`+cOgQdPFn&@{dO{TRtsf1>zYs$qa>3Wt<6%I>aW zGkgkA5chFu87P8m)% z`Wt;@@nL}u+f{?g@lj;)<)JwTF0PR3g@r3nJ~nTlD|OSi5Jl1S=3hu<7cjns{ zr8k|_?1sj9^LET-E;v)3-zcgTB9Y#$9yEwX$D>!@Ief3V6yjab{&vK6hfTgPgQK-r z&3S5t^?nLUM7L#r`M{+3cR-}$L?|ah2bp5%9%ITt(d-#@X`BGkX_5i6D-ZZX&?=1Z26TbG4CgxRx+B0>a|dysEo$WnH>@mxwH(BD~UTV5l;B{Q0@Oo(;qjj;hCd&|E1nL%TmRF7xR*&N1+vvM=x zHUa?w(*0SMKwYZ{ZBs0#N^Az4r#Gcu8SwDz#tK2HtkS$4*>_#zpJ`mmU+PH+l>Z?i zzKYzubD-(=Q0C!sWdl=Ed|#ed080vaTjKNS0N*q<_D?r zqJ8DZvtnR9$H@Zi&%8mY!dS=nhEwh!ym~QE=EV-!()YI_(&~bB4-51O$ad8Q2>;*Q zCb`faXgO;7I3e;(=Y8ohQ53=~2uR3|PTKxFx#mX#mimu$b?SAhqcv1c357%x5#1$?~fO+Jw$5E|D-fJ?1jNNDy>z!?6(~?ROC!2Qpef29!r~TxM_#ae_+X04|hLJf) z@IY?{@LB(URILtt=tn9&ikDN`17B31!V-3mSR833XF%QKg;|vE;~AVC>T8O({8UFp zl`V8?5haC46Jc2N&j3IL+j;EcQDTFhtU^{&=VcKkBj`aDILwe1nMsz%Azn##wLOm@iH zvCefE`hP(7O37Wc$fG2k4TCJH?n@E9%VgOUd=&4BVL;AL_z%X&wDHlY^Yu2)eYgNI z-S`tAolW5Ul$(P}lackabBNV6Ky*s2i9wwy#fqm(SwnW=2u0AqQrZu3jF%JcAIi#(@}IK=Z(aMM&HM1h*4M(u2ef|*8{?lrFyIt|KjGfMwC}7P zJO+i?xPY{?1~1^EtOFv;RL*|g3>#mn0@KmJvdt3Df!gBoER_FTT4N_7bvSW&7nsZ3fWxNPOyqRZN{4Y>WlKX)w z`J%XvCa;d!Am@QzzK*OpmZbFta3icuJ3GltNB z_Hl4@8X!iShgfy24eR3*Fn_&_q6K+F6B)K%n2Bvj#UqLj9`(I>4V>T3GW!Q`Lw!!v zd%xZ0y~(9|S%S)ZAhcVmO0)e}VsKT;d6cL4eb2eUt4$HN$IxcVMP*jxsmi9d+-P7T zCwu_%^!nDRAl2`S%|)|6JmcW(7q%KFw?MqRJac}I!r&B;8}WfNiSvTWIuwT6Q}^C` zt+%7|YHJ@)gYGdk2;MN+954 zJC*i(3%>u_4p}*8z>QpKUF6s1MY5J;M2&?7aN)?cD?6_^0PtwRuC)=h6^(MIFH@Ck)`jiE1er~yGqS@ zWIERC3{*mw7CftV+ZWXXts)a*2c^>b_ORB3-KbqdQXmBvUP~rckw;(mv^%P(22OCp zBq_8D-H?Z5Y;IKKT3>{NH^NtJ86c{w<^vc9*UB$q$D=xwHAuix_NMNN>XYTsipHvJ z1k5qgC-xb?+%Vpp<JB zjeRX<(${xod~kAoG+21EX%S`8CN0U5oD#SLLDqJBkAx;f4P3}KADv9Lb@$Bv7CFOp zN{hR1ayIgHwpUwx(!|_q`EC$Ix!MS`l0aQkkuP#=qQ1#LFN$9-RD~KclNPi#=W`V* z)2^^q>W&2~6NfoUWNb(=)}>-}*8d7u5l_BbY`Zt%zJ41tH;gV|7%$xAcTKPbtG4y0KHA`#1 zHVfKZBG)wem!msNbkwdZ<;V9SE9}43zsaj{YoDDy!_kQ99OVc06Y{uD7^K4i4g5OA z*TvQK#tOty{EQmT9^Ha}bTj8Ry@wQIKzV{iHNct}xs zSq*n5Iue;ApL66}zI5fE%dft4Y(pqMnOJXDB%U!Gr4!y8Z=1+Dk9pE;3a%1QxB*ihY?5RKf|%Br+LOeCUJ3r;Pa-XOhzq|_iJ0Jt#b`1t-u_b5<>@X~8C zh2>gf>L!Svv1HFSd=|a*>nG)-W%B8bI<1`Q{o(|_%yrGu!$r>3HA=58dOy)W{u^+o zq%Z_2aRAj;OnSgh?%7it=mJMSYgPlxs5tO0PDFI#;QU8;eWBR&fZ+n{mp@mf$SZ)%+W&^x z0q`%L`=YT}?HVH*+$$RGHkeO|;4N6zp7Eq*&Y=D!q!0PVf<8Kt)&v_r1lr&3-u_`F zHFLKX>VW*=&Y)UCt!M*EeDu>x_}D3&79|1d!FW($kiC*Qw|j<^{kN;2Exd0Bg@9{* zu%#rkr9)ahs>pq3jsUara;p$$gXpWg@)n5>`qEHX0b}b(&(;6ha0;-m*HG0vI054P zrcTf&`T#cHCP~hT;=3^r&g0jd28h;bEUU&5@%k1=1G)F{Q*Xn?AkOdC@Ra8RCzcBGHPyflG$<2O3#>pcBpx#7&6$*eW zlkrNL^Q#qSXB$qb%KYo1U6n=3t~mBVF#zG)upqQc)JV53iha8G>jdwvOOpo*#8(k9 zQk5lEUOu*5^b!(xdcLi>Llc{exR$KM1tDQ9 zxQ-fRf>rqm(pwqYtB58Py6ocGLTr(q*-)~D0k?n{r% zw>9)0TA|K|M!!^O@EH+(I=!JQca~POdqjLRCw4f`@85!1q9_pBu>s9Z@9)`|!z(_67_ zp-eA`ltJt)w{#!8l%KeK!ihX0esnnNn_;rlRvuYQU_E5|W^#9Yc^>1gjBH`%R6{ZV z49L`^|A(+Q4`}jC*T!i*mNRvfc3PkZluQ?^M^xIWXk8#Khg$6@_Ec7ZRB|i`D7AnH z7}C~JDikvvt4vi4idM^NWRo?mESZr?SjsL?V2~w52qYn9>+iasK--z`{J!rGDus}| z@AE$I^FG&oEw^Pe`7X_I+So3c!;cnJM@LGVSKvw_6d*ADr+qEZ5R;EuHs8QMtP{HB zjajoGNI%(Y&`}Z8&{pc>%_2~sBxHhA?3pc}Of=pszQ(^;LiUmrk5N3oIswZjIU0L% zuXyK63!Z0n81ULgt#gT&LWy` zmJVJ>36+HBEMiQcJr|u$HkECEF6d)t6M@wnKu_Bd#*VQ*rU5^qxeVHtAstsfWwC$J z&=T4B{1WI?2_q74MZyJQg)L|SZK&9HbE9qvoMF3i;P567d!fceuS76CI=y2@)RKy& z3y%)KG1Idi=@_Coz=LHlf?DccBVagZ8?u*vs|E5v_V^q?Bw$PBh(#ZgF)*3zf)Wg{ z7cQ*CV+f#l2}L#nuAqXubea*>fw=wqSfeF&gshM-1cCMjc-!AKMxEN{O&n+Y*V z72FagqdLcO92khjuv)q)kwy9|jt=92#srM^fisdqOMgL%v+0{SzGO=uEpU@ey*m5< z%uZ{$LqO$YmG2_)>}8ZD{x!4Ef+2skLj1g9(18C!e9xsD5_O1Mgu0i_7KWB%(pd^t zHY9HNmul9({6Fak{;#k2mCVzN<-E|Ege)Hn4NxGh&frZk7tw3sBOq9qP=zq8hSiZB z6ctMhgph=z(&fx7!?~ zkQ=wDZX58>Jk6mTOE21q|Uj>Y$Ldmqjr!>%ekeBIAo$qskmR0|iF%Cs_b zg_1iwRx!T&Raw!=`t|9_=9Q{dqH5yXv<9U^?fv6@z?48`d+)Tnc!VC`r5c(MxhkM| z6wa*r$oyj4N{F%U3Y*OJyp3N4dv+gt=Tw_*05VOsdv99SJ(-t?-A9_&mF%EC=$8J( z^N}A;OultpwD?#Z8KE|~z%u1xPDi`N10Fp?p~8$2Jqico0M(BJ}=yN1C2bV{ldOwP2I^S}`qA&2gu$Ai`2qXMW#8Q=4 zC=&$eR+PgFF)ctr|I)+i{dqpE`4N#+#K*}en7<6{6+&;8JvIY;ZitEIc$~U>vacI8 zaO4<;3eMb}9v}8Fcu>iOdt68dh>Nj{~hUPrY$J9o(aIAG~)qw_BNT zgB2vU{DAehdlk1a3z9?gImEm{ae_s`XMKZqEv2@_XdO8^WMEu&Xm>$Ep%+M6yfq)a zd}yN~>xo9ifNyX+86{qy9RJnKWry0Q!XYxaHHO2#O+u#@#8q;QQe17@hNynYx;GOL!r|8yrM-|J8v^?!hH}8id@@Lg z4NO3RYP<26>1rR+>PNSX_ovvQ*v{KuN!m{>IRUYm>S)c8Aa>dVIgO@MbtkS4+hiZ) z;6=Eq@n9lOV(aO|J(!y!RW}r&YEviw%W}8HYnN@U6&uSR%BJz45?{kU^)wERf&tUI z7A2f}G_K%!8*EOIUKrUa{`TiY*?tHX|LTkz+HGk2gS{B0EA z+<+&_I{PnJ8Yt|EN}msDt70HqSl_Odsbr`~Aqfq!ntS4~f^Y|^ z>PFi)Y{0NLLDCzyDgKBiV^}W802PYnHcQD1I{w55VW{w4_j=euJr_ zX@_DiHm&zu@Pz4&+|WQgSEh&jwBiwA=zZ$1^EanQ#h9YSun)Hhs3m}F<5P8K<3RJF z?Hu}7C2qt!COY3AtG!NFz47My#2mr1{oLRADy*R|uFKUTO%rS3HK<@jx&*%}d)|nT zX3XyLCKPbaw>?mJ!=qe8Yh;%*5CXNyt>ULM~)_K(SrX`N#1;~ch&J7+TOM{O~!6?wlyW; zvAF}2DQ#n!O0cZXmi{{L^R=(aQj+hA_Z)XLWx2|&dmwCIR%LF&%`?~yTdez`@X^#| zH5WLauPGNqd9<&}1KBwwA3ygPIaXXJ-(F!x`?ce4x^GmD{XxX1As&3=A9UaJ^HQLMu6P|OU8tdQF6`8+M<_=p_ z0w2t32nF50?l4Jbi7hWl2s04{R5X_pGRuE3$s%*C;XU)WG!i2DS@cGzgE$FL48;9G zpE;{YC0>n=mBF$C?K=}id!B^S6OfmBq%a(62QUOchxhv>DsWB?M{pGaOqNg&?ub5R zX!fIi{y2Iqu|%Y^tF%QW|K?n*1PIbR63&5J^|mm#3|q2@dDg~;d}QnsNd7tn^m_#y z$L9IIBuKyBq106JsE_W;#I5* zmgN;9792pA%ko7Nhf+x?35M3lKjFw2MBI$<0{BPpquKdufyY%%EW?zgocfUx8y3o) zE6|StHwTGa*)_RJ1S<3hSJi_G68W+#Vg<$(qe1f#plXOw>?}EyW+y-sQG+x=$f=1F z2T&!F8>GM$cp;WjI1fT$4uN6=i3ye&XGYFj%UO{JGYdf)0)j#W*y%+yRZD@)X>Q}1 z4+H13{!j&3YV-~JRXnBhCCZ@(^*CD%0RP6CPT_`_egCr;KZn@7G{qN5B=sSwHA^K_Ctc;kfKj(Ib`WYgu(PbpuR@g!c<_|Kta|=T;;}H zx^MVD{$!>5LdPHP>!OX@w_$5H+`1E)ds|lHa@vl&P?=bL6%P}IJ@>J55H|z@$H4at zQT~Qq_oJvrAAUiW%)MoFfq84m)?xEDhaxPp?MDv6paLo8<74?7pF1tlz8O%o{|mnI z1aehkCIHn^jC|r6Ass=iI+pjA+!)St$B#uI1<;&tkF;-~up@cG`pP22xa^p@wEi!` z@3*bQq34YueNUwdkk&ZYJ8qQ1y7hhxbe~G0VyvQ%)1}vdX~6xf8Gr+6tn;u#t;j}@ zv>hIY1V4QhXi14V9pjfO>Po!trHL$w3b1bKkWZpJk&iw^zbo|E?P=wO>+{Cf2E!-> zht1w4Pn{GScPKyf)O$ogrW;|WMjT1VxcG2oUiIC?dFBr0GD}Mcn?CY_IP3DO&d+J~ zOwOE|$QKUYgKMfh#dF~d!q`KxQ{lc3ubdkkZ`eB*;n|qKL3PANvAu2}%DF(KT9~d) z?N3PQhKvhuhPFSB8~v5$-;cFle!1+#e{ob3huUuXALQ^`+`A84neG}Zy+0{0l`Cm^ zSqUahTSyBfZOaV@a_f?Um%F{ZZAS&l^aI7Nk$80O6P&u2t&9IYR6oB~WULUJfe)(g zdx&pc8{!OKp*K`D-2dD4n$kp{;cr`>qYmpR|rG+Q{2&C&|rqNt|=RiPJRhn!UAv<(nWJC4S^g9{Ac-8#Kb<%Z-5-zRI- zVWRk)({%Bh_8yY{!}^dm0X1gafETkCPSzo2CUJ!-$#|+rUa$|-`TBLJX7x+b^E)D{ z?xQn7VFjKCY26O;*1~X((z9ewv}yk(Ge%Dd!Z^v~B%z%sPVE6~rh@YWIv*QcL^Nmu zdUSqc!kIfRR3khPN27JzATKm59A4ve=7oGq>oY}gi+%B(+lX4MC5lLXCQO=C)f2L% z6x(+4EOClNad99wKRG~g?Kq4sx#9hSR>;BlX3-X3!RNwfkk=wMS&t)arG6ASk!$ik z4iZL$cbx=mOO$KxK`sc&`SYmOxKhwrFft{ zbrbjNh4$44!*%(Qd8^L`w;7{Qkfd}4 zbm&lEG8C_KMS#b11SdMM)XVN$uXr$IbPExNXis3MxnBk;aEtaSr_jCOc` zQ}McT$0KqMhgDujTLPrBgy~g^mo&ux=qIed z59STd()+X)Oc{}OdE#@6?#Q1Hj-ZOG)(u2H7!zj@~0h>;a3J9y&Y}> zgiI0Ze{;jrScnicb96;jZX8M*x+#kA#Z9@%hntOB2~TCx##AgpA zKn~j%$$CZx8XgTQi~3H{oA9NWPR{7U4VE2zI3l;Q9V*jPFMks$$i*>V^G^Cqcx&QN zVh=y?_x7uT%tSBa8OGHaIlv1 zXl#DFpeP4jot5#|eVSuZ#ui0wtATL)0F6BoQc1yayA2xu59;K54GXQ%vKdwePLb(T zfJpuXGTlaHQ@y&f9YHv$m{MryQZy#LI{Id}x)SE4R( za9CDngldCeU+%XnYSTlSh=DF^bmSZwG;igMz-Q?#u#mQ40D$^cui2Uy*c{WDv9d)`83PRzJ&Nk0CZsD!c$GTlv9za*1u!ml3mYTD zsOeismf%y|Mc<*v0Sa~Y@iJ{K9Q;EAh*EUUz}24`La&OM_6xl39N|zG3UJe?yPChi zK%C>>O(dMj4Kf^tXai^arPZ+nDlH(@s>0{UHN|2m;6yroW(^m10o93j%nx)z!J!-B zcG9WQfS;_Aay0cDFQzW2fqp@CXv@&$PY%Yhj6WalwQ2K-1hhgogur~MqNjX=^_FYB zf`aQ54aCp0NPR;EP;DsN3!MaK;7=_>*JWfA2MMV)pJ(fpgE9)g*oO&A;is=y+VC zd4<0qX&;K@v7fC10&SMAGd`=@;|Ji2JnLJ>5@c!ZNuE`P*v>rsy5Uh(%7Ww{bl=yn zbe}9QU?GCIk$+sLT!1LnleyAQ%)jq5iTB{9+9-Xj&)-9$V8PQh4|ET9JC5bZujT~3 z`$rEzw;RUzMB>m;{RiAcldPZNre!}%R;6Du_vW_!1Z)VR0=~=!Wnjbl>%Aq}BkB3T z_PDcjrH5^B>9+`uB*O3Kuh~z=&UBt7-X3uef>w*05s(br2ZhPZ879$U0ZuyLGbaNy4g$sm7Z zRuTL2iQ+e;$V2;E!c5PTC7Z?huvkt6+5kEj@FB#P+Nf=AzpHh@<5_<*lXlHkkOzz+ zf|L0x=d`eN(?hLym3q7p*1%CT;?OL-C~o-Z*#iZ5^{}FC%?eAmGmAWP&fi#z17a{7 z5RR%v1o6?&A)P%*ehGRlDkPnchO%Z(>27&@78BX^i0Pk~^rEN(_ zt&-jTrZplr${{5suOU&1)U$imyGkE4@srh)_!&&-^g|nFvBWqLuLt_^W0HX)zIB*L zW6G*!MFDv0ya?2jpmR$Ddo8h5BZTjn))ekV*2;m#Ibl8q-5cgg=O$cTZ18BN5iXE*PpOI%@FPv?=Y9Tn3njhNKlvf? zk-wCOeu)>g)OpxMWp8e&x1bmUHrz5HX;S{5T7qBy@_+lDH5i91Caad3rsE_wTN6|7 zhq>TV2ukQiKT*7>P6wbv1cYYcYIPl5CWZYCR6<%gvPI$(Qk(dD`~8&u5{p?e4WSf*CEJB zi{bgGPR@Mp>N)QA95drTDii(Nu}Fw8M|dM`TB+)B(CTH+Y0f8}JpYfBl^&wsjOaPl z0+^wIFEQ)9s5`-eRjUW|`fOc69XFzNHFvnT3)ukm6TibA3{~^Y?rxv-(>3Dlr|OIF z&>L9zuykNof~O&}>rkO-H@th2*i7MFPKN2!y|R>Yu7uiv*|k%nF%efElgF)_`D@Xg z{Pdau&+>b%IcGSvmtUzW&q8+qe&eC>T|R~PAAV!LNY6P*?NE=*_FKV2$AwprcR^%B zdw1$gXO>ipM{{c(Hlhzxuys(6N&0tjlxQEEh+PTC{mUi$=67IHLi&n?Epvx~3#p;G zKG386lc&x$+!i{p&Fc-tGMBcB>K#xE^6-a2O+SjPfeTVhzr{u_p#7o_tZ1v|o#Q@@ zPD=_Z3yba_>$v10|GYuvAy_s}{Y@7t;VyNTZT~DrRD3LW&}5BFg-SY%?nq(t3h8Ij zf{&)fkjIvaQY$-$v3Uf`(B$=K!X_=UWJ!ex1$n>|oH{|lA6jR{Mt`IU{Z4pYa0(fSD1L>fwdRaR#sr05 zdae7f<`r?H5fk>|}OYnSF z$2UJ~Bk}WlU@GURG-uQcepkQtg8QkMcSOawX0ldn4*zu6upQaLh%LzxM))cm0xQoH znNx2HI2#oMc>^!7%}P80t0vspCch!M|mOWa{|dC!w03oT#}9zJjqnq1A`C@plQmE zJP@QYh*CL{F;aaP-SXZye|IM*aizLEzR|$xcvOZoRNq!Ky;jXz0C|H~t&eIEOaN%+-kE*eeR#_-`dN3ck*fP1AHCX)lpz2f^kFy4Vvg_=w2^uP z5{C7|ozRUyy^nd)T=@(PrV~}IwW*k!BZ9EnzV51a^T|BHiy#FCq`Kd;i3T-07q{wGx>A|deNgveGY5>B7j>x-fn54P zSqD&SI$GQG%y#_-o;8Gw2P&K zz|?xyn_Iln#aV0!F};~s7^twZ4?NQjRJ70<4MRp)NG+Iz%cAGXTG8Uru zDvt{G+Rgd2Ioiz1b zLB22>Es7$E@36M=<^GA96dV0x5Wlc@WK3#r;pQ5=SB6~My4wAR1yBn+Id<<$aGcQ1 zz%SyjKcW55bYo?VsKEH9E}gYUS{+|vH2DlmKiw3!KNHbgSoJF7=ifNJGCCm}G;dXD z>8~iLh-R zw)&PYKw)g2ne_J8$bMG1NvPKH#gdRpzh!GKv8eRX=)H%c0v2~*cQRfQ@krfC$kRBF z+NKk@{YC!EG>M4r#}d10Y0JU}zsthB z<3=b$;Pc%1!U@dmvkYmIv=-*MpXUNot+7f0U0Wi7>9o@4&dupN0@#2wbbT2}VSMq= z5SBbBgkiVqh18gu$^pbe5?^VO`Y)#mf~Q}R@aq98(=^OUZhx|R^>vFr7z8Gvd_>za zGy9xLHI8U=CbRaXf?X5t_uYGOV0JjhE11+Q6O~ls)LK57QPGx3fG0BgIkGy5rD#Lv zhLS}Spz_FGKzn*EYCnY-B+t7aFOWZu1zr1*rC5kT|GEkJ1X!w5;iqRWVYX!A&XGSs zAILaU%)HWfH{*0g7n6yp4y4vpB*n%{#*d>pE4bG=rMum7vNOzCno$v$=)P%*Fe2;@ zjy6lZ1PTF*YM%A9Sbt9wY>8D^P6@ws1=#1$O<_7-P^|3kWO83jVWQ0$PA0TWRR-#Q zK6k(V?8W}2j>X1q_J)|7D6_=~c02tnu3)pCtgYU-VScu7lG^vzepW)yxce>TGfOe+ z=aRv4s{&S$`PC1w3a~_F{FwwKHwnEF?@DzeOiVm}E%zRo$Y+bVA$phH?uA$oTZrdo zEIlp=(e6a@23n!UL35AJDlaO7wJTg9Wr=Q_A|6nsWD=nu(xE|ffO2_da<%ga2VDZM z=(z7-yTSP&|10y0$Npldv*#VIMCyNKjMF;R(it?jP$L{6FHvq@X79Vq@F)j+8uFZ$n}b$o*K5K7ma$o|?;ExE z5v%Zsa4&iYyj(3fc+tVrxO+9FhV-`O;6eI7&102)+NJ5ORVVvJ>J+HZ(DWFj>_T`B zp~?KP?RDj*9VWCA!xZ=7sQA8y50JM4b-yO>F7wz~D_j48bc}9$ujxCf^78uE{)W_5 zy?2r^4twn@UJ2!%<0;t*i}wkxK9nn6dESQDHEpuyRZr`SexzKF{H(^w%OU}`y>>-n zv`RpKHtMGUIPk~3!S5WnTK6<)Z1+AoT@(Bc1gfeDK@J{8`@EBql-8eVxDr2g769jX z+eC>MFS+Ex zs;@q+HXQEi#nvJ(zAf#uO>zAGa?fJ1Au9h1^Q*l)n`C>=vxz;3?}Y%Y?P!{CO0m7| z-RaV!j{lKd`#_VrZ+xv#t-_?j6Ws&%vXlEcalgV@aes9$nH9N1Q#n5CHKzr>khgx- zma@-f9OA@{3Yd0uMu3m~If(N$m7QqzKSu$L`8snRuQp8_0SQN=unBp9&8hngA5HJ2 z9W9CvjT2~PAZ!|O`+TjF%;unK@v*2s-f0|{4h`SvQm%l_RHzQ*j(e};jugi<9(124 zDJZ=Is@Buw)^-w&5EAPnP4HnwJ~n2I`2H!`{Ni``935qwED2r|c`54tY}v5pbr6bim> zC_rWl_F-_RA|?qyi>2#!-`ph69`kKJzT-+S)Np=H?)NFz_6pj9>r_pVRV}{SGPhK* zUf7+ReRDE>xEr%3^tW5zKstv6%?JD?WE;$cpKfX>0BnzDIC(X?-Z!DY9qeh%L})MD z!fM36$jyN#7Oqt=VyH~*9NsjEh+KDBmhg(;La9NDsO27E3&S~i;Slc!Uu~&AYDY%o zEme*S`P-o5Lpm|~C0$P^4!L$6032?ryc0VrCBWJMBRB3n6{1VRlO=7zihfNvi8a%V z$GuOsNkA=&)JTLLGIu6u^Q5ECJDqY|WDo8Ni_!Ce*yr8o_lA;D?GGbchL(6E1x9L< zmq=|W%a6mTaf^UB49cT`Qs_upH(!XHFdG(fI?jE#0I!AHR;zRx8#F!Dm5t(w0_{7r z@lTb!0g%yzyE_YAFS=K)`VQ9uOnZSL#8Z7SS*BWu8@dD=#P~&kz|9SH4GT3Vt+s z!@j%Eu&WZC8NE?DAT|tHx$*O!m58!*hpTVP>l5=*=N zg<4G-G>BC#0mHA0;`*hOACpq~Qr-8avSRlR9Ei5A3&vJ7lRuO@!f9J1oYhwddZ3zs zn>Z)2C2)REdMZjF&)|kD?&!qPDWfi)U$s)+8I^#6whbBs;TgaaW;R{xL22cNA66x> zqiVRKqt2Pz5GXM`oy{2`?XMaUejzOy4n1@RtH#tKI;L5K( z4>;MUx3ykhKh`&zj!r^r(P2oZH+q2F^w~1HU+aujLzsya7wrZM!6WOjzJ{Q(^lEbrxn6@bh9xA^61->o=(r?WV*Y-y0fi8mwELpjLtExd9QUv+3|DAd)RY;%}+^_&lEfx?vWPT*UmRR7Cmq!qB4$=g#US5W#=dR>1muOJblc$y&|W=DZA> zVqr^?!Xw@1YD1ZoB*foAVt>8}(~W8iBN}*zEwv5QA$JXH-C84pe7OFI{NraVfQTU+ zU0buT=5bHQE8m1o|INa-MUwDjNyz5^z=nWhjPpwIRM!Uq$;y>! zUSc3}Zb#n7YvvClzuVl6a`F#;t!ya8v=BL|pFnz&;C5Ti!40*ke?_`+4nuJJzg1z6v9&NXUrEYqCRKm#R!POhO;zETZ=+4%7|v{V9K2g{t+>)5(_ z4fi<^0Ih;$HAEgCzMXfXW}SHQawr%170(o5bK2R<6J%NJ&fII*fOK_t;GSPTZ)@`u z5+&Sa0ke%((U5id`i8-`0k_rd8OJ}kp5SC{uisjIyUF!3q8Q*3++F_Z&_eZr@8mks7uIa9;H zqFk?0#5t*bBX&&w_0qalKQZD<-+P7_EK9e03(B6uVsks{fBu*51=3^!5g7M39Kg7Ql*Dk%=oJAJnGt2ma+rkm}fY~DQo@xh_e>>|Ar4Sw9Vo@;$TNGwF z70l(`L|)Dzd<$_%c1r3z2}%-mOC`ur!XhZ!q4gsEM`ZkKEkjl?W}?(#X|%#fYZA?> z@peP({i3wI$6BiM4I<(ga3mSsCfh;%N-a$dhcG0B@+_Ceay%Gp9-3rHuQS#)@56tJ zMJ4P(XT#tyD%eSCcJ(2(#R=`CvPwS03oNDwP)cvhfE9BSbM!A+Xk}nzRy_I1>MxHW zFW`l;zr@?nz#y;zz3ya`U9r6AFq@`laz$x2f0c#O*0 z5D$#Dq0n!mq7+7w_Yj5!IXW8)Tqo!y93hCt>Usp_DS8&~gD(M={wK^|vMWJyXhiSe zLeCx}=>sq&X!6XuIaBUYq%MN$FnnyXSgIuWbNi#w`0uPKgwP^kMpm3)M@$L>dLMMK zv{84lNkYnIP^OHR1-~b3&Tb0XY>hWYkY6*^duUtMl^;iDXQT$BYc1&u*!ckoBYD>+ z6Xl!6sPnk-uo0)=DF=2j>58*JFU$GPCg~DUO@Hly&xLw|s9ek2(P(YA%nah!kt5}{ zqBa^8HPkrR7|o3p{VV*D4K6ib0T10c`+Js;h9hDwf%W&)bAN%^OjDdr4%9FPj3zY> zL^As(vzLk;6&=~^7aPT2Eih=rx+RE-B9^1RGmD&|LV%78xRO*ePmmuLtDN4QB>H6Z zved4Yvnfl<31$}*FAV9oVoJ&pKp)C_JZ0mV&_i-{s?w3^xo1dpN-Zr3x-h)KuH060 zC|LHLK8QOwFfdlZ8?YB~i#NTCp5YU^zk17RMNYiVE?8u-m^vV<6(Bx0&V(i(KW^kH z%bT?6TgNH_WWN(kEc}Of`q|H%;oZV>*;UeS;^6Tn*Tpv&d|SDV*%`W%1Mj$4k8tzke4wN=!i>WO`ml_Bn$M8pN`46hg1giYt{ z$cG&VPQI07cNBzO@4;Ay9^XQUl9yI0(bvQ6Lj*v5Qf|*d5b5?3=oQ*8o~}vv4OrLq zl}Ke%$LY2!;Y=wb0}{__0b(+1)>#~lnz$Q1`#HFn2&^)ybO!EAaY4-s4$SMOzciHFU>C$(pBVMuKyg|w9hS!-$ruMb`x9Wite5Lu;~d5R@mUQoN(3elS+!=avq(+|a%7l-niOTQ}v1o;<*3a>gbh=}ga5E4iwEf4lQB z&D@*^GFnyDT{4wDxRw3rbsiUPPDi*7k~HD`70Se&(x#igUAyePq;;Qyj@krK-S!S} zV5-s*eOr?8MePPRO6iiK^G1vEoV4Hcl6I;isL!{1PXgCZ^0zRp4+l4Tcl1#h<_s`rlRRe3-KxkwZ5 z2VKcVQwpG8pVoR)L4d=Jj-moBn$43DcjWei9d2zRzUih*6*Z7_LQ9}CqOS!>9&7tT zN^`Iy33qrXGM@ld0+Yh9wk)5F+Tw%*hhS)JUq5=$;a!PJa6`LPIn$}Jc^p-ais9kO zzIDaikQ~FtdIdna!w7dEE+-W1$&n+Qt&^~-7q2dT9{$OOHc@ft(WAvW0M)niK9z58 z)TFwYo`x6u#P>DljLS0nf&^Ff_Y5YdGW7Xe{7_p8u1DuyzrDT4#unU@P}Z(r*Xewd z*l6(zq}Bcoy-cvj^leK*3wC8tE;}teL8<&!8;R+0=heceA^PwUgI6BR&oxlSyIox34j59&M0I%Mt?0LS>X`R zMg-EhaI_9>)n&TDnhQhet(77z$5h$4X{{r4!!39;Lx{>cStB#7dsUcpZLE)d-;^q9 z%Xp4!GB_YRcgHfrZzKi}pYo>s{MA)G`<|N8PgfRIN=TcWeNH!jTUpYGkBgX6L-vbYcb%eXAAK5qKYZL8Dq;1lnhtsA- z=xiSZlW%cdf$=c6*!7Q4X_Oe_8eL1zY}%pa{>MJWdE9B*dyT3PJ+BKYnTG|?Zj*Uf zDIIC$o1c>_Q{Nl6mwte9THYJqMEFi>6aPM|Y{9JphKJLNZXLyl)523KlQ?n^>xaN( zZ80pgelZxbhS?GG>l4<)KhjF3`|Wlp4IA9R1c43&O7wO=o7pD} zW9o)?2OR4a&Yo)Q-7yu572^Lf#6pSsrlUgcf>;JIw5R!Mlk`Men*Q$RFp3LmP>hw#2A{>5jtSyQ!CQDrNTLHFmqLe z59ulwuEG#uQRr>lkIu4qP;4){BQ-3%e;=t_{yrRp?Sx-`rT9)&TZMB}W&nC3_QZ1d zRTbFc1WR%R{tY@jdM5O4*6xnaL=kkV`{oGlBd5Lt*>wE!HK@1?M_Xrg^pRLhQo?NZ z2k%A^4a@A1zxS%gkb_Oz$gJFn$1#hl->^Bn|+b#}4eaGK594WK6Ol6iP z3d?1!7^vp#G@!%^hMX7}(Lzm3`-sj0sIxS|Ljdcp&B6!#NHw2(Z% zfz)u{Hus#!e4xpe$n0kKKgscolNXqh(KQv@$3o-LnpkvO;81eG~JV#Ce_1cZ%c(=L#J%$?Wt~#V}pV*E}L*)GO;dv-F z3NDvfQA?X8xl(lNTA~>l%f!$js~Ub-U&xC44i{5%D_Ob022q6O zv)hx#&E1Fv*wPS91+Oa_%)||=#}#M`hDfPRy~3;L!Ya4`If=R+r6$b5+=ad=^;8f; z8&%5P&`kkh;ncc0Bdt7dO!?C{)~*Qxw5xB8Ui^?*e6zPcnvyjmc^ULA-|#*Pipk>E zFB&2QaN~s<0@IA3SX z*Oi-l%s&_w#=nJ)jg$1_JRW56Z6O!otaV^hYHQ^;yVM^ts6 znsT=wthLII2@z=XT%=7%*2S5&)Os5Z%oO>NrMT59G!Q97q>LzHA*FamgqTkp*o@B8 zYWH)E^ybldLetv9H5cz$HCBLfrz;m?59JJ68DEcPJ8-083P*~9L`A)rG)gO9Ote3a zB2tiuCTQioeChbDe6ZG^c58bZ+Z^N{-jiiNj|$H<7`o~IhSe9Ph)@X+ZYXVdW1BfB zu}2R-(ohu3CxT}PYZVeuGKSaKAT}S}e^IXJv%M06_IPwRKJmlBb?4@aH1*0&@2}K& zve2&DqYiJuem1gRzGqSks|ydQBz0$Vc%l01anhtBlmIW6FLd@b9LkM;74kY)`AzqK zH@+`zGRw{Byy)Z4!ltKsZwrhI=9VK^qR&0%!tgER_xLn({w;A+_+=ZjynW5S37r;G zV5j!OJ7<5Kt|&ANOl3Ds&d4rs>TI6s)@A<}eo4i9j50s}s(t8n2*jy7t zodu`=Cv63UVjIjk{ zmOaI40@jup@1F7kvy+6WjFXv*sp% zCM3cp1k&^HP-3;^SWyiizIgk(w7nJ!5i_zjvFiLBNFnMUMw=n_7m$`f!i{3ac0D0E z%b|d8C(OgTKe9pvMv}&mFd`-VS{rd2*Rz4che6iBc zmdgbEPzg{TGlq&FI(|1hehfQ3O*MJ*N$=$ZwJGgc3OX`~;4g($N5jPHF~pg|s^+u_ z@6B*Oxv8@VsG6K_aQ#v0^I#2TEx(-C0p|T`jt>cbhX<8~@Vg?L7hMi#_4-tLV1jG{ zm`F_`X#J!+E&+LhvGl?}145Xn153;m!@AfKj4+suOk`kfntH&X&yx!L=gJZE?!$xQ zd#M5g*>mx>#(w zJnC@0HRIpFIjAwxOk587Rm1|Y2oKJENg&K;2a33IK*bfvuq`Bc0}~NoRA=uko*9c@ zGk~^5{%s2_0}TMF+%Xxv;)$cDzQ%ODvXNeM4!0!kT$s#90FdGPU>a)xbIZ-K0+k-v zjfHB6Pl)}71&ZYI#GSKg`|EDTZ1DCx!8mpDNdI$hD-tRE|9)A*??^2GgOSK#3nVxEBSGE^b>-y?A2tGP0gj#g4p>UHQc0Cb+CZK3HW%hkzH9tQboyJ$3h*=)W3s~u?QHa~mlc^?rni%D09 zJMqH9n&4+g4|E>-$*Su=8lfn=+WxLdWKN0HcF86$pPAoUiS*%KUxj_F-CF-)y+?`O z_1K*g&0|E<84l%lz>M45R?%1c*NXDI>KZPBMI?C~@BB6T#Rb2$H1A8}3{#U!wWa7T z=~r^hZu032{VGJS)MG>0)egDvk|X-cVJ;Jvb*QQ&ravDVUzvm76{u4=W}%1m+Qp5N z9>X1)@dXa*6whaGPS%!IRzB>`jWR!nV!NvPFDAUpQtTWRP1ZQGxslu46ZPD zKqj1KOlXP$iG?1N;)3p`fMNN|EU?PAu*q~25dV*@uOA4U|20M{(E6>HfA`T(R-LT- z6jFOPbijEvb5uBG(mFg5{70}}$h7V&Y>S!rB(XK>qVr(nz6~&Z05ml!(prly{zzfA z8S2;cncUiVk28OENAcXn7MzA4L>mwNoaprGs_Iv!L(wa|dK9o_KOxr;64FSfLts>9kM&->7&Zs2v~9ISzUx$Ate%pa z@T90Of@BnODZSLR9;Kt)s8dPr9yBFUKAo2(HY%KhK19M0gKe}sSR7HNI%DuTFnCL^ z<6wC~2Zi8~G=Wd$qDM$tC&I(042JjoCZtHC3pV0Fq5T_S2?`(FG{h2sJG3t#jxcj7Oqk%irwdYfpYv zVmMriT8=&eE9f&9m7I65Peo-c4P-7qE=H@2z<3NHGRcm6pak3#r?4BheTjal^?z5} zKAM<+Jy+U+mqT&tGO&+^lFq=H`->k?xzi zjq$VeMQCHCkL?Iu-ypmXZrQDyZk89?W|b$^+B_>%2nBr zu_`_gd&&Cy@$-jG>%HA*a|64ln|l2{EG&4M*r0z+?bp!>`_6=+(QD%0Beh8J#aO0| zjcoWWFodCuJ4m&R972nDBaQu+OPr&`I5|oL#C{jJuz8>DqjBtb^e!D$)n*au(H#?{ zD&O1?1fi3=v$)(4156GY6(IvTiA^|Et(#^jSEg;9(ZOPkke;BJ(E)Ve0KeUKs{XxW9r`df-6Tx_@&-fXaa0 zOWKi-ve}>Atk++O9pHiGmZjb=>4HpUf;&B}y&x|)EPR+o7nvQOapDg-luHRuZC5J)K5|8<1uU(9O z2%8-_pjN}tzV@8xV2jU1LYA2`By

2@qSkJk=CnopI3G68XC@*dCXLS{mxO+Rt)Ko(fuuy}fY$%sL0*ez(`pw__xposFd?C_a;*e9Ei7E@?L z!3&<8r1Yq&QWs%T(1P}*17KeTmTxTl9OH`d1J>q-5|9bh5U8LQQiT1`nzND>4lKBZ zk@j;XfOO^`w-b-11%?4wAyyieY=+&fo`*euYJx#Dtc@&}n_(-`8Ni_nR0BainSNOl z88m?+91S!|Y$i^97B>9?7YVpoE*LWfSWX6_UPOO|m0&OQc+Y$>!XW4}LhVlksij6R zfmbe&gOO6Q`^o6#^QC{Mz(f-%Qif@a0=v(-ztl zHV1Cw>M@E5&+dJ9vr`iZ`&q{xJW6kDz|oNz06uamm9yc*>Cj#qi<$#DU#LDHz*S)v z3;5jrOGq?O<#>lyNcX)E{^F1XB9nbSXD7u^Q$*NRoGBiATz%~9H6H;Yv;b3F-jyTx z4;Lo3}yeHEwj=>&;#yKgT^uN#@AqMCyn|wNXZQrwvXMLqHJ-7E=sc z37*hyRNPz6o;$WER<@+WbU?8VRDz;H7n{SVY1sx(ee4WTD?4?Nv6zr})8t_*13&`* zF|BA+jiW2?;ke`pU+i=$_Co*-`V|?ufsknqfrA8R*=`dX2T=Xu{cvoZE+UL<^uQfn zF8{XzB0$M8h)0e+Z*<+cVSQ1w)5^B_N_b<_S|?UU;of6`uy0g|?+^DEV4)NNTqZF@oL}u?_Gysyx;Bs*+zd@AI z-O-kW=Tp-mg;*70)M3vi%Z&%JDIG!e*Muv1^Aez$_VxtGxcq;#x z&s|<{OGtTTm0^dg{LZHTeme>B>n7|!Bj6!!73rpdh<4?IAXA6QTeCB!&9^$mv04ZD zy1(KVTXSvHhMm~#OdQxUvj4&xnn;~yRC{3%3TKHa)AUs3JhmD{nO8z#eM4`X)~RT! zb~^vOtl2{a016r5GlJ3>#V6SIxV)^i{o&xzw34G=@wKOs>0D9u>(JCE*UfEl z=G;tEeh{`!8o4Q6aJ!J(4@vSG>G(#)lAXQ1?dG7dsRp`SsmXN)D9I@oj&H?q>a=~=MCN!?k_+I<|FfSlLAEM#dcRg zk^X2+?#o+SHJ*~tx_^Xm_=)=#yy}Q>`);+HFKekA=9iz4epIfI zR&zK6+q)|osN$b-3W)g z)V1;8hPso3^|{-&;d9Wj2$lj=*-{t5h_a^so=x~~fnOF5yGpKyYF}k47+T*m$-MU;fV^1vd=q5O!6d?(7gR zQoJYnH?*m6-#ek6c&y@vT%$eeIS}P=)$WJ+TGlEYRH$ruq8M;dK$z33Y#4xO~EwR;{gn(zUddOp#NSpR_hoY~@E|Q;xqtG_M#jvGu58ktV z($>EL&Zl$YkU;C55Aw4W$YH+V8V)l(6R1a?3!XM#0%g&GQ&0?)!OS2!h(|}HEf0$E zoYc^;u3A-@;X}$6FZTD@`XD?Clsyddlqwz8Du-16<{6D72z{m|NU;{u>yzjhl_5uC z2O zPom4hvEB6s{$8N!D6LpEs(bfZFAtvQ5gv>nhn+u&c6#x)gIjyY4)njmwS$;{OLjO1 zp=#|Yg0_oC_d2;dJ>=p+G&t}rrJ#$-I&uthbYnWfM_gKu`Thje@Z)eH`61!WC-2y; zMo@rOW~>^>5yH+z28Xs{D@GniBRm}KYRd(S?#WCqp}L_nO$c=A~FV%=Y)dMuL0 zR(#yX?09?#`m8BY{HFYLw!^NW8RIbpZ8mDQDXpd7y38cn=!rFChT_aZb+1GX$&!y zAjGk(5t{lHmA$Eg9ASGcoKby({PA^AA~i-minj*YuHF{~K7O}E8RU5lmJeD)@~U^X zxp#*%P!Gzyj)T??`Jt2I(-Z(dq0{AmVO%!AgEdV5a(|j=WrPwrIcFSycl_%ECrhEd zKW%thaI4$sthzn}I>s2+)81&m4UNLGr-jP2F1OsO(yM)f%Yy(+JMs&;@Uw4~4Sanf z@=7>w>KUw0Eb}dV;px)Jug*BguD$C`VKzhh?Dovkfs+LCX6@KS{;`%jY}3wgFp$RV zzrZM2QjopaLeB+yQ<|4G0=yaIxA1^bCqWkayopXp*q8tBV1b2DXWM@X%*UO4$@1I3BOFncKIR<_fiPt1kCI>TNSh$flThn%yX4=l#6D8#M@zZMwIp9QyGbN;pLzNJJ zC_s}TOAjd^X3A*8+riq8zsLt?j-+R*RQwa15oQSTaEjfykD>|YFp?Za14)QCgv9n= zJ|V_K3dbP9>JPG#E&!{LWN8}k5zGOPyfpSPID3Sx2UBT+5IeJFvvL5c7g@mibSOAV zOgGsK~grdQMr7!Wh{fJphCMJQ}D3Ytf z3FsREYPfcsXqqQ*S+Ht}OCj~nItZk`hQh+_qJiNT2l+SAIkL)#RQNQ@TFCp5XM%|Iv}>B}|*E&EozRVG6?imcW(>oKn)T53AOlJ=rZZ@iAVAs z&5)E{s=?>C<+0Y0A32yk8-uBAAYN(8V)%LHx(RJWXoQ=1OE@)Kh`;k z?QJ0gzn?xL3gHaTTGfrJGMW>LVYJ(RbTt@SEBrz4{FtbSs7c>H3Vt?m8$3Npt|6!Z#pC)|YFp3|J+-P|!3-~VEbwIHng zCC1q;EFrpHRlD_i_U=y(-`}Qm>&YIM49IkpLM7-M+q;&t%XpA8sde9R)Ayib)ZFhC z+RCejr%{EM_@LQ;=>5&ep*|GK6P5>gf0vNp>)V5k$X)WrQz?nr{d>EveD1_;dt_?a zUjQat9DXM@J<2>&3= zdHcqD0vtSBE2nknE^qB9A(Zg;We~sbReoT!JALzOeP%EZUiC=C1<5c;vmq*oe*y9e z^fapkikj!sh3-%2HuHnsjfX{fnp>mtcV{u@{jD$9)+&*qg=L_eX?p&%J)us06LV)r;yEE~ ztFN_6kUHxvc^}3`@2wpwA6@pIAg6OQ)37Qd{dT;a9e@zH1p{3tntncN<90Y@yg`1m zt+%r?0Dv(ZwgrC%e9_`W!lk}FQ){P$rjy<%wl$Jg<#NW$p|n_Tv_(Fx&M7fRO1 z)10_cT`f+H6-_5A&;v7B?nP2I6gO^1^_lU?W91xVF2myqI<=>E}QP4p!vPdu0@bK==g-(dcMH3C7- zu{P!@Q_=u)Deg{SC-~?YGv>E{{x@phbrTL~(dc2Z3@$}E^$*|2NM(VmO2pYv9JzL%rwrwCMq^s>HHX12?SV^Sj0OMBym!|KdlE!#D+wk#!JJM#_Xz130 zX__6P>^J&hQPjjRcK3Tet-$G3YTOr!UE?m^d!O2+VKER@df$h3iKGd(tI?05%mq@z zrM8>rhA>yDrr(!v+iF!f=`YWyXhe|C5>LNFPEY*r%DP&eL$uiKhf&3i+dx~9;Rg8^8zovnn-@= zIrTsioKojIgH$MdO6x~UtG3y?yqS;kjz9qwff5)NQzFut4BoD-V5-U3Cw4!L{#s-<4;aLLioNY=S`;aNvb;6sR`MHxPgt5eKPT|BaQ;j3>+^-lA@Gk6N1=d~}0dL$He1AeHd zR?xN;W-HWde&ZWyd>YK_^!V|K4^ocIPKEn6=lWz{(XNQ2v8!> zA?+OSa_9~#rW4Co|7yM;sqVO8U?ZRlD;w&5fg zQw>wt=#g7NvIrT75yt3g=-=^!@<0tS8ORE-D}Y~K0DZD$X(ytQZzFfH2a{;$c~S52 zK!EE%CjrWa+sY(BKN3nrp|!x?d6I7jnC(u2mQm!x0rlqe;^C+XOTgRs!fLAiiK1ad z8VKV#YiPl+@-Pxng2t@QLVKknV|(1bU1uw>oqu^Q?aJKctWL4ijfbLRGvuVUGrYb4hjexE8s7!U>SHu-); z4W45AoFNBaq$kdR_c%P{)9!*79(O}yS0r5Mxd$=I!sK(B?XOWN=ApYjV>>@uH@~Z| zYRuWm_~B1{iuJ^?tRR*%wFF%cf(Aq5Iyf5YwbmbPUFtDsFvgJ!{PTj`SjlVmXPN&{ z9qrrh+tYHYd0M(>sD)#BeD9%T_#a{#W`;SB&%ee0W#R=a+H%!KRQM;SFRM_!vV^oT zG|rS7B<&-~S^@V)c^P*ERnQHZKIul6Xa7*gxy*T__Vut}+wHc|g<<9DuX{q+!+;;uOOAtX%git7snmXI~p$~sS6C4bMHObn44M`$&ONx3{;S1T0Gv2^hdZ?kX zvSXHqO?GMK>pV=&W(yWG{Dz~j6hv=g-9rX|t5|?T03mDSbmT+d&?1lS{o!Ri{69L_ z3m<|vG)vi*?yzjO37Q2NIymcI$gw+M+_CM$R{8`9g!9BoYLESvPVx;!%|(LdM#?(^#7mz<_6gb+fv_y*KyLF-*TQYHhlMg)Z|_o#0quHlcKV9r`6Lg5h$j=tFl;#cE#Dgt%k zvC9Wqh7iJyrS79XfOUd*1ckS;be#o%2}&>rpvQAfC7(-T7&I&pSuX`@5)yRgJ4y;pb+#-XncjeY(8 ziiVW42^$dfiNB;k8NtOaa8)OVa-rQnaaY<1VZJ8b6FOD!a3XFPeJZbS2x7Hoi5=;R z;AN=}2`~B7D$y(LwmlyyEaRUyIfBi;Jn~plc1;!>pgiZF?W<~uJpWQ4|H>Dwfc6d|g3UkLgg3|E{pRByd~+hx<4E_Hx0uDr`J5`sgc(Mhw>vmiIXS8!vS=?l2u z`P9E>qRP&_eKk!3ea?e3<5@sNS+{1=U&WqUt?qLcb-O5n(x7fg?Niltn~pWqY2WB7 zbdwY$e05^Fdcri`keL;>c7sfcW15vHFG9b`FY}Fn@*mPe>s_|pzsanb+?QToG7F7( zm2zaIdRr+);e?dz{~GiXNrb@coqw&JsB==)%T(< z;}C`5EF1>&(HT1i{|Is-673msvV=c2F`A|Y zpQOB8i(&ipl_WQ%OCysS6g|0VH&B;MgD_?8`^F717n=Nh?sGpuaj;?5w4xF4un3lg9N$`u~m zyn*!IuKBji+3M5x`MKx|!zg?glpe>fvhpH>ym!CyhJzL;S&CAle9) zM20|1swu~{WaW`)xD&QSr<+%B=Vd|7A0ji)sgH}8>@jt>2&Xm}T*UZO&N(rV^T!lh zm}@Mn9S2C;PIXY1%9Sq7sZ@x*UR3GzqHEHeksISeF0}>%pp22Vt+)LQUB56`4&cFY z{_qwk5GGsAOauRIn-E!DkDXjohRCnNI&G>p7`|X4{B+ZqZghh;WpXD?r-!yoZa$mE z$fegRuN!R)o4N!;tw;k+#h2O-A%! zMKundTe8H8yKQeU^!LfK z2a~rcrs){$p3%0!J+ls1uFaegeTJ~VQ4tMSd3>eP1(D1d`W4GRa~$I6w@sA#D;xeU z7@YHhg`^0{{2z+`y05_;niH^;w1up*?fY`nTw0 z+A1eqHn$Z{-Dwl00z_yIE=dUN?464ay5mqP0afqv0aFoH_DP-eaa`b~5RMp~tCH_xwa;UDrsu>Ejo5T1#=S!O4P zhezV85M+E$YXp;nyu(=L1JiqrhsRTyFWkGGYQDp?z>07HSeerWr><$uabF1WxwK9t`z3`kQ+1$c$qux1n~L!fZH|?m$U>!6 zUw}D^7=F}Jhfs2UzA0F%%JRI(Qb1XJt=h3t@JZBd5ocsWW6xYPJu-Jd3x*SiHU@F< zC_e#u{VnDK)i0blNAS40v;&VHPw5GC|0IgGZb$3(#!+Ql$C-SW`eY9s*BECO8gwoD zoQ}xVg&7s?$3A3UEUvv)nV+O_a)7yI04(g1<^s>muM1L{EOWiZR4$0`(xkKpE?OU( zMlzXGQe?3fu>?gEMXI+bGeD@AGhs+XhPATp2zhF1x8rrr@J!HzX1#@95dVxy zgJ?hfiXz(iPLOzVZWhH=xMGtj#1U+Ku6g=6&|;#MI1Vh4Po*rsdv|1r**+8fVCE803ICw353;GPC@;T559zt#%}s)S`6*3Y zI$~UZgsX!#l=(UwD@t@t#TJiy_^CJbo4C56_@ImZt2j3*H=c+tOmM=6?r!9CO4f3R zl%tph!s&&Af&PzBVn_B(_)+3ZHPZ~0LMeh-J}qw*w4pYA2$5~1akRce)q?iZicMeRR(E3kMbUOY z-@G^lwbQ+laxZym=@+O6BZM+Z0j7U<9NvmEiV;$<1Ru)5Gq z6RFj2rR9w}Y9~F(aO2+oa@ zuN2g#7KFubvNmM%ANiNXe~isd!1+lm>}*nPYi^wv# z2$nx<@%1=<|0``6wSfh~KMD>2lrYx+^*R;-!P05plJ;-d;)7ogv_;5-1srt6Z1OSU zV(FWxi4V3GI*<4-DgPJ#0$>OQkC_nG${rL43v=`G*Xzf95L}04It%a#!xq)pq_9zd)rdEU*;5a|&YsE_X5&J0YCDPv3qN5AdYSa0|DV5z z6An{9#PYJg0STJ0vQM6RL^f^Qu@uDwFj^~Ead?kJVrVuA+6EWys40zQ-XOaSi3hOaPvVXkb?E-H_`CW0h1Po-=vC;uqKVE{ zS!fAhKh9cn0wBHld(unfSbjjy8D1Gk$Yyarp-=b#kmDeI>B4PrMo^T-dnlJ2O51dW^ zCZ~g#zp}*=tfygA0mui?c+eQ!!S>|oiWVv*EL-g|_Hq2(=rEiWo$>KI&|_jspzSQibY8AQFgXSJ7JM)d}X%l`_)wBX#0-Ue~7Vg*hfD&4Aoo@5NFsk zn-7-XNXa81W-GsH{AqGF;1|@Eh)z&V+9X>1-w;;=JRB59HU||0?vdf)5(w;)vVsl68VP8x&18v7`M) z&4$UZGhcgW=4avBYu6*tf*b%|w3`u9LPI4eXYy9*OeA+M1E(nAxk+J3T@=q$#20Iz zOZ3_voyZH#?e1h%%wYT)nZQxmCn@A$hofRvvj0{~w}YKR0vh3d-0mY}U9{n}EXOZ6 zaP*Iz>QI|6?dmmW>VEXfd@<$b;-mg6va`Rw{x6F^uAI#`I{&AwPOJGYly=n=k&t}Gg6`caKzLwS6-v-X#fF6q4Ux9=7yp#vv{bK#Io0Ta{;J|#b*hhNP1Mdp_{ZVD_(ARi zM&a<5Gw}m9H>xa8Ck1SMwYcDJcSpSOwWND4FC2g7{?#A-_oi_c_udtJE7HkIRdD!g z)8ejeoS)nf7rCSxTn_9~8x(V=ni{5tti8CQ^Y%QiYL+wm=dJgRCwqplB%7la_&O5U z1l*FCiX6sk>)jN8bh0#f0N?x02FiwdLCVyMm`Wn?Ds(G`wti6c4ItjEK3CoyhWc3W zd;(Jvh_jq}z>DQCOe+GR?pKrva$58ra~bVIjbIwORV8=m)(NSV_A&63f5hOx<2*j` zyf_(|J$l;eSRhX$8;%pg#<_H;bA6QnEzNqK3C3mt;%^S3 z8g&Ly>Ur{R2QChJZ4?;{c7+7r-q~Y1r9Y7aZ}kZo3V*=ffOm3u7$^HQWVsu_0#0d; zF@^plp+^;*c|V&ZVgR?D^ufb~cs_df;EYEl(vlV+Zbup7anNRGZO=2j%?H8!7&0-} zwc#{Bya|pQ$GC`+9xW5FL{69rG}_{m3RA`vNFapdHTGcq7Ne|)pDFGN3IVvlq1Fz` zUgs3#%Uuv_ccViZPHqbWOpra-w9TE)nz$Mm#N)pnHz=C|wgN8QaCDgdL0&rdW1;U{ z@P28H)cCf;m=JDGe7wG|kSgWD^akFKoP(3E%_n-hHi!9f%KS{%4iTmG$b7SN?a5M*culA1TA|GjfpOdvZc*EC;_5Ou;46sj z&^QXG!9Z?{7S+A!fWj0cQcs$&{y5LAdj2oBNlF)C=B&X;|6dlTgRpn;#aSiQMG!}o z>uk?kXC^?bU;{h1%uq1}tc zQsY{1b<0g#yt)0hTXjWpFAfj;CroeS6R|oovgYTMe1O!Z%kW5;i%IrnUIaMVLk@1o zYM}IX8B^WR0E*h1T|%m=#nuAdVBnIsE(n@yTVSKxq(p`AJIuM!!?gWCydDLd4mgn1 zfaA^|#Mc`A(4P}w%@<9(Hih=|!rT3p)9Th}2SF*fAAkcifW)u9c{7Mr5#~oIk8uxO zH!hn2#~iNjT;(o0^m&t{I?R`B$uAx`!fPS$T0%B)l}h<$+B2^9kwcwh9@xr3mZcDl zy$*UKo?G`8Ft|!T1v=(-@XVz+>!yf%nZg8h+LcY;M zS9gAlGrzO|Rv1~QL14@sn!gUa(uxH^KB(e=UZwIbS@T# z<)u~a@(yTO@E(H_T$8lwLKfs_Vgt*dsW{=R+#0s)OkEqQ@yu;_ii(56Qnnj+ zR}}>F&+mtyUf%ZC29{ZxUg}fa9J9^dhWg^hBnbR`!nblyM=>^cm)QhV+&Mfh;kK=H z#5Gh)@2hV{3g-&fR&R^OJHs(cFv1=29A4pZN^y#@8MRvj&%7>pl=P8N_8l>zCwi|3B|+5hxMDzNjg&0dj3wO zLc38AS)4Dh3^UkCBfs+rj?CH!qZ-l3#yAvrC%sQm+<4^fdqYMKX8ZW?A}Goj`J}tS zO4vEh9l_Bkk5bwb;i393i~FUN@`$JT!GEFhrYXeREI(`!*KCi3bU#EvLev}T+JuIG z27xjctR_e9G_jU2&Z_wY<;kmt(0LjET)!B)jOqE+k6nk`Aq<5pZ^<xFejY2f+y`Kg+C^)O(yX@b#s9efq9Hp_WoD z*$WE(6jAVx;=Da*H4M2B5b(p3@NfQ)eit2&sWk^s`cPm0YX~!os>vRkIj@J!@c0LF!5KDJIYCrKG###8sE zM%l0l{5TeJ{*Rw#X1#kk470sNd;(!Q!ZWdi`YCFR&hZb{{d(yC&8@;1{<{SzV7}*i z0#CSaI%lxN=EY{h742g<`#`fyUHUNsQULxaBH&Z0pN`>z2;PjWSTI7nI6BBwlU(@Q zyx9yud~n+>JX2|DvB7X+cDn>!K9=?pbDe-V{osSsYDNjZ#f???kY^h8CZpFcOoX?X zwaiC+`Q*q#8yC!W7>1#_M_Vq?`vRt8nF;15Iw4at*pV=Q)h`S&+P`h%!$fn&y+c$^ z+ItLLwpG)T5?}N+Ym!iYz+9Or4nRj}#v@Y?f_4;}rA)REp*Gl3+2nIL!=o6Rjn_Mi zjPSKn@PyFu3m44RCN8m36o5V{N;N1IQScxB3!I#GOBzV@3Hxsd zh#yE~{{ZVr@Hg;KkmCY^{>&WsX7lnka||dOjvbHx{eRY;{$ULUA`Z4b#y;H`bJlQ@ z>M$4x9KaAE+DteVc^T*`z!aOf95!?iYjIOWKbU0|`p7w?$_6cO8fk~&hrjK@0)(X< z@(7@WF(+QI&w!cBf%*!&ml6wst})wUWoHk{18zhnrJ@hM8$%5c909rp)nLemfCO=S zZ&ASj^8pV5YW4%aWx;3?WeHDY%E3qw=8zgRZK>YE~7iE%l%)n3#Z1uu!3bRWF!K2;r_ zsrGX2#oNHe^Zxa3-ZC(Oczplj5ELX(XhY}J%*%$yt-Utz>hxJ%HSP5?;9j1S&z-~t zIzC#VaDS3F7mqJlh9g^$4ZEPxn+$*7L3>yefs-gB553|l3bN19XYKm@em2?$$9k6q zt{mkauDJGN;qEp@kT1@;|MoIP_rAp)xd|QG>a~f<363t^ht5PUhKe%_2Vx9G#K?q` z&6}m=IFZQzQMe1?L6z5Qk!yXwtC$O?yvlw;_3-rf^{S`103ZtW5gdfe&sko)vC`B7 z_t;R;0|x%5uwhxlUsBhl`sA?VvRskqyhn-ev#y%JSB|ai{UX_PQeS27RN^|~2Row< zg_ZD^Y>q$mmittWDL4h&VPFdD-ZEM~xAy0CFMd0$-{{S1hv9m*w^Gp>nbR6IdUS~* zWYadwCn9Fmn9sw7AGf8ygTAk#Q(2!_=JLk6H#^AAXtNlu;JB(TFyaMQ5sFiUS;N84 z(smbbL0TaJH&f2n1{f~9M?D&4@jq776gt4w1`%9i+tAHJTNYwNA+sGX_sZq@E)6M= zJ1{5Egd?SQ91PjTyR5kp_#J!$xlxt>q=j)IdGq00hH|Lj2sz6J(EJ#!>6IR2gvYsq z_i(Sx+<_ZHo#i!eXT_m|UtEN@_8BGHh1=Q^PL3d)Qc-;3j(Wcfp#TkaPj>Ie~=&N8oNKyN;MT1Pc1D7?JT%3$d& z)YWaY-P$~9iMRwx1NL_6x7{}8yoU1doQIuPu7m%UX^l`C7OY!kB{IRrcaOZf@QvYi z4Sa?)u}7J6E^Xk&WuTkHYsL9I7y~rpB}W^t5&cXdBB=BdWJ=MITGmjuK~U7$rQ)*X zk1@oXuUdTe5V`-Sx?R-|f&zJ%H|K$Hp`@!F`=TOxUp875$h#xIxp(tCZQW#0lBLab znNft>h`Kp6K1^!f{`Vfnc6Q=GrDgvBDhn*$02blPTUC4;0qfoy%~072g$Hd6NnLP3 zqr+QY??w=$Syb(c0ttq=TUkHI>+)?*+I;xqHq&xtZ+?WK9x-Zl`W1xs`Gxc%RSyLG zQoateE+}=W0}a#p7vn0s(SF0N22ASQG)vZWvTF8AAkzK%RaQR>Kpqg>z&7-_XO=1X zK%(9Wn!fRAW5_5L4Lno!hSa9W;iy@O1ZQoQS3lVMYE7IR;qR#CQil1Pn9c`}b*p{% zFL6cDt~fVz4i)S*wP#^39TlWL(61;*n@;K3ba3V`JX2vKBZ%bIr@SmJHae$gb=I~3 zBPusEZu3v$JPEH5^n?^W34-~TFKn)E9pCPW80InSO8I<^>TO$IOovN%A*y6>sYGV< z7`Ohy>#FVUdHibrISi}G<|@bUuCC~(W=6#}FT|!a)QIvFAVGP*ol+u*611Is%tvY_ zbjPDJjXDY%$Bo)0t`}+NM*JP5*N9gq$mV8@vj7=1nCcipw6t(y&S(=A0CNg1b+^Nt z+H%v=MG;*Msh0&;6GQ>VOjw#((V59b-ihI4wWX2sB_!uh?r zZBSdCWl{>ow!dTw`Ioy!R}5+~23V(dg1_V?By?`O@LwY~f+N-UNZhST_gT1f_={p! z3A!U(79%1*_`$gj6<3UrP>dKi9J3M8mbwNss!)D%oajM(;WsYw9O)8HB3$I)@6l{N z4EoGwXb+zGTN7$AkTY;m05jOX#Dc!f`U`+qhP7b`hNpF8{#D>2?%zy$b$@aXGE1 z&P}~L^v3W)BspnJ#i$WWv@@-X>#ugpir1Sygc+D}mOD9wli#~=omHIM?wlW4`?Ap` zH5rj`wMx7z9*L--R}qdr6F&9r^`R{&K{My|E1Jo3A=h6mP~2X!CD^nvZL0$*#j&Y; z@i~zMQw=FFrSJ-_a5!8Et zl?{zTVCpCEkvV;CR=)}_kTC_*mDCGmNqe%#^D##+_NrqJ2QGQxPT*W!eTpsK?{I}Af>QwNqgF;DR$wceX3cq~9R|8luXG_^unmd7}-iiD?(8xWMce{pJ>!;At7 zzffF^FfIAqjHPA5bpJ+a&M>pNB^nLs;KQN%XwA6?|FU47u8Hw2k#`p!aRGRA%z)i( zh|#?gkAO6~P=HXDDG1N~rd?KFr_DtuEF&HL#a@bXo1FBVi7pz0KV4Zc*;Fh5HhL8b>}gtvAof}Zb)RA9N=HN4KbZ9y9*McU7DhXAC+}zQaY9QaN-4k zKtF9@%*}%3t3$W*7Dx!LNOLO*@I!VTQv1eK*4*i_-r{*~K4~b7aZw4W%xf#?jk4r| z;!TvTj_71XmiA`8}}v4HI0++ zFjlgLy~&8!oXZ*$4XZ@Dv0(e2xwFhuPA>qc9j-O*lXZ;U;r?@7PJjSA%H;ro{C0NI z=D?QeuQNHjbeFyI4ShH;VU=X5pHKPSh+%NZu=^t0@vcv{=MC6>1_|^#zlEp2{vN=5 z!Kogg`8PRc!@v0o9_+f<7Wlu1>BTvZvIbXr>w9`1x{y-?|BxEQ?vyLHyl1J5@j&?Qa$?1nuzZ$1!=9!8gh?#te_*OobYAl{$zhi;? zurN-S$V&xzX3lKr9$XAif$8S9R>qip^r-^DlBxLAm!uaW+d^l#&Qx45>%$YV!%?~3Fc-ii$xby0x(uU+8p+v7K)<%(gFpP^1OxXG@?B_l^=xJaBXH+@A41W@I9}G zT=pBMC2Xr3s_f@5#$>ezM@Z^>H_;3OUegT|VV|8CQcbXu34*2|jHAW@gS(k#n47Y( zk7zBoFz2a;g{(R#HbMi5u+hHYG076!bu=*i_R|vfD?BkU>@$oZB4Pi3dsDK7A6i~C z5dZeqf&c}Ah9q1%@l5DP(O|Rk1h(2^u9)B&sVc!PPxe^Q3`7l1I-Xyke)i7DlAfv? z_(dIWB77hgfz5m%8^Qeq!An?^LG#07cBCKBY_1UaE-X^nzU3vKTVA)9$_ag&5l=+4`*mWY;zG|iHroBQZ7Cg~O$b>qJ z?dkqL#>4Fm6%fZ(OTGmAeWb$9cJ%rm(HT}~c?t`dCWPdG@{H>Mey*%Y=vt2IO$MLAwtFYk6+QhMs{z_>1k zWQBBxNs^CB18sjWJgLfxihI2NYNvLsyWe!dt02bxSq&NSo$aq(HWlx=81};7J(lo< z#l^Kor*2ZIYDH6o;c-S1=TzPwoVjY+?tjOy_zPbAiPOu23&Pf3J!Mp4-hxAl(U|#T zJ%=Y{8jW#yPTwqyk>Bh;!X6nb$DnaL|6}yQWa<XhZrlia&iU-+gVM2)W;zM4>g;@LUUnGQJ4c+jAd0*`DTK6VZA{!24@-boXh? z?1W_775j4U!|)2bVR4z z_HuH@#y=`P-)Qfoo0xl$GwS!kYzBPkac&1vq1Y}OQP2pO)VOGV(EBgX+`l6DBp(t~ zPwjev&ZC^n6=&z?nqBJPp0i7k0qw^cga`!7zeY=E6IbD)VS(u_e4ZJY?MWFSmwf{y-y-Z zXYWQwjV2m^g33D>(W_{Qz<#EF5^42RK5^Ky<-RJ4a;Kq?)ok23qy=|{#dU)?AL1M- zh)zDl4}2^;aY<3pPmUU-KZkCN#%N z<#TUO2P)SdagqpH3YbnN)!z;-gngGWZVxLV?kW@fcj-KO!zpr5m~ve5W1o{nVPHgye3Yc_oHU`Y zV9{Q3+|N|m$OJ`-Caq8!)+Za0B41H>i6XlNIOhMf#I#(lX>X40D_e)gKDQ;WM$!#D z;!^K9iLyvhZwG{$=*=0tDb28Dc+$e)pI-O)k{=Vl10SA{6m8#gL-1kx^ z?p6GtfJYDOPn{VX*t`F++~Kv4om}a+VLD`Aa)pDlxg;{`zyA3 zu%uvxebi7on3TY8Vy&D|@|&E|uW+(=K|);cNmP8A(lW+1C|xnv?l zZHTwd5`|P=mv;$T!#0VgERA9K_Xwo6_V0xs08$4lF?82VRo3cIsJw&tsm6Ci#e9+W zOA*9r=l928MR78@p<;^7LUrdC6!B15H+UPOO||%9{a5OGx!`+g~}BZfxn!d)Ni$1 zIZ6aI{3%4Te~zEOiUSOtF_Y*F;|Zq8{yNy=Es;6XM_>CcLYc(*1X#KkKTF;YS72;0 z3Bj68vT&mtNK=xv_S_rZi1`Wbh~#?R7Z^P2XS)kiT4Dm1H1!J$xd(-xa_;|iemA+o zd?ol)22YLA+8;3CLh#vF%**Ev_3!Q8<`3o-xN!TjQ@%XzJb6-hQ<%C{a5`TwINDUR zecPP}f092L1`H+nySq$Kla)qWuc5$eHy|MZAQn^*_lA zGK*dKIz+7#c^3UY>M0T;rr%>M@L8q50s`y;@SMO6tbSCEr@MrI&DceBYm5eaN(VT7 z=*&hViV4~SS8i}u*>NSGEc(Ne1z5=95f?d=rouC_gmEq;*nlAdyaSt@E9Z9x%Fur4iU}ae%@C;rsp11~VXI z6=1|(@Luo(%7~#Z!Kve_PbMnPw|Cwfoi0?!fBQgf-xr-shap$Nzvxv|%YatJ< zdm?Z!tywz?2UzI+)7$RA4r><`)kyGfSfpSBBw@XbHQ(F&g48FYBQChIEWEdJ>Jq%jm-^@b`-YknmRT7<>QO?CTT4h#$kPErXg^gi!q zJ~MQnzgv2FF60;0H-16EN!}ilr}cApyNAmH$G$2l&KYhzwpH+iDNQ39;YTh4FAoS= z9HL@&SkI(OOlK7DlwWK4!5XiAt!sPg+$Z+D8^;8b>t_oE0GWe<6DQ}_q*=6UxBj@( z+j`BOHJbANs_O8}dlh6l81X>xy8@0INe&OXf{>x~0!1hrRlJZrHI(wf^-Q`qqWaFW`+A}NrKwVwF?U?W_rrUsEN zkNI`!5=g<>RO}M zMLi1ehQi}-4-Xs4Qk%&ms1gNkJ&)ulH-vIz9)t^uE8>)To z`0h3hY<9il%VqDM&?ejt>nxr4>v4~xmlD1l^0*@#Xd16{xmLw_wW#CgSDoJ3*``&A zhBp_$7&FWe+$|fphv%((%=sP7&U`{_H`|~&ZW+ry^3I)(sw$@AzDz%unHqEWD5g3q z?%r$?QnzaKwmnavds1T&s1EYyr)ky0Got=gzrWez-(f22>L2L+?CqakhtfNLfT`Dm zf$51oirgkjc{evAuonjAChOYxXPKYDP}s}1$NQ{KTS$;U^=jEIbSy`EJLw1aCG*@69Y5jW@!ygkN%GpZcUB4TO&6dro?)~F_=BW3G+T5mKBpe2*^U+YS&v-0)jk& z)1C$*KhDgEdGl_Knv8@UMj`*Ib_4A9>V^JJb903$ZBV0A<9WrfrTDR)zDr~hoMx}~TtAW*Um>DS1P8FQ_4NJ#cYRm8MS3EIiYIQ@$C zB3%dfPPe>Z{*BP*03)`c(w&kt}i^AK^2{z0H01cq4m{# zs4#8SsSd(f({$o!p|_Jb(nghP+Qc}6Td^e>hBm~Jf)h?4{2QYp)Ve`m^zk^6=(%C* z%q%5&Ph@N5GJlgLvmNCt4d>9sBe5%MdIqgUN2>d=hE3PTOV+LzNL79v!^<7Ch$2G~ ztQL^x0NJLsMSgzorY+~N5m=&7+zZr4Rp(}K1Qwt8hz;){IwVm(!ML4v9n7RN(ZnPy zdH-(Ilm?sWy55^#02`yal492o^F{?z%m}Ce!&|6_ZY?`fxBvt-8zKe}D(+G`a&UH$ z-4slIFudTeRy5<4iOT`S>=u-|MxZA0KXNt zNI?uH+Zg)8UxjaXv4l( z3w}|Mzu*eSrZf~>mH2mq_~mHcgIFGDL2c%1r0Hl3*1bf2VsMHhD%@XiuB6TI5=iWT z^fDc{*mQ>EYaZo9g_G{EW{dTx25nt`P6Ce`oJl(;G3UG*46E=ZzvsMx;;TTr*mM0h z4lJ-Oi-8OfF*8K|d_s4?b9{mbt9fcv1-?VU4%IKm4b3s`MQ&Yc^+A4qxL|Utw5(vM zKay{`*ev5ie>oZ_N>zGO33YlT9x0ehu?24`u?X@M%z^DB7|F&ixD0LqV0?kcPeB(T z^H606c3?5P$ihkbt&;E43MPB2$Gq%`4ir-Q!+i`M^GSF4YwA(a5acjc&50dnTVZgY z7#ZzPwMb!b5!B<(JS{R4Y6?$iQ#B|Xg2(+@4!PnKQytJ4it=aHoQKaQDDNaYTG3YM zHB?$|d+N-Iw~Wg!)Lcb@N7cqG@OuD*IC4T797Kd|>VL53{)ZTV zj^;oJfOT7L&9LrOL^%F$&a+}GVkOE_6xl>jVtKID4>IjE|A=*e4A$TPIN^rM9_gr?AQG#5z8vD(}{V|}(EGeygsYK05jba<9IW1y`?(*x%) zM-S%}|5wkWR6}D9WJC=~AR1z<(}%I6hJPw1Su3uK8tKGFUhoRRxw)D%t;MXcWrsoB zEL$YNFi@wJ4!ux+@r5RP33tz$3jA%_gz_T*8kdc`^GQn~wie9{?6B+e91#qTnKoX< zs|05Y!v|!({YbFs3A5UlMu|2LkAGR@(msKCBL6KG*g`lRa^tkcMr!lZam8&!t|Wpl zg^>|A7~mQ=LcVM$otSSAaE~V69s#b&49%i-EDlQGoS_I$yAG{zOK_4=Ib;_Zm}wbkng5^%9z^UofQjGmF|-2@t%@za#E9o*oZlAiHjW$5_8} zPzz96qHhKnhEz_)SVrV|W=4+ek|v`-06@ZdfX#y-$uIwyH0Q!S5|FtU$_4Z&vEKZw zC&5GeK^9U<-S9dBh94uIH{11Qry6)Un0_SggNWlZY&D1VBw(LD^q>R8WcKg%f20?f zBp}&`Yt8(BR`UFtzZ`l3L(Eok7PJ_cLogIDRzSxj>Zhi2k!WfXPYJ&zPK2^+ODJjE za~{R(2g8mVVoNAkCF~k`3@K+h0=Wr;MXi7K+t@EK>-Ew1W)`*J+fY_xx0RK>ff!X{ z%hR}=UXAk|?AJ2P4+A?c>>XTwZ)ufUlN?;`eHOxFo(NLQL?O*rYvg|7$u(WZJ*xMvT~q#K+;?S5fzy8o=Q0H$cI{34{|LPQ z{pAa)nM3Vy|Cn%;??FdG+SQ)!)zr}T?)7V%t_)~;;Tj5beWLY-_@$iIXctv3e3MSZ z;q?D@JMXQ_UOA-|qVUk{KlfJ6lqgHi2YkFtHg8Zp(x(ZWyVP4{6jya$)IZ&NLimT; zla|(ki_mwqhemXv2L88Dqs?xq(Ybcr-hpI=t}2=HKu=aMQSjRi7`V|_36U;(xMT2w zPEkcOC$RRWpD8@?R&_nR;c`Z9t|C7YJ@p$JYi{Eochc!#4RuiPYU5!iE_*Cn)$!f- zeNC6fC$vd62m%ZCQ+ej_~fwiwyaaTr8IAa2gL*B-~hQlHh;s+h8ga)grmDZ{l(b) zpA-5#Mf2JXRoNddvuj~8cNV9U{+RyIH9*Uc|>Z#hg`Qr>d+gq!N(`}*{# zn1r%v#c1T)Z7?J14v140I+i*FMP@2*{PR8U1G)YQm+q-dHg4BNqv2f>yBcoRdM5s1 zX$?;_$CMe}yzey)F}pI_!bM9b;@%VVy;4;6v8Ah*V}9)6SKC&&i45+M{%8FK?{@}F zGZlY3()i9-FhLIbx;_hxQ|vIFUwr#T=oOEG^PJ)#hP$R-bw(4H{Ee;s`Q+qtOZkOE z8h1_m$$ap--rhcDqbi5h7YY)=lS96S@@~mm$92>jm*$OFzZ|hxKdXTfE<+UFzjvwg z5^Z~bh)47?X;AC5(GD+o@@|~*8=Y#n@2~bf5dVHu!C^$+1~lArL9<%|$d2|TI$>^%DPYWZ`S7MrRSKlep9)6#9BQ$6cl{Fk#Q+>g7CU|9bO)zL)$ft%XqA3%<^dWE_)ug+qxWY`$GFv z)=eg4;%4Sn=W_TtyH0g{_C`g}G_^`V-*z{+g(f(TH@-F5^hH5$q$qrocc|4augO*V zLPtcrABhyU#tPWBNb`Wf z+musYLqWZyBXY>&vQOk!hJAx)q;R-(KRv&Rcexv58|=;1D>VH5KBt@8;c9(*joP%$ z^!pPnXI4tEmlGaT06I5~0iw5uO!yc%+=CTH6oogHA#boLDg1?52!%zHgI>^0Bd7TG zihbD`gg`ApQfIA?g{LZLj&4@bXP5JR>$7v~Huz17UJp#g$$;@Nx~T<^t# zE1*pdF^kZu@r4VQNeY?_=Z>YNRQPSLSE{~dsK%R-Q{=U{IWChVsi=SFqcV6EK9`Gj z-ALkguV1x8AeSLYy-kBIlnI{I;1+1SoO=e-@1eB`Mrw}3B!&PVPbiz*>4g!^#zX1Z zC}qMF{TEXV+P(4fRn1F3tf5+vY6F?8zmmJc3$*GP$OrS77f~ycL*X7Lw$hygmBQYSnNG0tz_QGtP&ZdVe&0 z?Y>9XbzH-Ax;A}?a}KWQGr<~4A=#wqv4@r4EfhWcnb)kEV_%rXQukJe<(vH7$>4CO zjF{6l@Wud;p`O+78(IgbMeRz~ly^+j}wZ+*gm^26E>4W&Tdz?+3gM38tat7!6WmqfXdB;k@681w2az4Z?y z#&8O9$>2dF(-|SNK2`!TswuC(j!6msE8FTWo{4}3xfHCoJXm#guLXYuUS_Dw6lSsWyzTbL5qR**1)zk8dn4?;8?q5&DB zB0&XTuF8%3>jmym!%r>Yp?t8K@1Pb9cce@h&<}O9IY_}p9W7#9SQc{uGEnvW9HOUY zM2P?*Bex950&7BtHG?64z?0d~T>xfY*1%L@N{jX+9#Q1ni{|Did;&swEUMTSW-j(Q zbC_qBN{Y^**mrRz-{{{sv8=~m9mXbCWxJpzA8H zFQCo=ch1C8FhQFepMpP<70pu~&W$*^Syvdn4TmhksEY)LBR^kt60syBVWr@I8o?Rd zR`D)vygoj77?L0mf0Rh5#m)|G(rxQl)-ZM~XJ|czg+(g!ebHNgtcRfanK8|Ck6Z-B zpa%s~gC}vG{!}`)!%FLfmJq--R)VmLJ>SNn&DNH|9VnQ0MH}-*C%-PCB$C zC&9|7NzZ`J-KRB9sLPWxKgp^qjVWmt@7`?}SVA-pi@Dle*waJQK>FxAOI?rUU*>ESEdywh9jw`#$)omT#9`#nUpD)&1e`Pu{&N9o9 z5uc>shGNV1cI_K$v~^Az*ahI+bTbo;GwuuFv#4=R#0C8onA)@k16iLL-?e3k35*0y zUN)bse!_a_zsBwCT95<-*PR04} zRBHU~Hrn%ayk9nb#_f9@7GiRPTWOrtv`phZ`TK_ZBl_P1LytDqenO4~@qNPdo8vg| zN&{be{Qwbt&4*-Sy8`Drp)I^}5NA~C`=QW9eLS2?^F5Q%9YTW{zv%dF#?;ubuWntv zbns>H9{FBrJjPf^s5;mv}U=rESY9#BpP5u#1DVPCS)28ITG zZq{+IPr}_@y+!3UkV}EFSU+LH==2LBU}}29?92=-ytg!)aUf3DEEdr_hH*j>Zt(;f}!{U^wA($ zC06$WWQ}E|rEESlT_uH|(Z8|Y+@u%(mHur8E#VoZC#kD3RaZNbWuk@Y84k0QtcNV>nL?v_N|&x5EP1lY=*S8ib65d zQM5%76mesT0V2wlR)ln*5*8sW0s=*rBF`}@OUC4?vUbKm!K zpYu7Nb2LY&PKSCs;daJ@%24K)G?fC#4q$sB9y4i%Eu;I_(d18)1CJe)A;()Vut@a6 zBkf`f1h=s=uF3Z`#gF`qArXu4zLd> z9DX$0oY!9+PYUz2SJ2Q2>uz$Loax;LY@+*1ziTB9b2CO#=+DNzCvJ|=sxb}oLL*Al z0_-L4YQ~@HRY;b?fXK*q9p#*V&Tuz|xlPsGnORIT87g3FJU@FLffd0^sj9seCx|7# z158MOh_Im9$J`d&8sjFYpMG)19A{W1vI0X%87-ttp!-b6^Q4Q_09>^&8|b<6ce$f0 z+77^i7(%`PWbc%?!Eh3DtJe;+xGnLfe@iksd zZ2-~cAR^-4((D0LPnb;IqSgRc2I2owN`K%;0# zTf@EQ7yf=9FnFM><4tKJXyhdhIY=^aI{Vv$j5D>%+`Kk43jIs~6k(`Z2>fK?p0dAB zmxu1ok$JtSm~bARP#k+$S-RlEM**=#7dlICpIFgg?I^mrsHz2%>z^K;32VIVYxE8E zn_d`s$3G9}O+^TE1xr;Q6lVXOXF4l$AoWNw3)>YZ)8ch4e1BV^hb)0Jc@VpPKv^Nw zoqHA;TM;W&CH&naz$9=$^)NlTA*({zZOIqHkO*hMirYBZcJ!%4CC@C^^qLO|IqeV3 z#p-iyqrBR|V28(uW&24LizDLcI2d06XPb$~I=xW8v#gTa#{d$IPT}l~NhYd}!EkEd#;QFK<;h%-E4x!} z*SB($a3%U7@8;dp9_fQHw|1!bZw^m`?}+2Xzx`Ez$y*Yab@F+UN+H|Yy}UV z&;`>?cN~v=`gnc?+GWA&=xWW-u5T)ToyFs;C$jj91~hFwZKY>8s^IY;abS-2VPT3a zE`L^6=swktgS=XOgZE-ghq(CiEz1}D+O6F3*O58eU@c3#II4pqPaZgK*;%FYkt%%N z_q@bA+&FkMBVeBHeC*NyReiz<{RO8@i}LP7hI%xJEaz-~_qRE*L4%n$wd;Q!mOoFt zW->eSSfw1J#y3HhOjSce%(1x67;D$RYmem2-6&X@THdleF~_s-LZC`}qUqR8WHQ}6 zzO{4cwekbc?5!lX)=rtN&z^pFnPqcXmW8R6y11u1B>dfp@E1l#ebWcJf{oPMUp)|D zRHCrnVO!}F$^Rei2c;#s!10fxBFdcJIQ~B)Mg?$l9SmA~NR=QKVY3zrJ}n$RTR`4) z{vG|SW^w%*&G7`2Fm^>f#K8Il)qK=NJFvy`!o0KY=0o6jeYkiVI^(v*vqR`_RK&)3-0N=4R<7oxB2U zW=NX;LTi0gkCMk3hZ7V^Mv{3v2gaoA1^Oq|-_*Ma^9r+G82u^ooOZt)7W&9+hCTRD zfmcJ~3Z02_|3{@+bU?kwy;h+jO}eP9@l+Dl6IvWTb&j7rcFYcD?5Gnp#Wvt+hWw>! z&FDafEtK^g{w)*bi9^#xt_f#0l+-x(;Bv<78zTpH0ZoD$=OvKc1|acw>u z8opQVyIal^qLY{Um$jyR6@MJnL5mkbxa(LgDVSmbgCqe=7~cp!e@dGb15Kuf zY-`oT2?rTveg8d^HT(lBmwT!*vjrR4VQ*ZjF9!IOzJd*Rx$(E$8iXoA0ZO9=oQ8Ko zH#&ONSwZQSs_q!yMtm!Uywj;(7==U4$mG`gQ&vcf+a=XT-a$ZNA3oHM=_p=R;%rkk zHm@#MmDXmQFWRevNmC%sYH&aY-zwghJ#WPhtshdYJ~d87%=8#WVo~b#=7Xpn6q=Z~ zyEUWWis-a=7_0A=US)@W&gC^VuwI}kDqkhZ%Q*qg!Ox?2M*a{KW@^=!*4zbV3~a}Z za4f6%`A17`QvR8DbHmqu2{;M@E5l&%)}~)I?0R}QXK@n{l6*%9q3pU#>~Rm!WcnmJ zD(OzaW!*utwFeFZpaf!5$D=!?<(pY=MOL8Iw8i!;&X#X=OvU6}8)7&VJg97?cru)Y zf?_}I**DG}1rkO%6m)K;(W6km$(Wxl`6Jo602k!_oYx( za(UIJhogA|kZAXgH}DvreeXY`=0ircINGxs3uo=$gQcZKhF)8{iSiR1FfERW_t7lH zfvSgtW^oi1vlCtQ_PkkMx+V|+;bEP8_JS|92Xsk|HvV|!D)==lJ2=plDckn+d0Bf8 z>a<{a@jP_C8U=<1$g1{DOgOZL15m@Lu;Hwg66Ha7*N3Net)AGQIUpX9tKYKJ-7`F0EGo+!oEC_EQpjPyr{(CkWCiii@?d0;^f4!6 zIF|9GKc>yqdAtNqaJ+X`Q7s7OEXB|O)>UWWXijB0RwH^Y-^Q_m4A@iQ))fUgU5b&v zd#&nM4;Uf1VblPNu*?Oz=$hF)^)$vA&U$b|v_myItrsTUdWkdLev$q^9Mzp~3g?Hi z$wLOR%Osy6%*tso?9Gw4q1NY|_|Dn}E|vKXty~qlee6@ouFy3kGcu?``mAI6eGi-F z9BXn;WA`_HwMmPv2Ip~_Ni|xQj;%ir7k9T_Gm+0Agr7qc!)DpSEmHc9578M9W~9F3 z4oOV)1}AsucjlQU|5Q6Dq@_tA9DA@xFh2i-xq@S_%H%|H#mRK_DC5B~m=E!o3`t&p zEPX@Gl3e*K)K1~jJkOS9_!9JFM~V>@$wD>8mz!L~H;F8JSZY!?F~__^{ln9MW4zJH zC`_9t7LndKpbaOsz^YQ;sx%exvCCv8ETIU7EN?Gv!oz!w4XLVjS;3Q8SFr`Mn;>d} zn1hdV51A}6j0H}t?kE|pFv$nz5dD+jF+U0LNBGhnjd zGtQ&M)D46#BAn423$o`ehqtA7d8jINWH~J;(`OMD3qZEWvWGJJ_-Di;|2S^*#G#x6 zC0B6n(`MPPdW(_etMRRG9NyC&ya7P;rr#pjQ)ge<;I*0_0$ykk5j##5>u7t}xk zvsMB6VRXzl86E`1w%p;4@+Fd!PAx&}w3nn5K-Omr%X5f6hRc`$j?{6x@ed@G2RDYM z0rJ;w`IJ^0YH)~9x(38L|4S;VGa_<8azb7(vLY}`n1`S?Q_4hD&CX!_O(X+;j0vHB ziQ@m)SCZROU{~6!OzoHeB7n5P;{ec@%SZkVAQ&NYLqmndd8+v$B-;?zREruv(31g< z&OjX#O{@-DbPoBO8Z5_WUO{?W051k7jT+bwgIP6Nnn}m}JJy@c6z>R$8o=TV7htei zX2!!HAjUs1AD1yJW1^R~8e*4?NgV(t1sx33YfWNaU?lux3#7e-{R#PxCREaYLEQab z?FV9~u3EYf08AeS&N4cCdb|?1F^+#{{#%Xe?G9S+fV)n-6}1Fqu?M z1eO3d$gq;JA8B?>T&q{C0|xDxnPNL;W)jAMxF?30p`QnVlR`@8DCYP3VHV;Y@hM1C zKu8!gCJ8V=H*v6lYnfL$qM-{rIJPtF;F#{jL*S{28B2e2q@eD=%dfyaLfU@~Jxc!6 zM$|(0d=*_JBLtX=#z!CY&r2s|KK>Q#+KhKtJmfbY&4$~_IIccrAv2*Er)&rC2@ja| z>6r&c4>}=;v726+CfbxZA5;|#cOJLD4L=w~ViY3VV5W8x z&qsXrO?ib-$20UatsXE245`AffiDRKYqng&tO+AuVUxBF>&xJ}mXA+oh^)M*zLdqw z@Z#O%{qkXT(Hy%`o;jZWIQQMenX-rRPjfIYVgw;zPe8xLCj&-e8r3FZi2=``MQHH(w}hRCu!e?>~w2kn781bm_q{st&f|nd~pmiY_g)^txYM z&90H(5*2CR&zfw)oc^S5)0)4IS#Y~8=h!lZT>;PD&uBu;PSd(c>yfcF{-j5 z=fA2;UuSS_2rY3}?t5!*uGOIpNl!d_$&y;iI5&DO**(;h&8rvO^1QtG9B1cn2UFpn zB;T%OM;i;v6Mxu<*$|Wbb#cVL>vOsHo_Dk~7Bxx|&XnyR+5W;PvDhc=XTK;;h@Ri? zRgq(Jcw!H`Hq%1xRx^+$+f|_-0g9(V>{!!1K!&td%Ec{AB9FYsL|AeDaX~oV&*koW+PRT5!iT&+daJp99A~+06Z` z@M>a!N}_wG$$6gV&c@;$fx*#MIqbSULJZ$6Gm z?sp5$wphR7!-xBVDp~W2*RKmNFg+FlhH3<-vrK!nfsc|KSJX%A?$j?9Y_++Q-Xr`l>T-^s`e3jAPu2mwdrCI` z=fQ(~(R84L{GnM@SD-Gq>>~VozUd3~*x+C;%pI|Z&5PAgUdT~U!sna|bMqhEFBQpE zPI)rRXp4#^t0z4#vrmeg3p^8pcib%~(C*B-w43#D<7)BUp~FFeAG_de^lh;2waUEd zzaHXt+0RrxFs=BN!xlM`x>%9T9MN`3vE?c0&tV>(5?OCaZ+ORYuV}A z#tn&K36@(`=4D_9#@aNd`p2Hak(*t6IzcruRHdH|3RfuJSTzi1!GP-5r2RR2p zjeI;hUq4d*!f5)q7q2QM6sg0A6x5&a`5FV#wz^BAfYp1M`rW?)XX(n%I!KYa-Qi-N zwSL8etB2A%8q05<8WNrLujAFO)|Yu>I*fA{?5N4HEjc);FR@;!5%%s6Surc>ncRGH zjzhi6YV4cc?f14g;+ebxqJ6eYUTGehgloigK`S+dv(Ky@#0$`l9h9B8VuvtSqzxgM zdq=Z}&sF(qBC`|N>WKo4?1TMlmfP20i<7VF9YnBDP-$BXa6?j4z)$WijrPCF1k{-9*oP2jC{|c?sU?*HR zHQh5&CI)LO;MraDTmwD&WbPdmK#eZO+jv}3kdsGcI?xL581{H;&Av}*o!Lyi0nU<96~yn9@}-pfB<(vmHmj`6Hni-re> z31|^k7QvR7joP^XE3kP)oxi3o77sWJTVI8rk{6t9qD-CA3Df4YyB7VBTZs8Pl;`ES z_fThERK`vd9!`q2k>`{ZqK9zd@<=cuR2PJjtOvtuY z`@^YMWgn`H(!r`eVlqzIF*@X8!|tyvG-?uo#p;LfG(45zC+O<1?jTM5gX_k|&WI>1 zYJI!Q$sTGL|0%w|Y{BU@N*mbYq0+sifHm5Uk#6vv`As+A5sC2>kn*0RD}btWXPPCYGQcHlX-~|q zs#ZH8nYv-$<7;-iYTwSIDLc@&C0Eh{ixU@X8)KDsIiIw7(hLOSMNCK5GCys!WVgRJ z@>IyTNl5Bxgdc{ZNoOr#3q;^(LgkR!SV1H2+HmU?aVyx^6L2VqCK}aYQ@VoJ?;WLa z#$mrI35DxROLc`r#px_joYy**xaO}%Gm)(!_hx=5qkiGlpveTP3cr4QON4JZyf9i~ zR0EqRY;REA?60c^nNpnub)3#LpL!I$jS#h=ci`7$Mds-pO4}%|%w4lvc@gK`y%^Xq zi(`KtH~9lO_q7MYR3eAfseGoj7K5|ehrI)WV7%fzydv_P-d+)WSgah(n|6mzM&Jby z1A18f&ZK_%1vN%2y3gS+Sbm3jk4B-6Tj_Z@GZdrD-z+G2Q^h|K`<`}Lbt>1#BqGaK z7BU=)pXT#5;l(+kyCZM*X}YzXkMkO=hs3=DrT@~M4Ns6)d2=2X6m*P_`C3lfk1T(X z73tF@fyYOyqN9J2l9jvSyLqYQ%N(@&SaGeuGTr3C$b$>ZQa>jHt4!zUai_}wI43SM znB34|%~)*!#WV1lYGS`P@G|%t3<7#wsVe`!{MM*058VQ8J=7>n*%oiWU?Cs_Lu&wm zq2G~_<01-sVtVEG?$Gd>(Ls13ZPiS37DYqjhv-)@Ad$cl;4u}0@vi!lK;`+x*NALv zGbj|$@5?Y6RF!`TbPOKWgqN5&F>s@_gB#v;5eEbw!JnCHqhCN^oxwC=Yf!WB2H1-|Hb=pHnTq3H#`jG^fmkWXI`0aERT z!|PAcppDNhPA4_Q9t@PBe+NmMHDLsQ4ZIBRD*QAcI>Y@mX~`Y|P%o|ZV`9T<0E8{h zV-=-?@i(5n452LEQklBM`wj$a4^M?I?U^? z0K-EXgIAy8V^bWb6>LA~PJ+;4B;H0=efZfHg0^JMPop#5Vzjaq7azv!)A;84axZ+f zfEiNYa!1Ozhr9wj5q<-)N3co5uoSZNM0YaYaYXMh=4l3s2W%3PE{s_D0vLArtDe0{;j4gAw;Yh^jeayb<@C7!rRPB?ALTvIvIA zAOi!eD+2V)%mvB9ycAo&fBCxjm87IVU;r9o(1byZ9O-lzw*q1Y4YQy6U1%C&KEb>2 zQ^x5o0IPxtUaR@0O;i~riP6#oUUyxG+{YeqHhJY{W({LMHohb2D_(J`rfK*GEybnBwx($s zXJ_;zUb(2h#KkTFnYL&7$($5b;gr|biC|&sowkI)qk-RJ`9taFssD4KXU=S7>9sje_VC6vME78Oz=aP;vXKPM3c_LhGWhZl_gD1}ApYc|1rF!t>HESpCUH3v zp6F0)o*FxQU3MAzP=^oi3)!Q}D4&}#U0U(%$n9ODza9){^?g(9RHiB@yjKzu%ju;7^@8V?}!1ee)uzu&vI1i)Bgk)$6>Vz$LKD z64z$SZ`7>2B+ty$%~xra>skNo8n_#l0;=n9=Eo0{%O^4#%FhTNAQF8=Xljx6^#}Kb zH~~aApd}4PI)-*Za*4f|S&0B#1v?}3%2_iG2Pf3G(IYBn_4D?j3d$*6GT-VJMER79 zHZ&EPwr)L+70vkEJ?$hx*X4++&#zXOyp1}C*6|A@wd>#7ZKJazgRIaSZo`vJhbO2d zK_YF*EU2^P29}9!pWR0p0V+Pz^pa5M{uI0dY-vJY*K1GvdLKEgzn5#esld#)KfLzj z)8Vl#PTQlmpI^sau?K-_?{ukp$JUkpwIR0t#v(I7?o<(HAmXCT<({TSe%o7j1<*w* z5(TzLMnBrh>k9l?^<7K!F~L<=5nmd%M=>Ud6VDq88Z7O|?(iDez#5I8No?$Sm0@Ix6jWn2t6oW z_k-gbkBpdfa!lr?j_cS-g8vyQ?^W3g7WI@`jWnL|^S3Qoth=e*rL(ARl$C^bxz}EF z>`IRHdlz(9?ET|WK~`C@(wn+-x(@~N_IH+uekvAj(tdQu^sB@V+ka{1-Hfj;%r6-p zyt#P}TJcoIUT(>x#W>`ERwV zuP8X*y*Z9985fW357^Ddn)w;i^3-{H)!XL+jX7ygiZ=xwy1f;q^L(RxrJF{7JMzM? zJ2q3v&pG09J0nr;yta#by&4dpCLNc{SyQ2c;02-2hkrPoU)a>EWOp5P=i!t zk;&@w#WD0|{M6H(DmSk^acs!FuOd`hB|(^wTYRSs`8^LZ1+v*Ll_bz z_!dwrKxUxndI)omz=wV*)#&)Zx#KZIuxOOMwqkDdM!#brlf2>c>T?w?`c);RXot5> zZO&36+gFOy;-pM`=s~uL0`Yg&R(gq-(0keC zgSbMq}J2{GDS;SjyCiAywtL$!QlChxZU%Jv4pXLpFb1 zOlP3}q$+tzvB&(grn3D(;raz5jiN(W`E#CeznXjSq6@Ph?a5(3jFyml)O3t4H&6eU z$cKX=rVm;YD^xeW~b=dIAK zb(VYJ+2t%}`sT18)l(Gg*3_|I>DZPs(tzxW7e-$tA@d-I7Yq+PN?q8vRG;nV#ymP? zd++Gtpc24(YYyMq*ZEyck(~eq_bQf%`z=v1IidAdK!;|^4$V$XAA>hh&IwevSQv(z ztNKuo&nvARypLNMa|c}(pmch&#hP#s0%?77m{^9Ao}@{j23~rtA)(-7m)shK5(4?y z`i~Y*^7=27_=5((Syz}RB2R28eZ9(g31{oKqWkhrw-Oz~;oh^+WRa2$t8(VB;?K>I zC`8gWY?jWx$S52Fx~&&y8~Fz`@(znAw@-1bVA0ixte8e_2b2u9HDUPZQThaTSj!*I zIq_&iN>*Yhej(=gsy@>7#uQ1%Jp#X6M;U!ia1^5$D~=q^Mbpu(x>e%!$HZe}-hb;g zQSXH1s6IK6<1zwC9(Lp2_WOovN`CJrT9c;0>mzG69@!1+M6_?gsOs+0x;LM-_iK>LaK9j-6#KvRkVzH)L1h`In$d2xxC6VUQ$%d4k!?6O*Hn04=q>o(%RvJ;HR>W?y4x9YgK)stfic< z@@)!$rUIjo>Sw4w+hop$gM4>IruvjrZ;z8N3+Hs&QEql_t^ERH5?l}~O7+0mz-^+g zV3YF8qGFg7q%M&n>Vm}oIT?4kZVXXm>jU%Wi`~xh>#qzcY<*B$dn)o6Z_3W(;JXg`ET%T1G7c>TGwWsklF7WrMh8T&#vt^UJjm1KbNp-B$vO2sIRS zIy-1uSnyiKQIuC7TkpcY7o3F=ecn#K`M%TQW*m_sqBuyX-@2c=b{xvUSOB%*P6SW#sxaTAwaD({ z!5kKO0JC{QUmu<%37MLiEZlnjcRV;t_tvZe_SItqYeG(Dq%ab39HZDlI)C!B;z@d; zwD5*{ck(>!Bcf5s?%|wq}+eh8e5cX;G@5qw%JzhosZ@b;peDbh(ffe?oyC87e!^pxxz z>_g!W@^P6AsrE;tt>T{1LKCwE4>)BV2tnuzZz7XP-TM1lA2Pl3o}E@&EYi-ieb8(D zEANlk%;2dISCHB?>B>5i&g>VYO#L3^)`x``BrX>m($r4fXby{R*LurZD<(ly{*Y?%g? z!maSrK>FHCk42{Co=Lcx@xnHo&I@r7X}!ciq#2s>fpIBp91vrsU)TVEfj|g_Fn_2# z4++T{hFq08sq0Rol^2+{c|MRx->6 zLjxFe3&ds^Lf05;F+s;erq7?D0|q2i$Rzw=;7c%E4EhFyVi*I$SQUZT5efOB> z8|fF^nb=RwY=Cxj!9LYMOV|{Wd`Rb=Pd@ilDx~#Fy%kXT1jDEt8z$`CaA)w21OjK$ z(u81M>jT8ou^^Zxr~?D2D8UYt!dE8KuU{L4>Vi>`7}-GNL3a{6^XkvVqZ9f|A1%65 z&a?&?G50ab5cbZRvPmWzb={kA+D1-U<&07T#p#>+Kj1Q1;XbHQ<7?{J1f0xfYR5px zuatLSe+Mk1dxN-~mm&-r2m+ym)?kA!ZDrR$7zRrT^m|J+N9kqAO{d2ZQ3`!1vCmi` z%#190?{RR4XSjh3GDL_oS4Q)cJ?guTqTfijU=RinklRaw4jNJ-gNQQ3JXvVu!E1dy zW8#k}KTIV92wAw-V9ZZB&Fubl%uvHaV0=bdF%t`5#_Qu^TLTk8L(25#0CLZ%#Al!FR;{38%0M;kqQ(j3B z74m$L0i$r2ptqcvMk}$0P>*XIv3~=Ri_5a5ornltfKN6SF!{~1q3I_K8 zv_5`2Ba|>ADS;vL5A7Lq7P7`Wzz~rvfuV+J5rBh%(*;B#u-u{9?Mw^IIht(zu=u6= z%eRQidYsY#Hc*m*&wvRShKdVF(@z+p0&{_xQU;Y#UfO(KSCCL;>B-ZC=LuMK!D|t* zJ#LaXiBOXLCaA_=YT;dN(cO<%7x(FS-wt;uPL(D-1A^IUt7LnXut(<>Yj-qT=8Vkl zXkMr0y_xYWK^T?s{Km+Qtb2Zni^u$ur^cEB)Q^s8CZ6f*OxdcDPa{3#+{fkpcl3XK z0JwgtJ7~AKF{__VQDCoUS@heKtl*7)_!yr$oDTOZ8mF?sNh3GftF_rZddx#} z;oP-;SC=L4r$H<(XjxsF^x%nb*W7<^;(cf48Me~?aR}Qa3+CTmzC39|7;Ev6%?=ee z@fnT-r+oLJwub2$@bh3yr1uA>Bng}FxDy17#gu%}+{U{h3qd>=D-Svg*GuYe6E>qh znl0V2dNcnGiYj`9lhFH?X(Ci=PEMErI!MaVG5dzV>iEu3mE6(>sHHw5dP96zk@=DN ziz*yy{I@8|Grs%Vn2D+S%Rary6_sjoQYAaq@lVmME%y4jBl;g4YF)M*$?^?cqAWj| zk|=Ff@_Gg{b##OZt$8@TBILT9HDZZLy+lBs%lXCZ1Cr^OTH0S~zAUMnj4q$_w6shX zMnz%6TJP+uOy#ipTpJKJ&4~(E4a_Th$Q!jN&>o4NNNMOjY5Vr$G-=C&@+khd<2-e; z7f;p`bF{CcrNwJ;t_3%Gv9~lqy5UmVQUv7>$5T6!&e5b#vdLmfb05bF7C-TQQH56xRIsN8bNY+426+?4yo&XsUAU7#W@&=D_NA zQ$_GvTCNyt5(4_x6?vNHgtV7zSu@s@VtV76mxu1e`nbuq%$=S~y}k9?#_zM&@l0BD z_vBy8y-dIF(k+nKoo*c8rD#nk9Eh=4>e}cjtW(PriJesm>x(5Vj*%TrOG+ap69WvaHD)5kq!m*R`;vjj;i4sr%cmtxXIeEiL%QrtiB2rg9* zcOV)U5}IR^5vRuf{^Wz$S~hlGIl96hr^pb?&R8=ib?r~#=TxnpTxb1yLwQM3(%IGP zq~pQ42TNGbCVH+2LSIYh`e-g|^!dD~*92YKdGGXULqVhP`JZSOsP^Su82Sk6aJSaf zyi|v}fxA^}*d7&g&iFQb;A*!lsD8(=D&4sG)I5v7)a{7R8XDL7jjlcPU1JLYyErBdf-4uRfJeqBR z3BR^Y?K1gO)4=5V$$Q?bdEJAslj8XKTAq1yLwc`lU)G88WVnIISg<`uEkzch%Tu*a zNYx1?QTO9Q>&o`ehZTi&3}Qy{HOD{~O+6Iq450^NIwvLD7p-z60R~5B=4- zR(U?WE^}~evb&M(P>lMvW={)| zO!EB0+k0(LMzYqoIxB4x7Nor}+NTri9d3=x>*1?rL$MU1N^Om5?EgDT^}Y;PfXAA zC6|HJRnoldIZp7|6jg_IMOcyW2tMCj3OI;N2FM%FxvZo9gVMYPcDF?D;Qt3eio;0g z+K}3$nM6upjBGEW_rmKQAl8m-4qq58_qSiV{%6$fqx30UDsxUy-f<1gH$8P9%3aI}Z63uz3t2`mN9EO?M3QhOol>zEU}HY@#hF`Jt{#V;IvF(1vQXwxjN zr(#P3vd^e_eOG0_=hu8g-OwxvJ^gr;;YXO7XO>%gsQ2Gq8AYQ@?f?^aipF}qeQcZe zpApVDFI5w+w8MaWOnEE`Jr=h%#b&t{)4{yoWn*4ZK^Lr|4lnY3v;i52-fmey zCQaZeCu|@1x~zfqou%L|;4=9g39DTZHpiwnyHVHHT>pv~g%fPKJk1W2I55mf zW=Ze?0O(+>|B{d}r$f~q!@I=;KY|yOz=jugfy@X&|7qC}QwcD!=^@1u5%lPBEzUWS zCZs|3-G>>#obDg|BrU!(aWx95@1Z9%di*`w;gEoOc}5Q!{Fwkab`78uDQFkE z<7{0X1~37xhsMQdWnK6s;L(wweZFn*TfTv&;*riHOx^U;&;{tL^oe<5b}XXfKvzZnmd#L!kXd7*$7sdq zz=_L}d@Me=`CxiJfpI2ByWLo0Z+0HXvCtF<20-%juDVECW3l?=*q_kyAQi&(&`Id7 zOIO}Vu&Y5UyC3$i&Q-aku({OpgkUym4*j{g%{kFW3d1_2j6$D*_60i9_IiHJDSPA& zQB`HhyPyj3#L1M7U)ntMiDJu&)?vX`s*lfM=YyZo9K0CpNMZ$OYqlXi zmgpJ~tP+{7l$)Gy$!5^xXRp>dShTZ5p0c}#>?EyWMKO5bUFYT&vwmjv@%lDgD=B*z zaj^r;-!?=$uG}&$ImoTm{sn1iZ#l)W`&?m0(=fm=FfyFK?gpQ6kO%@Sj7xCU$(nZz z`2a&@tT)a)B}iPj=L-f&F-Ltm?h}gA0l6`u$iE|KGH$1r1Gv&|!ocRW{qA{qcW4$REE8vf4}?HtIx6alI-w&-5H8vz9w2{mJ8WjK*j zU5`LDVw7G2B&NUG;4;7Jr6GXfdxQfSd|*Ymicm)j6#*H1$y^l*&nmHoOGQ<-;h(+R z(RgQzP-zDl29quF)v=XfgheSX3=Pz7cw{=A$A{9NM4i!9>&dGZ8G(tpxN1F?ONyb^ z$rPLEN>N&+8@i4_En|t2(=j&k5mLnAHUP-Mtj96bSW&#BCIa9D{3cge_r>GsfsS`e zl7iP79whz%1N3_sNDq_^jQW)oCNjXkkGNZsETlp=W55TqG$y+owu1(K0Dytsj(|1d zsR|jyiz{*rz1-=Csv-sydIIPQuwCGF=7vLFMZWor1u^?)WJBbO*`1omz+wt<1h|TQx>UB&81&~t_gc&eB8TZx>4rNr zbEAG=n+cT+4kw-j-5xXpiJmYf9Tadm3f-8-@l@$I7y>Rc5Ef*^|6c_D@=BPoz`)4$ zTcT*Qp)VZdph#NEP%Z0--<#?ALllD6Ok5thmNW8y@QjM_-Q4-Hn;`p)@bFjo0} zr?~0+jz~B2U0b0D6@6S$;I^PwDJd;5!x5>M*C3D_fk z8U$DIdb;(o@;XRTIessU{*tPm6gZ3+tKNTp6V0Z<`B4j(E%vTH@odlef+gPvdRm!y z^p|c5_{pu+-K48nyD2B|VMIacAJoMzN1lVYUr;B{JH-3En{IA6a zs$+-BA{sA(&t|LNK!{nOIeRKZ;8?$9o?*V1pH+xlSw*RiklRanIp6>qvz`5wr zI?&?v-2KYEful@|?>^F(DyB+dWm^^@7+C&c#b z+R-=Z+!Pt}H?SJC?d`#(1vCv$qHQx$wtexMjVbfLX(aBXt5Px|;0%A)vu@K6Hlo_+ z63yzsDu=}wpTm(TN$z`G@;F*mYMW@jO&f65F~-gNPgB)Vr~mQqv`{wZRiAHO7y&Aq z+49T^cS~DiB5oHu9nfVTnO&m@khF}11|QIu#>Dq|UWu^nzFpACo{B);MX_vOV7l(? zb7Ap<>plsY$CKOCx4S%TUhQKC<(ooi`2a*^o^fNz7^#uwVhh?zicYWt+meAf|t5IE;h+--cSHaDzq^_+e%>t@pPc|o_}6g($ji8_44Xp@ZzP@n@UFh zn5spoLm~TvE?pt&HS;kyO-`ve6LTU2$B)%X3XMqCRQe7|P=||0tp^Y5t7ot6vMEd0 zEM4FxPu`PoiJzM)!lwSqxZujKM?12zP0}Ae=fCk@>SsOTGo>anw0xJD+V-`1uDR=pbw}d~~J2^g? z4(5dGlE#j9K-q5g$WlD@~Pj0@d^l#x*~KTUeP1 zq*0I)D!zJ^0tp(eW6NSVssPD|7}e@~>#7ou7Q!KKwyp@$cs>}f#kKm{1KFtTq|3It zefC(nWv5Cy=c)gGiIUZyM0t=}i`&6~*EVLhGIdAUwZTR>N3l(Y${)e&P-SiT%HXJ^ z93l~rF{yfQviUDQzIS5dL02Zxsr-PR?`=H&-bHZ}yD7P0vc*~{?o~J^dj5=-+Ovm_ zO*VS;wgg^8>cXBADm31zbFf#Hd|2Mj=jv_;Q&hzzJ4ndQH+x3+j;Frm>l1K6vkYQe zcNsjNTDneTEG=o)9%z77qArRl|8h^=vkgQKb2oZxeJme83X0x%6-Y0Pcq@n`f2Fp7e*>l5gbW; z%o%9;xzPkG8z1y_S?eZvG5U^T0&hzj|FU%vNR=d6YR1gXCa8K1LVLGMg7KC<``Hip zndCrtIPz4LBEolK0$WpzxfCvLC+a=%=|EX}k@>&B zj0d6|+vFA(E~&F~GK~;_K3cC^p9u?>gk3mQ%Ha7Xfkk;@*etZSjZ9!<9VQ5v=g5)9 zhejXm7Z`(hNq()uGHuS1JvQtb#|bwCmq;+J8$hU|5JkzokPsf`-o;ar2n%6LE$q2H zzXlI`xLetP#)BcXKGWmU)C3rSaTpE@wd7ESm;|FYh9 zREMOsU7X7pbn?x(n2P;yYl$#R?xfb<*EnC=Rr!S(kMYOuNuAa=B)(_9`A*-aN+?FO zYBY@^Cr$SnHkBrTPFyccn$23-y3bKk;SPym0-<_gBtldAwE8)dm&5CZH~SWca+MS@ z8#pX_S}e%ZH>FD~Ot3KoevWkk`Onnx{IvN#)l4*baARgWYCDkT^Y7`fCSgB_KGY~s zHUba&xaaA}vxhy3JZw5SqOGz5T!67W*}k9^W_vL)cOV8Zq+O|u+@az7MLu|!i|oy2K8lWEtnJECnO;z-i}&b z5cLW zgeqC$iu`q#Dtk9onWpyn0zM6e8_vsX+bgL#rwf)G;Gpu0XS@RrDe>?UUB!fP%AZvO z!5=&!e3hAZhli-4Gfq6#9Y4XKf~8I9XDnV)4lhIHVKP{}uUkGQL!jpzHlh|au3LY? zzw$C%PJtuE;5sd;YDaQIn1w~d<5c~*PeYO|78Gk2$#gNUcO#wLP;uXY-#n)Du{jni z>9#z}?nZJP=UHN^N^){~FDDu)tHJ8A8wFdYT_g>xyyf`(uXj}JhR)u5Vlz8mCNqmd zf=E+l=L@3~$zsf%PGgxd-n{j-vGpKe6`f9rk1jB$XH0q6ZVhbO1xkTBWotsuH6Z<~ z&!9Yq${c}nfTcaC@G=1kdkMAYe7SBVPDvya=P=?R!r{BA1=*P~w4ggrVr$0g9A*KA z@;5C29M-u7@I5bN_z0%_Y$NhvUP(wD+F&Y_7vqG@Tpa^NGJ0MD5;}(icaypvx?=`; z5MaX}gTvk^AuzO{4Lc87gp4=s!_c&&wPoe=>H)HLH^h1Kt4gf`#K_2^0KOUiguu=#MR>lQ=95~Gk#fe=7x0-Fj{ ziVcl7?BRD{Bb2KvTF1ava6MwGT|+q!eX|$@C}ZouyV*j~fufGCV*nmS6{zJs@GO!PVs$xKzfo(uhH}#Qc*g5DT6$ z=r=59WU(YMrOPqfJ6{I z=oBDj585YiTv-JwgIkMryXSMJZ6ow6eC)T;TsDuPjaJ|S^mPsBY``;;%msj#q-XsB zjcJNNK&gS*{p5LPnoAH#RGMhaU}r!W8VDVP3C5E&F_WQ3Y77#YV<*S^`l|I^v?}OS1;bCaYLt}pP#=+w zp1Fs3R|q|r&I|+GhJ1p7zr|Pd%?#Jxvh?QOBblg4PXAWax>1#+Z^q*iSS$(6~u1yk&{QVn_*QC8O(WCYEC* zklWg9kTpQLiD?61aCu}&VTOCoz6C zzVOg4a|ADQz_@zdqCLT@cPpMs0VUKyKtJ{mU9P14-*`)AhGE^})wCT@yam%`2`fz8 zrcb$e*-ZmS6aO+S6B8DDqt@`R&?=58qUZfojxvrpc14fYCfWKG9D$W<#k-PexV0m! zy@AUu{m<>z=0?fsFy1&0;1v&w%wks*>}42@{tF!ge%{7S*F~P_IflB<%3Saf)FM>_lMH6rW=1(8zUMPOL6i zsP!|8iP^Fy!MakX=7AdpU6tDl>Ouc*56ShXIsbmQsL1T1>mnreozQ;v zX=w?pgZ;d8jqs4ljBgrFp~LOz{;l<=%Xqmv^T zGv~0^%oj|29;&x$+~S-bxxY(aiNkkG-t_xffywJn1ncr*LT^3gRrR#B^+X%1?30w) z6=$-Vx!2mC|GW2R{bINh#>mmW&I6N?t~(#HwQ|Hx5v=z8Twf`AZB#P3B=JVjfcftI z71b@zn%!Cv7k^a$m*N%9TH|R=ZA4T{WlHmJTxTEEI+!BqlY%)1^yi$>;W{)m@_-D_ zIN8kFKbvti^$% z3tDZBO|QDf#VJk{)4Eo7rr+oE31xK9YO!p|K-uBQrmPJO2V$Qm-uXiJ@8?-+UfGRb zom9dnPb^GXho9_Lc4UOrx4C=#{ljQkdfb>p=z`OSniu9M6o;qY5f*P%*@!#`b(RNd z5Bj{gE-dsnZdO=7{J0a}I6CkpYf16!m~QdxfhFMy+$&wr>veApp6j$zr*Bs~-|Q=0 zE{3P$-RA@Hl4CxW4MB=*=T^H75;Mk zqr%!Pj^U~WSiz`Q>nz_H9IcSFl@=qy zVobpe+y>|HYA@B@3LJ@HWYvIBIjL?{$HHar$9r@1TX~7TMR~%BJ8C?%!i`^f+6%4) zr3(d~%!srs$h5E;S~~J_B;`N$S;G zjl7@OlU<&qglTv5js%cLxen4CZV{xy|Iz=&$KA)9ch%KVy2JWOukyGrXE<-0g;{M@ zY;EvuURB#s(00`Gt93cqi?r86Dp)Y)x_Q2vR@6BA`NiP)?&*&K=gf`+M9Lp_{|x3L!T%o5UtCz!|2b^y#(jYqt4cyu7Y<5c z+!k9KvU9(CJ?AbBM~182hUw!W-j1*jlmrIj$hXL+eg{+c-hCrf=MOFiM4o?s-eW;) z=B*JEa5SCfBJGzijCP{P_;`++s9zP(^T@dIJ3Qo3OvmHy6PWr@d&T2WO+iyIySuWu zzNRc79`{6hAlobgP*(X+rf_i~eg~m;Ee^L4V}fV-I*YIS(V%c773G!q9;Hq2kadU( zv7K8{d8}7P^`bMw?b?vCW3-2>ie4qzaPf1C5T8MnMymJCyjTR)`HqR4^hb7*CS`0H z$157qy0#DcI=6xTsiVNRdh2NH1e|_UX#u~t!#yScn=ys&sMa0!IKV7>Cay;K>4X_5 zqoaNjI(fdadrlFE9qimz1F+-jFPlI>Qh%y{X|C;)v)`0==y)z{P}cn_upsE79xA#0 zwM1WYA$C>Xdd>5M7!^^L1$I9=jeI}y2((z#xQ?Z;S2u$@qF14@xOAY)XUchF3c5+Kpc>zx+wh3dzy?^PDGtBM#n+1+UPy+jR^ z#68q(uyW%Lxwh-Mu$y zaH_;Yycumob{Bh``P;LhPnYA+514{lptl?MD+Vb=;gfq%brVjYdUocmSJ@n#xwe%8 zdyPyoC3^!jn`P+caAmj{q+MV}@)D`KNz~TH$I$8UGBU`J;NwC$J~z1$fD*E%woWM0 zJys8`L|y&>!Zn-fJ@`ocKlu2%4_p9_9r9$E z+OTVpj^V{^reia6ZX*y*upWIN_x9pYp5QDuoICR5*v)EY^X~b&H=fEKdDm;i;v@jB zXbAp=atHJ!6CwG&KX8EAO*D3(aoUPx8DTiGkttZ{B@*59Kby!hVQdUCUuJFLv_@z* z%ZJR0DD9c(GAt7dnq++?ku=Fj2kqnD!M#IPc1#v^@0-L^7{ASdz^BBm6?Y-rXGpPb zmRCFOK&j0OBRipgY;&Nh)A0s;Ly<*7-3uJbVbauu{-`QJ5wHt~{2L8~s!$6z1UiEN zMau>H3mCFeTmkPkD4KS$?|Hb`yXmoLHj?`Sn9+PohdLw<7Ha1)=?AFI!w=mhPGj6` z$x^2Fm2A{BBrYG7D3WT7_ag9S8-T12pJ+k{=SRowYo)52t(o`fB-mf3h-^xHSg&{E z-Yc5-9)87vr#xwScF06c9F?0&D3O^v_B6F62KY$S{IptgfTtY`4L3<%Ib48HBDXzUPe?G0jW$7 z8Lw`mxJgX?muOBK4EikG2l8|xWwYNz#-uU9U>vUhPg}$Fn9^|cq>*F(YG~=Fvp#mx z8`P3Q;${*_h=4a3++aA*;}{S6>y>y0$fIj{M*a@G@Deis%7Y4_wi(6(V1`715E~#L zk`7$@?>}6!aA^->NgVQ>qY2AdanB;=9tz^Nuxs(Nf>_9@#59d^oOQZsW?qpO0@PcE zkDJ;g?y02ah68cfR4M=m>(FUUQy9Vu1Aqmbg5;SIhl9uq#;nkVc_U8-Z-Z*M-7EMH zR7RM&)tJ5vU^l{rWbtld@}G2_`GFa}2d~03&ICF`Ai$R}6UOEe?t@lY&QAsHh3QWzc}=f(uPyr_X74u}nGNV{`FFGCv#eiAFRK4~ErAtR&R-o{(O zcn}e|s^sN**h>JOvQD(KFZw?i@Ftlqo;L#D^fHIRpbQ{?25qqZRh>Tw#lIKI!3OvK zQZs7@V_ZiP4M2Z5?|9uQ0md}Y5xoHW{7-OS8OTob3v3e&-&(0<1*=tjs$A4^Gthhvt&Cch@^jo*Dg zP4G73cFHv1g{6gEk%7 zELP6^l#3lI*DF*4psBL3_92M}72Cb5z4THQM|9{&s{x`4j^6v~+3oe#| zIhp2}1Un)rR)0?W=xp#K?ltHwp{)ShE2JveZm@#&m)Rb@2$S);g-$J;a+1*pqxZRG zpkuL#GPET^C>Y96$JVjn)JnUvEtVzpLLg&M@E8!%9iG`vX-ob6&$g$1Z}v{RGrRX7vAl@?qe}HT#WP5P{WzC&Z4&=8rj!Ubu#a&n{4m1OJcS9 zXRRYzuwkeq^Btb%D_XU!tLA^V`@-Cx)F=Ip+wO|##7eTg+Q}AjFY%|w?&z&pqb8+T z8u2w!)yBt3#v7&|#x?27@sw%k__Z{5a4!_TFq-Z0^$qS_)>Dh5 zQuENqkBZ*G83Q4Yv=NC?;qt=^;C@qCAPU^IdG`W1N8BlNvV#yl8?Y<7OH-LQNJIiM zO18Me`#V3xOMlLft129m2G2WH)3@#t?6I;FmrEg@&SXK~zQ4K;BdZ4W zKgo|Mt;!3UE)0F(`lAQ1R>f^FGDCU&DpB9kL$4I;2gkMzxBsZJY#mZh6>E1#-9IOo zn`~$Dhd$WsHEMdNr<-Nz%LV&wuU@@U@XmeSse8AvLKjpyO-1G04huAYe5}KZJu2V6 zc&@euy3;RUw4=7A$g~w~MM$Skt3MGKN5QI)RQ(~X)>5a1uv4YVs;GdhA-0ZM zp_q1DXjP1gTiIe{lf4x&trWr{VG|H2J7EnWBqV*$bAvPUdq1D|{X<2CKhC$PXc{dMPKYAUd;}zlNLX(GXwm?;^66)wOOMaP~X3_|obQ&t|XIZj&w_+ws@hqDH!u zV_<_CwPq$A?NQ$ZdZhfcp{!ro&|~pdHfQXdm(d&eUt}ehhHZW*`cdd|zSGyXSo*BbCktDbd6y(N>7<8IVYNj)B0L9R zGODwTN_aH%==GtaoQVm`uNxGxLzuD%K59Fx4N(5wdoim*INo=7iqcz*dNNA_qV7#d z9wz(tyMq67WxpBqdWFaK6G;KzJVnis(iSERoDwRxT`j7u7GTe^g|dpc0L`H+q_eF= z3Dm6=Lly!hlSP32>}EuQmh62b{VZQ|s31?XE5suz#4w$266NQm*@d@?k#5VzlTR*a z3EnD$Aidwtm9sx9#Ln!~W_9(+N9f_RhP$S9Z0mF(whD4+9oUw z6Ki<)G;X;tR;<I{c0NT0aO@NSo*mRu4sU*a3 z)EI?bIr6j*J6T>tDaIf01j-(@y!}e-dj}Gjy-dc}Yv==ILMcfaq7{A4#H5r#yk@v3K1!pe(l^Xh2=j`nrsGt7%Qc)pDm#< zkGnGd*Zgh_t4xeb0AW zZG;kKDI)rOFt0OYfuO>n%<#GJWh(hROfT?pO+Lm<0yA}h#H{OyJ2#JdpqK|^Vcy0q zhAysPH#Ph3HN;j{nsFFi?#2H3_lvRNAMkJC--03sQ!Tt(lfwxrEzRPi_5I2V7J5)8sL z2X^=nXCRye@)Dd2^v(+pX)ww{ouLtgQ-Z5lfacqQWnw(}h3cVAm_S8QkBDszTz%@( z%9JodxLI;09}hv0Pcuw};?Se)p%#1hF;S=QvDpQoS-ir&-S-Rd>Kw)6@z^p&5s0Y% zjg_!$*hc}AXqo=69+fZOm$-TJXfDzYK%;Pi2#(s!Mj=ldvj13_7fHGrEs2o@v4!b zbH2}Egm{i-ybcGeYSPOD`y>=9c^+mwSfwE8eHU3?u4KCCkI&R1Q>ag_K%HJ1Ac z)!=9vUh=)$NbNR~`4&T~(c*)u5i(bPOS&IrtW@W$A%5`KE0~d4nBKA%xi{KP8N6@W zVN{c}a(KmOD9tY#ObK4Xg?K1x$wS#7zGBHKmw_cgn|4ojZ=rF@!-K&%$0D^!0MM;K z(UHyN+-QejqC$k2&FGet1&HHh)*yXUsgjUEVesLFXHj8U+YYjfL&G8xS5L%Zdl(dc zP8HIW@u`k(?;VfcI6h$X?xfBt{+e*+DEM5_RnF%&uer6yy?eh#hmFTZ7?>?e(Y?uk z*uC?9jdb?m@JMDlnB=bxsUB#YH0)Ecy0f8Z9|aAtKYfk&Wwaw2l!eAuVlT`A`&CyR zf!#1r0p!RahyQl&Gto;M0QS&7MmLWt+x1M%O_=MyU=0SKKoyjv0LL*DsEMAYqcHL3 z?*o9q;6oDE6aV3>U4&=+69%h7EaHqx92V(le(G}-3HUL^IMv74)!2GZ18PCe!4|A7 zX3n3+D;)bEbB(W=M-Uof2{>6~&lw1U6#;r& zlx$bE46y^mrA+x8TfesD@grg*@Z1%!c0hfFeNI>pUlLfU)y_?9Tbx5<;r1py#>0Tr zUPcyj_6=w!79_wflv(tQ9Y5aF(E$F5|CT=81MCA`V8GCF;Uo;-mA z8-yUXZU>Q50CmqX&xAoOeV?4cU6_=>7?ng|tG|APf9U)rulA)xfL&=%o?lE z#eWTA_k;$993NA-bRMCH%D^BMUp+xjU3AZfE*rjt?*V$lf6<)z^E3fw4j+^vIRSLH z13i+DEE$)A(=`DpUW<@Iq7_Io=7NnIqn6!EFyw_T$#)Ot)}iaOS*5?}f1;yo!D}<8 zWDZifWv3~Kabl(g?7g31V_HEykl=8RM5F$d;Jn zxUl8X?*gf6!OOXpnRGJ@9B&q+Z&b zM(bEuwqx#_1k-$ZH8E%ZMe;EjDKtI-1W&)_XdQ?Np!T8Nc@;vI@tO229L~~nlM3XR= zE&*R`#Xu90*o?ZtvB&M|%TrkESP|iK`YLcZSe5!0V$O-REhCqsZ32@>V3hQUlO#>S zU*@(p(x<@Y;6L&O=IgoTNaynZ{-Oi=w^PRR6G{69rLWSrf^%C1rf?va#zLem{2>|c zR~k{o+fs1L1X^Y64Y!z+i_TO(E{Ow{Ib-I5Y%DrQ^FZRlMFK{k4GhbLoLJbcfSbip z6N@!iK@CQ%2a;*bF)88yW(!h|O!N0(prJJDD)y$_io?_AdwczJ%Y~!-t$|5)cWPh<4#eug#A)cpkaz?&VfAFwS`ZM(Y6DtW~n#T|Q&?$pgzwM=`BS|sNZt$$&a#J758inJdy5vuy zvLM;lg!3QRsv5WSt-L1oR9?KBQy(wczcFLpQQOaLLN~o!Q^%>dKFF@n2|~T!I8Yt3 zqqGNe(pr(uzN~K||JP-blkv@RcNA&+e^i=TE-SmyX1qAScdxW%?Ur7L#Va0i1}@(A zfMo)%6C9_rsw>@08SSxCnzGO#8M%o?NSlue`S>L7X3A8bIwFaoK0ThfdwBBB)I%}8 z;zPL0q5cW~OUdr1QSbgXgsu1Zn8;TZd!xzS)3AJ>CNXGo^q;2u)}^f0$V6qF!sMw> zRGa;#%KaEksVli`E$Q3Xec_#A`;RfV|Mjd%^mW_sV3*{8C;QIatd&2zy4oevD1U|j zK%tBL;q<1Bf07=K7Up`tJTSZAreUx^+!i?{+Ud2Upli?Fu?L;Y4+adiIG(@$)aSdZ zs^|r`1X;n$GAqvHUTgL8MB$Rs7AyKjzWsdX_a&KYeO!W-Lz_Kghg$j`rW>z$*k zFU>oyV=oN8S7%_?@kQ3UPLSXu>y?JuuLJ(B%73Tn_qc5+%SntZ z6sl%L8;Zu?nd*F< zj@EqB4LLkG*bg*ZQ?6Bt?lL}T{ER?8`4|dfR3GhpqFx_!d$>(<#@c?Z;Hg*O+rP(0 zgmszxP=~PzcnLso)BA_PpNf5XW8D?3&+V3O;Pgj4iOPPowEeFz3zPF)y}$3|+zW|o zh_klxU)mowecl)QGe9Hz*GM>im}8*cAcH0MjgQ*wzZxXYsNZeKj7b;63x8GWd$ZyeLBnuW$aIqHy2Ke5-!+8jO}== zfs$YMB;VjWoXtNBbZLqs4O2Io-@EMjbN6c_ahJ#rWzjUEzXc{y6S8*QwARcSJyCOi z)yZnf!;A0+EB#p~aKf;LjLFIq#Ks0${3UhtP$usaaaUz4+yYM)#1?HHs`Q0vUY_-6 zd~<|ys09cZkgq9NIP}UuIU)k3*d(tuqsJeBvE`oq{F^0l1&f67=kd1#BWH$@b z-8=X<_bUqFct~`pv}wfhVoeJMVM?dmByGqQEb6(&f|}(Eh*vfM8UwGJR^wqh5dH6KyczV*HcOfXP|69DVD`>Jr@rkRU5T< za(e?(2R`aw&&gv;0-aiMw(!)DC&3GWf72xq05QxPhx*m%Fbl`MHPhJGr~j)V-AafD zBHXd_jqdYvnL4mcT{qaS2EHoIWHeSRj59E`nj2OMxP|0Fvh%2tgZ1EJjziL#4jOL{ zYXHqvkEQ1YU_Bzy(T04hfC}Ok>rXA;n%t%HWGEbGXii;HK%RYpAw*iPM%mYBoLQHf*9U$3`t$lZno&Nr2w$F=yB72JQDV#>m37qQ2H99FT_L^6# z=Tq!67Lj%;H~`;=M+D%prHHdb9Epo=&B1sNn664uKCcj#31G7H%y3KTTK0wDU?d?pPmWq^G@jX)F zf3hHkqd-3mM_9{16Sz~5m~9EaKDqS{3bxf106e-A2fLQQFcR z3U%bNA+4xJ=olqiDAU^jrLb+q}jdbW#Rm}kpI2EhlNG9SZ0kztU5X)cIVI9AHALasVnyn160jft~x8jOOKAD zZ<4K(6lz|=?vNm@i{JywEfW#oZQL^1sg#NbBK138))lGF;eampAx41S_mH%5y1R!X zp2RsI`gp8V7$4yIWfT84FL@XY@A#0voARP$_WsLF$c$9G9GP~Fl*O-lvQe`^y)Pp3 z%(|`BlRDp|q!_32w!OHwhjG!zPCoT@#}Lm^mT`QAGAlUw)5~hVPPSO-@Q?PA70Ufs zE}dqo+&gZ~_{6E84tHmZjK>`NV(Wdz$Ark_9ls#tK}CPQ{v4TA?oe zq7P;sPzRAvw67b}O#B?k%qHV+pmYQob2H_5VkXuLGXDmu0niu_O8OmxvN7PnI=}?B zGuZpr^Hi1-MM3cCJe_JN6ZFpXgo_v{04(3|&9F+O+TT#`l|}@GlWr%j(Z2-|bWd@5 zTH(1(t_zG}M+DbA^aPi8UQu$NFw+D*%nCGM_zgfzLMKkt+^qsZG28A9urQym=$O&o zyiWFo?=Qkzz+eEKVLe2lmZ;to@;`@zjw+h4AYGmxAFu@0*qjOhQlNf1vQ_%`1aA9x zua$=7X&i_~O2~zunH_PaTtuQK2&kTi^el*E-=@jucHQifol2?{_H^nD2=1D@e`X3n7)hAny$$ds>ORO`UFV{hYv z;D*Oc6re0M_yM>_(Y1#s0tnAXkLkSZ#1`Z`kn2I0Wf&h%PzVwctU;dvjX&yXRPZ;j zBvPQSI$Nwg1>lU6_;0zqW8Z8M7NPZL=-%Rj!@ve-h0-N}`Mukr6Chiq0NBn@tJVS1 z2`mJ}Z^C4d;jQa+jf@_kzJx*fy%R$aUO@Vx(5iR4$lt^HW+@WZFkB&Q0pZ9;Gh);e zw#GKCA;ixxTwqM*3`yWiNsoGrnuL@pfzzZ@{n%tMil#O(fPWOgUZore7yAw#F)-{i z#<45VqlX_Pge8D|O6iPtJ)|>3mE(932;uq(Lt?0Z0P;Z!9Kt@~n?$RPh&l5`#V!!Q zrzyBs@u8vO6A)tv?H&0-&ZCQjF^CsngpYGuOYegnO(|6$!YR>rt|8`_i=L!!hvNW4 zgpi8B6fPW48qv%GllZWhraHqBP5qpcgLSPQva+xh8sk4B{T> zG~n?e7b-f{pwM6f63mbggq5GWLmDfL@9lOpji`n}C2j(sY9xJ#V3epI`OuSFG#mu6 z5|fm;_*gnPLRdE7MiwYmW9X|IJ}h_|`100%<|ZcbCHadMh2*$nVM_y zUoR^dV#)e5a2L!K-wJ6eGbqC>2Id4&tm^APcXUhA-3OqLy+c2p0F44NK|u_yA-Mvx zeK0f&J{xwkVwwxVx+Z!^zul0M0XhTRxQsGuPEBKgd-VC4f)?pNC+7c6UFiP~na&-& z8<7rk-+;UKIf08i6k~9K{=6YPi15I|eT3VEmePDZsff=Y*a3pl0H6U;95~f*mIN_~ zktnb?_?QVuO46g<_8B`LEt_5WwDWIp)u%Ql?fmmXGRu_v|0 z@;9=|QFd>uIcO;z<hU_3mhIw^8$FV>jf+?SD%fW>x!c<)$l{+RUI`-{H6q<|4zgYe-j`zIW7#NQ3w9 zl@`G2O!v;Rlv0YEYj(3}H`}(YGo`66F5+-(>@W3>!ExdDYVDnV9GL#6_Da{YvEX^3 zoX)?$HX}}wX6kZ>{v*&gHb)7?-4IN_gr+i;@Yk5VsFg?((Q~G*qvrk8S#86(O zDc9pKIT}{Wy1;r_$@0I~KAjn^TvETw!X+|Q;}Xj;+CNxiSYn}S^Fj*L+wCW7Gcu+& zAC6XW*Oi-FuC;IV{p-+I<=a{JE39q_yQtXWuMO)pEX5^$whwg6Wp4x@^;l3cX282$S2TJ`t`!9z0GR*yddWA| zcV>!R-~8+NqC0^bE=DHFGtq(SWd*x!%5RO^5TVVF;lDgkx}?8+hr8sH;FH?Bxs@HK z*nhe|?s%eirRI}k@wW0-?YF9r)|a)GWE6fVD*6@J3B1H9M; z(}wTHoE|M>owfo+AP$w6N|hXyBCF%!(W9<@^7QtC=}r8$wS2Dlw?NLs5|Zm(s&M@x zD#)yr18T#p==}qkqvBt(QUi+W*^gFj9Oo9U9790h@we2jNjE#1R2g#X`r*BX@38LA z7Y!ag5Uu3j^Xb0mf9dO7Nz`$!#yrn4tnor}Vcw4i!*qfJo*RztpJ)V7gDZHU$<46> z(0t~1xFLU2CKL#}J-29+563F)f9G8(-ah*CwE8_nAEx(fV?TW5`fo3H&T?-U?)0&I zP(9I+xuT_V?WzgmSKWm(i&2VIaqDsgc`dS2Ux#3>=lo$@B#mi_?}feG2KTA&4}5%` ze8*g=&S;jMJX_J&6dY&$Xdx#_!Ot5!?^BOXD4sR@OYX2{kVgy?M6H)9rTX|*K#QFk zgVH>>;v;OV#vAqWGlwvBZgT6vvc9eYv6LKn7Ff&bx(V; zfd6MaU5CH zNf$RHvkg@N@j%6vfQ^Wg=Z&}nPkt(QLxVKf>R~^S)IT}8cYF^rAS#RRs^a=(1aJ;w zmHkv$(;%{yz7})=cw;m)*mG&}Db2!;ZRDrFuEj0WnNu{zpM8tpzahVaK$S&g1hU#U zz>8)oAgaAu<5qKEIACn5=L`%t%vz0S{{&lCb@d4d*A*U;23#2KVcqfQ;7c}Ğ~ zI649(m}0pq?&I6ZgDspAuI?8+(pj=1BVQ-YKL566?U}mb?B?>lXV^3lB3$9hK^fyQ z=Ms<+xZa%OZnV;=RLTj|IEDmna+q!r>&`l+gamTPl>IYVU&`q<-YPccGdLOXQZJn- z6|?c8*(g--ji1{l9RUuV(8z7YH`)VYB+bKouwE`t^6W=5cy42wqTetpf$h55A)wX6{7aUCW;{PyzI5+yY4b7k2nJ?Chox|tV zpz`wtk?=4j?a^6u<+Y3C4bnjw{|oeqZpZX(9A-rD!WMRGt1kWysXmhjr2x@wREF~F zsN97w#Qc)B9uuMRH|-TnzQf7udGC+39gM*p;BpfjHT%}{7*`ijpza66jZn*tEV`qg zOP?`^h^h8thDFRGDlM;K$P8K?_XHVW~LKuD-EEoLMr zHcp%lB_&`1Sq(_xHSw(_s)B1%xH=O(|~{K$}o)Why*~Vgh7} z)-J=tXE(z{AO+!>Q{C}zYQ{squ_@lcE_a*qqxntDW5cw7AAm)gAZ-}65#dF<)i`yS zG6x}n431;Q!@MJtmn_XLVd-l=FU|!Oh4U7q5d10bb8+qLqQjk@t!9^E6DIMT-zucb z6o&Lk{Uw3WJdP*OZo>2@ve1F~ul8ADYdB9D^IDv)a~{)p=NN-U>d@xFIdgs(sdG@m z!twL*Tn$dphUw-)0n2NMSsb<%T_ad;Q~j%iuZ(1pO0M~3YFKTOZP~^L3|uO~`T(JJ zyVDi@$m8EYT*OhN!6IPhNT%Uq4BmuVm$rE?x}qO}M|w-eO<~g@&S30nOf}=+$_~=@ zVX}7w_6B>>aJvUz0K!GKI#r@B39DXU*GWy2SqEVB_^Og69|e|-0mV{HE*{nj-ZxM| z79PKhXv6@uc~3)$5)4u3c_9)nI6ajn!f0q7(+sS2PxjWCNP=@VG-niL`L|PcTCK?FD@|=ciOwR zHQL;0=Vgr)CjIqNWt#uSA&VDfOv+VX7rZR_2a5~yL474B%#?NZ&GNi>!To~c$tRVA z$0ffx=ULeb^2jgfGY=nE{#T&jpSAK82tZ0VsfqhxZ^E1~n-N05-cV)ijbb3ck!hzdu$RDErGZN;61k9J{e-sO(nYzhBn%kbE4XC$X+7Mgx*a6)A zMaB_<#wZNj^b2^4vXX^Hpx@~GuLJetobI3>fNF)55T7hK1K`-L6S&X>5VJL0KenEF zoZ4(T+yN-@!8BrTc0W-N@0&6-|f^=?^CKZUN&M8&rbN1d9xStpw|AC>%GT8+?b}0XzZ& zeaN&Gz1MJtq+zln@kj+=BDP>o!3}^;@D0TYfgnAnumnkfZTf~M^AF6?GmTMzqB*2K z$8=zd2tNk2i4%yE1$bwpUZl0J`4a(b$1Ox9^nZ4aGBfYs`p_%^L%l2@2np(j(XXuH z&qU7I-_zhY0WZ)p2859yGU3;sVj?e{nsCBm#ufi94j={tY8X!Z`8lBr0cJQ7NRnso zW4E#X49AAUB8r0`75+xQI@IQkO`-f{6cu8C_fC}yCw~v19fg1-ouI{s=yyu=AeNN& zUZ7EWv+H%FIlu=6l}qO>woI)<)&XorP#@D=m2xI^*79&r=$$`?IQyX)bwkn`d zT2zQtCism^oSHk(x_}zfflhCr(fwgQO0$7lUK z#eyZfeU9Nbe9r#QKB1ZlGy~&ioU@zZASmmD8D(MQ75I3rh@pi~1buoe9%vxwl)U2iS!PU>4w>l<(fsTC2BD z!1~cAXR>f^R^i?!c?#JN7=R=J<$`M<6y(*+V|X(L1w;KYb5vst=?5ZsZg=Vx9QZ(s z@VSBa;Z%!R3lPx9j1^!KF&hQO%5W1*J!!b&5%VTQ&%w{}WyYlhkE3#OgJAqX9K%wu z$_H>>7!d|BA9I2iya0_W;Rxyakd$Wbqmp+7PLsc0BcXp4+NnU|;4IUbW`2=K#bEa~ zJR_f5-x420@O}>bJHp|@vH@&`ez*tc@W1#7=!$T1HsSsM`=6gnEcAK)He5M24>G`RZ^}EOz_|2Kuq1G0S`x^I1|8D?RLJj*V9Gf+eh(@e)SB6|Y@g zvh*CEGz$MbEwzwbFK4xE7wFzhA%vR{W^Ws#*{(K`~>B5PkMA1xC z7OUdIQML`M>4I>8wXeAMcx~l&)uM-+W%HEA+2guok7aq^!nnG%Ibgc!e!+6tE=*`D zMe@Riu^8V0{-4)GT>4oVdt}q69yw=*eG4zA7C1fhu+F~`B>Wr(T}dmcy|Y#1sG1&s z{ef=vxp#i-ZOpQ)&g9hSq)MBa=pP>{EiE5*jk|5`GD%9Z;z&eoVfG%m|*$Pz?%eqjBJR=G*G^lN_i z6J9XqL(NeE$h=go#c|t0E= z#-SoN?2626FUd+}R++jMj=Zjf^&m58aqMqA;qX)+>eP!j%bLpSf@{_cWAi^S9FOvj zWjnCvOi1Mc%XuHuqHX zP<3%=s+(%G%TCw@KW|~SyY@=SWhc#b&ZY;Q7prvQ8t3NJ^1PSAlp0G`c55SCFYA=X zul)?N1l6(M&uQ&%jP+bOb9aI7is9~rx@Bo?NjYO3BVYY?*k);x@|TLT!q&qVa|{~f z?`1{X*k*b@Y&6|))E9FJqeI^D$u|vs6?gw% zom117aFB3Xv_5&{mBg}qUK+1seAKHf;im?LDDO+y#PKp4il`|p3@PgQ!K`r&ib!RHbn241uj}My!(A)n7xzgL z5YO=)>UV33UjIv?uVS_RO0-m8VzsOZ5rG2oXsQc`pgyf{<8N>=O&q<~3t(0syv2;?g6k|?pT`=n(4bWvD+Owl%2p%0#=TvjExpGTm-VpJFkfL`A1{~#J_v5qLg0) z8GkvyNm+KVBrB}U=xc=MNyQ$mjfZiBmA?6VK~NdHqf5G13BiDJv&Z7N4+Rnae#$h- z`2G;Ne)hFD+TEIKkfnB&Dh@G9`J?ReeRw@W zpngA_0_dL?@+C>!OhZIPOamc3Hcs_i zn28d}4udF|*eGnuh^^$z!0Iy9s6c-?RO1<#&EQum*;UG*^-PA~Kz>L6l&@-i0iyrKcty%9A*9ec7f zVxPrl#ENQjHa9hj!$P)6~0e)8iCE z3W&s{QwtaH+}wnUq1B94T}D&_s=D|#cLfb(-{oD%9+?t|8xkZC&Htyfk5eo>fzb2g zm$AMuGpnqXRchZ&F9XZBr5X-^zA=RHBkCC2hw)T-8QyxTW}dP`4030P#g(xo9!n!H z*r7@8rY>s=wFLO7q!YFw%zB;spveWgI;nt^=f_?xkD&WsIj3DEWCEF0nf+(j6 zKb|hONF+6f8#80OOiF7{aY3>LA4fT~?l3Bwo^_YeP#=;vb`03=t($~;%{~ZD1X}wi zID9r-7<@2{k86E&jitA7uU?};$pR`3f-}oxP5oXSMDSyHcuRf~y2IC0^Ma6pO z`ZOhlL}KYxnvtXH;xLZRULN0}m%3ONx**&V=a)Z_=K}G{zhIz>uEk)QQaII(b9TxN zI$HL!X~W0TBD{3jqdWlN&FNroDQyHl_nG-Z9*7#$CabPbOzQXnjJAOaEl^?1cxhS( z8kUz_QQ8lI`)HwB)O8h{FNr{;@$&ZUGF6pe`_a@YJoNb{AqrYXe?@=|{Sj@jYbzUKcWSb>guC^*My!6NLsOWs9UH9hydq z59x6TRuR*Sx5;7n6x5Cr8LoQw38h0`p8XKSiCXDpjvblQ zy(>I;D{AKS^rCW0Ae9#nl!Ty6J_30mw$*2w`Rb#^g?U+sL^<~`gaO}FAZH!_o+@t#dsz1t_-WFWMnQxH#-u;cMaKs()C;d^!e9UD0Qi{O24nBzrD^11XXefB;ZK)PsI;5+GSK zbb0=nkdZr-B|=Qz18T1VFHo!8>yE+P=cE}C!aYn@9!MPg7sHN>(wh+|!w)T)^V}rR z!)$*LQs^-i&9_4(4~l?+G~}-zq$hgFSh0ruCNXHhB;YW>cr|_M7Z{=lM*{ekgLm}i zcWV+jV-|?r!QF+)1iU%2Y`C7W<;z@Vki%aVg>p~BoY1#-kqp01TCu=Srq!k z_{$D4eVVGC2BZ*}_!0=G{TkCYbZKhoYXgGIuFMn-Vg$fm`3C5JeonN6Pey1KIR4jf zXn7dU3218t>_r9~ngy6m9x-17k>q zBf~M^BO8dLDer`Vg!3!5;MmX^;(%S{&wh`Q{=ImdfstXX4_q0CQ|glkA_pQ=WEDil zt&HVKIKmL{nR_rvHizFZh;|MOn>8`1WYKT~DxQx9oMtsmx^wdsguV5=1N0Fw5Q;IW zDMT(Y;FCX4O?Qm`5|O8aJ|L8MAAM^a;(r3_Qr(p>g3-DY2*_qaM*%x&D`l`fLw3;c zzven6(=ZN>dXorzKU6Mukr*=3Q&do6K1+fJ^aXgp1&=I7SgLocgjMP%WS; zz$L+&ga#_`xtB1BobV*355$;2;hS;vSR2952WOb2!(_kESICuwa}xQb(s>Ive{TAL zn2_hMNk$UHNa)E78sVWQ(gStE{@+=*^oW@wR+zhsgbTQBo^KgT zc9VJJzsU;RypT!Im4`gzzb7L8zwkWs@3exkM9^bFERV>Q*p)Pe0raja&1^uLf`-!= z;sLA?&IHIyX4(dB9}8|W7yCA*An`$9_CT?`fy!k#+hBnl7?#+P;V|GjEkHd6MUNZ& zQqyZ0BUC7hn2*iu$&5(i`QI1&bxDv^q06m|iXTq-7z6j?i8==>WJuNB=Nq@X^MjB$ zS>5a3U%XM*>#u6B)=kK|O;S3wKEDlo>+933!_qjPI_}FsHN34Pi}UlF#bae!d)~cL z?yCjrci;3~=r(TD*MvJX$$#K|Gv9o7L3ZoY+Hpfz=_q&H1>OH*L12-+Pb3GQVl%4-gb&4z2Ro)lNLdst5$S&6SBpZ~l2A0}|SAVmk zi2I)LpUIt}OTz($C44OE9FnR&Tx45@IzPDEO#9j}?&#ZU^H1u~l#V!zhl|4E8=q5K zf&&HFXPqF2CDIyAAGH(4FLWoiKK)6-KY8@}=(eR7{B;gP_X`(}v^>yY@*R*nLK4R= zT(JMNCR2A^`5=Mev{DA z5J8L8lx5|m1exk==Vr4}BVK0KH+>e4Tb?~dXeHerD)7)UaX7qgx zCrG+b;*>Y=rDyxnBF%x@t-Ib9Z+g4oW>2tZyL(IKBP;tE^a5K76!~`fZ?*()uRZqO zdW)lPfAz1zy1gs`<{Gyzc^R_>|Hpz;!UmP)g>OURrfDG|6AOy+DGDR zUT%xgU-B{Byi=e)n$hu^r@QLZuFF>KTNGJdJ4#P&fAdb)NL+W9)JTx)#jh;!YXg|R zP_nJdRNxmIp=1Hz+~BezQ5jxu?1^so^bZ?7bpO(2t?*=3?{RGbD&8kiCt-GuiF%Ho z#xd!oQM-%1(c>Y!A5nQh^&$0==T7-ZdeoZ>d6-IQ8v+=c4M-9V-MY4EF zgo{F`y6jzx*0aJh0k5?P8d`Z{VV}0}Q_mEw5v2Z`IGAK8(%uu~`SQ}`Ee+GkqhLBX z5sipTc>tMZ*63*NrB{>IalQa=;QO^G>5Ksq2bir8?B3<5lrV3)O|sgJhA7i1CRE z)mjd3YKU1hy8?sE`K9+tHIC>Q_W~Z}eyMmbWXJZXV)0mms3{~QSXi8m+Ep;LdeDB` zGtKC5nOr&(nFS?9$e;pQY7NkCka|OeG}=bQ}V`Sev|Ly5ZZ=4 z;~Gb05gi zH{6;jjh7P{t%YGNt}x>4_uTD;WumgY3U)C~NTnrFK}IZ?sdUU4^dHigZqztyV;Z+~ zZmxvihN4AU2X<4(#)Sf((d6CY273r4fFungbkRm-i<9}%%GuZ^zNCcJ$~iFV<)1X= z$9ZVyr?Uq!g(CkU=W}|(DH78m&4f2*IVDq67v7a*U+BjSo-i8jc}Ll~LaLPu1=q_L zwZ`MK458D5dRoxJe6YqccrsEK^w=K;9*fUX%;!{w22rE=rJ2ZJu$S>}3e~58m$SdMF4y*O znZOaiSdL0X6(e^`O+X+Aj+%g~6H8`SWv82-LTt#vCG*%Q-B`X~-&qS!EJl)me>H(A zkI~*Cbl6-*Vf77RhA!>|mOc<^8}`cUvUvQEMcJ`TV+5?hG2;#NPF+AbuctZ!giYn> z4*iq`JBN4I9gZF#=<;ktR?&DsgO!NPfP19w3g~}VL`7aFyPlpHg4J-=8@0UplL$F` zgrA)sq$3Fem$iNql5Q-q;HS5^g zp{>?Rd48^B(BVGihm-6Qn&%sfsz1)5 zvrL}!=`a`+NDSSv!9CcJ#!FD<*stu`IchX0JV>`lVvuy)|8!{ahiQCi2|_*?D5fuW z0ptp4ay0|<*FY(lQ1ahF=nS1m0Rtme>kDOcgM zeH$8{`wcu`E*e^n=G{7uNRsqztyuEOV976 zP>uy$eFX@M3m7XD8p~r@s_pD@rLiJmaTp7D1fI?jdyl=$>7IZgXadY!l;&4{RLb|O zkFCRnLz$v{l-}+tNwirLYU`|Ew& zaKRU-p@iX&c4&;kEsnmGMW#PF>jjGH9|Cgg3y5b+NSg+EoRL0U6$H6e{jOUxU z$+};Xg&|`HB-(iH4ObOUdj4TZz;66su8@WkpkTeo^o@%%aQ~o@|H;BrbIJt9BUk@u z22h7vj6TgU#SOxJD)rxA#G34mVAvcg)2?DL=L%Y+G(do*S4a(K3Y3o?CxkI7ePy({SUzn)%;BnSMC4dxu_{@eL(x4;l|>kLIc{ONT|Wnl5*m1_&EK9l~=wskbsn70Dku z`zRd8#_U2p6QS3Li`0m=d z+yutX6MLOLS7$DbQym}5>5l-YcahZS{xf4r)&rI$f04@L0nIo-|EwV zhrrX~_G6e1V($rh1jnTmy>^CPh#`z<(-FZ&Sz9(#4bJ?GiYQ{(e^NIZ+o$nW=KjF8 zf<3t81w!Aoz^{uK%M>sS40*)dK@j8D0G~3K}`E)Lpx)Thxi9$f`DdBodSN(Uutsazv=m6gZz>RyBiWjzo zC&z7qc?TzqY2tX-(<=b&>tEqxN=w3*bFw8d7}x6=F7zNE>kQ%`@FSl=Q-r^O6?6jK z`21@eZVys2^euVf;)oZjCDQHr>oepXZe5X_a?m8o=q8A@_jC z|1g*b4vo=4Fvms5#{6Qa7n;|=^8fcgKL1bHPQZwyYXx2t#N?FrxqB|NhDF z5_2!UVClKJcf;({Zhtrz?M-iSuia9EKQRzNJjR?83Z6mM3akrzz@f%W(+J|F7&Dh8 z2WcH)2SD#I94`jlrJhN}Mh2+}@g}%2m(bCh;z&V5Kq|!OrRdhjzTr>OivK#qx!sW& z1-u#@maO^}+tH)@2P}1eRbR0&p5<}<#$O)uY4dt{0%~U{M+Zwk{X6HQPU-aDC67!^ zkGUs28k_29QgFmr*=-@SZaGVZ{v}eiLzSN@QoBwx8^@LI-N2qS)cwv?{nN4UcortI zx>~gs1%DX4Y}oZy$%c-A%s&jw+F$QX@)BsmvtB?hd4JA|=E|}*_b1-&Zr)G5eA@Xp z48QYiUtE`wWjp^T?z-q`9Hf%-xG>n;r}D!wX{$i3N&HCgxbV+=&*pwtK9ql=bfe+x zdo+n4G6ZRL;|qkdyB2b~yRwV_C(tjt+iI(dmBYJOonQRvjds)HZbrXv;dsf6y?52` zzE^PG)!w)TF=C&N{>Q4i*h%xlfbQk&Y*Yw>#|q<@q7I?oy@!e={*Foh0*|J) z;4(jj$oaC5xy|rblW&Ur49A2&_&c|Y?XEljEBX$q@yS*!gWEu8TUe;DjOP*;#h*QPQh z9BF5z*-#MgD#`jP%p}4uOI)}9faUIA-`HT=6y?JiPXEL>NK&&VW7MjwZv4>Z#%({g z@V?um>ngF|@ZFgEnZIx%F5Vg$8EhTSi#!x>`@!=5FL>EK$v5 zQA+iK3%tSD{Y(A++PFpV17{Yn{eNdISL~=;6Ov{0Ex4n?;`N4=TwP<1%-P|HCFdUv z#p}9S&-&fKO9kX9ZXfJDd?9bUW8bMK!5$|>kidvuRs9gi>#T}gp>xnR`;?`r3Kdb0 zI<^*@2VWU~bguO6u?DnQVh8f~x0gC^dY}#dg7>axLKOGMUb_pzardXZghxppzo|k$ z%P%dtd%gbQg-z__!&yE}TF^7TR+bILD?2;>6jk2oQg*uH%lrAQm$qTwG8q>(3zo#X zU9Q>_67Bre8ij)eHY?oar%zOo>cUY`)#lM1F4?ba{~z116}nep78g6zp_(pCP2B962pf`tk$*j^EO%XI18S)w z_b?)Gd7VYR`N4Rxo2Sc-_PPbNNVM-UNdg!k>KK=MF3X>NyG7DGRkakv*{Sg|2(CAX zJVF0SDoUb9x7{{6`WSk*rYYyUvWi=ReW_ZZ*S%6a0v=0qipl@Y+NP|#rGpvqxa6=% zq_uC{*nb%yx6KdXxf0g+lvl~kj#jG!RsO2{hsoq>VUj!>+Wp(nb;UH_c_$ zH=FypDm;B&<1Nw{Yu|0@DlS8RJx>WYGz;{+Znh5nfV=nNIL3FNabIr+@uomzMnF6B^g&}|~oC&%6)Nm%ag$0n6?a;L)^o@+NB^B~;EEu_$wr5>!Ick`^ zMa}U)dtgJsN78PV#go`XL1 zn3iZIK7bkib>(~=5hG^0^j@Wz8L^*)Yu&O!*G9lEazxkWGJ|>!Dyzl9^tS%x6K!%Zfx-`<1|NbHE`)nGH^dSzj|qg? znD{00geaM`ND76I4k7w&x%ac;Mt zWrUeVK*>SFlC{*R$>Y?pe?=(g)mJnatbm>$|5j)$ZqlCB+KESo`#rxDy6EsQ{@CRj z#sJ?)9X`sQlI6X?`orLzN%V`Rn~`Rjqjz6G+Qyz+3*i<-s3%B5ip4pfvv4pacrKS# z>MkM90#@0W$Mt_jWz1nREx=mvYt9A#J5_Ixlrd}1v8v9b^I{833E#r2K z|BY_z{`c0dG>*ih_hVb~>a0k?YWBw&xw&Yq4?6fvZPm|=jO$+t(518{?1H{aCHhI|3EKu{K9A|I5P5_F^XE;N;!T-^ZX?HR*aZH8a zrmJ7?h*r( z72+;)xZM<@$`ly8YqPa~;=Z?x>?C&Nr$Z_rKmu=BPcO0vZt+`l0|M>*HUH?A=hlRE z?SjPzI|rGhYI8UY!Xj7?7#1HwDobrZT`2559zVh;kgarJh0e=qR`L*-2Vzq_l!@DoY0H=8B2YD_FwcW?DAw28%%Y=8n)<D-uypZFdJra z7~hggXhaX^Q2^sRAsGOc=s`1>>7>JN4{!+0THN7)T=bvlAk(*`gfH&_Lbf5ut+%QJ zaAj&`{E4QdVm0H%LAvNvG_GqMd%}*n*P;^TkM}sDB_;A$MZ?(F>lt*Rhvj4~sWO`fFT70+nqWAPbgn7BFLq^1cLEeClp$J+CH*yfr3l4~@V7F3StvA;G4BqN9G*?~7 z1B}QA8efM#3LA(1Q~;25JOJg@(+5KdtX^6j&^;I`xPELgcwQgnLBE5jqY!Sb{0>S|7A?Wd1tpTJB zKy?bHtp76{F!gy2Xb)*1FhiIdBM|sBbVm;9I0dx1&#y=!`Fb$OCCI*+V+FjyR0P3& zMH0sZ|a-=;XQ_tg9Vg7L*As5U&JS43XKt8~5NCv4w z)cMj}&>6%&UF|c%a2+xNIa6}So<;9Ne3mO|#bIyNlU|H}0s7~u!$L(5*42`E3P|3t zenjv085328<7D^(TsV3?W_gjq1nbr}0P9vS($`OkP^yUD6A-5bWIU-?@EzdB$QThA z8v|$q%oRr312%o&UU&e~u`mw_K>ax`gSmAWg!r;P;Dr23XeNbWbMVLf5;sO+U;@*E z6s8NlI~u^6Bn|GhSg+KxCn5dPb6~FY$lTE_>yTwFQ3&{dd@7=EmO^g7Ko3zVxgix+ zSKiU;-FlV;&!*tZ^CuGMd4RQ|zwzg&Lgog=Z@3Ma{}N9%f8kAy!E zw15WGuY1I)?A(O4MUExRM?HnF$vlKhn*qM-dEx|-UXq)Xef*rVV<;MNG?O{a!G! zU-%RlDm~0}?1QVz{)rMc&C9JO-{IXq44RGoRW3Id;ORFKTzjxEetz+2=Twul z3DvsC=s~%lIlab{_5JNm4erBy}^REbqa{fSiIKUr=8 z7ktSSipAeyykf{#9{;-v)9}cF3CpFDi@V=7+Fw1n zAo!N#+Npp28|z#7#x{*qSc{Y(Cz+s?}FJUt%SwTG8#akwh{yVJ!#4C+i9GUV%@g>FV) zGAuH%i}T_otEHYAd!<%uS?zdpK*KhQvHtRapMS9OX12x0mUZRpI@j6%{o~$UxxLn( z-zys0{$sFT(7J@`@yrx$@9odmhbiOv-6L03Uq-}6UNhTlqf+Iiz1Y^Ypa;D^gd8RX zw=|zk+??$tIkEndIL#|~yRNHS?vQ?a%(?uM`7&AeaAZDbV1qF5npb~??MbISfx-K3 z27c>}C)G60_bJ`mB{Hilqo#s4Pzole)zn;k+!O!MW}Pyq)GG5s6!u$YxLYRf1{awD@&T&r^W^G$2l z(<2MdMjn{_JYj?TC5)JOx6bI!f6yk#X+5qJ>|&eMa&kGHtBS-Gk5+Bk7#&^^EkCfS zO)Hd&)*|9C;_S6SozcCaAA=q}d%gd#&g5y-t?KHL-VTqP-xD&^9L^+%>3)dSbPsvm zKOc76Fm^CnkT7dLHp#8Gj&Iwi6&}9T!oN8#$Y~6IX8eNN%5s<5HD|TUAuRCljBz#n z@fjE|mEe(GOIv4NeqJs$qU6q?z z!zEiEDJ43GP)mWz8NJWX?wk50D2#mS%H0Z|tjUnf@-P9zdX`vew%Ieuc$njVU$Zan z%DxkMYo^AkRonbrF$U~74Yl=IRkrRm(QAQ6)#h@+Y$-Rxhs%ld_pw`Pp4xwuKOqmSh%SXjSXScq&@RTLN7O0`{uS{L%FXElF*ou10G_GG+5MoVKDKpM z=?cI@#5!~|6@NGuDVH7fEz>MD>1Leao%6Xj0@;6E#yRna!QIi&vhLpE;kEVOcNQ(@ zpDJDioL<85D#2-pz1UP#JIwESTrde;13(})hS<=V@T~Ikjy<}{Y6P%-Z`mYi$QX|Q z)vKiaOvu!zV)Y;bF6D`7eY0+%8FGFNBr*lnm+h`~3-Wyi_yc7Vav$e(UXjyS8TuP> z-rU3KnD79U+XamQe8Xz^cZW$dAGT%6JxDDU-0<-kT6RmgC({=Oj)OV&?yz(pT|V)MdU` zG7Q$CI!yZ+yBSCxHUo(ABTSKwF?>N{%i<0;e#z}1iAB~>r*(eY);!!tD0APbT zoOM&&aMFpD&c79r!34jH@g@IiBL|fhu%$#EzGs?!>Sgk&bMc{sN8Fybyu;H?FGS~R zB3sjM3>cQ|)i~kmtd^3J1hd)7;2_&zPHvlw!#u|SAIjc5s;P608mD^ITdC6C3RqOq zxmKoHMJfVuxaf7(s}Mkj3bhDfPyv|?vE?cjs;RX=#UVH#R)!dvWUd3IS1JLNDF{?3 zGhs{uCM5ms_dU?wZ>?{wZ~gwTR)*x{ocFxv+56eg-t*z=;exjc^XY2~(s-$tptfFr z3kHe-sXeO+ZOngq`Yw+R0=JCwdybZxIiU5G zv3^i0PZZ*uLhx;ts@N?g)B!4&3{p>_#;ueGIEOb5@HWXDe$dMgC72cAi=LeI&tKC# zmX<&h@fex~pMDRn&qXN-t0x81dSXk;gl%k_JAO30H4h>Z({PW$zkzO=MPW@fP+%ZwFX$9#Xr4Z*X)WLw*zv^ zZ^uh%Woyjuy>yTnQ#B+K;hME?F%Ak6y4t85IEn;0sKt)tvbT{sAL-}7Z&C2#%f$@x z0_PLyCx?bs7AN$>>m=GA5Q;>mp^U;vIqpnpF)IL6_#8$xb%BM@QV@+f&N>k1&U``^B0y ztUD0RtVGXUajTKMBiu;KG#vKGked-2>c!cyRSz!_^J!@CgHmxvScY}$2uv3o_Sh)G zES3KAgZwe2?zDT^Io*k{V4rngRby#$igLsnrzz=)0K*^0b8-JWf@uKfKB-F)tBAFy zr($szG$6QEK1gv(gaMDer9-ZpCxim2wW0ipYe81X2QD_4kGq?9xe!{UR$jq(?ea&Z8(y4#fp=Y(?O52|zVjygKn9Xl zOhGo{c+Yvhzy;15;}2Bgxzuhg(b~iArpB26xn`%@Fe<^P^Y8}%piIfXdMhL#bVDPh zvEh_1)+vg1%0j;Ko{owDxB~&yy}A-VPQMolP>2;~mFEpXG%bc189V7{(OGGx_$Q?? z^M~}pNpb)g6#&IxF!^_;+;`+)+v{wl;{a%#GZ=j?vRlPU>KcG5UL;jet211hqA4$S z(ln!;b^HS4?hGSE@E+J9<}I;c{s{~N6*mQWly?BlL@njf%Y*|y{xLPjJqL|VM`FN% za-V{0Y8;7>JNO0?oajkt0xFeuM6-~CPA@y=2ikMjz^RV28qHFme~_MxiHC@cauX6^ zx|zI;_WP;`Mj!@#A#wEN<6IDB0JQYe)5;(m{&X<}P=mWZUXt|n2_%8Mo&5h|R~cz3 zlOsiQo{UA4%ur&yKzoYhek6#ytOhTJE-NF1PlQi^W&&@;%V71^N4?lpJ}ObOfxLM5;fV|8x)8*a~$Swke*%r&PiQoK1iCB$$Lp9s8#A!x}KdKnNb zLuFLB(pU@RLPBJLF2MLObxI&n zi-ey@q>}=K%$UVeJ20Kz^{Icm`czkb6VD%3eH{4|OcS{TD~$wBE|;kgya4qC0q^i* zAh6ProL34GnQM~PNl`HahOs+5IOb8x}?C^wZ?Iw{D2Q7wssMJ1)O~H zWShG0Nc)|fNus);(J0v=skZ#)I%ci{(;#?IP(p>QXtKB%B8fKKnV|$i&(Wm|W|^=a zWXyoBPd!vAk~Yi^;8aj^j#~T==(RubI+Ua7w&b=arw%tL)v#0m)g#UlJSDz>{yA8S zL~=_|>H!hU`q*+n94sK;`hWbv;SwT|t`J8T4Quk{Ts_fKUF%IK;d^*X@kvQEG82YL zr8)zB4^Rb^>dn#G$QcSOhSkPi}QQAwd0P*8sI6&e#fec+Sp;5uN{*MDA zt^ko8VSB_8KNJgGPqsRktd(&URDz*P%RmZgi+$tfv5#D4Sfc@0x*Q>bb8;vN_D?#g7`K4 zRSw^H|ND-eu;P49ZZ`kk962#v6500I1vv`Uyd%0dHTLhw74?tUZOIPC7jSf z#0%SR{*~UN@Z*9ab^u(TKYWVRR%%22kn4&eJ&)GrgYS;S4e9>-B*fvJ(wTcTM*La_ ziHGE#g=r;`N^Yw-T)reU0g{B!%T=|SY=NwFmU#5XV>=AL)IIoV69J3d_#VXB1^pax z`8T10#K&8e-{#()Kug^I-|g_g>c12e_NCkY`7iC(XwUwX&|>m1a^dfs-D#eDf7$QO zUf*O7W!c$2@n-YSEF1hGK&*PhGogA?mviKFQ`U`^rsjRp2&;%e?)SqzWp#OLP6v_L zI&Sp(5|-@4N14xM+H)73O>$1We?#-ltRe``FYTNxzW8OKz@({F=66(nAX1RS*?FV) z-A4_EzhaOX-8SRImg=7SlV2V^cdO0Qttu?R+zF$_xPIxLAWof!^Vq&|oxQHQmBW`* zd1>;T#ZScpA3GXlA+Beb)Q2D@H4(iaPm7NOOFqu@GeL3HSaAPU(grha8yn_cs~$OQ z;kC!Vfm3Gh9~jl>uR1AZm&x4)ag&>(qa0d}mHXbEGcX=@1=HB&?`c=1dB5gxYbYdj zSlP6ytv4z3^4s~a8`m62dtvCaF~1|lZbQ)Fl)h(OMe6#Kv+B|kI}6lHhK70t*Dg7C z=~rHPQu@!LhNfem+kIlb^F5tI#!VwGH}E2dz2~ZvE<1OHQ-J(f|B7DcV{C1L_8Zk3 zz7+-jIpJwmO=S(2xyMHipf`PMUt=s8hP_a-;*+iY9-AkfuD|)y%xf;+*3W7?Il#}e z7!n2WYt8s;OI9U+I4Uo%j%iRGc57Weoc8TGzK@J|Ojw*<56|C|z5ib1-Wkvo{FVzqJ~ZE1oW`$en=W zy<@~5dZ)o=A&k#MLZ;rs4!aZVZkaW(bF!54sOzJq&L6!cak)r?YY~)^eqp)ZHp-#S$t5&eJ0Ik0>brJ*%poC8MY&hI5owx{z341?o z|9!SEkT9r{=DNiHtj^4@5WKF98z?a8l#i?JkEkxchg$io;F}``%>!~`ZN`GZ0k)OA zyJ94*=T(H7);HaFqp&JXTun`QT;U4PYgUZ;Ua{?SdZvQwFSZ!y`pvnDnm9nk67U;g8X-0B=-qEVQMSCQb{(y4SD@*c`nkCR~uG_ zRA^ml&6Q7lbZjoWA_>zLFgyE){u*~L7wyuI-vmzoNuf5B+eo|0f&<;tTtip`~ap8mED-~0Y(FmKFB*G(RqQ(+Sa|i4^syoBdDfvI|1Xb=$V>5 zMLQ8g3M4pT5a0l=0nD!gIuoiS&l5vuTEPXGc@GJwAcZCL1bA4(p$uYv=~cix6n8k6 zY@NLszCvohHOXrA`8uw8*;`0CGVQCM>dg8HRE7WSB4I97T%_s#hkp z;FXhiQ944{+BX*#cZyf8z`o4a4Ly=>0|*nK*d&5qg#(TUo#UR*?qmL9J(BkzpCFQ7 z95_9*aF;+i43BS^r4#Z-rj3)=F$3VFnAL+FnPv&-g>cL@z5F-XG2FD}3vhj}D6?oI z{uQqW4axPGEWtxp5HCiyl?L|}Kz?6F9UzXh7X#c>nUTNpdviJ0Go@uXEQkLt2=wQaB`DQ6YsBFV9)m?Y-6TvN@u3%DcNVePQvk66<3&9p+Y{>dU>Q zpkI7N4Aadp%;w!0)4k8uMoDt(9VDqpLI8u-fUc|NKsuV!0e&_}#YAcZhDb8CA4(xU z%>KG6<$3oiM61SeRVGB{uMnwK*fiOzP&M23V&gJj; zLbCsu5#8VHgxtM{+!v%Pi(Ve#BZBz`I!p#6Z&{=Fc z!B2m=F8Z-vHTXl$j+k6JC;!7P5@AOWCg3KzcwPrrS5kP!{7@c2;J9L2h|eJ|D{)g3 zlUHKi8~-3*hW{C@t(VL`!GXP>ynJLt(9H<%soFaOqs?$A&JbiC-T~@-dLarz7<22`wVf%MqXhi41X>7fj%sV?#jYM^7%=ujNEYv4?tX&>4C%r420F zF{X7kG^GC~q5z8LAqFRS5mD)AlOEqq4e^QAo9NDwrvPC-zxFRS<|5yXpr=)Y6aWY_ z0*FG+0-zmu)JsUTtHEX$3V#EfDpsfy7?ZisT9Gj)W+;g=F4(vZ0^*Y0&8rS=Q`&<~ zcuXECQytlvSVE+Eik2#Mok`ePzzhLg&GgBWpXSiMEMpw7tXy{rsO-p^ zd*LIHECayKP9Q?UJwUS#b5(*Y=~wX|2EpQD(UV7m5Wun!mno}fROZg)9uUY!l>OVM zIst?vG5!G*N+b*b$(1H@H^QtlsS;Df5xE}3`ogljH_sEQiQ3pyd`z$gSD%9ErL0Hn zMK5uqrtS>C`%!&{6UJat4X)GEi@QXyh0%*3^`S@4iC})>4@$^Vl9vfm1)K5X1O(Yk z5AkCl=ZD?CSRybWT!cvsMs*JU#7H<{02Zhkv;-nDbV60p6q_(DYM>w7kYl`UCtCt@lvkxXPjZyZ+yV3u zG~AJj<%yUh85F||axzHYy`u__Z5XP@qql)9NIXka>@?(%yh5ZJgj9n#sFRLHV0qJ} zO-VeE5QK%o!{B2gUz&T4l7Emy;3}uL5H1RY4o*7i1#B8awXlC6`y&}mpoSQP{-D_& zk1_v4Ny%vqK6L~Ai?N($E~$!+vIp?zB`KAor(_MeYqUw zaY}R}gbX4!4uq3Q!?_VDWb+Rbev}C_fhdHJW5+H+4bw4~Tm%zcI2RF9ea3&yH=+QLxiKkBHVe zWa;v%ViJeBk@O6P}j&ZhWl&$|rN8>Xz`oKDIB? z@5Olq7o(?lQEZw`guB5^uwem;KdIXK^JDgK^!rJ7frmBNkcfNW_1C$kCaYjsGgz@k zxaJ!#r$;UC{Wz>mp4@6_o})Tb_w?8@Ze_WBp!rB}P(&u{Rhef3nxeu9m1yIzVSMk_ z{MyPHNyZ@;Ry-`@_}qB*{pgV6s^ejXw=Dt`S!?74$=AlUvW1bh@={j@K>D6!Yg!2FFU6o%@^ytLl$mnN^1pzzW9nC3} z%FbrEU0C7xBsC8rgfNRbVbiR8cUUJl72{1Y4^%^v1p+^>P_N<9uiJe7i=e1Yb;|9l zV2=LI;aTPD`rS{SjaDuH-l68ON5P4bO_^@JIc|%jZ=X3?G~DbT8JSXH;r*Rvt67_T z&7mkK&DJY^VJfTJcn+RX#6IMIaaZHfIMyxk&pFVMVOET~{6|Z(pIG*%BJ%Ssj)jf{ z#|6abd-T_5&)t_Q(;ITPx~Ey;>vx!+*(%KSV7-jOaLG3*_VS_sY(FI~D6~D@CbOFL zg&*hZ0qFtX=sl;x_b2U-JhapE#*?&50`(@;x*e^3IaztZCNP)27ZG5&JlO9SWd^&+ zxWH#sc%5~tC_HWAa@&TS83wts!`tVj-HwQPT49uu-fR^xHv3vL#;rUY({0GsvwQPT z9v)48!=HHNZfMQV3dqd<4~xs%($?2~;nc(4qrQHg@6LHxV6vp;H;@vBrJrg3IVr1S ziJN5oUO`63E-R9^X1e#SAFWmB9}DsD+>}!H`}vKHtxeoRE+}(nbn86(lvSmXA>k^+ zJD=fMl=-a4_mm=#<94?upm|)?79|Y#4@z$zzNff)@7If^onq;}3E9v#uKeBF``x?l z8uMm3-ohR#QyW=r8XAPkLnIRnWCb1J=p6W+eW^ut6&NY#ROW@`hi&|LOYh=1S+3Q* z=6w_Ixs_=v>e^%u4Q_2FlRo_Gy(!LlzxBToa`?z0hs0ke*K5Kj`L6Ix%kmmdKC>xd z(mLcC_$*09f!mnFoiIDIqKAt&owAN*8Bbh(4iC8muZhXb;8P}!(1Te2cPMwi>Xc=X z%(+VRqS!#bRbc4q#lP-`yEx7C?UJ%OOa~9$)(ZQ*K0d!+nsGl%U_-?As8O~XK0uQY zWwZajHea1R5#ggf9OTg4TGFOrX_s|q*9_FX?D8obKObb$BX4-fxok0VYGk80N^i-m zP__?}(t@m?_#=!A;H}ql=kT~Id-zP1h2C6Vyk>ydC~{3N zFMKhHG3`Z9_F!|Kh#N5C%UJvI*oQ9La?;wzH|$VAMlyU3)D|H^$NxNkp$zoVeRvU8 z14hC|XDC!rEJ5%NOFvHYO)v_=fn7Xr$An@K0mwcqe>w7mGcsSC4c~{leHtHGS8Q-b z%BtjKne~%WW%6WE-RXS*&2<&S8f!YjpWI)wF)P|95_{!dLvN|fE+Nm}R%&APIMUimJXI-KR^HU6E{C@^6=WD<8v%d$h#`_csR1Pwaj zDzxu2MDVbpDCEJQt#Q2MnT4i&+e4Mbq+FD?I&rmP4s8hlysR_C}FF zEZHvluvtY4PK#hF+>~%!YR3Du64QPT&xi2IRU70l5T*-ZGu)d3pLMU8iaHkrLmk!p*2odqp zInTkKy0ZJ2-ilf#Aw$zw!hc(l>BkQ$O&uFqc#fr=qp^}VWuiX@TL$>Qsplav6Gafj zA2lZ$#hWH$_!2n>S%ynyOVAmrR&GrgDsUq1is{<9C}1#|iaHKmf1qS9CZMtopoO6I zr?*N8nyyAq9tHzgBOSE_u`?D@O4T2F1^miOhzJ`ZGYj5ZFep#r&tY6~n_XQdrAWNz zljV2|l|alB{GnF|q62ixRY+DKh=rL^I0U`ML%`a%s-GH&$%eO9aM!9>EqptfMO^$Mcy z)adK?OdP)W7nn70rSKQQ%@ENl}346Ina9Lo~?Eg}fXvR+7T$jU}LYPDvoG@qY7 zw>l?tOcxh+7}vz1?g~+*l1>s;$xh{;&~^Tb*ObVW`~>a0;qYM3=n%gQlwt&MB%GUQ z4JAe&cpRl-0B*kbzq{Z9+;*L@|(fFF(ves`P3!N0WA zrS>qK{Fh<819_ss7!Kkq9#>y`q&n)bs3B`0KvdpY)$oVjjGM?bt+=9LspkbyrqxBg zJ=7?yA?-{^YHc*QyN2D@8>}#|@Qkd5(CiK0_Svl=*T07qAg;;cN);&s*1^PYU5hPF zQa5Mz`zk>!85lW{_d?S~mVI3wZx_2m)>^h{;yvtN9c>0}ykhOU^<}pa(}Gd92Iqw@ zf8EhJtyXYo;>kTWr4VKL9wka$hOUTDC2NngLSp#X#fClI-6sGdMms$S=`5ScrEeG=+**x?fQgz9mrLR;G3o?L z5`3wcfGRyqdN`5Y>bN=K-LT9+rZD|!{Q4+%3PRQpqY$D65p0TDM?HEkUXfEBN+GMuB0(cbtz zz8hl?QM)6Z+L#<13RrM6Zotu$GW<~{Mgyu}3HQa@J_CT~0l%JbgeV{la~IYig14h8F0Dvj0`#`JTLO5^g38t zR7o}szscX9<>V2#%zPMfwe@h@HcUp0G3>@%oH+7rXeMwPDFv}M7myluj$j=aeuSP$ z;B0y-E%?<_BM(plVJ$`B$kM4ojm*)!Tuy!mp)VwR7o-w1@DGlBJ-J**aH7Fp0zJp3 zT49_63yuoy=*X*WiQqNJTT-zGb`HE2c|H8W#tE|lDo6l@R>9mVR3L)YO}@h1 z-aSNqK%6ruh zhiyEKWU%1Ookn59k;$4D>Km?zS)TE9 zAms+{mV|ZSZmITTJ@)VqXV{%pg4bodoDFv|&BQJ}v}AK%HD5fSXQsqAgX(|lEZkxO)`S{C%%WE`*qZH(FELkB6yTbCHQXJkPj4P?vr9CF4 zgWs=eGst6Yq8v7KTjMOxz%2H)b|*#{wR<0OujwqQ9y-> z_BUPaLM)v9do73Npjq}W#qfCHWj^Lco{Dv6J;F~a*M$Yn8uI4OI&nJn=Pdp#PZt?l z-ri^GYEO@BwzJSI$;(q-suEhav^Ajvrm54;} z+*&<)Y>b?b`bggu=dxaZ)#C7r^4#;~-F<^6GM@E5c#&o7QY zR-ErA|NHOGv&+i;3k&LlBtt*rdb&MWd;dk6H1aLo+@kstSW6h^yL-4Qvo(+Yo#0`+ z=qyjgw^QCy)wDD>HAS2r2wAE3wf$+|3C%;JJu=>%F)gffcUN0i`0Vlebf=mJQLN(; zbwRVpvK(Out(ffJ7;IsoYuX>;TITS>MDzxmeD=u5;fnP7LV3v32i2RKzUyy))YlbV z8{%TZncS?@2*fV+k|ryDj>XVsJT`jU4&2ok3e=8~5wSI!-R=QfK;wX0$k<5&yr!$f zM?jNIl&`(jro3L12RZGyc1^>jZytCoaXEUn!_-DIpWh)FPyix8y!@C zbSlpbHVQe-m<<{c#Oq$yd{`V4kjbtY@p|?Af&6Ehl5=lXW5~U+sUO031DS=KD`Wd2 zp;Y*W0@E5*kGi!cWqi_w%&IuUt~Kl3*=Y69)-su2z$rBv_Hb1|J?sART|AS-JtH3s z7jCNmN1@CuBKZE*zK#uLVv!9rsl%)j%7lI?k?9GxD2Gm#Eh#T-u#U}qt+|hrY@^4Q z=0!?RW2#~@QhR0ickKnaiSMKqmCY004xi)?vyXZ1i!b|BVg(;n`I$smJlSh~%pDkxPf0$hHyGLNkKq zpeu;HAS&P>bw#)-=^{VD%0Uu_GQHCUNfOPr;2Su(HBT-ltNK|J6ITLwNehu10in_r z^ixS+@Tp5#NyorSbeZoIK7NL#A2F3e)$4`s9L?Gxnb@?d&GcIpKA!BgAMq2!G!XQV zpsxpR=LelRUO@=L=&UNN3fS0fm2q$!9o$}c=((4l!W8qZ=Ef$3mqyKrSAt3a=y;*z zRxUB^AFwI!wOKHGQ3#tuOj9ZZy6yFB0$b_4bsam$ta zDiKs6t$;?u2Wf>+CdSnBn*kAG*Y)fV-H0#^b6ulDi^>ETQ_?&~8f%`Qv$OW#$0piId5n*leZU)D01V_ft6tw+vPQOEKo6 zN|7`UmbIw*`Yg%P2}UOF1WF4@U53L`?Q7a1Rlb!O+ZmR1^kv6JRLi6q-)75<4fX?~ z3{!-`hKHf!fYW=vHV2h@|N0_5%wjVZmlGr^^N$+)$5j;?{Ha+?lk zqWPX#sCi85m0koT%vn7>jUgn`Q4c@{h~uR#h+AN1W~3X4t9-GsU(pJ!z0v&h^Z=sn z|Lp!?l^7ZYBI^in=5<0aLuMR+2x>gu1{;$|=MnsTrzR4X7Hj4mB<2#ZaBu_Y?`Iu= zSm)YLVENBVNv}PP8PWcxCNd3N#B*_=Y%SwLaNk&2SIuTO4#ZTd2Hb0;lZ7tN>raY& zeWJOpDn5)=l9!=A1ZC8)r3dN>B|fD{;oprpEwwWk-|~+7w%&EpZ)`8vir_eJ6y}4@ zp52i$3`synrn{vQ?$fIT{-V&~TS zmyakX;=PFfKQa}_joF*e_RG?@U7eF#&;?(a%Iw@%#Gntr6sg!R4?~~;^9Df{o*OQ? zIb|hvs4ky%uYpiil2xkW-W(pE(q`SxQvN z7*Zn;jQaW4kc9|B64`?wN5sQe(r?L8Em{P)r zLyHp5Ar$D5DQw0?2MZIJhUDbo%Fn1E>h`)Q+-lY8>RFVdz~kwqNtW&cu3v60ZAs&- z;#Ki0%qaFu=ne?;Ljn$PQ)bg|!i2#iTn{Q5O-U*83Q~QLSnh&5h+fB|kx!~jifGy*A#{8g10vn9@icSKq zr~m`YjGaXWR0IkGg#d;DL|{M3dj?UQ?!+FgfFSHGXo?R)G%!fW$_; z;u&>>L4rbowD_rXg|*Fzc-zD)NY9;q7}?Ss;@46A$|j7SFkS*szjSugCp@0w%_xio zQILI6i$#khQ5cQD6bEP)>jH~?>e%Vi+Ca+!%Ta@~Ow=oY+|=@cvOQw3y5x5-EP}T| z4VG*|4q$EWHL^-nQuQH-Qi$#Z3%>|A53V$MarWp44;lzuws2oZ$YZ9=Bv})xyV=L* z$Dnp~Ae>YbZgvs)4aD%Qoz-%Qjz@sT>3uL|zZq{uhy`2nUrO{$nM5$>8g>_CRm9-p zoW`UOxm5TD82S!icszXQtI?aBRB6@BYz#<%aI91CELfK{!~%l;&3b8-0{caJgMhRU z^HBNJ(Ww+dMg&c929&>|9K_DHG}aeljRpQemO5ST%ozmA$5GLZjWSULzGKLW;VQ7a ziJ>W-(=Y^GG4&IF@(ko20KfV_ulm<3gLom|{_71vih!ju3MGCmtWbPlyfH?ma7J-r zDNm)qO~`Dqx(j%CX1t0QptHCTge;iKyTlL|?%qHD&|f8UB9JJ7%=_yb|MNq;0?L}W zDpmpSg4}C#+|SV^=eo+|`G}@`UR#7gg~`dd=0ur>Q@sdmM{%Uf-Rf9U;w z)t{wQee4J;GIXE%N6U;qeXiYO{)gU?joK&a`-aq^nFg=7By6{Am(&&o?g>}9v%bjQ z)NYvGs91D}-#uEg|AohhZ}LP!=@GS4WAH+4z7)@?yLFkmCa$}Ei%{k8Q!sCx`b)R$ z(*~#5K{O7VH6jN9KE>~-Z`BxzGq}b;Rk-alH5qF#G^%J*Poyet7kyfG4!3W4^Vh{% zw_(qoU9NoT6QRs3X=3lI#>PbzLgB2Vw)yA0L)-FQ|IoXc+V!i)NYZNk(nT8YzUSx( zWJQgi2+kbL3Y1P3h~62#VjOiNpua4qv?RCM*Emk3{_M)Z$ik#W8=lIo=7}y{C_*$j zcf`I_c|JR;YE

@DxZSjg4hXQkZqeLUmyFg z|9-x=AosavD=XLC`DoU))&$p~fDHaa)A#NeolB@n{8bil&RwDoOnfg#UzwU)id|$; zze!1c+~$#IEFae0!ko+tIXrwU^ZKFVVF@j6eH|M?^Y>%9y7q7Px7*|A@twcgDKV{~ zIa_%xSedoR-n-IB#ktzmp}U;5ESS}kKhyEJXUog>u7UYx{1@X7GT8Gc4tex7j!7n? zIh;Cy)td?TC9j6}?C09}Kal5#7slp=nCuX%wq}?2D6b&x!#+%ZiHS!0ZXtiJp9x>~ z`7qLq<=h_>!_Uk27WT?^<>baDY%^Hre9Zo+m1U3>e{NQ)ALppg-=3|))jKHGgyn_y zMdZEo7zj6AV#Gx+HtSYhl*%O?x_*2PeN7@}Yk7 z@t>nv(8cd=YRC!zO@gLJA9-D1KGiSaiwlLd$DLZgh`4#1jWeYEw74^B+ zg+-ecUx;3`)%>2yw!6fcrCD&{Q@hE-&S7%T33u;bB6W4%wblOW1EsxpCe3~}x& zpt{<>&A2S@g2<(@PxgEc$IC%K+PyvhUnRBQsvYkFixVfTybU1|T1{t8o*l(Eg9-3>Z}<9i7S$4YlU zkm>hth{YN+s`DP=NYv1cC8xCI!n;z}3i)w|JWM(v8+<%idsvv$aN(N=rk4#{E|ymW zno0+fFcj0VxZ%5qgonbB5g(rCS-1?SD+g#+LV@iV?Bpg=o_l#VJZ^Vt5fuG*<)-A8 zrp8NA?P>+*`mx2sks2#iXR@|Xws-vU>%*cm%FpJ4*VJ@qKm7^|Sk3g`{PHj|XtWonZnGJaF6`tgfwPMuQla?~16#B`(* z*l6FAdyRDOBM_IpA9=X|+(>^Klho`#Bw=UZDI35hv;grz&`EG*)*Dv)qj_JbDOXhG zRq>1#f_`YWV0X$gw1R=~y>7gd8Lb#V0n(N{I%o@jMgqC3Yxz%Yo!N~;<()9fw~y2c z!c-=(rUqX?X7Mra?p!k@3ftlE6zLOnB{tgAe)Y!@)5qG}?;0cfLf$ZW!7 zq}ML}5P?ooR+9}x4^}9E$sff8MM+nobX@>P2auI)j^_<#EiB+H1SP3U4pY$+Q=-n! zp~;A*-HsTYvk3W0KvR3D7M z1{a)FNM|uSauY~4UaVrHU+#*!%nMDfMtD4B^p%hFPt3(Pg6lK-+Ilac9kyKQg`=Al zfG9p5W05CSq$kpy*k@{L({?~}D3b>b@;(elj&-iGV{8fq9@fX!4L-;=;T5d0c&I#pjW8q=yw|EPo3#pGY~LX-oDkTr%vP` zsXzT;>_w{I-PEXS3UjtBt5B6ItR}`&d(8SUx>tm%l{&(qz9wOIUj7xGXep1w!ve)n z;t(6GF$6l=eeikM;5=n}pZGMckxZlz$u};-wxV7)RI9GziS>sQfgyrnbpWFZH>ggB z68dn?wRh^t5K00R8^V#HO*#PrAm=xr!#MGVGa}o@QB31$JCQnLG^gKZbB^Y%yVA** zQ>q9Y1Olda&5Eif=nt+p^(QXP`3${PYU+shfi{Ozf*f_Hsg^922oU9Pct3?y)lg7l zWi4BZ|H$Safr_1o(m@YFC{4;yl=;W!O362?rOoVWE*oX)>L>cPVkc}Yo$aW;uE`4; zVh5}}hY0199h>~dH|D!HtJVzpfBp4VBcW0Y{WEPjlO#7NOUVFw3E>$~l@U$?o>qps zCZ8dglnhPAG}e&C0rZY>bB<=oH{|XtgO_5&ea)uYLGys}&b}%vkzR=sFeCskXmuNq z>}8l{PBR*l5bh&J-tH#6J;SOaqUCTD4)g7`wO5AH(c9JH4-$61oTZ~NoXgsX&zmPK zz)G~m%GBRh+HHtd+0ShUZg+r`h-=$DX@9Lh2`;W-q8#UnmM$oEKwONDsD^4!++|We zP&AtU;ECBB;23RDQT@*cLn8fxD@gQcTZgMEcHn4XS-EEWLEPs@EPif;jic8VoK(@n z`q}4nakWLK&CooZ=@I49y=rAj&&rhUTq=nTv*k=i>1G$sUvayZn+nAO#>oqV!9M(? zZrH=c4yvGR!WUi;L*9Q>F*I4ssWVO-1&kKH|8 zN&CT=P&eK0gMkARvgVzl%zI3pS6^k?{jomYGN`g4~q`w z_aKz0A(REKCd3HL^aw5YnXfFXAO-o9+aPLb_!44v8v-M!{y#GULvc%qc?SA>D6RoV zNqAKP}GjMP{p{^BO&Bve=gq;WWC?1Zom314CNOl$V2vP}actje;B&u^|C8 zfpDFm2r9k)v(9JAdBUd<%7Gy6f_^$5K1V&LiHSv+A|U`mnX`;|QlkIIIE3*J6cN$Q zN7Xj@cQx|%$fN-A)(8A=0K%^3cUH=+y~y=iiigiOECd;K03>7_Hp!t{0)9vY1{%qr zFg-H~m|3tJn8B+K0&Ir*A(yVh8Qp3^K9JddYEns#lRzQN60}e*U^4m10zD$tpcifx zw%<#PE{Sk}#H)b3DcaqhB$W>0`^g9b-!bCAOx?TI0R7hSsRBfT*_^JbzIbF%QNi0m z5RWuaK!A-H3~jITauVKyi-^5#e4>V+PBJi0^bBN{WvxD;Aa<@4;1>Uk@^j?(fd0Y+2mKWA{dt8vm2>ea=tPE73{3XO;7;lV zua_^G)Chc!6-thQX;>4piZlk^5eYxMmLfru^bp;P;pcqm>9f#rVE#)Xn$UH_djOUW zSfx?K?=TSy8jJ!N`f=kooW?3bZpd6!C`mJgfBP@ z|IyPyE*6d><-4!dy)MX~Aduf!cU>F3<~!3?u9VIg@0$9tJmHvzGy$ zw0i<3gv6=f&O~n?at=ufVI%NFvuCjN5K;3p*18l=(>E@`BS(OwMsiSOy#U2i(EtvD z45JXfj7%_$g0D!!@nLG9+xBnQ$V-x?Mq)J=@O#t@KF9Z`Tqfx6K(Yi z8O{T(1oBm~(5@cWXTOY~AC*}Cl2L`nQvw6>O4Db7ya(MN$R=UvJxcsPfV*+`Pu^ z;^%)|N@yrp$KJ8gu0cxZ{Nu#Rfu3r5v}D?bIkVIJ#Q*#6WfepYV(kgpA08i@2doRb z{?J>9ieWO6h*C>#DSbLHREfu|i6p1QCMB&LHQ_VP{qB9jTfYlOmu})6acFvRlRLBB z@W!#XA=mya`*M|3o?<>zcH3^Bs(JfALR41wOS}&WhoVdm1v)ew2@v>qU^E*0KwDM5 zMHuf;e*>{Z#-&B|aSycn z1~O(q8MupmyIT`DG-%099oC8(vvf@=+0cWaV=A+Gp@UJgliT0mh8^DT>(aV%Py3QB ztreMLb3(QTW(?ld1ai2kK{3sj>jo?Ymmj>aFl$_Cps?RI=CSHZXi&2D@0?lT?K{OS zpQrR5TW-;n9=vMPWc0;7oTf$I!+j28tFrX+SUm|GdG3`^cI_kXyL|(0`*?z9&ZpCx z%X84##66P?|_FK`x zQfJBemSZ+G4l|l}?8t3sUH`jQlvQ!W%fQav&b7U1a%X7%#NCB6ih^ZkKZgsAN`eEU z!VGc}_HE-OFNw?!e$Z+l9&SZ2C)@_N`JPoox|PkHI6V+;B;;~Kf_3NOI|A)&V9tRF zSiZFLXqi!-vD@44@3qD7?!(8r(XnqiX+@j96)(Q?<;mmovL2aOVx zt1Uo_-|gGW-17OE5lyjO4L^wk0*?RE?;;zqiHd8K@AQx3jJ%y|xJUkIXos$WeYZpB zS-e}B&~s?lgCoT`Grfx8XWwLwM~rMJrDphYrp3<2fPoC-?KM^B+U?h?yM=ikzjJn8 zgLcFHp7g9i)cs#o%jcO}#M|mDtj(e;#of|h7G^tW{#Ac$-N>h{x+)DBZZLf2=`X8K z%zADxF=3>r+#Vx&Xd-Afm$RR4%9`<@(D7i=60Z~Hn@*UE{mT!kZt30^*8~Upj@UN6 z9Md>$-y=!cd2(`Ta7t~2v1wvlJL?}yd|q^|Vfw? z1`p{zt2alPE!H)=$7DJ*-FhD45%@ePDp?;#=@TgOOB?C%Zn(;j06;8f+oEcwe)m zuHuw6uwwH}xmR$O!6|3{@Q)t*+FUtS?Tgq0{G0~M4=yL|J83o%_mCf;SyzDvV*tWV z`W;({Q$uN6V@#dy65MMe0+wI#BO#AC)TTE6buZ!*hwH6=3_WyFVbwJu?WMO|Qg*`B zmhoxdf(S3w7CZ{ly16p3YK{IqY|{~$#8v99+UfL1E-G}nILVT|x77I}tymX}oyrol zj!uX<8u1|5TV&2bgV1jae6_tKKa2FL5-}>i=!uo?@>1LRrbtKV*TtA^a2{?$@&q3E zR)*B?(8-I5TAUx741h`Uj;o28fI5t}mF9mvHFx0%WVb#QQ?QBD6w` zovKd+NeI#3UqQT{rBWzvL7?H|o|$Zza^}>r+rxZRDWZ1Pz=|-H+(bQt4;_3YY=RDF}YPC?cY02csd79k@uctxZ22^jR_3AJnFhSjtAC{9m*ByZzp z7$4;gfS(}70Q@xP6D`7K3$6nk7IuaF2u7sW$wzBD+IPlWeh?-Cu!k5Uojc5XGe&Z> ziW!Jb|@g%uE4^qbou#K}#o$J9uYcl?WL_dGETa8Dln>*l|9tnWJ(lSHo?^ zw_(fk4*jk7B31F2NmzoHN?FQw3q1uJ_Gmru? zN8n!XHmg6up=TRbp8SB+?uWOFO2j7fDtw#j;}Yy_e6I0?1DrNL4QH~non>{mPP;I@d8k`6BgY0&p7k?o=IEa920MB z==qX)ZYbYrQ}hqL@A^uqdk67b5QD+QDxWfm#BdBf3c>`$i}6+nx?rLUscSy%^cpR_ z3BWeAc^co7>Yl)iDR@H^449NA9e&VW)1GeBzMet}!btehy!t6DL4JG(H(|fgNh{xhky;)*^EuR6KLoOk&yvpGKz8S8~8OiH;#0I zs$^&t{5V>{g}br)hDK|PPdLeB1zeh)ZvE!_*z04+bw{6?y@A ze939DIAK9q%qxb`1|dy|@`OC~>Y|pW8r+~(hncQ*{Fgitf*1&aT0^c2dgeH@Q~&~% z|8$?9Jnk7Z3#%W-spFs?32z%o7R3b5*X0{w{1z*Afl!m}CM6z|C#lFhCC{~Kgp1WnrwOoN$qfMG z4(gmhhjwsg=t5=)D_KGs#x9O>AU8wXNqT@M|aExyhRsYO-lE zc2kFo?(LZa<=#vByXGHTb;oYQW{rU=tAe9($t}E>g*4PfYck@ymBB9~vZS@c#qa8} zGJLj48#9utt@lYxPUb%tS6)+XMLPQJF4O?#hpAD4WvFS=TOVB!+-mhQc#Bu`SfXn& zyE&}w3My-MS7k(cTVv);`sC6&-*$Ho1Z(ZvP*bw|x|d~gaC@N1L-#krYcnTA4^ECY zwjFF}8|~F4`*G|RweJpl@o|>LN`8S4u9BjhU-Pfpjc2c}wX(1cE6fUBcu$*nV?(51V&b#x zgf;HVR|u?&laeK?Y@D0!c*o@VZmzcyaZe6!?-VoznFsHL5%Hc>e-#V#!Fu+=@vPy- zb6=MkesFcAb)e~ERYQvW*vE(1FH8z@AF42+COnm?|N4gIpQQ{$ZXC@uVmgIlc4dE=vTpLnO=&3_BodHp_JZDxs0mooa2j~1EBBV{F( z6-oSFuOQK;5yRBF+`@qq-CkQD9NwAlAIa`DQa&jhJdnf@pRpWr?uKVqt7oEtY$(urO?30n#4sR4kUqALwq>P=uXKX!S?D?KP z<{y1Dy89*NKf_<3Xv7}reK>m(x3-a_9KLVn6J6|uW1MSorGjsNJ=AY_dR=g|Py3>T zgb{3;%F4c!?SqNg`x4>lmHRJ(2y8z#J^7a1^5V3<^G{>H%*`#{9lFB((OK-ztCS7B z;cLH|;5fh4+F*C=-<`$(Y_C<-cdYWx56p`;TeCC2b@AL(t6AOaTxF%{Ds&hGZI4Ue ztl;*qIp=<1yIZq7Dfxy$)b?QKoYyz_aijm6zSBQ2Fg2Ief3q;>jtx7YIA=_nZ8kgT z`OMc=_jdv4dZ!E!5xcIrbpg zcs!p|F0(PV5%wlK=doREliq4S-{$ck{QQvl6STc}BlUOQDJjmYKa=y~iIJ>5I>x~I zmzIbjt+Bk}>u#NJ8Q)-{OPPDuYR9R-FcE_Ol+<+eq8^D-r54z};;n@f4zv%^k zVtcOpWp>B2hULDQ+{^D3X@b8g?;e9v;AZX!t5H#zU}k3WQu58FX7gqDUl@L?wHy|_ zxsI0^R=C(XCL_`(QsfS0?ve4R3T0R6VcjJ+Q&V)5*-wW=Ww0`-- zxfbN1H&~3UaK2raEXUKw>D<*IuMns4q3Ujn}LELsnYQWH>F zYn;w$%X-iQZLk*ZGs#Pb0eGV4pj2zDv*V_|rbc2O z@)0ASxC|@dbAh!vZ}y{e?K=nf2%^`f>n!X}0jN_IeLW9?LK4eSzx2wIU=N7LYc_oN3Y=o-(084W&V%2fiC+pc zHqti+7t55q?qvgIpsP)U;?6fB}Ft0Gk#fkZof*)UVIcp{vQ+VmFKc zoh2?zM7eO2O7gNI%;L<7&%tafdvKkp{Wz}q&}1}*PqYFMRNM58FKUH|@{n(YK_NS* zQH$AcTDDNMvHCH|QosmEGoWQC(DxN(=8&T8r8fKc%HX8{HycRcdbfTKhByjLM3Kmf zD}|bxmbA3-iV0+XD(`#vjv>&~ouAIHyu|9U5sr<)4{)Xq#)9KI5C2k_egaR{!F(fH zm{S+k^T}Lm4^y~hz@^i3G)D=&10=IfedWY8!W(dx;eK+>9MJbmL_?9BqpR_=Qt`%+ zaeB;on~5bC>0)tySb3x|<%>Fg*DJI^=*<*K6g>FSf?E$QEhsav2Y_7wsIPL#DS>ne zyBQIrQr+x~3ghq}c1Nx??189&H1!bXg^>yvLgT-17K@SjD({L3UK@35t#)naZ9}&x)smj6~d0PA!V?K zsP=!cY<;A9gnj2iA#U7-lV(^}8T9v;tW&VoP+H&|OBl4Jhx6M$} zYu6iYWc>`5QuaaY-iA%7zdaQGGv*hJfxh$1T8e&*#u~{k_8?hC0j0a7I`h-zy`E=D z`JM9o<@1f5P{82-!D@93W^(@B__LU4tzvl~vGF_9xj-FQj->BBb3MApKv^48iKD>w zAew;iEPqA?+Sq&k6y9enSiGNk4V1~`?9+)_Xb@mjxqw$Nz}FRF8k&wZ6Icrniw<`? zWuGuS4e@gZwy&p&@InJ{Wv)9)d3Lt~?J>u6HA`E91{t~8TiWwz_oAI zIH4o+E{b&ujc{WVoC=x~iHi`z>r(W@zEdBKS@Fnggw5%w*Gk~lDYAegAEeB{Jzj3z|$nGkXh#cLW} z5eTgZPms}myoepn7Xx9{hSXAxj8Dgl2>@>-$`}H4u!LX&fGH?CzE9R8!2H~CFQhh3 z$=Eq?bn)t$W-I7Jh>`iA3I6{v_T~Xm)@%DX=Bu1Cq*BmanA55W$gxBem2o-fNmJ@A z_ANm}0VPFbbC`C~zw7=y*g5C> zzTe;P_lLQ}Jo7xC?YXb}y6)>Xo>1qp5#{%E%~Sn3kS5qYs9s-&Xmpcyy#dl|DR?N* z=LY2N08Ab3sAQGuxB*t_0rW^kNO%<%a%Xdmy)#-9ey|nU!#=f0$HQ2VhXEd}+y3yh zPf(n++%CthPWgbjE3*ERAhg>P+3?sRqmSc+-oTbt{Ku~**U0s)5S8HFNpIMzpV@s6 zCH-U}?(S;wbn-?#-^>;?`eDFbQuup0TOg#GNA&|W+L@xnX^u7+YD`G-5W}&@sAxhz z0Dwx?MSh%mmr<A@m+o^^O_D==(;? zXY}Vtx1a9*qeOa$N|AIo=L?_33r||CHm|jNvq_hAN2q?{<=w%DUEjA=vbxow zx?jfxujx15`R`V@XWn_QTG%BYzM11ap`J|s-K@;&^d(+bu%=q%%%E z!h1>b`zPW)4|URvwu>_jBOdCEErg1bcWQ$`SAhSD-42f-DuUcrTz*?kK_-?0OPl_K z_+R5IcR`3>8CbBVczm+RF%rHRDZ1BPn&5&C6unt_j_;2|)*lA%URLcCcj!|i_XVp> zi$+#77aG0i(ywVW-KzVmXUuOaPx~q&#vAcgtTnGm36!dTYtVkavGWs`-HC@Ic8WTZ z=LD!%@$!F)dSh>?(}d8ZSDj=x?cC5k)4S>iU3|0;I2t^B`uvHaSB#IR+!-CrclUNv ze}hPsZ+ox8WNd8iSV6bf^R-58xj}=H2Y6|>H>8N3pqT?Nt6{uUu;>qiJ+oRWjI(6# zhedDIn%z(jyiyq~aR0k3+2y;|GskO3bCiMmLdClu=v&o8s)!EN#fqU!U4BD3tkBw= z1~&gvwks_{nOb?Hsbfw5TOr%GTL^O1qozptjyo3p&Av6ousJ(rFe`X}bPMu@irAWvPpwW@zE)S^y=niJ;Z!iQ zYj<92sEXScslE~!?t#Lpl(C^F6M5P_p$*qO-9?$giekGPS7HkP(Nw!KcH#P+kps35 zhL7CXm$Wa?(!t%WbIkk2wx<5zMv7avn$_PjQ`a^d!&0oQ>~PA?b9Nye&bHtFwRZ2k zbKd*%U%pnb>ss?8Mf7^VK3mD;_IFYRm)dpjB);32TV0(~GJS`nLD>}M*LD|Pmv5UQ06dR&CvzMRDez{WG;pq~vDD1nola1NM#W@A)jBp9e)L#0mYPv)CfCpi<#bzz~ zsDKmM4HhA%XK4?O|NViaR@fy|hKLir0`e|6H|AVyS4^xoKR>Wkb*t%MBS;P?GLtve z{wg@vuCvV-yg9nL20r_H1nLWIj}aXC$DZMTy4U4myo2$cBWv&3`t_J~~#BFNYfQLeM4YH^PFqjL12mFM_sfdlsFju$Acq|TQX7vvXbzic*UQ<^@@ zYa-!&t$Dzf*-2)+Nsm*I(-yw1v3wJz0Bt$RF*Q7NoDUVhuyG=RVRK~K#J zy=cEjad>`o^7AgW=qi$l@CrQ)8-O8|=W@L26tC%KqBEmilC)8AI5J3Ol`a1<D6)!yBiOdnrD6)h%B6{w{&v*W*TT?7IQq4c25SS z2ov35V$rbv#wn|GewVuNwENbKG&|Jq_@@O+o=4d_m&*4;gu^?$nD4j}2NFyeKrAvo zuS*@fwqi(g1R-`ZI*4a9SPxD7)H};W(JyNrDyerKgwuc`O;VknBxx)`nAuNqWqlJk zyoT&$&>@q{8qDEeybVu*SuH5lQOhPqczy864X}mRUckbO7e%ODg1hR)AtU&YsRS1~ zqp$e97kS$jeG>Gsf5tLXWsPQ^ftP1QUMh^5anHij1`ID=B@ra)u0e|2N3snt zHiG?u6P~id0g7j&lG2J0<@3N-*93@hbK*q}Cs0rYr#%sf&H?w1&rn1^_}F!HkCVAw z@e8vM8(7J6;K>bT@mN=%>o_?iR*^t5kmnDKbJ~S;cF&YC`jVzBgkkV;-K+5I3H1YS zgd+Mkbn)n5zt^=PDiztJ5KuJ0;H~k9u^R`WebDSlcQgIXugx(6P6tv>#eexsFoR-b zZ;l=g_E4k=>q^Y|5Mr#MP;kfL%jS4pUJpqcgVj<)h!5v;@MhsT>BUp#=)aNZB-k5v zysEt7qslmxCGFM<-uW)w59~ghkmOb@jYI&@aJ=wV!ooj;Iu4*mxQNcZ+YBBZJa&mDy-76u~T z>&A15R4o<$AeNSYw5aj)-73WYhv&~g*EFi@L$~%d){y?XT;2#ruc>}{sGk&1N(9HV zl#mX=Q;`r3V-PT?_LQ=im(JTZ`!iDF>7K*0=rV!Ai31_+rCz)baQG^+Aa_|X#Kp-7 z$Yw;|udWT$H`SK2j6&cPm%PSjNcT>vZ-0G8va8^XTxXl1wBmQ}(|^gYL0$byOFk41 zTlEXt~Y?-IS{SitBCqa*7valLS9h?g<6{bV!VQXpF zsFK{ekbxSIi>PZ)D}7fR+9a$4TQMfq@tt>FYpM>uU@BB;J!ixpLIyD;N3U ze_)!NKR^YqG*zk#cV^ytVPSa5pClOkepHp|C$is=M8s<%q|u3A+QL@^eObtF5%yH& zTJ+k5zL@Vwvxpk+=VzI~Vjn@sSc3pL8Ne3ZVfcrwlXPn!8E~?+Yu)q^&_Rg!HKgxj z3+QAW`=y+b!!Ofl?!d|rC?@Plj(HeQpecsCEv?>4y~BZ0;9js^1WC<*3WTpetE>$T ze~;KCY!iYk2hTBtfI|~^Wx(ttNEK@K9bJ$l;DeBP5(H&@BuRc9!6ekh8|+0PH-no1 zq=-I{^dMm39=RaHb<(xD1}OWjDDcgWsRW3~%@46SCw51nGNF5N0K-OhkQ|`f!lK_; z&HkM(ioDS*UO+KUgk}`kB;*c_FGo&Y5`Hq&?2%w}p5(*vCCbIG55z1`PFb-0N+vu=(cI z&t^jkp8$hRJS#Y36|fXNJH1d2yJGTqnpb16ED0TI7mvjQz15v>!N-g#A&4gxO&Zd7 zaZ`^$#VyeQPN^CS?odAnYUqG<1FsQKt<2>OsZmmE(CxF2A}@{yAq&AGW;RC)N%Ukw zYeJQ6EU|+&6HGc!vU=ntiII+<2KN;-YNWOfSgreDZ0AhfFaMLrovLHn^zR6ka4iG4 z{~PGVzk!oxDL4FgY9j#gW=xscV|VvuUFXYNVXflYl8!nt9QF4~Wi=ZR5M)O&vN ztxxt_zmqm=C|i)M_cX7z3~{g6vC{f`vki*_K3f#fen4lQSDTg|s7zH1Yd^;#JpQF@ z;oHu(LC4yx7C4;zEmJyZ{mR!nK7ggva|+4y*(Ft?l^>@qL+GmpiKgcYf+M-~{}?Q^>Oi#mlzO_Z+6 zuUr}ZKJf3CR$TYnx53Tb!LQH7?Ah#++wvS^)sQMOpvoLYq=oC#g818Dkrl4q@!wt4 z_&Z#Ep=bS)$ah+bEc3K~6P!t%C7Zyy^$RoSH+^w`q;UUI*JgKA0=hicYd2?ZbI_)$ z^ZODygF{3SzpZorMA-3W{;JF2^I?6G*7>MTc|;tTHPBoT^kSb&LFI3iEB2%{-hSA* zAIC#&vT%E*me&kvux5wp(f)6n%yKexUsH&_&+O5XmwbhHSLayX`8%Nq@6)xwON8yzuynd^$mZYM8lNC}c;D;22lZrE}& zFm-vbS^)>Y+uIj<&v9=uesUx$$?zYUSufvSc_w8A;%IE00o%T8@e8Tl*XHF4j6Pkp z+YNoV)HzjW_k>?}P*=1SALzeTdEDH{G-TfSBhE*fZuj+hXC`q8Jw93AZsh>SUIdcRxXdv9KZCX{Spq+6Ed7isO=0!|JG#Rdk2@NyTMwmpofIiFFzmH}#aSScm15%Z0W95y9)3}E2jLJGFH+yol{hgG zvO0Fbc3+jfzBhJ3PLIaXC7a)yH?JMV%1Ao__|5i*VU%Ap%0O!^2c{*Qs86?&;r|vAJFpfd|I5hxT&!?f5AY-3X1vuX z15Rb~A00)gaxz`zgz{!MUF(mgAk}ft8~PAEe}sFy}Xi1S8jEAOF$s6D_ zYZ36vb6TI+9hxfINR1do+hn)1QenpqyGO^vKXy=Hnykx%8$4`W;jSf)o-|Jppscu| zLC#YF*8!SP7@dIH?ZW~bywE16Xbe!KFX@AJLeO6v&#r19)_RS$WiD5sy61B@ijtQV zAD-qNyVfmT-Hr*-z}3zoa4=QhLWvVl&U-^I+?aeJA*sPHBsFUGO z&>n#92O2>}`o=|%POu1522r-3#)!LaYj;TMnM1(PJU2@#fajrQ#qa!Bji1STK$v{!eU0??mrKNFVm=1 zhj&o&gQLeFf`vvF9>$RIT7DA}azw<>;=?9a)TLww4jz|80 z{vgQHuOFR6Itr}7va3ajiYq0+8C)VbN|4}-H6@1x;Mkeg|U2)U_O z9P*rw8NLN(S#Kw*B}Q6WUwp<~^jNG3+D5iY0D4W4=2gi?d#dBOupDj~W}pzv`zcPe=ivxt<5&G*E&C8-2i zm^CogRBH`j4z`*lY`8*JGyx`6G!}dv1<_S8v)`5?OsD7>A_3v3h68fcShPNZqwHod z#5aXYXCM;^NY9bN7Kiog93s_77G^TyZ=*E9Ji__@=sAq`ZG5c(~ z=+34PBL`Or3NlYd!g2(HK~Ll}k{JsAxsH+W$;jrYK7y$q5Ce)nWmBWX@(R@Ojuie+ z8!A*mj{safU9rliT`wVZ0o?Ee8=Wv1y^A_H3RUO{CfnNHXkqX?Ze zZh091wTKV9E6|-xg;Rwx0B)txfXpSwG^7dXfna|o`qTc5ocYl$fC;K86=A?}&IEhm zD0iRWE&>o}mzkU!-;DcRS_JkJU|K{;3c3Swk7FqoCN~Gr&#Cbv@3on*Q=s~2>M}>5 z9CGeQBmGEk-o=$KPbZ5xX2QmF=5xIOk}0)9(Ai)fL;_@1M(7+18GJFfSn2!Vw}wQ{ zGq^Bv5Ei}M7+{d_hvkZ1n4Jsq|j;ffZg>$-=9k9*QBO7?85!Y2M7D z_Xd2GtN&Z--LbvGykfPTT%NOcpKaBt&h2Z4viAomUvL*z)L%*S{dr!r)=*iW(Xrjq zxb)-3-dvtv9uAIe6CPW7YJ5H{5OoQ7$_t*zxXMwvbHlaOSu&9eTqu7z-Jo95>5wG6 z9uwfJ$UoTYHF4~kE~{fmUHLUerRag#tlF)yhsBxJ<3rl|fz}U#j?NKuWbPmQYr)LU zg7o9NT|ItBN39DHJp1FCo_belvrsB;NlDIsJl1_YJwR==*U2t6oGW!7n}cESlyl0P)$c=VOf(4_fR z{O(g}r&BG`q$Ub6OkXa}71`ac8A_Y9Rxhk*SADU@`dbv84=Eio7AVt?#a$g1y_OSx ztj??X=-=~ef&~2P0r8w__2YKsTbF7&+%vcGs&3C(n4uPD$c;O+Md%W9rXofWII2FY z-kG0zPFntG-&+ey<`w1Gywl`yDcIG{0&K=sX^v}B$*XuJo;ng_?C%$Qe235Ap*1C1 ztCnDA#pEWd1A3|7uVy*k+g@|dek2_?iIQz!up_r+#YSbt`J0QE*y}5``+w_d$g`_a z2MBYTFZ`-~e-&U((bWfDZtlT5)-KA7KKJNIv9CIQiy#2W-5zzsqXR>BW%}sY3j;aZ zuN+?T%t*mvx2m(f9j52re4PDk*XdP}Eg#0l27eat)8nBN_Sa85qdR#x-ELxYr}5mD zfIQ8cc@fL^{|vR_?VNO%d?X0&UwV?hg!jqf(}(+fFRlo_`)fzZTYGIT_zxDz?J9%X z3e06MwoM!wFb!?rhjl70RJBxwdF}f|-CxzVPg$tDUUZ@}C{~f8EGdh;+M>Jq=vJBg zmELZTiMV|)>&^5{^+$IcOyenEMi;per{cADqfAUMi(m3`|6x$r-CWdlxAa~^Zq~p8 z`=O@MDx;8D0mFZBoy_a(i~vI3yrOhqx8k(b3xe3K52TGL0hMF>{XB-3e3fG~yXyX< z<~BJ>ne*z4j5}j1k|VXPhnl+-t(FHp^dqm5vd`}6dW&JW4AD%#P)I3w}OP;JhLca z?TJK(u6W~8@7YJ+wB%>5h=kAn@3V7m-@R=%Po4H+r#>xjT?~NSoQvyswkyT`=Fk)m z52w8dcdely^LveB`vpc=U6-sLmxwxoJA_7pis%;>mF4j6U^9M%I z5B*B3;L8g=_bztak~nVN=w!L>UEhQYWTb5wz}NAWAw8mg>8_3;Ghv|dExFaO?Wpox z_CEfu8bq{_9;l+X^i}3&FPiYM(eCQj+$?TK$fwa_l#)uqB!21WPZikr=)+P~!(J zPySK$w{k(ZbE@&ei38+?oxD(agumxtG#jh^z+yt_PVYf#IEL5deI~#;-z=+7M@=pI<*^!B@j(CV zXudMnMm*dN-KL$iEY3MQ8&~;98C;;zjwYIAJ*tA^kDo;&I@TljQ%LsJrvo6FZ+xuJ z5QHbf<2+j2f1P70Wc838X*fVM^4V&1I?KT9xilO{A^;ZhPN-*@z`cD9i))~c063B9 zVo)f$E}2giy`nSbm#KEwK0nPJArHKy-kLHhQt0Xz-TI-=WBax-e7Q9LM7@h5a8S8IVmy@$IoR<7;dodMss>0)`;nO5PHVz7{NsxufYdUSx@K_l;>6*1|(u) z{-ayEr)V@vq%1`)$x_ndUu=*I^%}bxWQAZ|XGw06N*q&q1DP`+x~wrLGXpn-?<^kX zLh-`cYtJn!M%<6$3)Lj6)x_PAq+QQ1AdhH9eU79EFAJr|SjXs)>S@R_BO=1GbBOb) z9Rb@xaWsK+PFn~fbi<4Wy9)gYqKt)9-j__q(=`<2;z1|0$#V%Ld!*pDO)_G63SKUF zqcXn+L4#er{)dfKdpWr`kO5C2(S%`KNvaMO-_yY4iYpqDG=w^$>|kTn5-QVx0@F^? zRi34nDm4wq7_7M`z=?rQ0(rF9EBs+;e+9fWLBTd+1y*+jTk|-ruYCDUQR6IMN4@RFb76$-`3+;k2Wpji5LWull zfOX0)3=uYBX%v@E>9Z+n1g<1b<{4-Ync9_w#nf*tc}max)RyYczml3BvH(^mO{5N zWmn@wnD7-7fSZK$Bg7lY4Jmcj0vchaQr%1-0^G;wYC~juSf_JnV_XQ%A&kyK-G(g- zq=~%)i&v*uiNKfobu*OD)Y&swL`~=kBtBM9IY0j2@H9B^{-E)@PBfKemdgx+^-uM&y6)Bv10 zVIy0(Ul9tzNMZ+0f7%zEpcFA_3WAsuEm* zsKo!|3pbb|%Bzc?N{Ml^a@qJM2W|-CQKq618WJn~lW3a(H@r|{a9;*UgaTmTF=R0? zmYsypP)rP=3+&ZQ4fCiC*$7_ArS#Tbn#DD} zLsbCaAwk}0;r5(77?&W%ouD2v4Qv(Qhv^RxVH193#VN$T<~2rDqFfzc%7zX`rpnOQoPIEtY^Pp!a;3yvT7_~{Q#B~dse`xI;c|M-2DDZF{S_?>gaa|2AR6?i}X zuD3xX$uBe3gg#IVl>I78>5z;k@P3{r^v%t$9)d6U?bB~Ug_s(yOGJ`0`4*JQo!AV# z`p4g&mfgO0&4mgIw+)n{98SGHqB!B6%@Oa%Cq5akExOq0nBEj}r!9MzEZ~IMbtuLn z9(wx}?4QnSk@O;-b+-u>_vW0wJgdcF&K=_jzt#@D>E8Z^&{CbPT9cu=BKRa^I(&1E zrW&~+B!%(3{;Y=tU1`$&O;;{n?d!`AEY!|8KdE;bk6sQpt0Q{#h>X|!L&`$$C&n!e zCt9o|wM!dau3by1ipC2+)Cn@Cp%6FCj4QVbv?h(ZvX8@r3ftlad-Z;0@{Q`{l}WnX z;gXsmPxIiivcL$#(#pT~&3gyBZyj&3`A_vFWqtvbhxBvmrA<*So>no+(+wwsRz_}A ze|6BSWZp+n;W2eD{}kN`5B|x3e~#^U5#E*M7Z(E?Ib=1-uU(Yo3a*xqiv351PQE$* zk70hxkWp}_!R8bQaM?)7+dRp!Xv zKao|CwLkRi*>{ujT2|J6S-;qAR1x776jYGrSak44b4%=cGp%xdU-S;2kah3bEpUlH zZRs|oi12GnDXh8dFaMQaC+c!-C@hO_pNu-l*R6lRb*=YO5Me(9$p zrTX<2qaU_hl<{@*5nwE@|6H92_ODcVd|WXZnqvE7!+7A~1(&rhKf_0+c1CwYkolE2 z4k*J@Dns=f%k#_@H(d+1Gp$jmBBQf(Tm9woqi)Mncl;pHt~1r`_}FXVkfI{CE;19S-qfiMx>=b_yWL=kpKthX(l=HrrKx*k`+As9E0)OQErlpK6QXVFk{ zR&DWFDVBcQf}FZBSspqFMp%6oH9xGSGw77%*eqdIpC*jgdoFLg{@eHa#Iwi5i#*($ znh}d72-0p?%pE*l{bHKS7&nNhYA}> zuX;?i=FA&{_a=5OtXLt$h`1Jrsq5)--pG*ac&He$_}+}n)YD7uU6@}xh-NfF+0MMs za3LPu`;!vm7c;ao7jz_hL6(@odaPYU)gE89U^HXNRdU0<(c!}C&~mQ$p}^8N%>2h5 zbG$`ceJ8R?y#uQcSYdz~a!I5pC@x9TzXk@y>rp|~kIR8t<>Edk7EUGUUqSAoRwL(M z8<;20=7k(v8@n08vIP*l0@ma?TcH&=qRhRwhu;{BPuG|Pi~5_t*IR&;Yi|rpDo*Uy zi8UcSpAxNi%gy>MHRNBjV_o3Q{jMOz(}~ZOL?t*CY2Rq|)7=uJD^+G^=5tE1<_sEy ze!H@b#b-+2rFu_yIN$5zJc_SpsMjw~aSQuKniw~(u@Anq_HMl4$q*6wo=Nc!2T?kA z7EZbR0KU`kl9uI?zBV%1J-q8y8S0Ly*(p^6iWx`+{%sksiz?lqPdOcXAADo-ywKyL ziZStq7=}%WPoPxm^RRNybrh0`#TE7vc1G{tapA6{@u-CvMTOeXmb9oECw}$vflakC z#gXhqkAx?M)hdaq8MZ(!tiN8`3RIH=m}`s-H$He;Itc9`(^@+YVSX(9^U>lWsG2`Y597f zea>%VXsbFcE`O1(+pnQK)z1Uyjvp1*mJ0zN^PC3k%obX3bIu7DX0y2qiQat~&KT&0{W`a$<5I$!>;dSTbVu*oJ!v+WR~ z^Sn=ytY+Q5 zj~FeP^TwUNnbX3;%?pT@e`lU_$zWWH`6TqwezV!LOA%(x-mg$jfl!b+=7gjB`c}HnwA?&(`x_l6s6nfzs z;owd{-!2%5gETmKIUPj0dS=rhHaN4el zmX?_f6C5Pukt1omiS)C`JMRM+8rT{yA|IS&6Hk73=YN7m26vwFeJo^U>k}|lg@rPA znxFWIKIa-PmENCj->wU*hHM%kYli+9r+bENJn2P&7vc3zOMos&09JeqN5I639Nj`7 z1rk&i&rShna>Av!nK6){8(Fd=g9E}8={{BLGH+)%-m90<@^_4QE4AwFGTz82I+wF8 z1g!44{bLK)8b4BmM|{W$1%y|qZW`{+FkNH%X)K|ghlz7mV*?|u7i4PjC0L8YeE{qP zv*F4|Bt)q1iKw}mSUL2}*sR$WG zy(beUVFG}A#Pqa}j1jz5qW~p+GHe)G7ax2%IX*WlqUt?<AkU!7o`o`%~t6 zPMZjJ;C|p@A&2IuT9`=?>f}HemNJtSk?Awg9tPCmJ3KZLI#U{ECjT`>0L1^pz!L~* z2eu4S<%IQ$=qo6XWDlqC_orI3p%Vrb@zmkqWx8Nfq$lwQPA@>3XHM!xXqj81@ZLj4 zuEvjY2;p1l@W&IOMO{%RalnM0Nxs6`tp8y!4TDBdm(?y;qf{6o;}5XXs0!X0wb1le zh=G>EXdBPi3;0P}jS-=oj+w@RXlvzEl7jmPl;TJTaY&P3Fw!Hs=l%`NqJ!lB25;HF zEraR=a=bnQiJ1aBv?+k_bv7 zyc-QS&~1V!rrEMZFEcV+n?mFXpy^o};6E?VJppC_`B>V%Pct!3W!>1>q!&S^j%#^8 zGta_}GH~I1B~N5Dk>DUfdWfIxeA88aQ7=CN=Lf{?Pt<=HY)N0c;tzvhI}to$fA>E$ z9(s zi3&EUh*;&RLzN*s-;4$n{}rHz@p!`#$$?74o%8VutE?{`LzsT0)gxFh!kEMnrPXVl zajkmYmPchbV3Hfle-fT1zGGVIn5{1hf{)T2PkdWFs-|x}9uv1%JLCxp3ytTja5%sD z#yrX6GyTC$ZOEYSc6JEuyj`*0cGicx(M>D$&HL)MXAbs!{&VVBu6C7J>aY4#{@}@? z?Xfw(TE01|40WC-kGns)NjFL}-tTz%BI+bu%wdwJo)+wNbO+t}Z3_`@J~ z&?3hD=Z`Jz-e1~&=;o@pntPK=hZo0n!?n!ev0-dfMNgCWq89>s#9wNpXlQm2&INJG zHiHI>vHb^Nc6}^(SLpghL(dh2Z2fRgCtn$`J#ygwFOLUBS9-^t(KJ|YGqZUp(Ydf# z|IY`57Se+kiZB1@?%{=irm|)wZ|&Ibv%mWPVen#3!cQ_EizwN_7NukQ4sg7j5;Cjyci_t~B;$}av*&Tl|-=cb>2OFMVtmiWas)O4Rv zHZ+XS;}7?!3;S+7)*Z{!pma4Dq}!H!-O|{Kvz@*P`bUkzD-EI(`q$8TtngA<&>sd- zwlQ{@M-MD{zNNbVW&2w{r?f`=?c9^`#QJsK`~3FnPkmzdMF!&amt9XUL|oFf*#59C zVAH3KfggUef6>-^3tJZ|=R~LIwnR8XKM)w^eD6Y{v?VC*fMvh5+$xVmtCi!E+i}!Z z)n+Lz6oFK+>vdqs+$iz6UHhZg?tiB4XQx%|yE2MKmS{#jWU(txKQFvDdZQ@_k-(SV zNEMO;+E8n8a)7Db>(vKl#Erwf&8Fc%$gW$J?}&#br>u_Jgs6GXyXdZV7K)x7SkQe5 z*5AI4B?~4^GbduZ7Dj3%bK=+aor|9laq_X*32blaxdL@;On-0iCu`2V^oN17(UPs1 zlr;j#c@h!=7g8ZnoH%?R`uLcyHNO~(4FFVLuDS2IQ@Aa=1ggTHGfUFi zJJe^7TJBfu_&OiSn~F?-$qBm|6Y+yl^_|ggW`oFq|cU@kIez6l@2bkuWdLJlAH+!I8ntpPw*ITP5 z(h}~LSb8q2n~8=j?_k5YZLO03^NoOuW;Vm8drkm;T&9E>_%H~?wCH=FGfw89 zZ%eFb7x2-^!?4fR72YpW?0j84@~cgK=xTv`>5)}Vk^`A3n(F#=bJVRS_ZVm1jho2I zAzzQOZofYMfzG1QYmFR#Mh&CM9tNBn0v6Wu;d$*2U!9d=WmuMegEOyprO>yrsbAb8 zK9B!|loRIz1f8i9m_g zW-?5^rZ&X%!uqkE$*@M<_1I!FExK^!t~|Pac=4FHX(;8`VKa;2o@2ty!If)yDq)WE z)_T#}7f+R0xOVByv)~vdMAS`3*wZz&p{k0q)8QVsK{ zZh2#X@7+-GRBGkJtCu76#eBaIG8tLTg7skp9)C&WI2vB5+ zba6!aeQ5ebp2%dDC|GK?@Oc-!b>Nu#D0^iriZ^Z*XHds5eThGyy@28c!}>GP9Dz}k zq(1)xfW{8s@^dyn&hhA3N-qxtU`Y}(j=}p1rKa66sp{LjoR5gyLVEf$i?9gU<|6hn zvsa3$M^C?mMq_!d!%k$E_4tG?Q?6|9trvywNi)uBcRF@Gi;&XJ`YYGkTw1|vS06S} zf+CBkq0&A88u(+Q&0Q^9oMS$ljj=@808{r+$|zCFaJifY8umnV^W7ovc@(>Lp;E8@K9DjZ^_ zpaafTdM;{0wXbNhz)8?f2FaFH!MuC?U1?E;r`-y)_f-wjRj;Ep)Q78|gBFH)iRBFt zOC#FG82VvKyoiPzmFNbL;ST$1bPZqqE8Yk!mEt(aT*1m(<9%Uow!WJeAj!gmZAxaA zuQi^AGha^^=lY8ekn(P-7S#=c;pi&!Hq;I;tN`KbNQ9_-W0mP)XqxNvUKCcOhi`4< zdR$-bF(xyisd#+KF$Tb_sXk)%r)WlwNKZ+0h)Ff7ep#~};)K|`*aO5|Z70X|Sr>5^ z?R>2j*_=x&3H2hwl1JhMs~q(I=s`uscLt7;aR2=dF)mgTWnKEt6$kK@=WODxu!KvJ zYEg$D8iY-8U9^mwxhCnD9xcQa$S_=fPLm^Ulv#U?Wb->-N8`6fdq47NWpcYI&db0A z?r~U1AYDL!6P^HZ1BflgT407?DCh+dDr}-zXnzugKCBoF*^@MsLg3~w2Pz$Fp+1*0 z(4NsgIq@hagJmFYXO*S5+FnTrmM-ojE zND^L#xm*r(#Hg4xM0$yRfEF{+OQLNQ&ocrErfO4~P{v;s0p4=GsfoU0R2!>9{eHj?@zmV2z;`c2uU@PrKq%1~c++5%M~G=Z0H9%*v2SQNbJ$ zs;N^KR&dt^PI!$0pUOH=^%Cbi1~E?)AQAu;H^x-BoG{h?3!-LDluo!e2KH>sJ8TN> z;ycj|5U3yf8nxtTk%ZD=9@=Da((PuC>7eB z6sBRHK|l_tT;Bu-JVm4WQ;y>p4t5DZrm18z@!r@d+?!$$jnx@>ZVsIQ1N`-n;o375 z0axM;+)J62r~i+g!(dFG;wt>!!=ZXy6w{1Hfk@(Vukn)*+_@PLJmF@t4(HyJnBVl! z*cCYCtbK9HkUk>?0vP)5VN=xJLk)9+nHypRuYQ4L$)t@p*|5?;f?mMS--#eAF-i&b&bp65-h?9rzmat{zdD%|-8frF*Ng#V z+3UURho6bT+88J&G)yx>=?+fV@&RjKN4R+k^U?}u1H!tIsc>s0l-5G(m2y0^2WTyS zN137j@r!z9v6gyBXJLXdsZWd9$$*`u(4g3U`ZAfahhvE%ngLWOP$qwho@WURE_c8k zk!8Trknc6J3?2Wp>jnNc-hJ7l@emm zDn(Kq^wYZi*Mz(M&uLoEPlkDnm2JFs(M(u()An%LYn#iy&#FGKCM~+-JmAyT>*%Am zs=i|4xWdb=rOB&4D0TOq82cJ+es*$}=lJT|J9rA=-Vzv?i0`X}v)ruX@68Aj9&9c!HiiXC ze17)w*pJ^|%xmx!Dm6@DB>!^|}7ViGatI4PRfuq3T z-VwLlYb!^hWJ6AtsaukR^Bd+JmC6KeH4n-sPIa`%7NJ|$OD)a)g9WkccZY3y)c*Gd zixL?>(D|nS_W|2uH}Ja}olhS1ZQ$3Ilv3esu5ft#!;|a3%AH75*qUiO{xDeN={0ev ztxBfQ?=+u&k3V~GQZ%t+Wx$HSMpz3b9^x5>1l0XD@JU0V{Kj6p)!Vlp?5NnNJtdu) zQ8O5ppYr-Gk?x0(x3}K&FK3EW>Ib6P3B)vGaXNq1-}sR(yM z0jWz{*h`(7UmEx)gAQ+(8EzEpiPWD3PU0OYxcn&ROjl`}mFWe~pKGEj)v@A;-gPK^ zhrz9v{6y=-U5#vezP^mVFCQm;06)|DW|)IZNoIClMlrw7q;FTo644KPl7p*pLjC?0 z@)gL!`lJe$jc%^BZdaK*_3{*69}o|Z0bI9Ol~kw~;n=hC9@bUv3a-^8 zW%QaJmu@9HCztwkGHQA})&u$U=6g2ZnF;F^ReUGB=5|9!;|a@@aeCQ0pNp99`5Z&7 z%gP{6UY=Btg$hQ^UP6egLcMp<1L1t$ga&T1CkIo>Ttiq_=6w0*1DLXk)UwmTnjF|P zBgMDbskp3O+L)d!z76C1|eUz{-S+VJNi`t#csa)3|_kf<8 zM3sxs@=W7s@*tOl?%)Ryn%&`d59ppFSb8%nExPjFMGj$+o3vjucyz>$=aa z-!7z+OkR|OtgcK6`p+2snQ3;nG37b40fuxqE&BO)k#Ry&JySyvWHw%p&XBpKydO9m zpg)Gs04gzqX}2-Z-dGe^5Dz|JWH6_|Cn~Djne>J%=K?|+V9yv`O!CnRTW#|2VGpE-LWw=9D8fIlXZ1?yUTL`d?Q2fZ=yyeU!b<%BdXNN z(pnTjBmOyWn|07F-FY-yV`@K3evfwUz9otylF`wFl)IrB(7{g@{GhNxG+E^H86jT8 zjd1JX;wi@%c5AZ_NtnxTF?9(7>4;wmu@!Fg|C&*iNBRfA0%YemECgO29(Gg!qpP#w zZCi7|+>VQZpvMR5g*674+~j6js8?Z%JxAG2l7M->I&-BQt<8Mubnhr%#`@0lI_PBA zq+XkZLik|4UYe*3rxcudbiUplmU` zc=90uY5kIMlIowouie}chZv~zVkGZLSYi6N`Z=iYS>OMS`+ zFGowCyTz|AkvaCJeYz4|&Pg?l3?-N*kZ3g&no_JOA&H336v=}am}L0yfaMH&7?BCY zK&6b(W21I61S$_A;P!zSUz(&w#E#gNxGHQx4ixd1n#am@yjZ5qzU*~IoU zehTP*Dl3DutH)2NcOVAq(JCHz5p$i4Ey(1bO#V-m)*i{9(z0SSEVqSMKs(isLTX>O z2d3aOi9bGse&iBavnAZn6@P%vSWjEg0W6SVHH)|j{XiR_kDX#~*$SlsyiiYRkhJ;erfZT3g-VvC7!rNV%8pmG503iQ;l_!+Kj}sJrN?{OpZp=0C zY?~?wr3-=Tr9`bLM<~moFp{#s*vl}Aq?MtJR){|a7Ym`o5O5T?9hW9h(YJvwBGpiUj`P) z?m(Vi&D1ImcRvIBP@%WukSthh`eCLi;to4IJ)SsmNzCNJOq2E5hkV@WK2pe z(#@n7XWbBKV@@fjsl1txIDu!5=Y{$S+4HzIsg0pfkD)aLUkwZaEd>sYvTPL7959HAYXP^iX_-1USY8M_ z4)a528o^rRmAHJsRBbo)0%LCr8cd8hSa$C3^agqQi<*sr@h9B{C7rNNraa1s#%6vZ z5Z+KeWa>4?6zEo5p?^9G${4i{k2R*px9ByP+9AxRU^+De*It0f1p#nKelP&26Pt@g zO6h~AP@I0pfyrPvr8Eeceg z9SYT-NP>e@f$#;_cA>xX#^Ey+T3#7XHSDU#3OXM-R9j+WQlL%yXefG%09{+Dsw`=& zhqv|EJER1mwip?n?lFz^*}{o9)JF;YgED*1MZJ{QP;wWgwdq;%HN86jhBdqwh0=7` zr$^SqCi1!pjX1T7&1`BEIU>z?!d>C;>RNQl!O{7^?2Gzmb*a1lF!+nrvc{{R0pg+6c&qY6{0YOlt zLy1khO6h6%g_T7RGLMp)zIFRMj5GIrUz5JSccxrh&=_{J->~%cuusNyd@73`8_d2P zVD*ylm(7YFV@8dioshU^YuBIB+GuPv>qDn2P=>fRd%)9X-FrJpz+bRbDR~hkbMG9vFMDs* zs?dh^y=kxMU)nwdbaNdQW5XvjmWCH~7h~_mOHUs4j$huTaN_SvHy$=T#qU`Apr<&q zdT_sTaO-5T{J_yI@=GsBGG#+jw^7jxZQeLARHH#&NB*ih^w6r_wb#rrt0e1jx!8wl zFdN4Pgc;r*s7MbHcdVsIgq*g`VZJ&FnQ!m>WBLZa{^mOk%Y}6TJpJp!;Vv~VyCG-j z^V*(|#2&c;KrwxF*1z;QC*1St)c@Mqr5Bi2KIpsKQP;7CVHFiBRMbMTt~k?|8cv(6U%x;~!si5(xSmehxc z<{VvHZ&<4I-Mm299Mw1Td}-SodMT29BHojubQCt|F0XUCw5)dJ&>a7}jXQ0h`+Yx5 zUe9YoCyU-MJLWJ4^*VbW_8ax6%YV`psCik=8`0=fDlzF+pTE!@FW0S=f6$@yMSH9E z1L)J*aTZNS%i=DbN7QU0>u-BqFRj(Ldi-JVg|%qn-IbEO0KCuq@>4}8U9uhrX-rNZ zhFT%O=T|KVO%EgXqV~{#>qpkhU;;2 zq;d2&;dhsRrb$2h5Dm2Ap=3$7lLx5#NB%H)c7Uf=dd_H%RYX;&d3nxjdVJolgVZle zhS#E3j0}F#_iQ1)K;t3Z)z2-K*`I!MRO` zqmlxo9}b3W@cG-L6Pg)`8-cc^>A$^2iF!S%N!!wW!=g$(dY*aMG3ow^#bDao=I3FB z%@{sBviAW%WTUu)5L|!bqDxk+`iSRyt|Qxg;<#)c3Z*l^U26~KXhT94lw_#yqlcSJ zBVY{k%j0o`YZs?`XQOh}8Tw!OW0R@~!;N@KtCrR9^moyEruy~S(2H2q;i`Y7zSnYB z&Wb_Kq)QE@8S@~9>{yuSyit*Q8pvGb#*yP7i1ZX4TR5*awKSfT2&I(>4pP8aM(Xjk zdSS+fg`|1p)h46cFqNvgT!n5q5^=>8$ALK3wR&b*VmvPZr)M)1+A#HkczUh#nkp9- zWPwjX{DC?!z-%>NBh)Pgd)EgpJbUJGDR4=oEwV6Oa@{PvezUdvLOrS zJ*qgocv|N|bR7-4)h6eKL+2af2CCnx#jeU+G!K$k+6-CFQ461+f#s!dETnp0uOYnz zt-OxDi&M7kCw0GS0~#TlnITE8-LkiS42v0xB3EHT3%VdZQ`{=nIPTqNjStzCw7LO6 z^=41=cFF|k48!SDqnCObVu))ZhE-{i7znC~4x5Pm+RbaBT@YmVSz6E5KsQ0O(oxJI z>BE*)U#;rlhqFdm0V1Sd8>P)9rIDUnPZYg-Wg=*Meg+KC4w$C~2K(5oYgdG}k7(V% zD~%!8oJc+Geoq{Lg)4fFcy_Vk!v&2pa?l+?h;-ThR$Lk_!RUG* zCQvpx2oOutgI?3fTSG`rF4Z6m5W0grranq32(e9YGxFn#ARG#=-j1`j^Rdk;M9u!X z8!h?@#|hdr;tRd&viJy{z58-kQfVx+ipw7Q6@%&`u#bvT&Q=tlNLs2EHU+D=9J4UL*r)e+1a##^# zGoXjK0eFNdP{2h+_9#J?qaOmg&{8Oe!Xr@qx_&Fft2Iu7)*p%re<``#v&z!SA`q6|JCN15?g<}b z&3c@ZsQb*OXb90c#YSr?40W@VCRk*`3@5uryML-7hvjGVi+2@kt#b@FAoX}fmCV~; zQj|k6vc4iO<0>;4o*nFa_eiMx{73%ny%9GW{iP?cyE9n3b{@Ys#H+4F?{#y{UG}rp zwZUpUyew4&53m}|`-bR6wJO3igh=yZ69!)T!M!>=qsAgTpp3V zKVg%g%lar!I}f(64U1e`h4P%EL)!Hdfx^3cZXjKb!A%taRA@fc?Ok{DUC?$E#v+-M zr(0Vc-pwL@5>Hd)Pw)VaN;t2R(Tr}dutGLj{DCXC5HUZ^y=3pjc{clVt_2Xe+Wix< zSr{j|U`aT9M^rRLcz~c(g3B>Mj47nh6$pCGU4qiUo6Laf14b${6S9i9dH6K!ftXB| z7^f*IIkd8b-#G*XaE720fhYGxr2PakU<5iv#%oMF@W~L%FR)PfJ?(nCoBt1AUji28 zd3H^-hE}7bRWLziz9e6xp9&<6xB?^bBQ_c}CSetr2sI$6Xb@#_culC1KyaD`X;lP8 zgJBy-_GOqv#AzX{3d5qHP-Iw!odIT;$$#$W#iZ?j{r`2t5nT>FFasXt*F*VJFUAwv-HrouDcoSd%aG3&#)$Ba(2#0}yY10+Rw^(R1VV8XS?5Ab0& z$*7S?V5P1&8F$^nNAr<5Mu|NWzG@!=Wd_&i>XC0a&DI9^P=GcP7httu$AP5wDZJ(W zSr&*A766;RRqVcqqQkR53HnLJm>!RYF{H$`^P@{5TOOE@dY2_1_yr2=hL7F>ltN{< zLl71B6P`d~!bP2f7EUnS`q>9R7D^Ck-l4XIiB_+QBQx)>fxnHSgNN}2UH#b5ZCs!c z_o!zT07zM2btC=<Jaz;6 zK}>-k?758@#pV*R#?SnV)LI$@cz=a@6EeiOz+6Pw5L30|`qFQB?p9-QoZ5Cp@!$p< z+i^-CmDRHsE%BC2N9n~7k<%#wQUE|}t_D2&?nXxZYUUbG9fJRxA0voeBKO<`CPxFP z=oH2Qv#bk!4~Hy$T+N$9U)$ z@h-t+ct-#;OJ}%aPxVm9$qBpLjNSoIteDZxLf@y_R7ZTQ>BjeP<{(x2G8#^zjDqSC zm~@*i0M*^M8HDE@zh`T>Syq5r4PuV2B8@j(FlZM10*0T0V*HIINx41t=jl_QL%9O} z4@(VU5NIISAIt&h9G-4k6V#54brV=@^kK`3GHA5WSo#Csh*zU3fEmD{goW2RFnJD@ zYSTKUkHhp3W}#@?7kMxwad0-xM~0hHhiu(ay5pBWSie7C@zX1ZPa6D7|LtW(-_O46 znIq=D_Xeojlf)87>+ux_n@6B1Pb}vEJRbzr8e0sqlr}@YJXI^HH?}-#i^LF1&4w?_ zFf)W&P4%hEenxGS{I+{?)&Zy4^trs-@}_=8BcHeP*`%{rf`s18Dfd`%rw|CyYG%ko zXSlBY`n$OQ;_Bl~cPG9H?k7bNW;+*Tor`{3cqp$R zThX;NCV1JKpN`(G`m(h!`8PPjx)lxbh{3_zL*)YBrpT7D_hak_?oJ)@l>O6c@yLJa z-I{~$jXx6}$n~D72sed7b|6xEv)%A*31=2|ZqoStP(h1X?y?|kS?T42!JoT*p3}KZ z2o-IKrm^;6djL1~Y*`UHXQ56P!P}oDwO(`cmtR5lQom3Wg2S4j zKjYWh*d8=Y3cuzp{#2_gt<4l{KU=jYV0il%Yi&`h7IXt-^NMikZHbv!`oC5y^})Jb zh5g=c+S<0r`D9>#F)+m1Dp1$$Knpv+PdRp832C-8|v6E>@);EVCKe zuI(<1ZF0Xi7`8@X{x{-u!&cEB9usA9pO6QtY(YT=9hKxp$#w@OWIuoCgoW zUh!DpF*f3lU3hn{;>HHz#3{Hwa$msa>w>-H&xqH zXB_0id6yF%Cb~5Rt=1(?wec9qV~@GtoFqo6OAo!9DGkL+>8;^#%mfcCt?S3aQ=yAPiL8KbdY^T#exD z^J+<}uxB*7e$Td;#?o*!MgolpOruOJNLJeo7Bq=|^Q&j_4qG&VVH>#g;7gH%kH^I= z(&_1x4(mIHgSqu}2UmJE*=VXErY}QyinWMUDYj zWcVvlkVU?3g&>8_y)<_I#q~#jGO@6=%^ZZt!iNhAH%Q=gg}JjK z;r^*NM}DupNp+5e`i!b&KLm>pI8BBhS!Vc)v~z{1X|f+-DTKhlaiw4z(>sD>dTZ0X zv*ioGa3r`RMKM!oX%~cQX>5{;yG6Cz`v!#f${Mvp1ySa#@TH&GU%80g5cfUfbPtRq zfI3QIksCtpDSgs^fK#tqF%Po_M$regW_f$*D21C1l#=N9F=3II*f9p;&fGU=01na) zxC%&p&57pB10e`bCjxODuLUj54yo4t1AF!c#fY9s((HWVN-F0wrjiH!-%#Wx@17M$ zlFEqUoGKBB$|@LU>O+&wYmOy3r!`}@)Il|%56cv<)N~xc3{xQ{irdy@OI$fv z4N3I74?s*(smQrcIV#2xSfD}?2>;apQYoz2f7h}6#nDpxp!68`Xy>ZpHKs2D0l_>w;jJwEpHocJ^WN=9?^uN5 z>DbkU%u9mimerDe3?r%Eqj;TZosmZ$dhGl#i5$0xw525ZQ&H&2aPZgIA(L+!6MR1T zC>Z@W>b15wYIPY%=fgS&;&F9O%4p@}H{g2Hz!-^H{W)5Hvsi>6Ea)V)d?U#@$FnaP zXgtWn4uFEF2occ)@aoRMIcRI{13`!Q79x@g^n9(!MB|<|az?}NXW1KQa;J?f29o)| zt*sggjK=#LVSo$I9^N2%X#%ki2$eS7&SdnnGZ{dk<#{XtPu) zoQtk67CQxI>~g?kRT`KZ1t1xZUiM3bd2=- z`&^;M?&)LPr<)xnl1~rX{eaOE_S!vQ55cRa)_2$Z)%?7{yyF_8XO+Nco6TOa)q+hU zuy~Wy8J5@m!d9932RWj4l*!iJP;y=aPVTE|w$K^cC3uVewS=X~RSVeyMDP?hYOv$t zKk;2Z2Y;Y#P!UZ!QkxHmc1F+$>^#emIMtXVdsGi?E}=eF>W3@Y!9Fab zg?E$5?VthUTJh5l<`MO?` zy7InnY>>+8*&f3B`w*!NC}x4A5eBV~@G#`I)WQkqo?N*C34?>i6?PH(fX=^!(_;A- zz?itJ0fZkheRW8}uqNEh4hqOWbFaTkKkU{(<{IX+cfqHmZ6Y1#kL?mpty7yY1k7L& z(!Q8R%#w~%PB0;i(Yvf%2Vnw;NC|GQ_AfZ4@zy^f3kHwJt+?2W5mJ`&qVDh&N%SVl zkfBjDOx%C9*Liu%LyRwy=n40dvuvrVAs;vadls7O(mWv~h5V`che%GtV~bo14K!5= zG(vTcK>;6&BYuK*zgCnTkS2!R0s5a&(vzkrQp`O{I}uA_m~Qo9^dMUTW-{)Puz_D| zE{L3whQK^PqK7;VSKkIUM2;sChTlF3zC!~R{)Y9@mvmUe-R-ffD6M;|%M?rN|&NYRg!2C7%kkK-^233iYr3o-!=g5%?&j3`~GV`>BP z^d|rF;g@j$vOGN)WHx12K~2oxz$$p1(L$u^bKIq2J#832GtZc}utY>RxThI1GKx1` zrz)MbYE;N$^*XE~1wP}^k+{G`u{{IX1loKS?5rg0i5bXF0ovwhEO7a_(fhbts;>T5 z#W$^;e;c5G9V}<~84!GiA)K05$1wdjL)WB=!pSiY%q((2L6-=E&&NOE!!XX;wp~RO zM7s^|8lIbz5`^xBwA(>v(569NJ3B;vh(?>r6WB(4STJpSLFUqrWzAMIW3lN2W_fYG z-23V9{fku^lbQo*{|Ekl3aUxNj!}8T_)>t@>c59qRK%akc6^vRH8vm|8xwuV{b&o4 zb#zR$X68u!Q0a!QHIPT@?Fa>Z{Re+r zI9e8FIJvOMHv_^=!(#Md#j)--Qm7Z5fl$A-sOEZ+lB`x;z`Ovr7WnnL>zunM5*o?I z66UyC*JW=i<7eLX%k z)OL8d?CupmZ^dc5AjALc(mXaOXXF<;o(uWNJ?fg^`CKl)_r&CV5Wsh9Z%@vt%$spO z7{7agVESZHw&;fws^Y9_Wx3o+*V_JwgF5p1e1E%!+kq#n%p2D2LIc~K;s11f@`>No zusv^%T&YL3MRmouN4fr{BE_QK&8k0#mc%a+iN6gw>8V(fm-KS&|HkdfZz`+iZnUEK zyS4T5uH*YN^X%>S=s)-hiN!@nqketMsMI?aMT!rH7-O`7ee04xx3rR`i({+2^xIo9 zYBj>niw)2*n*)kUB5L&qw31NmW3Qh~+;bdi(+vj&TfL{gQtE97+|COF3Z&~t%rD1f zhAnfK-;x!L8CBlGGZS$;AHLeIck9>go``-7aee4HVHYHb=U?%wE#H6H!qZC76d5X?8>B%<93$BnQ0W`M)W_n>)PO7;rRUho?A^YCzA)&txNLN zjj9amxNC07GjAG!-~OCz*kE*Gbvu&c9*0bsr`sk{e~$U+LleMefZr_5D=fQ}vGfGz zmyVvmzB0eZjxvKJ^w^blm10IXvSRS-6eOCPI{YU}q$UlVrp;K^e=l>qr2J8sjWqZt zlVe9wTJU5nbMu{?zcy~yDq`&x1{)oW_reje{s8C-4ANu7h~^|R&fN~%5A{fLJrJP0qXs-ZmYTb1p} z)c(ZcNo}_pY3KaxXkjxba%46n=-jqo#k}i}O2OsrDueCMm|@Mpdbh?NxO_KH?39cT z<+e4!G1(XgBWE**$GapJaQV_e(_N+)8XQ!6hpWoDLygu$c^PP5KE_#i#QUl=T9>-8 zNacm@%Zb6-dtN>}RE>Uy|n zw69Y)X*|_mJ7h1&4lDoQ-i*st!-e+texuOZMs4AJR+s;Ti(AL#d!it`2x#i-Rzr*i z@h_fFLZ6j&J<7A42iG}R+%4shbW3YB%cwfJ&~sF2Je97PDG`@UTt$^JU*Bp8B-Su0Lp%G+*mgoLpLivN|0m*b9y66+5oj&E2Ymu_l}>%T6gIg3~UC;wh(07QN0V+9JN6=Cs zugsU8)M~8y39XMf{V;dOE?yKI)+5uDo$II)YdyO)wzl%s zm>1eg)TiqGW;%=XCCJ0bAaW;{sI9=&Iy3D7aC%rfp9P~2w9hAobB>K#+nbG&_@*fH zoVvp`v0MWTpw2ES?_ZhRDvln=1F1f0IKaqlAn8P&ypzv83VHYnl)d#jqv#th3LS_8 zDQqeybj89k87^&4u8Al4kX1hQr2%*R1_`4kt~QK8JaQlGng7FfUnqbZ3O1FyCYsDslPO)`p#OWCp3bl9@qV@CKp(m9Y z$a)^DcE^Uat=C1f7JVPH5t}D1*&Bo&ai`b8?!gB=3eZpeI&iS;T8nCb0Eq1)!k*5e zNTKdysA!B1HjcT=3*&#p{C$Snh0G>vV0MQs5C7$VQB?5md(G=EZs|%FmuR5mx;i+C zjlMMqKHEBfU1)D=#Ra9##q>qLx8Zux7@-11Z7|!}>%%>Y+kzeSo;)JvQ(RiU{oXV+ zw_or4>%wui|5G5 zGnNz}ly1td3z0PTLNaY6L=;8-*x(to6h+fMwHjW*80BIR!aQyUS*Ag_jXv-iT_Ztr z5LS!+Kwyj9e`{1mIj)vo#2C_F5(0d8WxS6HVz{X>@N?PU;AM7&OcssO4Ij7nnisIW zkuoKs0s^^!4}YEauxI8J)|5!w5hD5NU(^hY14qBM4A!BDHVf#X z8JgB(*Fua>72kSRE9!^mi3@%QVrRX#r1b{+IdN({0Sw4$0(AIDi1TBK^V@;PR|8un z8K#Dd53sQoCFU^u5A}iMnSLr#oPoIZ=QdAFeUC+2v=}im=Q?zK!=xz8^z}0YUfk*{c3C*vzNYvC5g~ki#vicSIl5~ zcqX#YHe(dN1cSs}*U(7Amf(nu@=unx*$XZRj)ZK#C_2@l1VNQ;2x_(%_~79&od zObR8+r{OzDOYF}a=mCD&M1Y|o5#Vqo$1A$qXDMpPxUpI>f^qjGS$2h{FuG*dcl|T< zamJ^UdOznHd_QptU5)(^6m&S@S9EqjZVdUG0M@+<#nPy}XU-k`&W+uhpfg+GNR$G( zs?CUZo^27g*B|*?Ow+0aG{cU*gw}BaSG{eS-(_m0az?%|gg zVorbXA3&Q*Ntk6Iwz(xJ)1D(|E*;L@^x!ZZ? zn9Y4H+{6Eou_Yx!T&9D7PCf z+~>s(lF#i;Z@71GwaKl4$Ap71_5XBFPVd?~CTt!7mmRA&CS@cyD(ZAFiAA-i*w!cy zEQpU!)c(ox;a{+-FHCvuI4O*}z2#b^E#%C6+>RX#_k)%2 z>bZYdhXruYrkK0tsak8lDdN-~t-f|VQh#LPeqYCt;SOD>=Gf$hljWV&6`G-$$OkRl zmAddkv1BW%Om)^MP~D488kzh2jJ_84IpyW(W8$|{0y|pP=XCfW>Gy5GM!T|@zlP;( zFB`G>?wHngYAP=?a!-Nbe6hsL$AdfAx3su+=FZT@%wvzM^t*OPE_?km>lodNdaS4m zX5ZaDFU+^iJ!~sDlKOb^v!UH*q7cw^b!ZZ*9=3<2x#iWw`DtO56`~jE>_qjNQ0Z@` zJh|rrwZ03`n&X6X_x|thP5kNMxow?G+oYj8DZ&U0-RiX-)zT>ogP;s-dcR!ooqb9F?F?xfaRsr~)OD|VVqc<;7C;$hAzl%O?- zrMl=B_JD-#l#O4k(pU|)>y3^94u2ls!aW~$qP$#z?f;O_U;p-#T)PAE(&6f|IBlEc z6%7|q?N&5mk=*s=5u3X<*6|*>^@>inV}9D9DQ(J#Y(v9i>|?_55wsaz7q@KBi(V@N zGtncs_JabVq~U9fE_`+m*$_8yt;kf|2P^e1)Bjp({l zvB%uU^FaXDmaLvAhu|4ygOddvy-PsNdk&ZUZ8GXL995RW$M$6qP(+m*zjSAw1?Dwk zQRA2WR#qdg#)MfL+dM7HB!1!Ge19j(aZAK-D6^`VR{t5;gkIGDT(zm}ChWMRm@;`F0}tgv4lN!Ec+nRL@yHUP z?pTJ!`Oq4Y{h^@{sWf{C?0y@HdULfn0G~L7{P`@*1NxGU;~5~JxgOesm}~A0cv>SG z@Z#yN=B3yf$HqXX<(8J!4`vNzN)-)j1aLSaD*4m}a%nf;u4Apz~ z15Q`a78_pLZ|3y-{jgcswod-0Qq$pI2)T$vnAeWVZwPZ>mY>(~A)_bv+=lI$vBi1G z@+ZvR8j1!F6!U)uIy+&)9o5;tWNTbgvh%rw5?!bfBsmc2NL%p>> zb^6WcQW{ZXiuaQz?U1oF0K>kSE<@OA%^-^WIGz#(r z{RQF$$|3f;Fn5?-k4CX#ya%z!-KW&dbiNK*}tqXc!n^=VU5x-UotmP6{i}q0o2!Y-$`&o2B}>G`C1%gJ z)4q(vx&^$|A(wga0ln-*amflhmLIFu_CZ; zei3)z{N5L5nT_;SknsXc=uwAV_sqE_*L4)hQ@uh8Fb?n?1?;QlRM5XEhQx!LZL5_& z?(#DNr^d9b5|;8Uj3okj{fjJ;w=f0@;{U-wIr~ULkxc&$P)=bsDXi7Vihx-|u_pwo zDgSJG{3aMgD8bjvo0T%u(Ij57{1ay;C3ZH$#xiaKLW~3#0SE-s7>;0ga%vQp-^UOq zi*fnw6%)HE9TyQ-NjEkQ@SOvAz%ms8;IFqW?8&L?n^d?1NQ+nLel`6q z-~SB-=`cZr=?K_6W=}uKo3wkP$Q&!1m}n&PNHYrxy=#zZ%7KIssgt2Qc5zZ8V7g<# zmWU-paD&VK7Z`5y0RTP2fA9vJFzm#MB2v5TGni^bV`I!A0@NU7T(Ib{t6)KNZM8sN z=5EFy?z56sF&wM(u{S}Fv@>USOn1I<_|J%cmPnkB0|#@_+YNQ zZ-q{H?AX?MNT0xSn&Bv>nIL&T48cIIw%OcZ4|2^VTz)qvGqxoQI|H%bnK zqIVUVJYai~x%A09jB}Z{J={5rq8ns8PB{|@$rl}E*MJIXdUWKIjh`3u@9gR%2@UIw z4nCI^fFqN*yhjBL7WuSVc!>!->XAU;Nnxh{V8GINhNK-p?@Ku4(2N4G#tAEpd(eZ(*|LIj2JyIY%rG|qv3`BL;SHss9gKGQz1eyO!{>Zt`d3AvUE}sM9tt zrh&c00ec190|5}$s`_JKW$<}uOo4b zLyMQ`6NOAu?B{F$H(mEHa2Y-9{u3_~R-TWY5kH5eU`V{rW0;#c-Or!T9COfN4la%J zR(^B}B)ES$*Q~o<5g&hS%Gf*{+#J*_4A>B_yG_~uBxX)WRe+f{@k3WtU*+-o@ zks0Fn609^WJ~-t#@#QcbFldq6Av${Lv^7$9YZQUpli!oDNlG6J+2cJB8 zCN3^{YNljG#&?Yzh|F6!F?3_uqB}x07a&U+KdQ71rxqq;zvEACyT_89&q6*g!5Qu? zyqzQ(Nc@$@a`9i$UDb77V^YH3KD}Ersad*oB`R^yv?yt^H5*NzG6}dSL>)qwKBhH06 zD}pzb*wut|9_<+T=*+`ptpj|pZ|WT?tgPyX%BqfuQ90cxF>etC8n+b61!L0{mT!ST zPWK8(hWtVjRJrJYu@8PuH}!kkG8H{F#S2iN+1eN>_XWckC{J#a3hzk+>}R&WS(fE* z_=2mOHL?=c9m?by*1+AW3{MC|WP1EcH+fDSC;RnGfWvb(cZrYf8Fz^XGV==BMvYid zUvH@#LC+>lz``FvPR@YS3q59hscCI!oD@FuncnRpu6Dw&F2eoLYkb{|?BGYjF7T%g z>dof>+y zD|e%^QFyar5{<&}b!~!Pbe}QCglfC+zHwt_%*5ec3lyI(Rh?ISh|+@89ZimHL0-lT z{8n)q?ooG=n5{=Kawa-3qHxw5_2p+ktO2owI(rd)FCK@k_Jtj|Z zh}IJO<}N(D)-#COGD2CNzcxyhT;L|8Lx?;P#?#H5_s8n;FzKKwZh|#e6W2Oa*63`y zW8G{H6+d=V;)*GnV+>Wx)Pzu{60iG~Fbo1sz} z8rnz?5jq3ItTw#jxD^>MKyfjg%`sv*03&n=B%@t$fe)AF((X1?ebO_a7?}pn>a1a7C!0XUszwmkF&@bS{bEuGP>&UeeUf3peRGUDhyNB&P=trg z41xMn7s~2x{=F_2&dG_%ewq56C;?|!@AR6f7(`7Wcx-;?hK*rx&gKy1y;Iz&D?a)? zBBpC~&zXnaC^$@P3KbbPPa1{65biS9FQ1yY$~B-P^R?)!HkknKq;=72rC4Oi zts6`8Qib?ZCuAHrKZweJj*j7ko-H({L(diNWMnv|UDJ<;jX_!SIk4 z0CbLZB+^cDb6*NVieQJPCHyL;@D_?`1l!=JNH{=FT$*oYfenS`M2;6BfHKF)fTC+C z%AsXYHbh_i-E38KEIGB`>dUD65NYw2#&c^m0VWX9zJr+rH*(B`C)$kGE{#YX3g32P zvtHPC2nE+HWWo7ugEm$3OgDmgQnh8=#QV0PR*Iv&aYt5MN!8%_51s*}jR`syNedG@ ziSl6pGrYl*o-)whptk?5KguJ(&Eo+$WtZP}75Nnx;IT(6dLwb_VK^qgvphZ_RPbEp zz;P}|W!AZ@RwMaLHAg=8kN7%ldIz>zTIO}5Sj1BI%AW}**I>FrJpU2CCKpLAdI zld>BPhXxM8Nd;NZk+;q)0VQ&os$ChTA;N+j?g_rpbRa7(F(Naz*?<$=YKfSUDhXXQ zw!#L$1D-(j2W-P6FqTx5WBnWz)y08coCxadW%VRczurw|(# z@hMe!E})njIT3m<7#}i5!&JmDk^M5z2(T4@4Y3TG%z`F_tk@jV_L4alumnDyasU_) zY|=qTiQn`Q#lwgfqU=>&OOWR7|Kv&B8| zUSSBk(<{0`1CL>H=Q17^a>jYF){f4TOLM|D}W?uqTjg zn3U<5uo{7y>Y(?s>;&n4We;(iEC&IB@9B&slqkcjhxfV{wo?P7H;;A|*!jT9fVvJK z4dsB*L!U#=T>#@(QI!nIIDK0ys9Uzzxig70^8+JIz%MFq0=Gz*uyo6}=-~hg z80#88#iK{PIVGr5>73nt1q&Ew47Jl@NRAn36J(}~U@Ch`sLzU)c=vL;v;Ws0+8wZ& zuwG@qph7k#|Aub)FCJQ}&qa+?-rlEb3QSIa zmT*OzXqu+%kcD#|+g(n>22#|F!KIxFhZoJA?9+>aZPF_cJ0!Lu`NBhL5boHNXW3-R z0scb&5*Yr^KW?Pe#x}0reEtX8C7naQz-C5Ysh*>_V{NH)uRCHZ-98iP_IGN~8XK&< zg!$hn{!;sTc#zg|I=QR%tWg-L|e#Bn&L-|zKcKv}YrTM!sR z^6r_`sSan{Bs86wHY@w?Qgs(aN-hKaz60DCT;2W!R6}#TJecsV}_^pgg z52^00M*=u2~6!X^N-`Zxc z`!RB7WgP{H&IO8F82Jb#{?B$gO6of0_IP|MCS{+x302~UqUv{sWc=ZdVXyb)5G2*B zq`H%^m8u~NWxuIU8}#LuR~F*^iE%-dvh=AT zAOce+xqwEzb`T514c&BAmN&xIsQ_KZQG=-eb#{nSS0z8^b5GM0k=7=)C&`n~7W7Wd)ZXR^XVJPP%Pv#kdYf70*e z(xI~|(JpISmOLIjtR+3YY}L+=?+g|J35jo5qDaN-u9>mLw~4&ihRZAR*%EJrG& zu#ZsC#7WCNZBobD{aQ=2Hx|p#L-ch6I*yJ;lB2Zrk0T$+5aRt>F4xAP6#T*%(Of8U z2C_GG^hBjAbqTGa>YTd4;wEvoX@0>Q&_*^M!<%d-SmzgCO6sx#3_38;K5sDS@_oFXw{C{AyD)R$0sQw8wBjW+0w){ zW69Pwnr;mVKNt+t%p<7-$z%WIH1MN=ybS&qykAoWyiFC@kSA15kFBkg?!eK;hv+Sh z4dh|SrKzMJBM{$~NInxT5I+9ZePe~TLnF{dX2W)Bsc3)76Zfx$10s6+-f&|Kt|B+E zS#JyXTNbfIUqjm-xO}VNmw{FwhFHjTDOR#ZT^ME&jY!6u_qHp@W(kKA6wXK{udc^+ z<8dWg)*>1)UO+lpWwZ}NM5abnh`|NO`j&S+i&xe`eY~zjB6Dv9JvyNe1@aWiG)G)|i(ZiX%v*o{(j5--c##GBHv?jWFRo=s0)8 zO>3s@)<{~d`(VGMbAxD9Fnn3$p(cI7PbM?rtq(-+hIga!@<1XikfuiU1fC?o{ERW7r_Nn?q)})sI6_c_|%gX zv*UU@$*jHYb(9n{+D!YqAfcO{(+Qh1wjow15__I0G{+q18X5t`6M=?J8=*X|X>vhB zD1~n1+wkT~TRl-uZqqf9iJm7PgN1}@o8oN1KzL!_YJfxpvPN*C2tz5APiM~ZZj|mF z^sk-a#20XlET@1d&gz@x`6f^%1kmn$i$SaOqtJ?FJdKQl4N|1C7$Iyv^CUa z)Wc+)=I3TizyYpXOM7E|5bAOY%8!Ik#aGdZieCp?-^de=c16NS?f9q7!mcW#SD*sb zx{RlVpw4t7q~cs{Zw9%{qKdh>1#Q^0#U`~_AJ_*m8_-~1jE`p{vyn!#8I<$(egLlYdQt$ga2~_E+;9+&E2fA-9GN~?9d81oHatje zrE+*^=9FZ5GCR5)9o*f(l1GvA;F1S!Dc%6AX^*`We;yc8<<6upYEX`fFhI{9xN!jkpX)3Z%2KCTj^9upkrBvxERK!xVt!K$APB*ry`y^cCdo~5ZbP6AfR#c8kv1T@K zCughV(e8H^FyCE(HhyqS(alPkfW(Q;ZFf|jh(^|02(ubMYq%J((aE81xq2p1l{#VL zL!g)(Xi|qSCVMJllD`&B6VQl<0S-!Z0=eUnvC`}B@gFd)yer|WK^XQ_klQoSYO-*i z*p;AuH3tsLmL^F5j}9@O3=ap$RI~J(r_FVlYZ~hDX$0BG(t49J2#>fNnLJ@LY9)Pf z$LgF3F7zW_^^^Tc{2V4p^YNt9B>J#ks0h@qa7o@3&^$_f2OAG@38JpOEa7fdw zQ0>6{Ma4NK6Y;9CkvQFId53{Av&;ePT@zcbt82qgCJ9TBn{PyTh-pHI4X7Ni!9Xyz zFk^*RV_b(u=n1kWU%=ck=#osett3N$6*cWNK)s@iRq4#Fn_T%wZw}1v3Nt6@7bwx- zR6|Yf8IYi^J+w~WJudKKIRS}GjHt7bM9Gxa-qBy%K-WNE0~?SdGH?+nKBf}xy-s+D z{q^`5v~o(;?x5L(#N7(%p#X-`N=mT|2#UhgiE>2Ddy22;C*)#6Rk>68E3|Xcr$4oi znuUOjzILNj9pgmvMVwO0HGr^$oZCAb^~Jiw;N169qb}Gk!{oaS2l{bfO>|Y_`yKIh z;Cz!Q)eILxa~`jORn%$xjr0s~DpH!1l1PH{GP^uHqpRrZ13P}gX}`gmz~9vMIDuC- zFi{V`M_@hg;0t2$IPf{hy*(oS6WRFtO%o`D{^RSe85jYZryO(`PHSlCQ zCLA5{)46SpL-9VMGQ9cpUUD%&^Dx8=FD&Z%mk&1m(tL5x^K*meWCU0WV^? z9c)TJ55v#M7CVu%2`*vp)~0TL!_vksx_%Sk#&bU-*ovfZg)(_zXTPy5qi&6`(?SAE zEo|-;;mt|4ey`jyy#KVyrgxcBidhVhL)y+%e(b z$-_fH4WSG*KmC_;eX9M}3?=&a2cZ85etSgp_` zb()D3?%3gzm)w5axx!aDP#n{Ci0gVXW>_TO_V!3v>2O8G`8>aB1g&xFB8T$Y;?~&W zqtUr%;ueWKy~VlisO0R) z3%N987(mMt^iaY%kt)7gj3RKsfMUuxv*sVLa+8D_ihN|l`U>s#_oMZHYWuZAfYr)( zUhwoVe9_j7{ww}?dKRz1|N6#=>Dlou)yG8vs*b{;gI@B7*vS4|*`di7s)2te4?1-| zmaR0)@PK9$&B=~1hh*v2kwxcr%a8e`iqrm{Ex?>?p^!Pc?`G{bfl-KmgEriZRD(pA z(`9%$Z|42sA&=pS#+aG^E=;UM1;;9tWK7D}rA00aE??!+Xr3A}{3ftA7eNw9CY)Q&j^m zD~%%Iz14TK!u{|pMdnvJt5N;jKd@OfbBe1Kl<(THU%unE&Q)aA$)E|+T zQrf2It<|6{Pn!6A=iR?^sVZBnKN&UDZmf%R_pX;1qa%Z~U8sIao>A;`1?k<4HrDRi z^C!@mX?((IK1n_ zgGPzb(Ahyn!Mc4zIS<6=WJ7RKhE3TGlaWQ(9IPfgi=b!H?-;aq(4y3Q z;1#0-+Dc=iJjgAYLlq;D%&B zdsp%GgbCaMbGGccle9g~{$zHLVAN8k-*t=;yzT_&oHLolQoyF0Q-0et=N-ohYoQT` zVX0KG18AY>?$Q${*2A%rFRlfGG^}P4LAdxl?$CVqt~25 z`7Omgn~p6zn+4wi&mJwU)k9$&D9=Mzs^FluzD1{A7Y&?nM0i%vbb<_sHHj`ha3@Zk zE$2NXn}mK70gmYY@+TE;oi&497nFMcX5?_FD6t^6XXEV}#u*{Tcf(c$P~(L{r9;1;oSQMNaxXEv)512w2p~>wfL7%I30`pG^87 z8K$bxeV-&}Kx61{$mu}us7k4K5a?G*0e~i7w*^mExE~hgh)0TRuZ|Y)+44LhhdsEB z`S7#`3ES%;!t(0rzy9nq+K!oD>*}4sVWD8v0?`(Nq4T0cjVQ;f-e=c z$)s7;423S}P#5O4QUM=S5PA8WQWM}4c0^aTv!G0lB-M7@==4an)k{sbd2##Dy+;cH zG4dbHgXwaVAK^R@t3~#vPXLN)c3CZOGP~HLI`DJR|R5}adr8@r|Qdh((j5uHSmqUZ;6={so zu7R4nQOcoE9a$t$AnK#m$teh!DD9xOI%ozdM zF%AxoM3;MsF5Gk*K|KS`RQx*&X;{EWZ$f!5iw=3w-&4mZmJr}|%p||1V}RfXnN2g; zf$y_Xs6SW{^dVy4jkkf57tA(i!l0R;@r*0MN3_s|pDXXE#x$F(^X#0bNSxWs@^}Tn z^p{YZK;ceY8*KZ(0nbCcC?CmUoSgKwZNQDtMIdlpbQ*ij7w!;vgO_|3eEWE7;3OH< zKGVlst}CcHu;dGFGbPyQ?n@l7UpmGOTLB#w{FSA?X9ggX#6Me6hQ-%yS2Hn!OGeqhVuEhJtbc73WzZk(B zAb+aX#@CT_6$&yiu~-(fb%&IyDV}h^Ps?GgrwcOZL*)iI^4WrEH-PILHhLXwD z&vn3WP+CFQ(`B*bT~ifpVGIgll!IYHh@Qxlb2(xNAi6Y%;cz^GH?hg$FX|p_vKTT| zEE!_`iR@)%yfAN~gm;Ej$9}z?2?1?9W@zfiFl$zWlf+TNO56#wCGh4;n66|9%@zYz zHin4J=C$h5?4xF^&H#cr?@j#Drh;B%MyPQz-?9QglT|<=(F=tq3-`-J&Camb3jf)c8jBy7 zMwV&RAnRX*xY;2o1pb&YN(Lfc%NqNv^Cn-Krp;ptgqu_zJ4yt~+L@f2b&If5~( zWQhbGNaEY5weaKs+xH$`?+ zP`NKXuh@@ehDPtyRMEU<-N0a&@UHEz#a{$kt^ob%_I2-i4l~aQ&os_c=hsyu(N;g( z;U!I+2rdZFO|~hzo|J{WUM|QRKJPb?`Jybt!*HvqP+ziTu{M6uB)V3050+3)C_y=O zwx-nU?a99#dFA~?kT-}P+4QDJ&C9guWg@6TRYu34H&xqZjT*=D2*+BBc(FP_Emis?Tnx-3U8db7utKYBf=g2r2%3!+fRT5kzBGSMM+1)PaMx&1L^{AP2~S}Uv4rRNJQt+0t&O@i@p2q{0IXpU%{Y_&!*rzSH{ zmE7C|Q=GABPeGf7qy4ZPMC;6{eeWL6su2vF>U#mB9G4ES+@DC(cEvSjX2z$zN&N6J z&V@uluIj}h7N7%6DMoEq=$5Jdo7pO>cji016r)<4qeB}L88%Hc8*JifLL@U%y_nIG? z<@`^{iiL;0Tb}f58mrK7tQRgRZ{2Afk)cK{vE$ceFLktreaQWN1CG&c&&s2>PpKa7 zSrHf6u~>c5RYxAjU4u>c43?!`cZ9tcChiu26cjAAL$@cxIhEnrPh~FzXQpn0%3YNx zIx<`lUto|oo&DM$Q&)j5CAp>0Bp{;&WUty(9rWVldGaVinKK&XN(B8x+Poo z#r3z=l~TpzH6-Lo-&9xy6fWyY5R&~p(U$$tT8$zq;8f#edf zINs*8eLEu>cM7K_3Up29viuQJH;F@%=D}-k95gmK?0D_6)wOzo&dmo+MQadzqy1{2 zg2#H;tkf0>K11*rQ>fiKMkYYsmgpDylZEmJ;(Q%hex^M4vnv#7RoB772ucYPbRBm5 zN4nd7u(=00wM&ooJi5Wa23)9~r~_Z-*XvaM8HCz%MpXmm$r&C^71u9=o^@`gQfuwd z%*B*PoA9{^ZIZ~uH|GTQEA3*SHYe1?qKO;=n5(b?Cez7N;kT%8c4&tZ3$3-T!A>GW zXDESsJt$@|kPjw%KW?OVGPEXqKi(`HA9A%B7!U`_zXchDY!9UKW&nNGv)TwFF&k zg?$7lW6{Wv%isX$eDnYUMr9w9*Zby!=X9ieNP|egW1=a4)5uGtnH8h4RWCp!693dr z7(zA_6IE$&a9r4bdAT2XV5V?zzIly;#kdGsYVVs9aM$yoZ0I1cYKeuFkLglG5q8r#-8;05p^1j|?x{?q}NsYOBYviJef4t$-tDXKJl6ZoLAITyPz zl;&@jmdT3?A?t+X$aL?uc6Y;9=+SoS0aG_y97h}h5*P?@Hr8U$8u$sGFIT)e%~6|_=RS~-!##ot!Ayz$h{45d4AwG%l-F0 z?yt@Fe1MXVtO(^m-ebjp*!EqOEoFnr`2hy>JB&E@gm&ZGKV8QSh)oC9Eq~!_UjXff z*dBZ8Haw0VDbJosYrD~gX%y_ou9{ycJJ;XRzY(bUUdcVe~tdbfEUK9i?Vd1*6)&3L}& z6^NJ6Nd=^H@;K1&DUruRm`OzNw!(RyN zMA2L4F|L#R%5fu@sJxz-dD$Rlqv>jb1(?lz z%;26*a0gQ9eW&5AACO9MxOap*y2pTJx>`ixw4Dj8(MpMO66@;WUpd}5NW2j|w9wYA zcM4x@1#0#~cRhL>UrPrclbW-Lpme3|uK9A|)Oh}6c@$;4*}_mTFT z_(HUHnQ(ld4$S6B_}by#Vr5$V<+wqZWP*&MUr7c?GY4?@n#A@--x_SyOdt;w!hZ+a ze*-=JyM6#TAsGj_CNJ+Mus#duC28#l7i?`DOM94~Pu$DonPDFXDY zTMkXHxm36bCL2%{vl>E$snR@33Nd_P4%sttmar{i=*q?gayJJUH!^G{>{7yVBSpsc zdy7yJsfl|Wzb9G~KZEv$9G?yA`D97TT+o~NRKaojKq~44CMScbr7)FfeM0yIEH2GZ zgmt(orr)I$%&O9-*GNQfJ_6NbPs%1B1pv)2x(U|~9QT~WPa`(w zz|2?tvWs5v%yykQz6_6;bPces;|0vKsFTGJ>}gZhhou7 zUPb*nDf0gV&*I!pz>;O3PR*uh?cybLIIfmr#^?Fb7BPREX4PIOJ7&{uu&jb=C9V?F z9Zd^URRKZ$gCi^RM6sDaUJa5t?Z8ne&nX!Mp#O45Q1E>Umt=@oZz=) z@EnW|&bWv?x!$cUQ<{A^z3gjvQ|+*^UoGsZ)vWWVM`OlB?GPO4Pux89=J3PwH-&XE zI_In;gcD}Y+H;``mLvC49RcbkzQ@AVPIB_1!}}V@tRJr0{91E#?aGCqbKOmU zU=nAp*8{h`omIBjE7Cx1Uv4kNSnIRjXeobVTe;Hc<$Q?q>ma9Kx5nhh!f8uuV~f}) zsWaaC)Jp%N2=Q0Z7hk_~tznZsKGJA+J z#Lz5AqY_YKw6ck80s?V?X|)PSScCvkimW5D4ahK)_j5mkNzb|d@Bg|=HHyq{d4A9D z{@&l^wz|}04QC$U?zC0kxT|F=n31u0$3OI*0h-%m<7;Q;>EF6ioIW&pyS4(ru5-=` zE#-zEZ6y;kY;eI_gPDXyo?dO;Z)7&%+|4)Joa4JkW~VWS)^J(3xs3Uc>jKRC!48o( zgKc~G;T1Rw5tSP({I4F(hiM-$u`{3Z$o5>BxLs>mBHUMPL0O2;{dAkrULh{c z7Qf)zg=@|x^g0(u!tUk2nq;^*=~ggvx~pcT-v7yJ_bB}4Zl(%cnAU*TY?mydnc0Ku z92YfXgzUEdc?wwVd=?{j4_pf#sJi&ui*uX4ci3~cq|>x_$xYXXuXkUv%?~k@omw!> zb@}f$ZvQP;GV@$oK5d9|DYtKQpKnImU{$Bv9x!QOZKH?Dr&xW{H7Ry##LnLugs9AHS>bYN<<2Mrmbx&8;ekxDL{WFxc z#~Z6bLx~Ao+vuB-mlRO-TZwfMVnHDv#_V289UQy$uj2=^qRr2k8zu3+C9_m|0^30B zC78AoTAt}>Ve6&ymgMbKX4?Y}iuXWeEec{}$=)KOLM_wSZp%N7M(F65H~G{JtE=0= zj=uYfb&dm`7bp0|TceV#Crz_Mo2jf}n+zxL?v88NaIF(%NYwk^HnT5E>=qz(^@MyA zq+scuVIe?b!p)5VbB7=UgC7Oly406SsGNYI5(>%`g8AfUBTU=DCGU$4 z<%*3x$_DCp1dLo4H|yN^{+RgH<6`3x2kTe$$9w~DeJVWFLv);{`)=&0{>ynj!Hx-E z53Jmi3YoIQ-Goo|m92*+WRZR?NsEo$?WSVkAgXE)a3@;|yBZft^Q6bJSP4WIJ8JhD z=#P6dpJ(^g$1robxoW%RsH8-?k=r9Pf6Rp)tq16;4LD`zr8n(gcV2`Nn|}9HP73jw4$)}dU4cTX8)4!T;zNNDH8Na zE7F7O#*V@@==lfscH0P4r$r98(><02hj1MF37!zb?Mqp!d4>~f``B1Hjp>@~6B z(?#J(-7mQxly*Y)lB$*{m6WsN;JBLtR!!47QF=5EW{qmDHW`}l0Aj_aOznNTX8A3Q z_QhGD!Ma{=r)ecp3YFgLBmO2)!wouhFD@QA?U%^3E{BA$m-~hO4Y0B5^nfWMKnPlU!A3C&{Dev9{oa91 zK*>9~6KP&j(F^_6-DzYS)lMuSuhY zt&yl5iS>$@i@KBQn+N}BuM|{<-l^00p~84Y2EpB|7;;oKG#Au7uXBU0+yqj<;uJVI z%5fntMik#3(;$P13zKjgn9~4t3XPg&2BpUMegpe~ds;nREim5L9Ki z#o76_hq#Kb4O6A+R2?4t9GA~jI58*qlA>@GSben`JUH6@9IF+uNe%me&5gC4FDPFs z5EBq7pF@!k;N!c+u%}=SWh1)BVfCZ^Rwzo#UR&cr5aID-&~3n@;ZdUdQ*G~rDf)a- z2`+z8feM)e=oPAZylGv4U5@T>?0PJ!oj&hF-%8|_&{K>b=C=`Z|LYx^8BNW+dK3as zFh{|DM@5fAcLj7d@(b2TwbFqTGXDud+Z3+(h3`*v#+2X`O-@rRz}kK)LKK zn$ym~<*1QGGl$q(29l}4GJL0SdfYBeJQfzt?gp2F-I7V~WjJdJ8;u<^x0qA!n-Ogi zJf<`O`z0T1m($Q+i`;4*t@CflQYBu26vFFJ<#v)#LT;q-`;tSN=$}CYLTSh1zDc9k zVf%m&Y&Zc{3YO-90aa{oWBYJ-lRx1n{#5pDd*94q6)*j!dYMix!i$At?dFoXsWfJE zV)&%9#560IaJljgYphe|j(Tjm_EHHtFFhmd#O*Au63*B@$?0V0{V;*W4;KS1c<=@u z$3AVinoRmH2-p3-#cjwUf_UNDNRdB60~t0?LaIcFuIZ+{*L+(R%!lKjQaFX85$|Wl-;i<3pTzYVa3d%&yKnLhrTm^Yc7S8$` zp@-qMm2#SqjzD4omUf{@lHI>c5R|Gc2D(%$gcbdRRKmf~JlIQPjX`BnbiDRp#u<;Q zyjj~(WsFgr_tsrBf?z}iPi}Ky`no_efxkSP0$j&96n6L74llL8my|k0=z7^Uq^+cF zAxZ-@ZKqD{@@2&)+IdaY7K02|7+8TlG|ztAAh!$-6*d=u(?3uasK`=#10~~$Tad?| zNZ%!C4nKz+g`(R}DNPYx`=%>7*VWwYCF2fesFM`3U-S~P;TPH|YdeY-ByInaZh?Bn zk`k%k52T)!QKl~Ks$oZe&r2bB<{enH&D38guA(&Y@Vk}`UtSr!E zUVBatEX;nq$F|vLmrXfu-`DN!8hHYHP5ZH)vzVl{P}r-@U}(E_zUOn!Qwt%Oh6fRz zg5#3*3MV2~V%fUr(1@lVqV{XT&}hhwphaE8TLc60RABJ%&iUi7?ky9(<=>CaJR`7u z+c3lj;SVWyyq?UqB=2GNF55t!=hGT&@&K;P%MDWk;&;WCA^!=SFyF1<{P}!>ABus%C^oP<)_RY!Fb?WjGpgDe%c47%G4QeeI7#3A9aXbw2loRVqvkI?+CJgr(9(^O*A}&| z`SdxGe$uMe2uZOuLKOal*aTcEc+z>B%f5=6Eza?4@NWNW4v(;|r{81d$0>fk4ED>s zwiSLBl@BceEGu=Jt?*&%H2kAIxx>{p9Xqb66E`T{uhcZeW&m>TS~8R@tgG zT~^!2K9R}XyaqJPElpqPRg;E+-0D5al?b3lUolhvk?kJc1jpm=e>-b~@qF?~(X!%l zj5L%NjadClBv5JT$)zsDK z_fFt*3O|%Q*=e)3;OroP0=;i-+59J7HUBfZ7Yq9_#(*+*wfW)VPl>8Aw~?$Cup`vm#+!REjA)B=W#zp0o22y% zc<@f=E7sK9HeL6x{~CA&UYE&Dcg$H*C{^n09B#6Tl4RiUmWiJemmKCfR#?tbKisi_ zzK$=BF9H8u?F3^Sok)FZ`-iXNO25P&nNjdWQxukWRelDNT5}1~OxM>*t!vw)^eO|NhODZ-+V?Wy>#f(lJwifqL~P2S5#Q*^j&CR2L_Bil;cr2V_&hy-M);LIug&=J>&#DT{OVRj%~wS&}xaP@7Vb8QqJV z?lirmzNXY63xq}e9hIsrZ^4Ek-fQK_o?q=~3a9ogpDPOSB&4?&dq}#}ZO9&Uswr%0 zpY=wkNCS%qktDGQ(E3%h-dvb1vmODbt?mI!yCzqgw3lq3iW-?LY87_mN1_(ph4Q#@ zZ-+uiiEG&Lm(b%Cen}f{TJR-P(L9DHp&q+jIiOXS>^O(Bj+Mi$Zq#`-ZlwFuZRT-KX`XPG00*ViuS+% z_7K@zoe_&M;u^~LYReb?vJaR>#`x@Te_dA>XJIogoal^owZs(TR}L}d6U=m@gi%gT zUezDD(ejnJXkThlRQ;)onqo}4@)^nPrQ}A5Kc&j;mPuoEH5KRo_SQk9SrHwRZ4Fa? zfBB0G_xhKl-K*an60lm5R79hZeivxaZj0*zUUo$R!OA#FOCUO6N-oM|sp9FiJ0ra8 z1(4A-;K4wY%a+K}67I^GI9Kr%4kt7T*F=Rf7N55m+bEUb3K&~lrbNfW*9zdGvXbO5 z#@Yam7$GUyPB#I%GXIB0-z$$O-yirw>6xKo2i;!QN`~wzdAwR1HgI2y_>6=036wx( z_qMO)-iF)7fh?7n0MyTow`DO&+!5zXMlwgvs}nmsCjqgH7hQb-0%aWEs)U?c)Ffmt zCyk+v0YBuw4i~ZRh=VDr>~MC3u&dDZ=hy0`; zmf^ntb)FQu2Cy#hy5f^`->+hhXB8__TAns8k8(HHDqbx?pR(tiPlljD1AGJoVYJqW%?SIY0@zSkCd3xYw7{R-$!UN39_z4X__G5v^$ z#WLihgc9W*qF{IZbamWAws*`g@!ZKkp|;huQ7Vi>JNpuy_t|_RFL<@KKrUb*0t7w}Ji3zL1^NFb^>2gdc5)#@ z1yQp(C^}a7NQHIG@Uq7{-oOzybC_dP&wy~H{4_arXw}vX8M+IS5mY6*dFHp@BGj0Q z{p8$cX6!ByeUE7y?*M^d<6;O|7St{s8!XK-vl%R^x;Sjw^4a!WH|f&IoFDZ;9KF1W z#xPZg^rZ*XC~*9LF^CFXOIbn~#Uo9E&YidMfAX6LU-Ac!O6LPm`2YRIDi(+MeC6HCp~;s`T&b+T`z}1So!* zuEwuDE5kfB_b2;p4%n-0ey?Zpo}%h|@TW)C3&S>)Q>8;f790ZFaH+!2LZK&v>V&2m z6^JIdxa2ji59fF2{vVfGB3FNq6OTpl^6kpokanxhCKRzZA%c?eU^a}ZTU~G6XLZ4s z`u4NJYx2;Ce{FVd)_vJ7Z~kb?xkO4o^K?RE&>f`>Ee^!483mNcP07BH6>FP*Z`St? zdqYC@TsAj!KbB{6zaIYbOHb#FekX9G|Kfyr^HaC@p_`Eh z>XbPXrNE2llAWTQ!)glN3DNH#ePjk=W@LrM+f#O5)^)X?MAs`c=@xS=ezF!9m-M+s zse_yPwtlg9@=;^!WEiwHme7sNYw){nSyD08h}!b1xr6k{ilmOvsMS%A``BW_ZpZ(m z&PV@AzfG%O94ZqB*`UrQPRQPi{7_rMbM=j5<<5_EOUo?vNEbXJr-dGPwXx&HJjT+b zW*B-kLx)os9x_+^w=DtTx*Ic6g`xcf;Fw46UNKixzf{9 zv$+5aZC9U^ZcBQ)dix40w)aA8Uu?6Gx11d18{kuRx3Dy|u*_opBqw6UnR>(JXj4OX z^J)n+yG8w40oxu@^&9?5H__IKUQr2RPEp9fAVdSJ!2T5YO*0D#!8Yhi3d0Cupo;EV! z^B*T#6=juu#c5&l-M;9w8pO~|eeB*92RyJt`nP6EYV28jU%QWP4giw`Tr|L+w`Tf# zAnaq4Iz%I0_BEG7)oHg!xmZj-24ntPXro09m*4o^v%7nBKi}^3N&&-_;evHV(fgd| z&au256|m^qzJD5O3R2#(P0C}&OS?ZUhP$Y)9@?FWACR2 zXt#!!YP?+QT5J`5XwOgLj|E|QeZ@-wd@!gc(*ttfH zL;9629iQWwn>{|w@p~_~DJLl6jB+X9(io*A6K-5|K`Fa$xRyzdl4dN9SV>8b7hu*t zxI3YQex9N^$QVN0v1|6s!-6+cOM3S=o8~Vy=TUMmGNW^(Ln7YrLr7y$Qrd31Gay$@ zFN)tV`o9Z^+GWg)52?EWJIwpjPPc0|<)%cELFb%%#_|xc)1RNFKv?hJ)jlwR2YNXl z8jzWC{0Pksn6NF90NgDpN_$Rp(weDTM~G46lfj3u;;%_wRqIn*<%S4NS7s;=nsc$8 zr|l*7&ke~UOJ2=1F4}I`!`eH3vr9D-g3hyYrvlr}b5ngC0;Rl&2@qgK9Z1b2a@#-V zw%YZE;2lvoGslrhv!i`pY>JvI$q}^Zyh?XmeJkHHPkfn``((0-(G*z!%J>8OiOHWb z=R>Nx+N&E<;Y!i%U@Pq~$Xm_bK&4fIXNgAL^Ta`_@pF`W0uE%*QcHiOh`5FDXjVWS z9je^zmN{2?eFHlRTVsAG%p6LnYqZ2NIEgAY>Q6j}&t{f_sYOo23AhtE4GlBgJ|V18 zfQ%%xdPszdXpL#)Q#(uE)7P1tx@G`&i*~jkQ^KiI?Ro^5*QyroqT6Po-O)#i-Xm_(y0ycPUMVT=1CKtFtftP&m? z+9UCU?m@ovHfop|HOIP2pYDF{jtrHejhK>obUlhwXG3q3H>Gi_*I6w2emA5kLH)z@ z*?e)$F$08HQ7({ZCE7Bkwb7W7G%AGNMnGcP;*-ogg+hIr`*wFXPaAwqq95mUAd?NZ z)jz~C$|1bu zrrGls*giLZSaoja6J<#od-KQuton4|CBy}Wx&8F0tkE*-Y?R}pPnM=9K5$&7r)<3r zT>rscDC%;V6Qp201&P!*f+Gcq$+}?u(AXdK7*ll7e+gqUQ?ZH=~~ zImb-^Z)CNu_0F!z$fovB*RcVZ?(7;lm&LGK)SX{N^OM;VVVj?(mqx3LA-eo+t<5)x zR9s|($f?Zj+8{yHhWq{3_K=E9nTA?kD!RFL38&EVc0?-I;2%3d5CZaQ-SSTXU9hw9 zrA*2~MxlLWJxi+r6j&w|s4m3pCWvK!g!OGy?%^-0w5wrMX1OuTPg7b zT|UO5h4WObS+7K86HnW$B$`4(#CbA#RB;!Q5+uB8+G0v^6I2$po8AtI?PaB4WCWf} zspLC{b5Vs;@;UWYHr2B0-!@7i{wpb_1y*yRWU!Bv%JMxbHOUUK;XG%Zb7X4EO5c~h z77(*v(Tg%Z6fX38hfk%_ppW*}bfLb58zg^fn}~ve5^p6F2QHke`W_GYRJe;Wo&oy1 zSCg;5Da{78tiH*bi%!5o4fBn(BCh;Z8Vp#d1$vrxcf`NWPoPGZ>x`u#tUi;*5M~rP z41ia3`{EPLR^2PxoW-rXQua{Eh~n4JxW(*-v8Q>yNw)(AtCVNEi(=UoE}+8hL#y<; zcr$B6*m~Tmvufmr01q8h*IQPdRtb$HRDs&Mj0pv~EI3Ibn@SF%t%Bo{3y0mG&*2t0 zaV>ztcB{K6P6bVj>sd8%`0 zpf=K-sN_g16lYYb#QrZ4X8BmbuAYlgSEQlkp!P&=f@W(iK zaa1vU!!URPQ%UbrGI^CPL9W^>+{R7?E0Jq*mny=UaD`_4=&SZiG3FtaKdsgq-3I>9 z@eyTV5>S(rr`HRkgp#GpkUp0(k=T z_!{wp>iWT-j$l~~BRDZrFvAB(3PAohR42ceJr+ebW?Bi^7U4ef66CwUNjk6M3{YFK zU0RKP-?^|gPP!-zEwavL)-wOR8_JSv)^LA|Tq+8e6!{n-^-(@^Xf|I1^X$YGo3s*j znIEm%k^H|B@Up?%_dj+E)#j2_L+|b1c+4LpyiNZLt4q@*-S)eI5ZljnBfs^#>C_vw zp)}F?q|GNJIIeBmnk_o_pn-JKqQwyt$;wp;vm=0umcCCx+ ze{cRub2}p4tdw^(ED`R$G>Hbs~Q}O%oTY`G!u2r?T7apEBMvhXsbHk zpU^s(S@MS)1yAbF7{0oxKUSZEeD%ETr&r(^C8xwy^LWcx*h^d2g`u`OaEc+Sgzwc$8feHTT03U8ZwYU&<~hGc?tlyY|nBJYAag z*hRC^R^RTz`!F`l0k{^l{XKs_soxujL)cNLKle)bb8GV(YLjH}X5txUuw>1g7{Sa~ zV8NC73#|KYujmzl_Hq9kAu|tue5~%K&*PiEeP!!yzmsm(G{mFk+BV`l8mbMMGqjgH zrfV-Hd0X>re&mbhTqE4*T|;?cb0u^A{ORkW;3*s$T-lX}@hTlUB8(E%k35v!RCkChI)v zwK|6_{cnf*FBQ?HJe0>RPx$%zINc9R(Y9v~4ZRY+YU?Rbq*<5YAC&#QLtqMWz{wN* z2RH2~+y@M<^VpkwdtQ6VKzBk<*Z57Fx7wa9!yDT7va8Eon4RE+uF+}Mv|yh?bJ3nA z$Mn6}!8%i64U+WfJb+LX2Y%OW?D&kL_N&l@b%4NRC#Ut!x|)OB?<(yzu>rpvDg#KZ zGYuD)nYB~q)XZ~orkix>?bvbD|Laqj?TqIaXpvVIcKi8Fn|0;*%G2)-E!(*~Kbl_a z);0I85AIVkD#FgbOHVP_{37cHNh6*&&$N~~QXabQX7iy@T|c=y9h5dNF*>W7F|Euo zbwjMptI6136Irr^<8S`n;i;-GcJ^0fx(?BG`eWv~f>ho46e6r+tKwU9o4N2Q5g;CT zc+W*0w{pi%$ZW12tcY4xG898;gtaoQt~7LP-x_j*L-wcg(^2|v&M&o42mapIb)hBxqnmJIYKk;`^ByRfU1B~m8@m*+Q=9^E&%NtIA3Zpij+?`zvp-`zE=>O;k-hrUgQ-K|wudo`8p>tNQt(>H zT)0{YVerMuFbq_E91Eb+K{9+@3cIY7;kFSK2CK%GxR$Oyb6d-F$68%^B3%vON^DKr z`~^F36}wC!NeW$ixOFd2RF^u?EYrg?;XBd06wT%boKpj6_QGRtONk~LaT?WN_Hf^MS+3i z6~C-eNzh)I|KbIO9TY-N6FMugbVq~tVmbcLI80K~^t2Jw^j6CFgb_#h*lH`)haT?w zibJXl5`%el8n7hF_tJL9DMD70KfX1&70fyZlxr$gpvs&pw51w5bS)Ktn6qR8M4$E7 z9~7|6QP;n+Wlvq4=XqlM8I>qptl02&kS+_wAr?&va4~U0DFk^Nj`qC{3jK zXOf-LtPIyo)UnC2_@iq|D^j(!#GONx3*|F7cH@RhLwDmk9`@kbZyzf*XUzO0WX7q7 zi%xuQyLv3Kt-r8mMB3Q3H?FJlx}nj%Z_`?%$-g>$vd9MOq;0Waii0!wn5mpvBb)NT zJvAbV%cAQxiO4^7#;FkjmnsP(vxE|>>=F4XWl6?#L-||?i;do1l?xK&N;2Cj!?Lcz z@t3WdpAKovwW-;FZ)1LD_`YXj-Ae|3yMz~R@S<$qyRlJ+S`busbv-p)lM-kn5jS3C z?lK+;e==3N41IhAcqu8M4R1YQZMCcx^Z zWgWu%;WmMtY84Tl;7)-NO5X@s%oz+;1_LKr7|F^kTV5^|VN8^d#AM;sEl!Xx&X z5JC1Y7FcG0^)b037}WU`goE(*oTW=FmvPL$$YxDzsa!<7qP{R!1FPdPoMDaU2Jy*(>f!ASv1vb5mC zg`$xvT;RmYqBwarAJd~(tG%!h)2meK6%>L{z3Cut8 znH(ZM_G9&LN!bgz)me|%U8SGi1RA83TzJ_d5iWnvqEsN;8LM#d=`S49r!zEZdw z_Wf#%jFIlp^HL`vLC%v`QIX@KwQTRbJm^j6CQxd~mSIn;$;+EwNMSaRJGNdj~$ZsqU1@8-G9==6o$4OJU*cYVkq+31pvw1W^?r zASNpx`=g%#O?BJVRaM)!on3u^m43=HfAISzmd^=CVMH-`VC)pjsxX=I=pkwuIakg| zHW$uH5T`=1@|)p@hT=eky>>J&dkzSkptZur2AzGW%W_X;tG+_B2+XNoqzdwy)P4C8 z>XMAiQwq(ibi@Lye?Y2fhAuBXr z(SZ&|+S^?iU!C91P;JT#=9ftqz&QPYsgO7?e^Y2beft6O@nZ6o z;|N?VvtNJML%Drov1jvu1ePs?CfF;p&WYbNH2GcoQ?7vd12~SMU82De$_U3EQmMKz zeh6d&hn~Mz-8C5ySc+PuAh1gqz1!<$#D`1&*uFigXZ0__dM00D0KsveGJWv1f4*6z zHSX4))kYK_4d#s>-`lsUVbrLYx)a8CLys7sO^qjS2B)zH^M-tx zUU9UjO*O?Ll&zTFnN(2DCcnV7Y0JF$tfuPtWY@8GlV@FB_IXFR8T7b@b%~1L=Vy=Qq)m)p{K+K8_xecA`@)vz^nSR`T)bNtWNqfZY`>_~ zoz<1P)@Xmx?(FjWOuUt#r9Yh5=d#9j#dctqCF4TGZ&?%Q;ddb786 z$$NcK6J1m9??AKqa}nbDCb!I~N#oi2NyUNYp?3_8x04LP4c|Mgqr46Fe?we#>-w5J zD`0c4QWRBN58J+NdEDF{xn5zTnnhit-A&(BL3@sU`FR_j@RlYW9)f>PlL|--| z3qUiFWILkI>uA`JvdeE3tEuGLHkT{aMk2Zc1-y}>1?b0oo4Y&kXuAe}x=Ls6_pv1( zZ=f(d>GjI^wZ$I?>XYhLSKrhO3^GEwW$;e4?VSj4{3@GE*RRS3ShxH8B0}~@KGs$F zuH)sziBZAPXAQ@EYvjWmwZG|~zk8S>R9sU~9usq$I~XMs=+m5##lTyU|$xzS(MMdGXFU z{bc-yHwRK0|8PDnKQm=((dF^|krOTJl4T6lpnJIE`);E?SE`BV2=7~r#KlXtg-xV) zDe)p)~ruzKz0l`2}2HWBA%zF z6m99pUj2sFcyhK2+ixIvL-&rdYfwa0F)Z)smw3rxxL-y8VA0!VS}%91I%fHs_L&T? zC(J~Oe5E9vTqySm*pI2mgS`FLRuKc~)a%4mg^10a#|; zbau@24U`$1{*qmiCcCqy_iOF&L>5f^<^_*7bkHX_$Gu@-iK)5PP#O~ayb6+H?C6Q6 zabcKU?M5Tdip(gdv$ua7ego~>icN4>c70bt)RfFK^H%Cb<@p!vv;JmsA z_Q8}C3=Or0vX)~#rb5aB(L0pVhEerm0gf+qng30mD>6u$`T}~H6 z!i7mhk3OoyU^@+~e|)LMsixpkgr0rj+O?ts%;q5fYomp|dx$YYWQD*7WerY>9g3jw zqf3LYmg8L;owe#_P2Xwa<6`|=o>#Kr*r`%Vj()4K8fLcFV^5r?igQ92>@ZDH34jHm zBb~pqb4X3R1kHq%yY~jOFip%$+dt0`QFD zfXVu=WB{)WQwnPrKq}WyUPJbW9jFQ-)M8;N5bI@V@-Ee?+b_V^bLzJ$TmWUYVv0eu zKDpmEkGw*L(yL(6~@~4B~cn4uvXyt4R6>_T{j|My9~SXv)N8`(S}XoKp_>*@4R!5yQJTgaY%D4s^s z!-XuC7#*Pz_h0?q;a|m5G+m|;7T^N-3;H9E{@qYcgW}kJ(WjO^vYU5%)fkm-E(@%; zMy`;$5a_HMZM(FkEQ8d6t~~uS+_#0+veo!hJ{!}Gn=4k5e5|L@l{Mo0V`Zd7*pA4J4# zB^_$dYdpV+&J_*mvIU&|7B11(2lCZ!hrHH)lTRg_2k5Pr%Ur}m(G`N#(LU# z`<+{9I`_a5;w}SnQ+8jNP=q&@Q0%V4y*9Q#7x(yE^T>z$Zgsr7A^E$K7Y9yf`TaU4 zyE3V;${M}D=WyQD_N)G9thsq3^BCbilHglE?M)wjUEBdIwnIa&tsOCaX;5Ijdo;$M zdS9Pa^$q&NCU@~u+2%(NVqvxBmh2A>xH)^ea79~Q*ENv|8jS4UmVY-`xa&dB+C8+T z^=FYzf4;BcW8*-B^%?6tx`Ov!Ok1-9T4zC!;dlj;Uk6Kw{LQiCJrfwlRcT!CGp+Fy zkJHhb+M91M^-+H-|4&L{TavZjunD~~1QXOxLh05C6S6OFxxFlC=JFu;7Vkz|^|Zbf z9rIkw+ni%#>tn8u|MZk)(?`+unJEP`+qJ(gA03sneqgqBMRohf>_yO*_4`Z4 z#a126W{}$Q%zVeVzCYhgJF)$e<+pyjP5Ue_UP2CYFh*D0(6M~&je5(4_^wW$J=K;; zse_c&Jn5Q)S!P{N`@+hJe~Jb&x+I2pPgpS&&LZi*+0wO+*bn!i$JOAU`l>}8$p=Q} z!NAw)fzG~(I*;=OmD)1BdBWZ4Gb1JiEVDRO2gh>W#WSY-Z1$74Z2#cgweD+0<(XL5 z*59zfGKw9PqI7rJw+K7gXJ#tV{9+9kGrnA2r%h&ifvS)U<~F8MJ*ScpR@yC8Brxy2 z!utjJ5nFNre5Y>w$a$$}>i0KwPPNzjqkUeU(9#!GIDS>Vw^h{H&Rz|EsDv$}whq3W zX7YU$`|XJy^Py&~p^e!!fljoE2)`l&smlXKy=tam;ESN_s@dOc-?zl_E0+GD>U&E* znD2VLWiAiLAx+I@9bW34FBa>Do;;|1k~p(`ZUv1c+>Xt)PP6>FyNwnPBFT(ajVVT? zt)Vw|UFuMDQYciPr7~S|cRUxF=S{40Gu1Qc8ztLw8fagGil( zf35y2?F+76C{qkf5#BDp>OEU9)UN9w?RCoBgT?#C9n6eo-RgRw{IQZ|>tkgUK^@id z2v_N2s5pt_$EFh#2Bn>{`$Hq*`p63N2UvQBLI$nK+$ld<8 z^&NjZ(gd!t3Q<&UzPQl6Y5njP^M*mXOg`P0qWz4;{Jf(2mf(Y-t%kAnRqO-j!{-CRo9gH z9e>z1-B*O_q$u&}&5DXX8vC_u4c<}_J>8eRL;OH?kK3fUF$!rc<>FuJqM_aZ^r@R=SIc$%(B)GM&b7;4T zS>1PY$b9&{%ab%Be|y0YlwEErY@#5&TCs8G^2(6&OT)I@TCM5bq^+_}Nbfy7kML4N zHZXCtf0($%?v(*+p!5i7bod~tr6Lq!1(3fY6Li98AWt zB_N<|dzkvqIn>_Y8bjR*ER`-3ydJFu7%Q`X?=(_q-!bR9*`WujfrlVxFtR{(ahXNv zjDWYhQ#7Ff+XvOA#1`AfoC+-)b=7-X3ovl5qBTL;fPpbjM7;^oNZapk8^?%FkUZ(% z%R}sDXE?U(0rYHY5m3vQ9VURa7LIKht|j4_)8%he$fsSwI9&YJN?Vc&Q>sk$H65(C zzx~G7RNHUa|4b@ZN@+W*$?-UELqK+Nt`LKOKs6>?rU?mxVFINgi}h2iqfy{4nZg^C zf7gDIMEodwi9qWr>m0-JGJEaLJ~lUcfITOAD)MOMptKIB7N@xzhX!s=aGR1rv3XLL zF6Pc!ng8bNna*Pk+gT#brYJz(XOfBIXK2OG$rYcPqY218S0HtBx4Km_qi85!UugQq zkYG);eKGyCQoYlnML{#X*($SN`)Xx}^z;06Da!}4T(P(Laf-PW-ctGB|0;bHxv$ZZ zsjCdP5VBK^gCR@`#JNP6*vQSI)go91Trc_0-Tg4E@OdWtL<)N?LP#X$ls%6 zN7fF20@Z+y6mWs$K=ZL=FO1i0B-BC!?D(KcxWrcgwR`)2IP@GuhRPp zS{CWW1640}>*0=&ZJ9m%m?1YZ&c;3jnx|(f50oT3iEVt&TA9bLU@i}lqR*S>3Z*Q% zhf!jmlD?8JOPic0!LnRVR6kZ(0mH;J4;=gGHni5vAW+a#^$lb*fd~H)y#9}d09!g4 zQ4gNI2ft2wXc^m;Sc=&uA2gxtzbfl4(bGqHTqh~~y9Ckx@VBX}fxK8X*@W-K$J8wy zDjrUc>7tYftTmGo0`hi4aX~C z9|A1xz7b0Az~5;qqYBU1_dWHoY9X)absMlIJj_INF1cKyDJpmB3R2|ZeFrjL8En~- z)V8HkuYbV4_VA@(Ha*vb_SN=#&e)tEGQwn=8~%d9U~3$90Ixnc-Hq{}&u-UsgrBTU zLKWGT|AM`-)fKq6W9N>9Z-YXweQ?Rg{NMll)5Y5*9}k5bZ_s3TYN*Hm&YSV6#A@xY zw`*-cGmM|pgRj~!^PC>|=haaG{TGq0;Y8!(6Tl$@;p96P> zE{m90+t3?6IzR_Qj3()mDigwW-rBA86RQd?2DcoTY+K~1*_1kY;4nO7sBI4q1I)bB{XJjgzoOS z^w@OUCDLxgOS;~OHqToffr=64Aq&kWBO@tF+eODI!E!HuBiqKacqA(d|fdrY9H$twKm<1w!-&1by?F5 z?fi%9NB`8-FDJoN@S8qZ_939)eS5~!|WAg2t1dWuzk~LZ8y03 zQefyi0lpYUs!im}@@NupQ*1Gz6D>&DTC;p?1Kssg_r3_3?ix!^FfTd!KY}&)KH3n! zIbXLvJ161D3v;9Y6K_^LkK*zaon`G{r!Bv|5;mAbBj;%Su?GD~ll#Azr?PWR!100N z#*2k(?(8;RS{r%xl80qLaa#KsQnTSIh=I1eOx6@1p@ zxD;%h88cyd(<)YlbM|;P!qsz2@`PzxI0=X2M2fYV+V>Ks~vuGIuGdGC|K zk`Bt^F0*&Z`L5sPSQEDdM6BDi&>2QU@9IL>&CxK?x<)UY$OuuSccCi5?iKlMNikevKPuF42o!f6#P1^w944aimYFT9t(<2g^2hZrEZHCBlEKpKz@J@3| zoG@W|+(uHv&gNK_2h?XVhvo)s`MU?At{9=-%)qLZi6f8;)iu{Xm!Gor!xGCIn`mYQ zxC^~2YbGVe4|~~{b=Gu_f77u}qMzrV0v^18-ID5cxCKBkRh_$?Vk%!Ek;S17o)za` zpT&&Nk#(bMP0kJ7Lg|MdO(gWoCsB*~O3+wrtA0 zC*8x_S5LB6;hOrfD7al@d-J9Wpx>HnZRWb%RrMLNtVAj_4OpS@g!64#qrXLpHfMR+N;$y-v5 ztrT&PU3iQ%)zdN#DX9BeDToq!w){>%F6O9to+hm?C&@y_Y|8Fy&oKOjSqSTG`$BI- z_m0H-bR}1LQ{y{^n;}SVeQ#2yH)&eUF@DH=5}HSO3wee;>Tg=3-n5Ji^B7pBatQ6Z zzNV_7?UnjfmNjK?Y@&TYKCACAyp-y0t1_^g9rh-=^3U?7fB=vInp*LY2{${3(tp^l zCN-llY;w!E=DqQl;`c0Lgxwn6Z&vKXSdD`GkRI7Xf)@T#m>&<(L1U6~jt{X+mnk`H z@5cC{K_TrVjnP}_xJWG`PfFF3NLg=&dehEi4=>$_HYmOQNHgq#R;g-1aEuC^MrSWQ zp6ya-J(wazE+A)RcupETt?0ENE7c$!U&o<8boAyKdNsZD0d#VV&R;xf4-}`l8NDHu z5lHj^j?!z~9%A!P4M`Td|AXBs&?WRMqhQIqq*(8?r+Dt4y2K41P~0m|Q@mFsPLBkLkO+~%)giOin) zHM7Y^Ub5}+mX28LRFZk#an^Q#*%H{cv6FX8wExp-MJm>1MaWrrWtryT4$sdGAv{ zS{VUtMoYkJ*`*!9)moQ5ox!Oe$0P~{!nP=6&K|jarF6|F4~454J;CrZYR06?t&9!W~d(MNfjXp zToj86D~_h*ke|dC<`i~TV64=X6m^QcoT_BU9(q;%aqJjE$Qx2855tL8Za7Cs?(Q%N%8`!1 z)A=BL(kpfjS^t61K>YO4%1v^ReZ?&sjL^(E!cx33;;>g@! zw5$4&Qppow_eVe9)s&3x`#6+v!H>+LQTS1AQeD#+u^w{IA?Tt9(-u1nBSA$s?^3iQ zTY%^Z#K~coxGDj4bS_Z03&RPLd~Tlu-}I8*FdyCPlqsrMeVG|pdU6A&p-_cNrJ}Vm zX5v46DW&PGxY>#AV@dAvB6?UQLzgi_XKa96!*EeA+GqPplfd0T(J5$+8wHz=LHCT% zWp;v#GveBzs#z<5TRJTJgWnuIx88b)OZk2ja!*I8y;!AssIYW^=*;t^BOg0a2YgWoA8g9Z;+lrc&$@ zYAS6cVGxoPC;KeJE)_~-Sr;|gg&HyfF373ZO{7zC!_W5j4)2Wd=7l#uh_z}2t~%(c zOUQy|vXEz~%XV4)c9pK%6kSW8hAm~u@s6;n&$yN3JxCCigz!p^t2fx&olIhh(I&|7sQ3;dxsQ|j|VyTIbT0ua&wW} zOYQ-SowgHAi%v55|80fK)2NW0z!-^p75g}V5~Zn(Q)sGJ_(q)Q3QgVfyUY5OK1g;$dK@ zzViVzROZq$^|zw<{ZRz0fFnO)+av1ws8y=ga|=okJQ!1p7AXMguA{e5P^#L*Q7bHO z)2{=|iZBEfgi^|~$l3lmVR`Jn-p}Z->$0ila{j|$Z-lsUmBLtfP)=OJsATU_U+~g0 zFwJg@Ck2kDhs`PYO?$V=b5UiM+JpWL@cr=L3Rki|xXL#eg+bSE?r6j9klh_Smj+wm zVBE&`=q|gBY6u!Up?_z6dB?MbEfu%FcbHf(kxHOwd(Oe}mKi_4ZcB_x>&ZXy-PsC~ zpm5Pa%w31O%a|21s`{q&SaD+Ov+*s3)nQOI!Tik7u4}8Tc_Tl_iN4qGWiwXj&U9`x zG`8hBJ>g|dUv{p5Br_j!PiIOoa=Dv|XL@kJrJAqL-!OIlZPIh^7QQ=wDakAKh_-w+ z|G85W5%kQu0I79)Ek2j8NpSSve6FtB7fZOACLvH{FVmCAnEeJF(^Jgz2kS9KH?`3`PPslWC*LEw- z-&NBgU&oin8}rUu##?|k(ff^-bssS=<*@lsDcZ88hN1Hn64auF5^YWpd}YSHn}!rh z4ZH~O3;A;3lVYJQi}Qpay;(!T&#DIfSM|Gpw)B~ut1EXtmDpA@tB>HIy~k~z{3LY_ zH5CIj+FhN}t5$Zcy$(C&Qx=jKE!+5tiwCaVOI!AOM%O#A8Kc6NTx$4vOXR#<+vR5~ z8$vpHUS|CZx`9e9VDX7~^Mb0{q$I-)*AH7S-^y4J;Bjx#N73e}t^jkqH8WM$l}~4W z+qlTzGkpf4E~M~=Yc!KDo7Fd20x=u6Zx4!P zj7iDOQwx3@aKYvGRWl<(&OrHoRi8NYeC4#AOHOPF*u?H+$vE8~KI>p~@Rs<2{?q2z zgL92F+M-2wo_b?Pp>vp__pWQgjg7YI?Wb+OX5wGv##0c%)qN|@RLwC>|L7tCI!q)_ zjl5T9miWo_I{ceVX)A}}}Buo=VH z)rned9U@08RakpZIDVNmbGo*{9Ky^|$Wp*9eVW@z`ZJ^cWSwcBU~~D4ap$V0W#|R~ zwnjo9t*3KI=dz^F|AJGmh+^bG>X}1`uIcEvC$BR9B6sHe$&I77UKot8%?R#H|CY31 zB6C1fqdRYpuCoV{6mQ+rP#sLrl||#77HaMwYk0MxJdQbt zWKOosoMF4L-B7;4xUZlt(90ZJD(SSPFg2C;*bozV3ckb}>Sg_eK+N`Y=TUYajBjb^ zYE%NpqwWVLL%T;BOm}&f8cS?CGC8FgHG2089Sq)hZFuWQyJdR9Z^J>Lom~FXUVZp? zLD|?@x0cMDihGR&ium#6>llsO5D)_-RWD`wQF$FukekD5zH3}_`;yOi(TbT_Fz79GF&L~+i#+lhz6jf}h5 zTu>LGS0=j4A6>8N@OZHq`r4Xm`&|sL6{r#XqvLGzgHLSUCT}I-8`cLdZVH{gYx+z> zmb<}O|87Y~-`X49X_@{5gF}m-Ir6ndGqf|%rgeQ`DGj`r$kH%u98T_OnRAVf9d}fh z#e5T_kkZYOTszPb-MZ97Wb%?Zn>a2;g1AJa<)r)hRZRe>h49za6S6-f>&%VB=h&3p zCDVwz=dC0)M5f@g#g1;3&tFgSy$Pv^5>hfd7M1jUs^tu(rr+&oUyGMH(It2EpS0bd zGD5K;Q2L15$amBlZP3v#lQ*k;ipfqB2R?m46PR{3EurUnWT$mBOBJlUqIaHOCh8Y^ zc8R#~X3A^`!VK1yUl^2_Ef}w9* zUiI}J3|3VHOGkwNKo^*WAc_&s)5xO3K5K1x$LyW2Y*oq8B3+NublpGgqb`~~UIx65R-BGQRbxj78q!X{! zw9M7UT5}?JGx~)_(~aN&mso$3w0*A%h&yR@c|c=^3@&FdoMmj4dHm@?PmI`)CK>)7 zXq^WMv^2fo{H$-hy$u01*4)H_mrH7`8f4|~&Y$(Y!%z32o;2K;-ZDo}=D>=34IPHA z`T5-DsMCE;cE&$Ft?Zfzm{TV-&)@jsuf*sn4mmqH03V6=OGLL|(07ft_vtgNIo+#@ zTacZdzEfBfz$&~~;!k;yg%blB2=^x)ZiSYFW0dB}=^{yE21hd7gwRmIpimR>t^6BN zqe@XJ(H+M-{8>DG{2hY1M?Q)-_emYav8__m=WG!8058K^P_Ejot0Moi%P*$S4zL$*11Gy0+HSdVSNW^IPOTvh~9oT~e|da6R_Ia8&f z$dlC{NPP|Qd6Vl7XoTZRLOuZMqnqyiSU z-{ybn;X+Ol+M(Sxzp$(YHlz@%Cwqo2d91x7=-^2!oos{nsb7d$4fmSE$}saptsUhV zS_U1a=z1E%fmQK_EK2k^GD-E(*+VnrSVPqV_X@<)acI zqksy#$5q%fN$G(=^*6;9_AmuUW<`pY4_;GOes|dE;E=}Pkv}m^R#`Pw$no@Da!FCb zT9#ApOAfG4;+sO#AiV}DrJP8+E(<{Q-@bZC1;arHD&8znbKmWdqNAj_!px;;6Y(y+ z`m=-dEM>$Loxk^J508=T-$$Yuoa1Sk)SpQY(cGo3K%fU5cIxSGgoQgj)OXgh{F=bs zVI8}+n)l*;3>|U+GKI^3;f`CVtG$VZdQ2p+zwC_ABb=qRwixBgu4#Nug`TOwc4(ir z@TbDZ>N3>2lNO8uOUhhezx%9sq~LP?I8HGYt4`$9Lfe0U!llv%6jCD7y1>dxSpQ_2l zOMtW~?LWu@7ey(LV6LXO&hH14xW52=t|1ue(6 z6{#OFytDk5K<~#s5fQ7>(A!T{wh9ShkS3AvtHU7Pj?4G}luL5v_CC;$T%8=2U4kV0 zm&8=?H2vFj8HtIrix&h*4ma0Wj*_aRe%Alv*Wtt^o;^7*nRW`K6bLG|72cxf&KJ>_ z5{&>O%>9&?`nWQM%h^oLZ>}zsqd-D&UfOT#%@d~PZ+!;?)iWn#E1Kb6x{cNan;eMS zwnzS-?T#zQ_X5vBwyZ+=zSkb$+yC*a7*>ZzXboQoj#bEXGrt6eswzsVO;wc6Dzc}>uk~P zbboKnxCFuYF_Xv{M=@mQ|w>SbB)Z$ zslHp-THlsP)@q9O+Lqu&bvdmruJ`H=+=&Ho`uSr#10_l4`TsKZ=5bNp=lVaI8hT6& zr>RP+hB-Y=n$(Tv#3gQwCdb%lqNWY2ELCbiL5TrG(P5HUV}+P$o2ZQ&E-@MwaRfwA zAQ6!ssj?UrAxM=X>xc*pIKxc7@B8y$PW$`)zOUEs4-$l7=6OEPXSuKYy6)?acm2Kd z8D&HjegyV-3MUMMA?sVb?&qmzCPe_9{^_V(i~5p_7N)=!B? zYL9GA{3ymPCA`TpmgUDz`FMJ+Db=44QH0QG5Dj^w0?InW_O&QSSBypjcdVM>b;BSE z!6Sa~315dXNM2Q?ZC)rv4Zr>t+*DFnKj~a0Bw>X6TR-|kZS$c28i?v9PR&uygNeP; zqQzGB77aw{n^S`Qvs)?~0g2uD_x>g|vo|6{)t_Q}Bi!(@?SAf+7yE*Amb2MqbrHHY zoOWxY!(#oMKfPz%9%)(E6`4_ReM4+mkW0=aK3MuR#(4a&dan1|#?ze{rxMDzw%skr zT+hVouU{H*IzIkj!<-5FOOy2Gj+(Lhl?cs=hU4jB*{OdrC@{nycdg9^ z$#zD(Q)+%TUP~y)PBHyz87)s7C#4L_DIFz5LY1}7Yr>|RJr}XxK0rS5SyJfz94mP* z!CbwYW{QLzoLG+2W89_@zkG7Uc10g99^YKP0h=(Tho*~bgOR=Z@>AtQTooh?} zuyv+0iP?7}ZOa?0><$n6Lh~`7?&kiwg0EQ{Ab_%lyk8Mr_o#o z-mPY4)K4sK_>96qH|OHzhsGGxd(Msmw}C^r7Dd1asPynCc5Fae6@mkZ#(MTpO?dsG#At;W5fK@VV}8 z^Se{lxe=oW>%LZMJe~85O*Ykn;`+UGX^fSf{PMSt+plCW!E`gebLr^{poK}WV6^ne z5wH%BnDB}&wyYa3P^GaHGnn!F|HVTL_8~o(>m=G*tmB34=m*x{QrkDRV^u{-k z*agaz%)m#(A+Tkc&E~@ZHg>I~Hm!B{oK=KZ+S;4X6jVcVS=+ixth)O<=*PmOt%kaRheP*U0p zk7y60FTx0;CCPq86=!HAw&ycBv!{3!EwYW_0$&=_rEj~UqBCWI!GVoEMIeP$|COdV z@hnA{Kp`VsJfeBy0tANA=p6nX{rB*3ky`3^DVo#d#LusIUt?z1%!m<_+_O@kYn_@< z78LQ@Nm#SG(To>!+oU<;%Pb>*3sI?F?RVZjO6n!S9#Z>TUK!b$Q0m-oux^RZkcpqa zYxlcTuDx5R3CxcwD=^m8L2@BxXq;tPhBJ0&NM!iYYq`)fPOW^|4}zVz2j_!Y$DW%d zq(xp92GfNJyJGADOw&eS=;<3AS@Ao*E*nLE=$W^SmQq~p^G+GR z_Mig1ramL0a*FLO9UE(py5V$F%BG zDYfKf;^(0}gwW{;e1ijrdlKlmN(nWkgf)4hi<0rBlBk}5hw?(A zR4OGBZU`D>QY%Jdh*Ks&VvC3PLH;{J!4hU|C|d?uZ9&e>quDY}kR5>Hd5(zvVSOI( z^a@x!1YVTO!(5k8>`_UENSj}Zp;a4E2nki>l!0fw!j(dXYJ3e(h!_kgG!#v84UHG& z*TQTfLm&`S3+TRuYDkgcj^<0k;kEgV_zMlgB1j9r#YTq6?fdxu4dke^WkrY}p73|1 zuv=P#Qw&W|M7fN{^!l89j=*Cj!jxRudn12Lm5>Cts&lUa7swNolCQgMOh( z7s)M(Sqd{l#pp5;uuVg?nRzs>DCv_3Sp8hCB$Id6XhP}O2vQ*6H7)iO&=FKnWP4K* zilbe~+X-8$CT$Sm2c4$^Q)s<*quw$mM8(Mv?I=T7CIU#6Jyfz|mRJ-Ii8UmSR-Z$# zRX!|ik#mIM5JR2no_xOxo=G)pL1PT|#)ZgOdM{NKAy$8-b!HKjB~_y>k6mE%8EKPQ zZ8AGD$^GIfOx~R5mCDs2SGVJYBwBqQ3mGZdngqJ;4klLwx6IWAhwRezbn~mXq(?13 zCAV}NGxi7IuBEtDA8$5UPYiCt;lbMCAcRRq{sOGC(a{s6f`u_0DM+h+kpX5o;EN`^NNFJ z;maJ7Lil$tBKlQ1B63Q33LDocq^Q9=wc^ywR|{_*qa-Lu`;yC)<*25tO#7jHj|RXL zv01J?6^y;Lj~~L)-{Vhhoy%JP*DnQ_WtV1=uz$RFOfzw=V2_o$1P#}zTnsZzYg|JT zuEkED>ECl%i9-v39;LFQG5pjTsTxsOR_3|#d6PXumeZ^3OWYCx)^&98YWtHF|GgTP zSv|D#P#7@`MeO_O7a0Jhe3i;swv2!Jg7D=BlDi}CCGO5@2Ulq-uEXO~6fiHc7{_(f zI-+nC3RcsD{@e`iD%E|@Mz3eCdld!N9++PC%H+Qta(`;8(W3n6!f$yBpA|s|CK@KX>2-``fR5=2$a<&mj6cJMFw7 znzKcB|AD{ljpkj4c5g5+yY#D1ZTl5{C2MMw|IQp&ctlpO*Un^_BIliF8+PvHMeD{vQXV@l?%My{tw(8uXrY$@3 zZFu}ulMRqwlRS|5>Ns44tP`L3XvQ;wH6pC`#s{&dh}O|X>elJn##YfK2Q))76ifdN zXO;J$@8=<#-o11C%6yMm@t}I%$RPXSr%p0}BPwYTq_YQ9LC2LQK;~+Hf=YY3s zHCazXInCSNA{9o*0)6H+?B2Dzu{PS8WU|Gii@*#;U9BV4QjOu`uwX*h(p?r%V}9n* zE=yzG0W?5o15kwyW8Xru5{hE-bLvZ}%I)v0N>25!`HA0_T50Cmn}RDSgI%ANk-F-B z>do+eoKPv`n&$DRa^uCn=XdY8(HXm6*P9ff&!m}of)rD-*L{q~e{exjyjR(VphbiK zvA6LC-A#JNwj?(=&I?>V5#K&7J@jJX4I^DxbL|U5+w1g$=Wb`BU>z7R7rVatGklYH zALJwYKN!nT6%_5OC2ek6A(bGd>AoV;#Gn81U-p|z8*Wr@PjBwOv9`G3U|3zS;p1&5 zqC@vzotW$JB?A#U<{NUmWxK--XG++i7F3^{+t}~3c^>U!XQx{^3j31BZhf+#qW8Sv zU7M}$_Dw?njvePK9ABoFEgJQg1No5wn?AWc!*pwl@sS%L#!H3FTP3{nyO+Gvlgqtt zpXtc2`Ss~l{>f|;B^iI8lzghlr`ofR_6&6`ed}~L|FZJ4=$s7UjFL;ozbL zR?mU{vcU%!g3v{`^p`!khlOr>e(w1LETh2(-bsQ>oO4r3tkfkp#Cmvi zKes)`;OcxXKxbQ6GnilQTfH<8aeAD0a2G{mdFN($*9P76yJcnKqD*wLy3{P&lc*^XiJv4ZF0aA5<+wTjE@nS%Z*f-O^O|DEMnF|5ei6NIYz!pL z#ll+s`ho)TQ#%H2@T*2?O3vHrrJL-6!!4IG^-`Dl`@xm;sYOP7q(0Ec;LDjs5OFpr zaJ*-h(#CpTeAwe*A&iFN6Hy@c&N-EZ2??8R@j>582)6~-bPDAFd`{oo;N7ad54t%5 z0x8!J2UAhTek5H7D>#-^^MaTOPyG>#ItP3CIHiPoe@c>jd@-#iD zVvPxKdyrr^NWd;(?i24`79*>wc$K9%PRq_ugT0f^&(7dFXtBn=jD2lrml*yjLO=!j zxED}W)I8c(vj*_`AA>c(92P)DDq+59Qb$DM6EAvj5`8Js-sMucb97?VS|xy1$s)wPSP)Mj(Ou5GlBp=GD%h1XIH z1yHA14SUMX5X}IzsgVD+ zs(1hvetf-k%`4tVI(v57K^=LTm1BahI}fC-W##nK&o=eT$mrdD>pU)(y_M3~R@h@l zR&3ZD;oKLo>iQz<7bCVTnbQ@}KKHp>=f{`zTGrYfI;U6EB(KV{e2Z_l+fL8yq*UX` z*0a+l+%f)P?rVmoQNMBi!qjW%-xzKzyOX+GgNl z+uI%dr@!ygb%t8{{1~^@4go*atFY-d!GV-}s!V>XIQvLJp`^ZtfC{p2D>({`cqoH}2#|Rx zzLtt~grqzH-!4WxGF`;nhnY0-`B|`SQe(q|Bb+6yn{)uerCH$F-ihYE!&lYMLSr@!`!Ov)`NVz5|HiM9vDI_}*eGo*<`rj}rf{#Vn zsF=Nxa#wXq2CI>dA`4U%X9DEpAB5V}3*JnV2w6ndvGI}Q(<5HZn+zxJmO-hF(;-qqbx?)Nyz5S_>YD;v;+OU)r512|(P1H*IFB3Gc~P`Mg_fR?%P5QJS* zloEx#RUdm*5$a6GD=8F;xMKbY;aD}yC%w01WZ|nAA5^p4dq_BYi`~;UHeCAKD_yJ_ zeF@z?_3Its#~ibJoPh!>dne)yCH--dMn85bDR@g&>5a(1Io_F2qXvdK zIaHh{!8;feb0ZN5dX=nNUx?+6u2-ow@hyh>0rL*+We>|hJ>>*Ld`Y{PmnsCOxENbO zOZD`aQEsscWXQ%YvR#ce^~BOBj+Ywg<|3^wUya0WVEjZ4;d2x$`!srUETy+Hn*r-+ zSmhFXx3GSR<+5RO1Y0<0;2^73zaFP7bgV-E zD6q*gS!NINJF;Hn_W38(6}sarKqfUtt1mTsTLl5gx_O=C4Wpbbb4P=(N*SsQrg&Bv zC_$Htt@=I-?UGcJoo4hhm=d|xOSX*~KP3mTVW!KS(P}kG|6mgv zSSVvp9(!qw0u|z-OD2@gy~QZt<1u&2WuZ0JdQ~kXQ-kSvh)f$P8L`q1cVY72cP^yC zE0D-yl!V*0s-?!X8S0xkO5YyQF(8Tnyk=~c(^9BK**lJLU?G_vUhKv&_YSo$B406o z#PMd8tdPknMg!m^l@9yqjMuV=C#)p96pSXb5rGVs zJ@R1*@)&^dkWrU*61QR=HBZc(h8`#_QPi#|Q-B#~%0tI1+u{=E`#ij9$rw#bD;7Df zRGF!)VqtmF$qM~C1ls^8fr2SmrVzIG{pas(op-n*Z|3q4e0pSNvc8 zFMH4kAfLovNX84Q;+fHspdGU5rsJ^eUsCHW3f_e}t-I3l7}(n}ArPO6_GN|HqoN+z zj^2OIU5uE&RH-wT^c$Qq%~xo{^G|5}pT2)+ICOgGHvZ`%xk+{_?dksO7%W1)j1;>t zX|C?bK+l46r}HT{oc+Lf$-}K+U_;aRq)A25)>U3tH?Iky^IBu?DPxoS#q&CkHF542 zOPoI+n=RF0UMF_Ecw&2evS)jqzV=i~evG?uojt_rL5TX($JUD1=+j&8KC3&ma)Yn$ z3xw{DA3H|oB5HfEVN5Gy27OgIschbb8PD`=$@j1qA@4b(6C?h;rHgB;vNoh`n$c7| z+tc<&evF5Qvj|U_&@Dhkf9pOb%(lCCCmZkRq61sbn4jBUr-M2==WY5!NcT&XmcF6@ z)daM%usqUO(O!^#ZLf6_y-g2WPoMjw41(ZT!-_`#nvj%Di9NSHx2-pxja*ky@kQP5 z9DhC|-RGJbU(PQ$%Y~fCo7xq1!8Ie$aJIxU{o0m^HH+L*4SCmUocR%YS`p?+pG}ey zRREIlUJ(bZ@6?Z-GQ!fF!HTY2*Ph>K!%=4)B3+K2@`%%wTW5yeUJ~-)iTV}w^~SP> zM=njsKQXwJ4k7R}U6CD7BoIQY*u?Y9ykS8?gEjHr7I|7rJ7&{zfy}Hfr?72|dKHY> z+nz6-k^DoWZFL5Myz5gEF6!y+eUo6Ggwy30U-8MbuSZkoci)TQ~ijn(6!L)&?_)&u3PdWXpw z9Z0XDty43%1pkRrxt-E{v@mJfmX3M9am=(#dpg8_mVH@{F$(Xwjl@;eQ#R|u(+oeS z?$I;NvVJm!X*aiByENvmv)F0ENe0<=Lw-0Xlj{5s{imqUyy91+Zw_o?Gr)Irp_&zlk^);q=0A$V_`y;27xA z@1`z?PNuQxA4k7@O&mDMx=?Ug+lFfn0_AJ}hLg)a_qz1X$W)y@)baDwhKAak0m;4X zE>O{}&N254)g~%sOCyqYEpr+s7JPE7HNLoJX4OBMiuOHQo0D(t-6#ge77+Gtxx zVb)q_Y`zZR@h+d6mf)ne7)Ji^O# zHbhDt%HQ3ty!_~`DGf%5I!VG_KogS(jKm@*6jGY|BxXylcaL*t8JbcMry^zC)C0q# zYVZi1Lks%!VYWAEOj?@qybC`-E`&{FR<*wJsG6X%k?Gtcx*p7mULw{ zrFPDFZ5Trq)|S;d8-qqjD-m?S51b@Jx}WXav)8#D#{a-WTad%#BpgT6{I9#vmnGNk zYg_Fne@yj0crF#6d`&rXAe0+{3XVD?*!6cu#Kl(dT z84{X30lX$DV_@(+hNyeu?+~f!vb8qbcd0gcvT@TVj^P-1dDJCQ(Eh4rUF-16c*Rz% zq;C6!Z8bV{T9J&6L6Gg9p<7H-P@HIQ)42;TSKGXN`n*xNQP?DAyS9W>YNlO(g0NPy zwdHGONMvVfLRo+WpR#ZFkd;?A{AJZB(~g31f>vWK@j_mw*p`|?!vaQo%=p=pV}#O7 zn%x3|QK8pE#j3G1XE6Uw)%AwwO)o7v)kp7SSmI-|}xoG7j}?H%mN+YE$9cj|;6GxR(qkyTaV1xg@qM65$}qWiG0>l^p#d zWm`3Wta?+-ng{F<8s5=E7TP4?t;o}T<#QfByHs~CNus`{nnD+RSe z3PT+;4g`^jW)u<%;lNHPrK1L=F~y;+p$m^p9Fb&|@kx)+sm2yg;wV^_{oC9WAknLp zp;7+P$DDKN_u9@-58)TNqs?NSLT=bO_eIfWa9Q67VJVMV671-$dIX8_p*`D>T zj=7bv(-Jx($C`Rl!d(hU|F;qtGz8in8xuo-nj-i4)=+{)O;#x8LYq^Qtxn9BQj%%> zIygbtQiiN><9yvMQ=`Gk8VP*DOVzIS4}Ve7`eCyPWE(IX67ho$caQ3`!>f6 zu_RW9hLEQZ-Ex$n7n^ix3HCzVhI(?cai|73ipr81CApS~B^>doyj0qmK>kV!f(jJ! zQ*`viG|5v+FaajRRuXL8RNaAKx)mj>vKe5L=K;L5?}4)h`;NIL`5Ry4to&C|jE_k2 z9Q3LZyDqAvHqCefN;qE^sf(~1l`cnLXvoXz;c3m=Y8w(Spy}yC z5@Ex&^-Nl!dCaP$E5Of0X{#pe2#;IRic-5GP;u+v7(V3 ztWkxppQt^R;gCFQmu*trjz*$B!b3Ucc*GLJRq$ybNU)QqHw?6exKj$@S&C9m+uV3gx#yH8Ti1j>W|M(^0@X9o4U6kxB zPaf$cG#hhyS7@+al$7`Wye3WumNF#;*ovI`DG9|})$f}^ZC{P^ z4d0}lMY`2RpS27YWfJHmb(CS5*_Zyp>K$khLpK?3y>yQZbc6AjuZO4GhkfFn z)Sq#1ZAtrI0+&@U-_-au5nzhb^cl{$AVTatnf{yV@pXjyy&Pu#0i_~ z__$qA@L6B_cY;eM7>`xYYmQES<-_<1#w3cx2tuwGN88d{>Zcf+@8l2fT4Z0HzQOs^ z3g_iDhCu7-+E2>5(4UWJh}^W}S7}`%*NoHsq3^|`!4VtljE;FhHN*OT@!6w2?jfz- zq0T2m%(TjQD+m#uVj~;(+NS%Wj|G2sQ@HCk@LE*Hl!#Y~3Z60BeMSlGYV*~{DFv`M z0k5@d!~BA@ID~BVy56oRKietOsc>uIdi#j0zLl}E0&0rmJbv80V|$uq>g2i0LZdgF zD3AWm<-A8o+RgO=`7lts53c|m?!{1fSusYv2HNi=_c z6AR4i#i2Jz^?$uOmc4QGgdOr+B=V!C&%TPbL6_6OQh`%qi@mdZ{!hMhaof7`6ot+uwe-y`xzw=AeraDn zQbD$H;YM7dT&!GrM7V^nv_?k5cN~VX2}>O^WNZUFp5Yu6kU)Ut{WB z+rmYZ+iF&>c{`!h+BLXeaf_CiVP)*^{&n1!pR=}5IO^)HK8D8njH!j?Cm(sm{@dUn zbhcPDJh-1m7Zk(0rAu(DbzZgW9SO$ID^%T4_222ln9$)2o>+VbktA_m9ZtQR{rkjx zPlA<9QCjZUk!2isZEAY&9M4SFWcy7w`r?kHn?s`lE&Fn0&QQxdrAN_YGjksE1gG!3+&Dq3-^(d>n!($FzC&>Lm()fWjgN2v)Ua$`wdsWr7*{l!k(Cl z$I4z`wVlL6BB}>bcsXn8;7p_QW3QrU=O=z>;@5OP7ijmiKBaH%{c+!i;aVS!3`eGw z+l_SOCc!c?^?_~iV z?kOF+ch8KtkhZha#GJKyYI@n=mKCU=(AA{*SE){&0n`#w;jx(*A4WQeee}|G$87Z0 zrrz!*fev}N#_ox0>}leSoqL64m|C?;oPFR3!sKTPfNR1~x`v=J2BIzNEzRhiIFVEwpL0D8 zG5D;DZ0b7OsId0B%cM1>3p6F{Y?f`<6>li8>+4bv-?M`BBIf_Z`NMlpag24KZsqxx z=9NR9>hHLQg$B6CeU?pBo@T&I7j&DE=z^E?>Rb7^M>q7bgGCx64M$5lr!(RItHV-L zxC^u{xh`5I#`sqyn)g4glZDT_rr?4}R#VT;RDAqkSQ|}#bC@x7QN}uwkp}mOK3GDw zY)U^wDlY{iu0L6IJ);>o4&By?%mdV&gg9&6L>U^cYy1KU^G`HXQPL?nwa5^K zL2tkqF&rWQ@yk^04g9DU(lpP?E5lp(P))&kLdwD^0kXsp5R8RU5T&m4>=X)!Hx%M- zxv|BD;mC;58eb|6`@~G_QwSFj44HO?VU-aPH#yO(1l{li<+cNWqXr9rIa>Z4IhKUQ zT)GXA1RxOVjg3cYtieQ5`MB&f{sfO|`Y@QTFH{U7Q+jwJe`?A@Q`ibtH9rj57KxMy zj$-)_c&LRv&CC5OeDs!*?_7pGKYKW}0%KFzyH8x9evG10k);(=U8ur7ric~>itXgE zRY|jl=uVfZWTrcu`jgtNs4z%;RSSFdnYU#dJWle#J#M#Vf2(WDTE_>>c5xltT3@Aj zSHO%gKqK|{L#7Ev*W7!QJaH8`$MY8nC81rfDS3hr67n&EmNXom)#MD3GRkSH22VUf zhL;1Z^?#>2SYU_9ZKi4COzw0I2>^f4=gKO(RB>kQ8~Jv}rxj z^b zCzc+@svJ)PC|n|xo@9iWji!!8gXax9_!H&!#7q;*IHYrOQM*^srwox}v$m5acC2uD zGosO%K!%iFqne)l*eeHKnnA%CFaIWh=y0n3?PJcM!nWJuU;7&&{C03*@h+N-I`AZe zUS{h@VtxK&9HcMUS3_j>%h-jaVE}DgCrVW{{w@Z>ZY8SyoL_~4xRv5~A~r_73RMl;yik<@`%LW;hEx`H#zH?M&Eek> zadG(&8pGd+RRoe{U*f~FLlxW;O>p4JRCW_MJzr^3fMVpx6R~lN)s{6HW-+%QZ$ZVm-1|_&%}mv! zm!fR7t7!WgvIypcTqkw>LMVgWbBD-4rIHZB6F&_WLIo>X8w9m~sWl9E>u=Mne%(lW z0AA&b6Mv|>^9ho63mB#SA`Rh-vQNePKU_vi8m;|Ge*J&{n%qi0Q904dIvWCg%UQy> za4P^PgD5I`*>%|0T#)10q@JgFzB~#Iq>HzbcuUcHnZNR{82Z!iF{IY*`JX`XBP+Ix z`*DEaPnB%^{{I}MC+hy`f+gvL`k1?)brEhFuPeDKzpAV)x+lZk$9ZJqaFgvFFJN>* z`N4w~J>R+fVXMw?FZHtTxTo2Qb{fhC6POc^F z)FPer!5W{r{#82L@uk&{w2kAy-4q> z#SEk4MLuqNC^k$3yvd0oU3VAGK2P}#6fNQ3D}bMW+@3MGnU!DVde;1$CnaNZ0y0v= ztuyx&wtaDS`Z~DU_>9`vjDh}Kp8uS@DYQ(SNc&k8kb#&OgoDSCW zuSte&>`QY?UE1u1v5Nx5NGl8aj^?@j-uVbuT14?=H3GU4U}xiJPvg0u^Sj4 zOoN$;jjg>uEU(&`2?ze1W2&^byyW#lLHD!ih4qm!4Rc`U$Qb`OYsr*Sd$0e)jL6>c zN$rn&oq5-mQ@a_sYtt>RJ)ddThu=0o$jm$ZZKu~y28u2gjIYw)&SDrOdgT@0V2pS}PJ75HZpK#n{_KbCB3?*47pOp?qEMq$&8o>U#P?3Fm zVf$F@LhAc^LDhz`I!rxpFG{>HGs3webRE3*XBF**G<}e60~v9f9cS>|%vP>E-1Od# ziE;*FVk#Dgw2OT(!jVU#A?I#Z>|Dq5v+k0|pE6?Cp)W~=6`aw9)_9egsx$i{=IF8w zA6m~R6?V?K>S<|Q&!a!3WYsYqan-M@1}Rx2Q6=;l^KO2#CvAQnyhX+e^MS)nk1Z*) zZAe8e-0R@{X}U|R$?;J+7^k>?G`*Czkhy7&6_@tbB)5Ho@^X$Y9>HyYJi+nxuE>O| zLk1?P?(zD1eUtRy`JEUF%@0%`#)gf@56x$Um6`Q7< zY-$_Bs(AnoHpM?-O#g|xwzc_@N*SBwTK#(bGIQe=@BO5!Ew-8kSNlE?z0D?i+>Lq~v^sY1Ng$(0HQzbv(cv=x5BcoeA5 z6ur)EUrcT27}qtp=|+g*P@YT1MoJ+M5};;fc5M;kP;}aKE8y~USzza~l88y;T7P+I zoX1afG|)f`(({jSddPZ}S+6hrL3$#6XKsEPbTeWv3c>a3y-GD0N+smk<^P3#jR*dE zn~z9MDpN{qV}^)LG^%sP&q8P>I`-sCt9DpY{u-q7W?9y!X(dh*{SBF}oGL5B;^}jf9$7mbXa>H3H0(EmgJx2BB z>XO>7f>LW|IVxSL;-uMR_2^6(4(d8y@trcUor`*=LiAc_+Kv(C%R|J1Sj z=2>w+>Zq@CDa+Kg7Pev8C$3A^BJl@3v?EOVz2X5A8&Z5|o|%w-r|v_hNz{UCOt4U< zcsm$qX_Zv=XBHonck1fh*9ui??maxp?cWBXEXntq7S%_x!<9$2(la=-(phA+m5Ean zn#%sT5{_(x_(-)VFjjOwy z8nu#-qLq&-yg^xQoq;>?nq$bgTb#j@UKx@R3T>M1<`&*xvk$``VRdQC5WnaHnk3=F z-cdiU=_UM#LSL(3QCa9jG*3eOf{Fz#BmAV;>pl**WJt!WO4N!hCE|itj5wg=$jT^G z^`S%gr{a5Mt>WU#xJ<;z2h)pHF!BV74ZzXeU_bEBP{REfzK_d?8=+;&c2)Tkma4FI zY;35=&MEGiSb_jD`x+*;>>vVLPWXFK>)`M* z_mDQxcz77Q!6*`v6H{VDj+*eYa!_ImSHYOV8A=Q~#0+5P!q^IBGFD*->f5x~55!7< z$G%gUE){Kc!q`uJOT=k15)2wwy;#wEQW_w!;Bg2DZVx~xatOjaalF<_%L!($K_H67 z&aCOucln<5{*@Yth0VjhRO&^sgn4uR2TD^81H63LW>~G!F49;KA0pvlVMW+wq@5m0 z3!?JLNOlz(7;X5-kzwgwPFAJvW||UYeHy%6tVNm`h6Po}-&dALJjBEQkl306L$Pu~ z?#T{wgb-b$RoKQ)T;WPpgv*_P$_RkFA6U281ki}q64)VL-LveTgLX*x;poW_+-j&} zW2CF?-^_~J)ZY)4zXcP@5kR&iV{lXXFs)&b@aVKhiJE$n2?Db#oR`2SHE73@v7Rf- zb{Q+yVm$pipf6MVMx)a!T&wxJ#+J{c!PKW=(M){F6rm4LxNtO43}O(WI9^N#@iw@~ z>`#<3LJ!Jo*02))uZE)+if_ep(NZm6j1Z4fRe&`Xx9AMDLU3|;f z%TAfh_Oui=YbrsxE2Z*sqajyDZ(A% z5K2$F@_8D_pW;fZr;E?~=3<1@EEer33&8WIj-MW(RWqvD_Z4L+=IG=skX1pIG9(6w zO9cv7=CKjIMQA5(dIi*{jsncnb4B^_(rH>~db>avrJvO%u`g}5E_>BjL7RR&d0n-J zRoxhxURK=1{0w3yGmTkT+0d+}Jme9Yo3*c)VyPN+S!JXPylDl?TPUe7fYAp>D1UOUJQeaadUuydoY_4@=+NCI z3R%&M;bPYO3ePQHDXwMzz)fBy7%Hruyw-M7@VrXUAyj6Xl+V_GY!=$bgNIY{$@R)z$+~suE;UdarT`mWMc0y_~BPRm?-6D;2f> zf+2;P90JLrL;3&n^*zEyV)EWdX`n#R;s(trR%ucpvY=eS)!dyVx|)=!RTH+^IrS@O}eLd#Wm=gVUYA50I?w^vh>)7=#O^XEn;d|9-) zu{v%p)u-{r{U4q>1v1{!mDScC(%oV$lrqrU?(b1}`kUEP`W>4aYF0KGT_x##!k^T% z$CKTjgU{?KyqB~oxwG0PfVSqNgqH5jqu3w^H|Kj77=BI{!DC{y>XV4ZCO`y2dL6_4ZRlUrI>l>*Ct%+3q(lsMfVLb~~)&M%+TLKG^Rc4@Z*M`_t5m@>I#p^7s@;D7T*4>2JCE|-E?<=Z4lp4d|DP< zgFdwX76DU1N?&j#WjSk$C5h#h^;<&PGzYBO6L>qp`eEMlpMow@=+^s^sFUxV{QKtEz`>L;fHRY6p z#=!Du$I9JkwM)D^R#pBjhU9^Xk{dEVi(CCd_jml)>;vSsY&^;q|x171({&VsXM; z=}1TU;OQ*2dU%xqCijFAZy)I+uP|UwcOxiVo8NHIRbTC|ZYQzazIxX|RkSg6nrrn> zxTDtmO;U8o#;DFl+^=iznpX$8&a8etsxzr9K17JJa_L0kaP6GVWm>O{%%u<9#9FNj% zG(=EKctAL#l)Rp?qNI~vwFP4fSDq<2+&rnF)matNxzVdH)Uvl`Z_rxUSx#%XF*FS( z$DB_6P)m97;%#W|Yp(LMyR!*82wP|(3nnrG!%TH9CGwbkyfmjC_uZt3I$?V$pB-Xt zxqQ&GtEhM<8%s!`G}4vsWgf(*6K<`swySzX6-OW4_wD1PlhhJN6Z?iT0FGQU4w1H0 zleXtWy&8`MyV*7g$$agdEIL@Sr%saq5J?Bhjxzwz%R6qB^>);cU+Xe0d05B3;+pzG z;%vIkjvR<`{Gicr^mb-n)RUW1FT01VXCm*6&G!X9KJ7ZId~HoKceK>*b|amg`^ja* z-qJT(udfV89%2eP%_1ta>292SMCa4}T>Uy@c?zDs1}jAR(K1NZZ2P~T`r!H^8YS0f zEVIpnNR17(BQok_z-e$BipF~+@?eugr+P#z8yLWfoXU%H_EFRqG#X3#EG!+WMF$QK z?{t`!*xwPmWcYuxePQ@xbL$uwoRX4j3!AK$TbvEgzw|ddIwlVieZX^KvC_NZ zeI!vnz*k)~?{4IjG_A<~=N&r)1|iM0L1-h?ZsCU>}aNkMpMuAFd_ zoMl*dOSq-{mITtmX>hCZMHe>i=PkZt>!T`Cw6Lli)vWzqW(-u6%h(;LXQzY67hG1k z265T(%o??x@tq2`g3H zF6fXt&CJfNhP*^`80b0wV7PZY1TAqiZfS<2(ym)Yg6^Bin3*QEzeH6KU3*De-tU4+ zhLcnB02$W?1vDw&(Na^-dE3x_|P(K*0>J-XP_w z#Ag7i@5yW0 z0#IvE05$@2H7h~5hY821W0P?hzNlnD(hu0_Oz{EoA1Wm_hO?hO=4++`M`SIPK0>!5sOW= zzzpfFZx<(LeQw6#x%XtqU~MUb*D9wm7|T~pfGVy#Z=U3P5@$*+as8gq-WjCcD@m!x zBheUM-0Z9J7LUOW7h>Y1G*3V-xXawHl(1V;Fl;4EL=+tl!JDeS1Oa2&Kze2(wiZmk zjsxi7Nhu487a;ycRXkY`(qF1UvgoBj@-i$4qU4Ay#Hm<`qd2Kq=m2)vFY2Uor=f~y zqU=wag}ZdbMN#pc%Q&-w)-57scxltJ>$k$FXW1 zX!|Mcsh;|NgdR$(<*5EAvUg&>>w;B@!lnwTj+ll4fNX}0kX(*do-7p%|HQZc%lE>y zbLSRHmdSAlGJWW?{XME>y-L$s7ur;Y-X6-+@N`H$C7PLF`gD3^U*qHg&{Bw@y65oZ z<%Uq}Xaz2E7cwpQ6LxP_8%A#O_p+6%D?6pxJ+-fpk-7LUSNP9Yj_{3VMWipe6s}8B z7s~!gJ_ujo5lKr*yA}fzKQg+0=W^?rY;Y7T?)`sf9+WP=i}yu7b#_sPL zRrhVU@qQP#rrTZC-ni1fY;aS}tkzrp|FGrkn3PPX9ogygKj>O?m2_o|XUc%y@*yC& z=xv|5$(OvIp=C>LLIWk1*1aDr7SB8=-bxr&W*Y;FQukw?2mn z&9-;@VoF;Sf5${T`{UGcWrs7;_K@bOSSxul+Mh58_~T_=IQ&Q15~!Yyq76-h54e(a z9H*As+P?Cjf%PSRl|4sm|J#J-R6VMC~Hmyl_>46Ycy{ zkn66lX?ohet{2{YgY8#lyKikmk>)!dq4&u=b+oHUM5nQ#H$`u(;>4ulw;3tWU#`&I2wj!I)N>`IXd9j+zor z`x4xELu_$}M>b-SYh0IdE!{f)N?BEwk#?K@$jlwNVcd3B!KpI+k8;ha^sbWzk{Rt~ zSr;r@e4RC+)_{#a_^ZBrYKMo|632`Qxiov;c1s35&SECie)Pu2M$Y8(nO&V{DQ0xx||Iu`Qbst=CSYq6TWltf9&;8NY-51nAumKpyj+o*A<(Q z7MtXuD5OL60D!jL96oJMY2b~BJ&jIdPCT|(N zKyihWp;;H=A0)*?1c_j#WsW(H@JDnSoc95D4HnWwd0zjG4GX_8S+E42-0`x!4=A;` zsst~8JMN~xS#hA*2y4Gv$y61Lr$#G!p9l*;SL^6V`IE}vCBI38#yfTQf@mqWXwtugz!(U_=tQbg`yh{x4hVj@G(oz zgS$9@n|t}_#|tLtM%7_cEfVn0eOCsCCGWhnXDHrXhz01IKH9NOB`&rkWFQE} zXVEoM#pB?bF{?f?J?PFmJVIqPB!r|ib*RAr8c3+2p|$}-0ScH6Ybinx^~4E}aF6-U zrKR^xL7{+1!k0AEDko^%4!@B|=`_TKvqY1EqtjElatdlyCLVsGd2EcJ6d;2D&Wbqa zUQw-s328lf)sO9e;`6$mrr(7tXz(JTo*p95v{q9`5>*n0Dwc7~DmX(Gmnm#c)B>x7 zhLEUrvaz(wQSj!Xde8imTp0|3KvMzmPrL(=NSYne+x4J3`|uB>_RhPrl>lsVD>_Gb z8N?@C-BA1|92{|sl9_5?lt7cULqiXusV(^F^HXM#n9$CHlT^kRUd7(6@`}+DRtk|N zq-=F_qQW?;-b+e66Q7um1k~7MCzOg{4sg#$uWd_@q{M)k(;Glctinz)S(Ty!Ft3i; zoqXMAY8iuHzBV758j{K#WkmMGz7OFiQLXcRF!{2ft3raL9@8*p6^q00XDD?b?FGkq zxnd?wVN;|2ih`^XJ|pdzSfXFc6>{D1#dLxq`yDmVCm|hb@1u$~Ijm|DjgBArOr=XM zKmI;`9io3Fcn(I0DAkwrspXIhS$`~vN9i%+8=-GR*)0U6U@K}LB~TPLM|l@(k6j+C z%uF&9R87dOFO~T8=zCAlW>yZ#af-)!{?4^_MIK8yD_vo;sopWt=-_#RSE)#K8~ouh z$k7TKnup}9(zsos;COEDZp175uX#qLQ{d-rY>f+9Puy!bCY>N*-uVb=&kO$3#Z38% z1~TYs!4s3PoR5d_CneC^m2KED{jV5_ku|+XL(!YxfL904zbVBOhYbbqO}dHKA}MB- zijXJlBJcW2$3jKV5+JK;=#qX$B5Wavv{i_7_FGm}H+6=*O}$O4AzW>2)bcBUqZqI> z22Z!=V8OMa`=g?RJI~wmm5k8BwvuY~i3$r|cdHUP8<`ZqXKZBcVuBOHk0tD4-jZYgd`y;cSpNL_W(x9+|OOO#=nIR8jcQ z($RFQf+A$t#1|Z?FPjd@16hy)1o>PnYmHx2XkcIXJ_W)agV^~Ux?s6e=2yIsv0_l> zqH(Sa1h7$4RTLl8>Sg6K6dU`=Q#dwB-8Ly1q|cb^QpgNy;@-3vk$cJ%M5gq_+}N`7 zsT45C0HM|O8pQ`Yx@r~6Ao4&lai?@c&$vrr2tz=H0Z5;6wXui^3|Xf z3x$jU-^)zb#&U>nRia!*6V->O_zf_F#2F8gUtrk3zE-{azw)#!aLC~2F;wn*((qCQFK#h@gS-AwLnC~YtV zB(-#){P=Q?n}~{8p?M_jgk%2xOX_(^P73wLZY&^G3C0^Q3UJp3L>oY*xMp+jQpUuo z<&V<3qV{fTnNyJAfC(dD%8CrD>dud(d4hHJ9h+HjUtdV4%Okm4@7mw#dv%Xv~%vwyVP8^o9}k^ z(Btp!^mRICV05&fE$2gLNo>+Yw=lPzFIe!a42y^gH5j6!qlhj(mB0#@)&BT-)?I8A zXQ%ns%&O@7930u2wPyC^+^~}R!j5Z{gxGS-S5ZG^9yaNbq;{UF)G=*65piYZlznKjr60bw0;Cwf0VMUu`rSOy9Xw z1)^7vF=8re8Ji#Kd(U9HK`T(}+FQ!6cvST>W#74M|MHY!Z|AtGU19ASgTnGiq!RB( zoMB^<1OFZd*qd!u1>kqDU8?&QZ<=D4aZ61O^nK$ejWfqFC?TR03@N(#3^|X8{S(X2 z>+8qnPdeQ5HxdwHuS)_uEd@wUDHuC@Oo5pHed zGi~%H0Z64bIP$WlcutO*c7YEMqxXpW30;8O9H`5NWXJic_;NYZ3NNP`KDO88{eUr$ zy?C1<(1LEjjd+#BIrlGUx&6ZP&(mo)a&6t^9Rw2p5%dI~(R4Qt{Iq}EGAok?ZH;}p zyRmj-$o(ekz7`9*|JW6}b>`|h#C>=BlEz>2D#Gfi^?YW3ES*57zuQ>I09lm-+!{TLx)w8BDyzql&Mmm^J#ZmA6<%BSolA-mbV*DEw_N9S zNhCakGtyCp1n*6zss~MUlQ7}b0IU!=l)iSjH^O){+{sPD6EIvIXm`7wA1=II*ztR7 z3(~C=Alu_?3&$RMrT_+Sl<&`N{Z4&kIMo~tYzilCEvoA*Y_Xmdqow&1aQ?DTjs`2= zM6D-D7dEBPJH1H8CO#4E(b*W`?lbr%kh%((xI$Myb~aAi*cv|>ve9gxH~rjCa30td zLh!IR9%O6AMfRC}EjPT1th!7eV{Jx;YklgMk&hZRLO1nMF??$~dUgtB|MI2@xyh$t z6n-HLx@(`WmM54$M1b}*9@X?UZab>M@93RKJre~dL<_ZfPc$ouOa5r=u`eoG%tDU<}l)El)m_s4I zNF;Q5W`B(EqRg&K&Vh1L^Ts7BbLIhL50V!(9`SR0(d0~MMcfirH;QhcnYXtc@G3EG zb|!Bz_|x&os)va^diTDq$MNVDaZg7oyJoXtb8LW^I*GP~g|fwZik6>Gi(=MmR>rRe zR^Myt^bM)sSLMsMY5qaR^$|Y$zLwxW1%^HP+>>xyt_Ma_f+~4s;ul(|z;laMH_KR7 zY>^i8L5e(+&=!pxCh{t+P6#B#NsvUSM3->BUfAN?A6z3Vb%~Vb@d^SZqK|cU2K5xd zM42S!2RvIl|=B}P@jFTQHe>3SToOE?nlj}E%g7+*dt0Q0&un|Y1%v)F1y zeiXASz5xKl7#Qgab$LWv5(uf>xvIWZHDTiSB<)UH?1Y^q2ic35ffdqx}F;tOZp=P)gaT z3Mb)t3Eo;=&<=pjlIsEl<10)_NKBE2W<1PJcf|i&0Wb=%NxUebWZ+5plHqa@GyoZM zCAHkxe7mGqOpe-uX4xtv1mloO?(3q74X+`%238_xWH1-HKE;3h>H%Rf!^E$%;*iz% z8}DREP>oni>E}Ea*=%DyBqHw-9Mavju(39L+_#cNs~|D~++x&-53NnlQrk?@e<_Fh zB8BaS6WBIMhk@Opjv1*H&iAd5%}lM3Zv^l0ENE94(yMs#HPaNMjut4`Fn zG+MHlA;;ZA?spHNTnZAUsizDt&_k6SYImscthz}t?Hwa$E>xSW0t8g=Cu3<~{6_;i zcCS{gu`8jl^!SqET+}_ZNLBCznoTR46@=<`<~J+e4;9)-5~JOYtD}8?T{~P;K3;bg zrv4H7u2r%T6lU}WXt6&MBR*svYfI2D`DFnTU5bp$2*de75*uSeEaZEZR;ky?`heaO z(Y$Rq0WCPUQcw|QdHkRs{gJs z0#cciDPrTHqfxR_g7v=2V3Nkl2Mf=DL(xF*VYjnN*UomqwRraKlNWmTx*p>BX%CK9 zE*3zvYg4I} zMG()W#K_QQX*Ki;yQ;(ySQI(!s?-|sz5c$!p85!l_(VBHm0YAK_*k;SdZP;hOwzG%ML=Q&uHu(LsL=7 zAwclSPDHaZts-DN@iaxK?J{P|k^$s(nrNHf^(eSf4v`s2MCWV)|K! zvH=eL2`fiAV1`S%E;oxY9{ap7ru1Ufr0DWrMf7S}dACa)MqV^b zuk!R#vt4!vQIX5;@ULJ0-{dJMa*wo|-6~8h7~eo~9Ia|P$_6+5 z-L88w0GO1GHm2s=ac50OHWj_@Eb5+Ref-*q zLFf2^2i#~*1k$>EDUeO~%R28Yao4rZoT=|^lN#Lf9beO-bwyWZAlAOVS=CV&7DZUc z*LOtGyJv7;#`yNLUUy$Qo>*H_bz$a_fg+3U-bWz@;t|7CuIrU!9s4Mm?X3BH?alGd zTBzT>bupGuXQ95SJ-3?>gtyj@V=;aku-gTB>g32tNy3JGQa=0n^nxd86^H zzMPS`o#*!N18#vte@oBkIZiSb)4DVxW7>IB@3)3=W@C;t1!oZ>#NA? zXdnWB(o=6~{KByh`xwW^iqpEj%k|iYqgE(y67)jEWjP(P`TH2HCAK$QUpt}#6C9na zn3~vikR%+*@8yO#W}Kb--0fvly?u}?((y_vAZebmW` zupWaQ8iWw+-eHJtjW5V-GyDeac%$)pP0NCczE2i`Q){Lrx)_8+?Ir znOIJrM`Kv*{34$UM@O=hBxfwA7`&r?#g?wq-?FFpJHE&@{%BLAf4OJc0i%`~T0w!N zZHkg_DjDa;CD=X`#FpxN$KxcHF8M#xoFDI;R2NK9;)hl^M9ZG+WJM=#+Hz@M%Qo@I z@?R0ieLePgwBvVxa?@h-{~_zmA2KhC#vj0 zatEXxJ-Yy;c+Z~Xe+}L86Ujbe4a!717cxCobi**ofdQ!N!2Ze~Q=e1^4UwU;lsajm zB{L?)?L5uge+kJ?IvRSd$+BJ}oVkBl-NvI5pvfiSThP3yz`G2uiM`mE(l1hGmfB7B z8KTY=7p0Mub$9)x%sAItw=2?|j>RZ@d{pCkAAlF%5NkYxC~*jTz|bkU*Q`cOFrSBc z<1in9!{^#?HEMV!aZL;7eZuA<(upOcZwYT zC?s$w?kvr{)zOT3(bNv~KbeAXwG8(*fb!H_YZK>(mze~Uu)%A^>e#_*yr%)^hchhR zx9E190(rKF4*rb?Q=bD0!{!983yuzhsRa9Uqh`f?8fjIM^?DWeQDAIcyJiZ0UzO=? zHwDJy_$s0(uzaBY56PbU0F9zRu)T#N{W;Aj(q0Z8nhZjnX8>%9M~2p2CXmUubg(Ss zZpzB~=iZ!Iu5L!BTgtE6=ZdyrwPGo^rvUDfF#T`=2?1~q??NGvyr)J;NV1$OS}A{0 z+UQ(>rV%g{4%!Rh6Qs|)GK@O4j-b=gZF!F89YlM$n(95}z31?{FI;mq%v^jF7hJ?_ zM{~Df#UUDjejQz{yGkx7*Q?BD?~N2Zc;Evhnt7A|Ay0?St*1wjn7)7S%Zl5PI>Dof zEE&Qv{~BrnFwu95`W0c|(VKPRv8PtLh1e8UC1X0Mrw?Q`-xIJ|Da^sqEQg`D*n_FahKFzBU`nHoFlabebOuoPG*MmyXLW<2loFB>N zW8a!!qsaVfNy2sqzHqh7y18ldVo5sksdcabE_r@wr=ocRl`ne?ZSNML(eH0UW7Zqz zC4espG0QXj)Zsz>YpM^fsIrfH1r-MscRLeA#@Ny8X`pEE+ZO32*-3ZL~w>`%&orVVF8RE=9|FEAKo}Ka9_w z2fW7POTSoyXNbuubzJ%2tP~GG(QiM0hAga8p_Nh#pP^O`&5=9g!Pst7LouBur|2mH zc3W0^Eq@NeXX^W=gDk%RLPmsFP7SeW0F6^pn4N1l!lto0*u2W@(7~7w&xGy2GmMX* zCmFKYvs|ICrlQ+uhhOR=$hch_4wLb#onq_e7HLsnlLy1qM7{&GIeY>YFvX{2+$rBY zo5@H;s4P1}(HfJtu|zs~QKPsvL5vH5*Ad~&jC;saTUxpbi;9I(RJ+DfwJ8)PGFdN^ z5D<<)YJ|GaK=xYfW4_Ww%|-C+SyLIQT!(SnBM;d;6Z9jX5L7A@L5B+0ML{1RJ99=W zuKolrE@h#m53n63@4?I$9a@55+F3k4_%?YwLs3g0Z5*tb0U%yLT9TY9o(qK?5=n&J ztel&NB>jl{70hQU(FZ3lK|%+rMpDHRbej*10{(gzL6GUlHL%)d73pP951>gmub5HW!^MQNXvFj*T0}2FA1;tPu7-a7Zc@NK*WK2;W4&3+*uF zi3!DK!bge4r~WrxE_{nH-5EmS5fp1LIJCIr5vDNE3XQ1;8`yU~xJJ|P#05$au?AyI ze7InR>s{1KK#~r`Ew1Mq@S0ft{C<)C1vgQ@?u(sKVIQP1QVRK5>;FMaPmTf@N3hU1 zVF6N2OhrN`bG8r$l!M#`R}LYb7o&rYI-71YoqEyIwJ^j>?Ncd5$E8Yyd=N7`#(}pj zOU0*ppxrQ1X`{_b?CCA+(3qFhmj_}}dMsa{5fIicrWktTV2lj=m@naHoFkjM-t05< zOP@SSTruW@rWB;>bhUI7$iG|VQqN)W=7JQn( z_tfAUFx{p?c`blThG+;WIWOy~3MdGkj+dFJH z+2XVV+NPWx*eXvARntf72FNmsCC3XOj)Cc*2G3BsO_n2g&sy(~Pw%_&i%uoyhD{1pxbxc36gx z?Z4PnOQHz;AOt1PRIdTUtox(~(L6(j^nW(JX^@7GBae@4E=w<48qG43U{HZ%V?fGW z?80CMktepHeO-_w6KH0z^8e-Jy3NX^Z9Ge{45 z8X?Tm0fpI^!FY(np-|(%(*Qkz$ij|DT2x)p!v=K=@_`5(P!Ht7fG^|lwg1Hs;K%qO z4ZH`KKx^iIu}PZi51Gi!T+uY38$u{WBv0WHb+K|mNNgW$P9Y_+vgi6Sb>|r7LuF@xG*RnpjpR`mr`O|slU%ZKOQ(WNja8K~)``4b zP^hV+xk&^(($Wr=|A^X4{ML89;q%W1{uC67%;f3-{hqr%MmJpopLhW4I_YYc?X7O} zT*63}w{nXr7<||3ki09+qcxtr>I%CCncVI;H(XL$5g?-q8J1CdD8Fn9A$+U%DA!@Wz85*qH7%{5$A3cHxwwH+skQ(>rSQ*28M_-Lk&GC$V{T0Qa8$EKfIF@_{1Gda8b-WX%-p} z>d#TQWBZ~{n|~W(r>m1P1ZBGi6}Z0icS?FJk}_{zd8;h-Uf+khP0@E_(D-&lw?8cc zJ6Y83`tQ*n%U^Q6w-|5XAMDMqqI2VJB`+?>&D(hZO+c=ki4BKy%ShfMuQz}!deo+^ z5f$}*3zCyb0-QKm;8Pd($`Oa^LF0?xj9%t##^Kpj;V;s7*Qds z_I_!ckzQ6u;7HT?MlK2zwb+h?ey^95Dh^;$kq zDKopmD#v7-#i?-2Yy!Zp^{6%;(Oqojo8^sY-IUp`nZ`bF+qmXIIed6aphQQy_jc%> z)=iTsA&qCwlzpfZ=R=bJ)ar47{TA{tT+iS7$&ib81cVvEaKK&ET9aQp6k%;Tvm!G! z$R|V$FQe;dXB>A|H+ZoIU7eNu$+wfz%!LCd7oAeK-K>k)K3tA(p(C;~Y8M%D@s~b+ zQyIC%s}AjVFlTxOheRJ{OSNa4iW-9X7xJ>$1XYJYbbi^K-i!uCx~x4NujwO^LQRsk z6xhJst=qjAfdPeL2wK;_18K9!5cN;Ms_zq{wVpD@)u7!8@{&t-4}g&g28INpY_fxd zxp1XPboHs#PkZ>|#j>z0sQQh%OEES1qwQzq*u8xFS~R7j4yQ1W9ejHV9$*0FwVKxq z_u+=#S-Qkw&^*__@F6z{qDQH#j=LK$yZy_uE+pWFMk*TWhaOrtdISQK_w*DQ?1Bf> z&m-hUz@7_zF|?;gQV#YOuGnZTF()A%O=ZI4BYH2L_{qN==kCDj3@&+)lo&I+J!Q(z z5uw{J)oo&^mru_9l80^i*^MGyf@tDW0yjNTnsfJVQ;F#N1i*E*Us#i!G`h2Vp2}?P zR$*GF68dGe{ugBKhnxA?v;WC^DLig%fl0F6ZOrP#2YXpGb(RL zb$7uVE}W@R!BGk`EX(^%TsLR*4B~%12vTzNMv-myOIULG6b@xe^;>e$t>xs%8IuC} z4NyMFpc>yJDxhgg*kiaQ`|t8Gr}J0%7I)L{V|xB&jC#%RqpF2z`o}uGR$vTZs62O} zMqH1q%XfHlQ>7}b$yJ*rA1QeyO6yrDd`P5$+ULm_eM8c1boxuIi$YCpdkkxxi6F-?$5h#Ct~h%IX2`oW_V<)nG%$@7Yh z&nLHJij51OT0sGhwSzH4gaogd&R531#iPD8RPkURcDQ$w9{Psr)^cM;iI=!*E4Dl@ ztmV|6Vr9PzRNGuV&flBAvTO-@qz9_e01TXLH8k8R_-gOL~ zD4TODe$?T{nXR}SC1$ZG0UW&-Cu@+~BJJ^k? zcVNy$Odrb63G${{^kH?vhvt-sfHM$2h{uh=@b(&7j?T1MXb(E;3^g?BBTA3e!VW+e zQ0PIZ%k}*kRiwAbMX&9iNmOZZ*tVOl?qs+3oF6nV{3hRA!&mW_AWE5xcqvq}x1xTt9k|+h7iuxk-t#H$ z!fDYS!QT{x|2QLH>@(5499<-5xelVkFGML*{4r^(6AMj!poWq8iSi%qjQp4G$|3+Y zV^lu^ZR)Z$l|2`pW?Xt8#6_?JzuknEDs#U?k0MD{yZ3?4r&iX@2qwpjzR&^%hj=-C z_W1kVTPS!$w&T)4)EBU3Ti8DFxE+zYna_lavl#F)xPr(t=VYdv$^gUxbqPe~V*N#) z;eUaKXGmaj|E_@CWs>=1!E}qdi;)2ggKF5x$k&3}3RNQ&N=5xj;smUOCM=$$ghqM~ zSr;Kj#zsXD0v6IP%P>wQkT!z|U?PEdE4&KGV_Myf*ot8g661+`0MHQvyMZXcP+*5;{!#kDQ8LeYa>24HiK@@U)$N~mp? zbKksbY)*qZ)$X z9>_|X2_I9uPT&?nYzgFWHR#~m&yX1@dZ^zKLR+Q-V1zV18j8p<5X?dW@ofmf7bwJU z;RrE1xgJP0mc2w51@%6Xu}q~&kOP!gcO9(jW9I!y34J;?RqO~9dJ}>Kb8TKdtBsE<>zDyO{Hn1#bvn^6= zJ>Re?2_?S~pZ0EST8*>5=42+QoMlmhKo^U{;uh~TKK9NOO}{S4 zZ%Z1#!7~DUac_pn7O8k4zSmhXd7Kf>l#J{eF5-e{{`zYgb2HzihK#;qn*@= zFf@gs)hq5d@&V>hEK%wX6@~eBBRBFgd;pfQPg2g?3(jAlNNynUOSlR$4CboO%Cu?S zfYFThwhN?MFH{$hY#EOlxzXhk)_EEH2lGA!O3j=fc@&L;Mf)~IaCLxF$k!X~`B`R5 zLo-yeBGk!#&8{n6NGDs2x`Y(`A&eGNq+34%#>9Y&WMjs_Z~Kxo1ZORk7Q`iV zD^c5mN^{U}m?5avV~7=a9NQUEndtWeLvczh!h3b;5tQxtcwr$Z@bFxCn6(rirP5HCjrBeFq;iZKPCB&2&I z#!Uj&LO+qAlWC#@o@H^V-Wf&75ctd_B7$T$o~Y-gYGfS2it;sB&Xk*`a{*z)L=4EB z7!8D6@W=-FqCkx^%i=z}Ml5Lp@N9d-Cd!1>WHeZhb|=^nac-LJEUirw696 zlpkBFKoBrwjd7-aUKWNzpaBFI863nJpNFO%Vr+I{aJ{pb_c0hQKyhkwN25azSD!QiNWUQ|N9w3ZJMW!9z_8CMVp#SGT~c zNwI%aX>fy2vbPGqZlC)58l*0L9yO(W`kl^IwW6B8{Yl?lu6ZhGzcIQcXIbgGMs6Ij z-=A~-_T(M}m5Q|ens$x8XRMQEI>c8G9~HA+^rFHw$ca!J$6Dvio2B}Q z9MZ*m`d>o2*DTKoUo!VkPu9aYCRPjhvo>J|=!LJm2`Jo0?QdL_6xNVj>#K3+b&Fh@ zOGnkZd&!j{t&-C)6Li{LkA6Nik-HBp6?bYy{-cs6!1$bfgLm@v>F8J`Idkvgk^+@^ z6P$+boykH{9_eu}6g;ninEIAC+=Wi>J@(e9=+xecOiARo(QpLwHpOu_IAu0={^lv5 z>iySHW|>!V$7pX@*BTDIxqPpd>q?hn&nVh9YjAlF-oWGamlnbl*7$LhsQIatjS1nn zgR5h4eU6iuf1z|yRnUZ!anEvR?JE3J6@zWSM(y9Yvf|rLsa}uG`2Nsp*%|v*C0^9^ z*_n4nr-sxz=e)sUB!G*|GsDb2PsY|KNkt81YsMCRi%`y`@l51_2!B5NC^!=S)YV6A zP0b4ndJff9k-EJqc6PMnN zn^RU-@hD0~I*seG?g_5HGA+TvceeA#R#bnzJtb1dbt6&%EBHhKeM6{Ybng?G7U7xld-^4%C@+)!pLwpnREYL|~i@t1rZep@@SL7b;Xs(W%;w`~|7-_1!eN%~8cz}?z) zMCYxq1Q+n9wTLnIf3aA-Yy1svD0bwjRl>CqcSY{zeEp`e?^JwPrOI61La=a0*Nr^T zP|j!8t($M%xI;uq@uuc5aXr^}zg)?`tms-JGo7wj?BB-Kw=XrViYrA3=%v=b3^+Q9 z>Oi|dF0}Rt+Q*jWWKehVO;d&sR*4#YZiaevc&iT}_typ-07ot?{Xk&NV zZN&5gFD}AT+YrQW%P{;P@p~zfcNptyTbVr+l|8SwwX2Wc$T#^x&t5aKg6jrAe+2}9 zQrdC>y~0|Z@22eU?FQVh@)eC@7U#)YbHrT@gIs9Z;BU>BK7+*Mi1(0UKUz)|-N6mH zr{EzRj;irJcGP@%L|mijJQ6C?O{JMH zmh3;vWd?iP#j;u#9E*K7r{TnC?iNh-cWWaQqbtj2!JX?k=4sE^-Tw!^XOBEAP46`d z8BV1yV>`u%FIFyGth?uc=WL@M;+|-IXLTw}v3N}spp9IjIS=clS%E#^8^mIxnUz7~ zJ;Ay?t6$GQ3o6HH(T1FS?z>Fm2gYb!Pl3W*;q4LqSiBgMNU2SH09!m!d_|S+g~zd% z;#b>!6*<2#S;d9d zG|Q_&*f`IJFn@n*mhUmqSU{g4x-C1$=~ij&q^dwtYP9Kv$XpU>{=Iol@0VGyM}Lo* zGVVvwoDES1-<2ewap&#sp>^VrG-EJ-dF8St|H9+`&N`U5w6|dYunq)53Y_Pf(L(Kx z{966DQ)rxE#GTIDuu2By#h?fh582o>e=hJGMylI1j5cG3(FUnLbnYq@X`g@#9ObA+ z?`U`w(4xWBYrNH=cQOn`pRL}s7}Fy2Yk#c6>$41*LkDzM=FxVPEXr>@m41?_p_OZD ze2;v7Z@eDe35L*?8Sa4GP1}sFpJHKLCM_q&an=;!i=85`Rp@BC+?=7*xycHcnmQ^F z4aLG=BQQYvWhiDdwYVIYzPYbqQIY->MI9Feh%L>+LhR7|_XH`g43DI<{|yROtnUfy*L+8;H9$_+c?`wYah*s`_skn_zYZ6dVuqb>_E_RvrDTBMNVUkHZ7Wb~_A7KEb zZQyRxj9#Z?=JlS;)_slymH`O<`D8p8F?sUo)*eGq zbxIuXC8Z5sc||hLLgyE`N(uEqVzc;2x4qkl8kk8>xTDx&+y(XTPRU5Zc&5#5hD0z)?4t zWa>bv9*kQcQbJ~mtECJZu>k~T1wjI{eZ_F278E}Z0lt3}gz%Qco9Gmdh?ijVY=M!1 zsM{?C&obGu&jrj3$V7M~qL~8wn59RMpj3sOxobO3;~>pLq^jBXWLH+qih1X-kd<^er7&hr&5Ta+8_IHE8uV-{b z(ryN{sq|f3uce@xA*C!2j5#|MATo>WDSgVC`~Y3!?*!k_vWP!t`QcST3}Osv#!d*> z+Q8JR2+kRbr_frUMJ%q@dHBs6RWPcrB`>u~mXr&Gmy%G2OoWfPyNVCGNf;HN5#MD^ zcj5DZ|6^o@d_9i+cwl%rj^b~9g(4ESq5uwpa>KesHoYlu=Ns6>G zzQpQpFfdu^S|DaMR9szV&4E)rFr(Vw>#A&ndAD$AVq6HLP!#}4OQ@jLnQkflImt!V zW)}noJcNDrGaZ%hQgG-HTR1w`WI+Cp=5wuL<9ng z$`ssQBU^BHD22u+_lx^lrBGmi6vHWrje`4tRT*34^Cjo`%QvmA4XetviHt78TZk|* z=mXMkXxi}JjDj)%tLGaM896FICRd>wz0E`bUTNu_Ue1oV=PHKKu@*337y(Bj{4sil zgXjrJ5cyYFRv?p@#t*B%|NdFH4O~0!`k0Onq9n;1wl&Z?2m21tIO^O9qnATT1L7b_nW z6@2yy5HI(XfL)?9VPMKByZ@f~bEDQ1s|95VG^&9l=Aa7#cSoMsjQA_q4Ydj+S70)& zNzp*0G81ajj)vCi{*SR(KpkT_IhI5Ifjz>~ysr->Ph1OR?t+@1-gp6E)Udb3c&~@G zH`N$Wu@Qswv>HKKQuPvvwhU;7ZXtl2kQPRZaP||NY1=ZkQGmH%pF|Ho%U@VwJSGwo zK;W!2s{bW|m`%U12=z%F!N`XHt|Y+D&UE(RCQxF7p%dy3ur;vXEW`FjQZ_6NxEV09 z7pT_|h#=rVF33p0Ff-ET+03B^d_+vJJR%)Y5Q7PF41oX-1rP?V#t4C&Jz{^cITGedg%GHe!Un>Gy@X z590-5X29aX48#-z`k1Rj7soKzO7G&AkJ_J#dnGJ-!8#AM#h45 zu?YT%x=ALhd~e+Q&zKahV=e^Pp&acGHRikpUWMg|&JYGTc>_-fJ6><~`!AOb&{ujeCIiKvTM5%>y)7>X}Sw zsGLkb3wLakj->cjA2-yJ;kzh4cfX=O>cKiJ@*;7bf0L-o`LvGY=I6ZME6fdTsJNST zz-}rPy zB7%D5f{VCST(ckg1HBjprV%c(UrXi|1j;rVwDm5}xh{Y3X|#*2e%Gw;F97Yf zc{i=zQ?U`o$JlKKI!#-;VX`JUb9ccSrCzO+HS0jugTT$t^rtzxPf z#~e6;hs?Lg@*8BP6LF4vg7h!zE_M%VoeL5&69!wUjLUU>s%F#7CY{-wA?~T_S_8sM zqS1kAcKD;wk7oac;&krulA~hiWCz7&n+s>$8>E}ZY_&fc{leG5i@fFfh*ToLZw3g1{}Ncmug?8 zl9b;BeHOR^lj8FM`BNYHLsi`}}6{3zdamM6F(F=EI*Bjpt7iq+!WgE9*C?b&bj;j;phnS9V z-;Fj7gJl}W9RHQ3`)$;+Cn3~{LUI)u2J|MH7m1~K>f5pSCkT+^WnDKw$(%^4>Nxhk z{%e{mpSHFu%&><2G)k&X$dc-(vJjFTOkIw**5A)7?Jo!FyHUW4^`CwpdFNxEAt;Q_ z+2f5ag8{QsQYL$v(6Ly5dRz&+Ydpa}8&_Drz34eyV}ENUoX>REv7BPdGLR$|Cd?w) z#2!pq#2a93$*KM z$mgAg#QU>FIGIX-JF|VjKh+_#o+;i6$qb|%`3Oy~)enh2PSwa+8AAqvqiOhrUx+SB z3f^jbp1P$4`*(iuV_89wyj&cHojh2cS5gFyA`Sh8qk>VX0y+m}AqjTVc_r5dstx09 zLu);E$FKa+AXg2vipzcKmP4A7FzB?f85i51A8@9@gW0)XM`teMQ>&$pvex&G^}cR1r` z-p;V*gJa<8OL-wt{6$w)gj=)3^RSClze{At6^Q?xapXMm>2~t4vF`3fnZYjk`+eJb zJ1`aGCo;IJ;9_feWwdUvb>&^-x;ZhY&yVGGt%D}wm}lgHfVa<&<#xS}du%ga95e1f zH@5JbqdGbE3+Edp^Y*^&cXAUpjP$d=1io%sk+TBsf-gRZq&{WaD+-ITSY0|NWVbrV z9nYL{$YEWB$D=Tp$|+NEP;I8UWW4Fp;KlBVOI0Jj1FL1f;i?jQtYwq$hjd7a;J`0V zz}$2i7+T4HdxWbG7kQDx$o^#y)dM$ItdUNek&avy;WKqOx^)w5KFv2WTf;@cy;It? z6SMz`Vz`|_w(wqz>At*zQoAXMz0l0NYk(fOSH;$Jf>iCtjiEplB%KJCb ztx`Pk1p5HyUoeb`{QC@F%gu8Byx{eKTS7tY=8BBOxusPj(|lHJRaIM6?)1W(eW+}t zEaA$v4<&j2{pK?eGOc;-_Nw6#SA;UJtF#Nj2>=NDl$Alb_h-x@^dw8h;k4TW`!=0J z^71*DSGS@_V4n}@ms~{kndaN~R4Can4mD0bEZkVd_zqa4eVo{5It4le64ZGtR^Z2UAR-Ux8JaV|Km z71?a+2jukGlZt!@0X&ESSdS#`!AwewdSH^EBg+H9Y8#=pCqpKXhb@KYe11r|b`sFr zi&{*U{XcrAQ~7284_>PL8-<$f!E9lML&=AV9+;KljbQzkNML-RBckjrTNO{-g%$2QX zHs!e8+pJrl6bdECNeX^aQfKI*P+`zOvSt=&Q!exhXQCa zk-P1|L?{TRg^_)=$c90u_<7fFiKdUpQ6;drBftYCMVKsvkp$r~9rL)2GXw9IETkPC z`m3))goNf`WN^p49$@nAnzWk)5;i)3R$Re25|cv7B9>~bxpe3u0)AV)u838vJQg>? z84%FO`gf0P(Hab?X9Y`>&buz(%|?*W;Ld{}wTy0RQw;i_k*~Y8fioR>HhFAbd%R$5M~?0?zp>@uZgECxyz|H~77{X^b9;@{95l5o$_q;hJ@OPv z3AiVQW#9l}fRWvN#N%UlI(Gh?vY`CHiC~fvStX2DxZzK&}Mf+6;CJxbd;S%z`qZl&OoPjySb+8-uLq zs|WwR5qyLH!gp6KeDGQ;+M$jT8mA-oq`^n4a?SavvqPrtrmz6TSjmiz6 zcOtq4_2Ez@G)PU>w6FncNzY*^l>q<`MdNQZ@=vlDw51E?&k>j7i)?1qq!p+PXkePm ztFB{c$x?zj$Y;-sSea2lbx!r>Roa*F1mxjrUZN3_=@RsOD5rW3a961Kfiwfn&zno*NPt75hhpPlUZ-+|f1IQM1Fu=eOgG)bi$4*uoGsbN7$QKg+6(EEx8VWiU1G}IJAZe>U zJ`3z+lTZK#tUVr2n=>&OLcc^dfHRpg1`-YIIU2(AU}&%=nQDWSCk#!)*dVIf(_Kt}V55_bP3jghHHf8Lhd4eW#&9}D>N#Duf&br>z9D|OrVJ7Z660XK z6LB?cAqX;b7oa0yBm<)bh#FveEnLzwTI0W14P50j`2!XXS%ENV1)dg`8UAD&54o>1 zz5`+!=3;|?f_M>B6VU3=Yvvh0U}gCLdtS>xeM+$np#L3)7Q5no%QYpquyLe+(QBxd zN>%{2=yZ(lzyf%7;SyZl22ANzUIjE-DI3=Y zobnNz#1Ad%l<%-PCWRqG&{1W5r+k3?-EILSXh zVsNK_+K$cD1&@k96W8baiH7_>Yg*SJ4S7CA_M$!ybU;>sq|{AbT)SV~Ux%3P*Fa*P zouz(~c~wKLiRA^7-;T|FEdD+MGVp%&_zT#g`-bbvDn`CfR*#uuul5%qE$icBSiJU+ zyLX5P15xsB-|d=7Ck~a&s_Rh>B7F-m{7?O6AgPEb^bb;4AKQwXd-3YmVNIfm{#{dX zwxij7F3#VFhf|$NWc}P7FCAN720RfTth8UNp6K^mZ~<0>O@8(JxxwM;J4VH6-8!%j zx(Fj+I7d?TJ&+nW$1C!3#QnqLCofOTRh){$ZY3R=*n>#)@2*r;@qCiGpuGGbPB(@P z`Pe6j$#)b2xt`|V2nBMljr+aih8p^;&{=m%q3wv+$8|-SFCD-x$_0bD!3fw4X}Nz} z`zuVZieAbZR@Co3)OHae(XYElKl*Vco!d*5wU8SGmWqw}cwQqT*PQ!Ow@ z7+ZRZ>s`~886FGPAulIz4T%@}1HhcM6w`JWxWSe=Jy+9He=x zw>Pt0|4+y$l25J3&XkoJqru5XR6M{7Z!F{=YaWD{t;H=M9mc2e&T?L(Y{%PVFN~<0 z_hTZAc$FEqA3`uZ7_7T=QQ43y?$@16g|I?Sah~c`r8e5WDc_=HRW;-;nVUIy{C`p8|BBgi@tT;wnA-E>_>r31a01M2;~rP5!U&tRC*idhV)*Z6R8 ztO|r<)zp*Jd2#6LV?VSOEtz}_+hr!+MuTiV8vfmZ4FHZ^i(Y^(D-U9i4rrOWo%6Sm zyAnfEg9%jGwu z5?xtfv-uk81k{pbyXG+0#V|44N3Iy&2D=%P=cW0$d1n-}Hmi4Q1Ay#4WOW8d;l^?D z47zwD?MAKM$g(Gu`|ej9D4?0hx6d;iH+-wo$57L^s}#Wm^%o)~eQaF?-qWLJ)VnT8 z<=EOE2Z%G5h<3S)qtlEqT#y=$=A0bzZsf*{?#(0o1Khy;EHR2>LOOP-B^f0r(*~0- zOHQM<2av0G;(radPziw=ou~vq>BNZf5FI-awH^G%@L;0@f*k+cj7oJI{0DURiuo+j zuJt7CLIa2il=h6almR9p5{wx^T|^};{1konN?fy?vtQO-@YZpmmD1Mmx6tsV;;!Gq z9mglDI+CD#I+6he1+wDYbU;kdUWVJ2za4pMZtH$EdV;)Z`I1 z+F?W{IY8{SaqrvFf8VIsiZiA!x)iZz2nArcl;WK3ZF(@WVzdbBEM2H!qNp?}$+vuR z?t=b9xTQ==673#|4fd~!lo|Iet~jqlR}6S_2E}?wpTV!D_uR$0e*xtF3DS4@q(sCY zJGGahECv*uGZ&%9;$;a?IgIq|SNjGAh0IZE*EEy+hs6>m((EJ7{|S7Tsvq8%Pk3?o zTO-h#iT0y?8d?l&7kQy{EVy02|K1bXEiW93DIH)SqIc!td!Y0$zE*!XrY8lu#e)0q zd}63oC8#EXrVkfom|&o0)Y?771mo%{qWrs9dqFW~4tNLs3zs^R2||Q>^N_PGdWqoO z{Y!hZ*tZt{|HBy*&wo*k_6Yp>k2`~=Tt*w18CQ}U_@o)0QowG)B z!$tY9bxBLkAM25#6ARvo+gw{DNMwSF+Z46n%EB6uYOtYb#Y{&TM-A5fE_Y|@Gp^E3 zhQc>fSA|zzRrw*B(tWFG@TdhMJY~_4M2O4_|5Fs7+&V8J+m5G@gJSk9U#=;U`Y4~g zO!;H~P^?h9-N*%Z&n&k%g;72PKSEh4Qn0{5lzEGvcTi!D!y~Sw44+*pwWUT_EySqA zb~xvwv8&uI#dpV_m#3_@0_erFGv2{#Vm<29t)!Z2^A3JjK#qM!mFrCVZf zU-sKKtI><;9Tybk3jHMtcH>fP;hXnx!tyC#e9BWHY{43Yp(>oO>X01=_!i<_%0N-1 zIwVna_nruWE~6Et>#(B+8}lwYS+_kPS1?O3!`!)9X1Hc{Typfwa^Z?8rehtlccNF) z6F`MP5bavp zqTm){l_FNL0SIe_ZHrhDVi<5mOw}!9*x3@MAaM<))8m)@8W_9C5Mi3$Ngm}%Y>Le( zazZb7B&Aaa{OpD_j9>{({kPK?+I3A6vTf`Fk;JLAOBsYgJLC*A?KJ6}*XEL%Ub z8gY3kurr0N6d?=D&;&3LK}5#?&>sS@MSSg+0;a;kkg$e-!Z6e50ohTD1E>XPJwUHO zMGlnNBr6qvP}P6JKp~rs=tt=A2jqo1p@o!T#ZW=m!s23xv}Xx*O>pQRM!5Nin#vK- zTZ0bb#ALz36s1!|WLA%h!C7FHfAlYa|Frs8Mv`y<&1#m&x2ik~|6NF50>*$Rtq{j6 z)!hOoMrhCKBq|bWzP7L?c)`_92=z(U1c)e%ud!nKO#%b_C9?HEDM4JjTYJycb*uU8 zd~Et6hyDx9HEO+zytwUzBWi(w$t^uw-*F+3g&gA4bvsrAVY~6Zc==agwno}hS@+BnAJ*DOs0G{&muBrGyYMVu# z>BQ%GL`ktWuhw9t8#C>>{a}!Qre^3)L9GOVM7YLwBQbg;L9ld^ggX^G6U)WFVU6Ap zXOMVD*%+vfPi$~WwIoDfiDM`Z*q2B!#LTWK-%OeZA_faEfiT#RE#M%<_&v#`@J&|x zgpS-e1VE8z^^Btyk$(<-8`KxTBL0S32mjm4aJb+}@2bJzY-7K;B{_Y3IVO{Lo&^=N zXNP%#92K@DD2Uc#W1yyVNPMFPVLA4dgi>^Yz~`C6!_0Ep`R9wN16?2O1E9VzYm!?v zK)l=0)(GS&c^Jap@DrwOLYigp-hUhknMQ+fD9sMCg&_ouv+W^nlu#*N4|KJb2}vJB zyTjTCJq;)~vJkm!rq9^TK0WU^IMcYh1`mN>BY@AgNvB%jrZC)KS|$Y1mB7r9HiBS2 zu_HqLjDYP0LkCP=jHW;hu#7)_A5IVKW(e$4zd3r%criZ+qC4>~)*KR+GU!M0HzJnc zk=O@rq7M7Zvo;0jVPHA5Kx{^y0&R#HWWAX!B3L1UXr)`1Q#(D>pID4ZK!h<|j|vnQ zdWNlYiU8UjE3MyarVBfP`l5vUC{zJ4!HN^9)`xo5;iUlHV}#!%wjnkG{3310U?h(S z8%`?5iDXr9%VeH^!ca5e0zOtSf^|3H;YnQAkl3*G&1#Ak5~y4225ced!&8cD4T6Cc z(Qb@!^e&_4sKlB|P~3w0CHE*(aN)&kX{J#3WGE^S`2kp^HHvGoEBSLM^iXtqSH7tm zxkdJa@KXp2Ao_Llg+Hbjk1HyrEwhvki;QlQNlAqT=}0hCsZC`~(sq+m4c~BJcU=9x z1K@|mQHI~@s^+%&2R1nasLQ^~jhBrR^Y>@Gqf5}RVB(FvBgNAw+gPWleQIU=yjbF= zZ55jol61aeP;2{X5I!4Uwl4etJo7Gj0N(f4m5C~^EJp0Nf1pe0D>tq)OTZYkO!(}r zvX0*vSU0ZpE>W3&SJ)()>_4j8gIbHMX?)tNw~bdj-5Lw|PGK{+c$ef7>QDO7h+tG5 zkcZA(OL4jv>%2ae%7I<@S8lLR6D;NI5X+?@4-nF>7gq-Lj>{(_tfBw^A$+Le-XV!k zb3^W-4Nj=$qa<2x>HVHz)7Bi|+N8t+Dpf#OS?5@Jm1RrMWwf>)H+&s9qHRh>k+rAR z^)ApIpBUk_CR}WTX9rcqiX1105%-r^Wx~GA5}z8#{*{J(laUbXx>h27@q#d;YtaU1 z;BKhBBq4Te?Byi>8M0x(iW(BR=v` zHAD&fa#o5bE+qOw%Ilv9w+YmL=Xe&^L{3Ft?1i7dcE2vZng0R1y{Iukfui3=ywZZ( zJE%#Z%$!KTYz^wPp+nZZn!jRoEX23Bntodw$pBuEsf-X1-_lWp@Ub-Gi4 zPFNF)Tl$qst;?w3foQ%+) zN9#e~6{liOk*mB|@z*Tl_T5U-^#8;86qnES4-ZQ-;TiG9Q>$MbbE;?*cOC8GCib~X z7lbbmw{WBJdS_m}axk?a%XFYyl`knoYLAGpuw4un!sGXThSI(9++A}51be~lEtjfg z{c8pnT(KSVl85VF2-bziXf|#djYAc>_|S$+3GuSbqt?3519ph&j#*34swc1+N-uW` z%^f_4MWY$~?_stv_J>h-gV@!f3e}VS0dDC-)}wuVux~GTu>CU4!-}kG*dY8eF1B3*Nzqzg zQLB>-&m%q;EuFTIK->IA9rvNC<)?P;h9=r2QEA@P;qCDiA?;Zc-1JPnLn}rY@0bs# zrJWFkmzYbBF{-$DY^htpE^pkj1X&(9|n#$TdCJ&1S zUtO-h#J?=Phg|O~dpN!I`*=m7yueP9EVCI1(p%$8EyhPE{e)6z@YUD!VxB5lAP$Nz zNrwI>+W5M@3wZaKQzVGvs3|wW<3dO8ZIO3~yP+zpbsEJq|7J5=f=P68t2a&|1hX{(hZ8p=%hIe9zz0!|5!A*fp^hw9%HbkcJgrK z?o~g1*lXDFt=LARo*46mSBe}PX?J6v*AHpO~4ryXnp75G(@rS?k8n zAWndOm(WG_+)|fZ0s#<&FGQso(9lFhgfi?7VSM`{dp`99Zx387zb{K&Cz)eYe zd|4DhV9mdCGEGEpFZ-mc!Uyp8S%&&mLSl`b%$DM@XU;OL(XXo#kZ7eRPB?9zPoGJ3 z%?~z{td@?6qhodamYsUm;M?5KNiyifRR_zxSr@^4royFG!*q7LA8$gR$TDP=?J3gi z6`KWv+fd}_j-7&QBHdaR+zHqJG$L2W!Y!Tri>ljEGV2q$og!st^s)B z3ZFB`*+J+Z?0t=VqMaDM;iBH5Y!=>k0Gae1N>tG&oe^={@BjA#tI% zJ;h)Ljj+X9@et;_a{SS(|iH?10ZA~<9Wh+xIjeY z1S{FSE1OF@n(Ln64Yf93h6otZTE3%#mr~jaIXxBtQp_crj$t8vKJ==j^QEi{Fpt@> zg9g>%3qVd>D6Ybq&JEG+$@6|^SW5@xbrfS{@3q*rA^4Otsj@I|!er?Yo&!o_s7?d0 z6vT^=DE4GI@d=n2Hirt&8=88I1Cxcb-qUO*z=97kBlQ1dCbt@ya}=Djpfh!~lliki zUy2okXT;9*Jb0jVRF`tTz`--FvYrbIDBid7W+(9h2HR57@3riC8yz}PQmpETKWZoD zoWS!@e`v-(NRCcHG_peXj@u$j2s>~@w{LtGPdP53jnL-VGLr;7*TFc|TP`1oCjv3Z z8$sVp&De?MLC%L!T0xBzS3UzW!72FS2pr8U8F$jthkkw^qm5b2|&8fh@9gty+zz6k#8 zkJ;6qh{m)Lmcx4vk&K_|y8#PmULRaCL-#tE4l))T0vFYqh=sC!{|u6PE1E~jGn8gA zS1$tqLT}Uen)~LGZp%e9V$DCzrAUWa|`V>4Xs0S-Ge^=8R1n8+Kh^_$}3{_Yy zID~PT|NsAwQ4Cb}Ljn~B7g>gurLz&O?)ZYS0#uFF;|6^^Q3Pz?v!T9@M7XkUj38eL zDVF1gW_U5Tk~ZQ#aXKi{g_9nISs9SH4JZy*`pe8}H18uyFy|A{21%eA@uwBhQS>&b zL4S854&X*X#{_}}kwAr8^KmrtD`8iG1#h9YaJYB`JHo1DT`5_n82NN&Bug~`mU05j zzJFH`mh@nD6wg5PGH`XnD)R#3>u3TqTLPrKSrVJl{Zy2NDye7A4W#cE=sKqFCnEz& zkfTe(Ou%FCrMP0cZahX0+8*#(Rx^N4TW$wG#?yJTMnJ!qtQbM;V<2fQ#A76{bK zb>qjwX$;sd&PWYTV_>Wejz(Qmkzyu$mTKrXNMl*k(jmuWB_iQltb4hyZgzZ5o8}_22{Qnu0C5PZDB80 zsOm_jfhGD-qzQUr-2BVsSZExT3qk&phq)TRKCdKo7EKm+p)JF;l6kMOri19tfQqsC z9#lH9?AnP_G{o|+IwWTYCM_kg{89-5%^)Q$WS-`0S-U#*-xZ(^Vq1<-M)}P6^SUx%Y|_R)=dv z{70M?bpvI&>wuGN(W9x|)~`MXwQMT$LWS?fU8A{=M?PDDJoRJAI>OuNIroJ7H4o=O z>nN#yyrLM01-^&9hEmrv&wrny`pKx6q4y?HYO>7q9BMvw5%MyugmQUtac)hbAOEZL z$FWGcISup;Bja~zsxgFrrKm})f7!jEMZ;x%x#|f&0<%k$S}b{hK;-VG!PA3ly~J4U zlhZ+|_S`M5a=VD%tZqef^=W^zdPq5jPbCNSSJ;ssc#(^1j?!H8_u*sC(W4@Ndx*%# z#@y83aEo`DuiZn&r>o*I)u8LhW7+o+(-M8;*M1PwBhLRIFRv?B z@i*>;l8)v!uej_{uF{KpGRx==r1Qw0`-LguGL!5-(&Xuvjx9t!^$Y^1{*fWrP*h8` zKgi}n;vgIIL{aRcMv3~k$ulJC-kG3d06c2Z^z#zD=&f)+r(;#79}MllPoWbPIn|st z|Cnt!E^aZE&W9duxcZgJTAxVX5Y6T*9TlVb=1ZmnNAD$!=DvsfR7~utspCFU#3!Ss zV6<-tTlM6$dqXp~FReRvN6TlF8~3e%Bl;m&blZ+%uLARLoY&h0$ELx-gdNS|t~EVP zZHmtMPp!T!=sE1*U_Neof5n09BN?q7kXoS!tD$tg{N>JZ&g7`D5lsxPuq6OJIqfBu zXT|LDkzDJl_#q!(vIU+I6{$mW(zcpEH`%7Onos}WhoNk`eh3Z`NKUqln$u5Cs%}ph z#pW24aBt8$ATwzkg19vZDA38Y9<}UHrXK(zO1cVb0Mn8@!2EOGsK}pkZ#WyFO~YZB zVMXr5peWzp+oN;jxY};xhtWFGY}5K;qHWNSc#qkt?$|o26Sj66&AEh@*4I-)0|rGJ z1%xXiul2YTX@Y4J_yV>5>#P8C>AfqC^@j#wE0~}A=TVvI#oW#ZFf}MpS$0h+o4jH8m#*;eT9j5? zZ4+ja#`EWxK0Gyg6(+p4m-QS71*zlmD}(A^LFNs7c}BU~5aKI@OS8}l(L39Qz1@rN zuYyRuU1spy>m^Y)yH;)XS7l6I^@RQUF!kRCV2g+OJ(C^njLznAwx(@p~9E z4yI-p5!#%*(I;jRDuJ7;)RzA8b2z;iYpgmP=`unwD}Q3iG=_&tfC7Q4Q=jAT3EKBO z`Fn9qIzuyyu_8}Mkywb_#n4D{>j1@4Ffl<7ORLDK!`xmWj1NyO&7=N@EMof41Ch)XhTcnZ1MT`_yOx(Jb|x|uhIZLGPEYN zA-4J%Cq6@UnB))y(pk1psL7)CSq^wvqH&)EJ_8G~(Ktib%`SJ2#PW?GY-N)k(AW+k z9g=OsPW=81DPDk{yX#ciB%#fUl`21v4RR<$Q$iQO|2pT`bd;|9oy|ZwbX6#@{V%|M z9If9GFA|!g>x20qgF|hTmIuPFdOrFzv#SNB&fFrjdhih1`=PN!UA3~2K}}SB3*uvW z7)(yl>gng<1{n$sXnjQ$)~Z7bfnT6PJUHlqeJ)v z_B*RwfW(1HyAT)!AG8YK|~P=*UIRiG0h8&}pg z#d$Q21istE|m1#+_yBh)onD|ETnBCPQxF zo^$;(KXIzXWI?fer4Ix8;a~4P7=ExEDq0i{yRG_qFGSe_%ojZVDT^x=>!IbrfjKw3 zABuq8aeAmzuc37$6=ehXIt#%31AG`dSYLaRf!5LzxdZ3AXHTb=F3efDMNjlaV_=v` zxW9R(fnXrnY8RH9u!c}bJ`2qo6ePM$$dV+R=biivNgXzg$h-{A!qdc0^y2Y@*oIHj zK&k#|k)>jUB@G-B8MJ_EVRxBo)6H+nreJb5cCB0?lTO$8A3Zt)>Fv~#3?Vd18wsz& z&5@EmXg5X&gBhLoOLn-pP{F?PMv)bpZ`5oKKxnKFgs&IEXLL*Sz|9uiFS5E;{TT)< zP3>`17^`JBpib|Z@4fI5OKHas)_sGl9_W^A!1=%5)OOAI6g#nTY>@l%>qg&#=$> z%^0Mbyaxp(nL`PB^@*6vtUy%f@8}h}NLS2GZpd|6mK93WyPe_2ZxvV9Pg`IVwn4jlXpZ5CbL1 zZD?kr-E%^z%5Z=xm|6Wj))-Qw)PbQqMbtla2xgmVRC*&H&!-#>zvNma6ZZA20v85_v;3@~8{yCq<~pYztlh5TpD;TEAMr$jabg;X|zCwGlt z7%mZHjgUzt-4H=QX$kVG5C*U%=GG&)*ug=P*e87B*?+VES(=>cF1T8E3?EB@dhv*OuRO-x=n7d85Ib#W2CollC&C%?E;vcUE+7W1b#oI;mK9b1L(&$L7#C}Dx z5)`@kKW4J)G(G?c2PEs~LK%QZvJo(lkR&!8Ks^Ak2}i2Dw_}^ovK%}ibW(-O;kxAa zQFf1_3iyw@rEz$&rwbA!Mvs zod}-BHspUX<`(P?2UU8wq+DSldYcKP@GXKWvS18^ZuTZ5?syh>;q6XPOjPGHlud>8 z7%aA8De=$nGW|iFXaJq*^E;92-b>pzYc#k;F&}p!;N}_Cu%XYdZfsQxVZ$NLep0M%fyOpuL_{bz8f zzGLx+=p6&rhTtkHS6IdzbV(U9>U}~<0NDl8OclUyBY|Hg1;EquaGQ||8ADlWH+T*_ z24=Xzw;1hXguraJDcWl{L=+U6H(=yci zo8u=Wx0MLlE$hl=5*iE3qhyO0udAt~AS4X_?ex|-VsOn&BqnTf!G@HFVhD1Pjp18- z?E;_hGBls1aQ-OE$<#!x5E3y3fmtAuLQ(Kh zJa*sa(!7z60SO$y(D}Q0XG2lKzIuGs@X!FO(yOsVfR~?RnI0kKvX>UQ@QY9vf&;mJ z_J^Zz;h6n~o#=f{p*Ll!M0S7-Xd`9?4|RM4A^t?SpB;;vB2S#ty^alGwd6Cd{y{iA z)(_+0Tz}A*S=D&_b)Ew77#7j9I6QxbG&;nckiyR=M|CDdMat4z^G%wKexlu?Cq0pr zu|n*NxJ@_@`?wGu;vy_z5>SVdT_g(y*vVMrQb?~B^iWrJ6n}G%1swD#00u)4RYKBj z=LpxvKK`FXM)5anYO$wSgO8(!X%bqi{ol_vu#Iqr03_tXOPFP*pqWbNkaw}iI~wXh z%LfQ@(G}wDGsJ~<{nydZiBEVwmTezCjlw;${)6nVNK;wh$H6^-5+C6Vj%eJ*PJs8T zQ?598;w#}<(Oo|akhO+-zZ2N75mhCdVjE9gPoYLsM3#p$6osrcS{&p3FBlO2pHo@k zp`+9k-zMzmy8xmCv8v zb*iUUp3ma--9{Wf1H3g0p&{^uZpiDD=pz^_6PgGM5#l?x6C&wCTMI<;gEb=DnSD}RzBv3yLqH4eNe|8N zK=ck)bQ|(nAE6;mQ+~vNvtkdR`=Kbyi>nd$_a1k``xlqX=JL%zNZ93LggN^phALjo&GlY>5%5g)mljD8CHgUXYq11Jw`bl=O05JDJ z3dAMD9~mA}bP9U$;leS_G%oatVxa^LG`2sm(fB_pdr78@P=($Au-=C63VYMC0#t12 zc=$3jc%r_!sQ4W>yDe^M$44QkMMekl1IHQ(odPvn-qgUZgV31cfcAYGp|bshLMaIeKERDYF;NB68Y|kcR zc-wxr9!y?J4H*#6@)W0&HkaTEE)An|6rUl>Pf+hy5s`Epx3M3rVa@RRk0j;ns__s9N{}c z@x4sSPR-%a#9KMF26PJE{_g^@&qfntqX`mLmNSJZ*=H1^Sg!{=uA$&FRYN7IdAOQQ z%kt<4cnBhZC?RA-CuyO(8$I_v(-Gt%X`B#;F9)@$lTIwELJpN?2h$Li&|lT)Kgz<< za0(>5mTC_*rmLI|Fx3<4hU4%#hN*kT*2!9~1_FmMkngs~72 zOJY^Z5HU10;5G=5p`SW1%1wN$b~2C<3N|k=Mactr>!z@^2;tT>p^cM6C_csnnyiIj zyp{2HlnPE8`2Prd`|zskGu?Lq7tIuB$~AivXdN}z%)Y$J0Lsj+wn{YtyHn24X@zdX zTP2x+>4u`MwLl%vxM)4L=YrOpYf8$lg@%SvC_!loNJTJ?J(w1^kbsE50If_UyhJL5 z2zgna&wW4NwGunC&-o)HBx|kT@ArLqp8L6<`+2^*s|THaoJHeo3J9U~ z41v}SHKNLkFxOM!c!mb@BABuY+KgrDFS=*ifRTtGOpU+T3WJrA)t%}~?dCeuxMKVS zBN!QLMYxQ{+ zKC82(W0^9rBsdQ4W~j@>LvAp-`oH9r`N zF($pDD$s>{NL+&0Q5|!WYTl(CQ2+c(K3+r5?RJ1ZY;5Y1XTFWMh!W;xER3qz9PI(%DkJCVMDD4RalEKNn)RVyCedwRkTU&$(iFOplnkII9FMP>4+(*5LlY8*|=l4rgL zLhgQ)rwUHk+^Y!pOmqD<_yj&;(YMTW1E_70@z{TzN6V&`1CpnQ3Sca{#K6|O$WLi-+W7FVw*0kA>SqrJf31omPoaR^#c=1Q>MXYhcc(~wfdy;NA;RgrEP zz5odox&hAD;p4H>Z{Y$Hpj{NyYsoydhUG&f@)}aQIZ>Fr%KnwcCa{J#)*M;DK5re0x=mevc)w}s&s$z`~&uX z!F3j}A1y-_*$N1?h8_t*a!}R#ng5nYi3>P(`n~Nz%Za{hgl|9_WwHkM!gqCDV4HmA z+@xQdG4_!Isu_VFq1=c@A2JqWBvw+F5iB-oLP~vVGdFT_=6nelsG|ht=`QeqLZX?E ztA6NX`evd2v|%5zJP=^auTw7-?-039`sp-i^U-S z&!M7)92xKud1!+txj#h?SghjTJo6H(?ydlM@Jxb^(>?>Z+5K0t7v@f9ZhEguw8hB9 zsVik;7e@Wkxk;`DTBwpR(eA1d0X%$Tq2TY1!F#aO73W$0dpE-#bEE{phq2U(zg+V> zYY?U>z^J;#hI1Yas~TCX7r^=gEb;y9?=f=R)rX>{AoF2Bhb}PV?Tcpxwue|s@Ri(; zmb^i)GU?ieQPBlgd~7eIXvfKm7gQ0|^JZ=plO?yqWD5#69L+v`98h?qVX`;M>~)Dy ztrLRy#EHG#d=UNz#Rr!GG*z#=Mt4=$T0Y~ouF~TV|93(+g2s-_OX6oKlt%|mcCy?u z)jzGcFuMN5<(FQj{=@gDeFnmG~e1AhDn8 zPL>b*IniM`lrM<`+&g7JT1xl<7X+QBp#q@e`F+d1v!&3I_B1MVD7S(bC5=H^Q9uam zf?@>U%nqSy3 ziD4Cnj7g~3sWZ2rT%&nwxkHyg+Ma2_Z+{_$RI3%7LB<_l@>N5OE8>}8W=D=O1T*rS z$vIs@f~7|Q9#5Nb6_CWD^#}*=Xezb#qGV4|3DrDr=_L#$@y0_~sqFWIR@I=A zSl}&fj%Pm!m|UxffESg$2^M;Jg}UdVTViB}&|->kQ^TI^E75R3Kt>?{*+)5n(rD6d z{BRnV`DouFfI3s4{C ze5vS(>xX7JE*oQlxojwPVU_F+5~**P+5m2<=8Yqp%E^Tp_A#hqS~$h%L)J^smv8^o z2cJdJyn6wrR6}sznm_I)r}@n#mGCGAgO*5CETbh=z`r*Tg*&%B>D6aoT!|pgehFNP zk}-#m?5EROK2efKdy)O0A$f!$sEEYsKU)Y#Hz|{xOd)4ZXpo zTn<7EhyRB(;phqaSgl!n5=ViaO~hhFB^A36CHf*;YOQF3j(L7g;jYLQXwp@Y za&93yqcYnRo1jo+oB8xJj0O$(C(y*ho=tIaKN1w*h&1*%yg;poKS1P8ZT1TXRg486yT^bExN!!P{QJiUj+q4VYGoSJBQ=3H6_dw`k_sY zO6qv`K=mtx**rrx03)P;#>ULh#z3y+8#ie=QUB z)_A7;9F+$;t0&J)g;L`VAnlgB@21Y!p5Jxx8qPHOzLu~JBjbrw#_S_vmD-fo(gZ=henh3 zxd^!pw$%~vEy(bv7uDLTQU1-jb?#%*)(yi;a|J22>vUk-rjzu=-`e$R2}zvXGig0PLmjuQ zQel@&6D%U5ce1xScZ&?_bQnpBep+xgH)K06JxTBUYboetYGmYcsE6jcuUyP}h?hb*=Vh zHtE5Q!DcMKW&$CQ+3L0M3Q@0*IihU;t6Ht=hIG^sujs5w#*zW-`*g?js}MEPfHx|z zavG$9wF!bnS~awY^QS*w^3~HFC(qoX&_*ruww!IG96wdfNSQ#h$t{~Z7B+*g5CKGK z@f0DyJDsOw%{|52Hk{o0#mWG=*>$rOU$o%r>4bO3m%eBc?I|>Qs3v@f13QJoS2G{G z@a1Y8*1iax*7x@P({~xuW#ih1oY!_<(Kx?qqBH@jfKU*5fVn1>t`>DlbWa?~QUP_) z4d{4ZQ%6~zg>iI|Ob58wW+Qq@^7k}_2D9pUAcMpVQ zev%pTE!sYTD?>P5+DvNHM~Y5Mgud_tNJ#J8B~1jj=2&jd0^kiL5JVV_5R2;k2J+{u zN1bxO(|WI~>x05n_{T-xidt)hWEl?DY%8vOe&4snsBa3}V75<5dP+9Jg-BK&Y1u6b z^RTk(hKR2Nie01Z0qTC|h!3sYt8n9*yTya+(@ZC|r+5|x_5U%368&(O?a+E4z zjR6mR;M|N~KPLjZ!r#F}UfSECb0hrYT@i^(yWOQ}Sj^0PuuMfPja_=~$_3X(jXh386 zkT_w4^`i7Kzm_^^Y*RR#3R!qq4g6*ihCpaev6%qS$D{w!o~0H35?px&Qg3Uk52E z9f7zQpzdXV4nQ!nO`2kYhxw75Ov%v&71n|N@%{kvpjt4X*`d?7k9<3eB@dj3N|#9# zj7%@QtD)t;8sgv*i0A@GA3yR2BUON@!kl0|J&iYnLPw!2C_4QP1#tnO{dB1>s&v0z zotVR*b3cHBQCuZt79?i0;Hl%3m_&L8@G|t1YQpkgFK;sZ)w%=HdLhSMxaVYd$?SyG zgb?0%oN0Dc65!1UwsGReF8g+6G!8*iEA}h;Smnh4mKaH2tB^V^Jdy=sEDDO|ESb`i zvl;X>f=XCg0ZC6Ui^NxC{V6t7zCGB%gdgRGbd22=YgLeFRU9 zWHHY9)-cnSl0&-3K9;Q%!Km~<4JGfM=A*a8@k*re9wb$|K?V{qGESdDRQbKC?Ec3O zLqD{!TPF9ukUh9(#C5;(Y(?$g!CV$UtOe|0R>J?=yB$ph`N`f&Y$}_H2AO^Z^?;C`;9kW2IaaiWb+e=(8q{&Wtbg6hiM@sIU7xJx+}5Tznlt^xP}cM7PA1 zAKp8fyLTC&wG z;{=d7bW5AxR3Yqnww!@Q83V6YcIxdDYU07o-B3GM9b)9hO54jW_2DiAq_<4}Ye3RB z%dU(f5U$)zuNaU1iVW37yjD4Ex}#uIuld}BaU54Z4CDI3s3SXGY#6zEmE2D>m(Env zvUqdEZCvaJpS(O*iX`0_v;Dx89Q{2Sbc7QY3aq>1^{72_Pwo9z`egQQ{}%=NJXo;7 zFu_VQPVO#_T)_0vZ|!^IhZpE)^x8xSPJC|j+6|w2T_S<;WtUFf_cyeinF%jc`kO9# zx?jb-z*c0qBuPDd&KN`YMUUHvGw$kg1Ge`_$NA12d;aE&M<`fc4~Z z-AmSP=)0pJu$fr=UcN;+Y^{#r-l-nLhjm5Lae=v5JYH}WEs*KZf17b)g6u~d%jIKU zS^>&jK~VKwez&YK4#rlV8Q|DYQmVO_a6_=P>Q5u0rVh<4$h(vEIWv3!@SdLU;ztg~_6qCj~7|lI0;z z5d5+jQw2L2A&dTkiM@BzZuT;KkK$n7dxBqj(#8b@1ea zLuCyi5%{rX(U0C+B_|9Ab$iQv+#Y@H`$<6}*r)*rJgMTPu>lqhp5U*e!bczA3y9mA zAhZ^jE0M#U*~5_&&JCAmrmB4ErUwWiBO{`d2A1FJPPS$nw1bhypXFjCOwk6Io4lqtKI9To}8C_Jw9J z{(GdPYHw;Q`tz#us_E}%4nq*Bo@Cl!b{MJP>MlS*U~54;WStGKnzvrC@Ku4@HF`DI!|q7KQcnOuS4 z5NYo2l}8&T-p8bACvVRFww!E_J{mx1#&|Ss#U=N`2(250gLw>heAvj`=TM|&Li}WL zjw^crZ0;Div|$8Ikezsvsak*j`}jM+075&KJH!vhHZY&uraW)gwU&JUEtfl6c4~j_ zj*V8!$f&*HJZi7i@{-QV=)Mr!)Uwa3$#8CYKZi&uXCrIiQsQk%`$_7#i>05p8m z^l|6b{1H83RF7g}2zNRsq5^i%g#*5Q3)$2qWu<6ACiP=F0n-Xd*$* zVy(JiKKt0Z7(QzdRW_0su2P{Xz&?Ap`8!;1!}uLj2hGQD_LLXtY!p!Kah2T^buT&H z+a$q0)pOH`hxn+WLu#KgDQ6u1}M-UfA--MCxKlD z7eM#Ye&tG!?Wk_*H;){W-*g0d=;6;zpqap09qaM*0hu+Jy`PYOS5!e)%iOnE4)(@3 z)RJ*r^c|(j&sSX;?t&TAihgnN+#m=aJO9nHOSuJu#+_{@X3niGpRrZBP9{)8p`#h3 zNa$$pE6J_ZucSGFhiGw;gO)44>A72GSDJHEP7CJ5kcVRtIi#hod;XD2fY4gdLkRld z#3%RmRU>rE5e;#7DNTNn(^deN=M|gM&S(uOPq=)PsvCa>1gD3ta6nDp zZa$4eDm^WUGNJd$vaxHfv1xJV(&i3Zu`L7MggBInlyhq984#%w_reN0o^$y+LLr-EC~*1~fs^H9k-FZ-aZ zKe@808@j!7en(yVtwcIo)T?5HSm>mT$Z&Ed5lVnT5Mk?QKRs(ErcLuanj_Q>ZZF!(oK?ZZ*#a=c>;dE^yDw5UuH3)zxJfo`2q|H*R%^qB)4$apntR3gj*7IPlS zN{;5;#D1q>Ap7VlY~52BQk9PRj*<6)&GVlFt$+|YU_nw|KH&EF2DD(l0kUj4at(DqM!n5+(C`DV{d!u^g z`9g&`J4MVk;UG!fh=o!rqwd2E|KPP@_O21)(O z&Voi9!up6iD4}{0IKtUDYzmE<1%Zmd0I$*$?h{f@U$#L5X0tG}1KCc2xq)B(>?fsv zunvQ*3>HgLOnER5!;PS<1k4t0N>m1!5dn773ECj~SwMcgs0{v6tNkp1BF%@k;xL#q z=sY+%e5Ye+fL=x=z2r8gN2Z=DhKpsv;TlwcMzf>*6S05#SFxQwPfn!eUeS;O4fL8> zYhP;cxPV8YWOXpqI5B_qv%Cy6h~97>6A(BBJRBfhGowC7`LT32+X!K`*;zh3nbJh0 z*t!;G#+F`$Q*3k1=0@_~IvRGTQvXDWI$C>Re1LNfRx`8ZwY8)zw9Gf-O@?nb`qd_K zX({TL+h1;AG@ep8$Q8-)LD2IXGO{G!zioO-Fo+SH_%RZY3K<|TJLz^a7BcjZstp#c z{SXH-NZ6TJztoBiILyc#RxcH$Q_0Lv<1?lUk_AT@m?(Ok*>jcr12v!%af?s>gnQ5y z!QdeZ)YcwSN(vdoNw5M$jo#ZZwZ=R`UCIo{{n^Vv!4A~z`5BVwTm}iM;&hN)zScuz znrYK~wn#uOZn!JULDn35bgLcesM>uAU_+p+Xq*GtVJcDr#!U$$deD$>&K^;z%=Crb z8xvN?f)IfY!fP-c^7e5}L=>!KXTVpbmUOdcom>`=77vqm|E@}+IzZ%>N~VLg#Q*C2 z0-fk4hkG)T%Mlm5hk%yciV)llceJ`3rF`aAQS8%4 zcl8WEx?r%xahz)2b0zAdQ#F)wNe7)y8lK%z{Um#7eRSta%qJmxwq>Ipwsx){*me{H zjERa68#(UL4(IjG4low#K-tr{<9l%<6)#|{FL+=i%m?a5ExlCe(o42RI%<7gFeKi>TC9__6;M?9TY zIXo4PToyOGpfnh^Euq+ek>MQcc z5i-!5n_Qy$dMSfTH>T}ad!0SM6=KmVVhgW)`JMfrs`^^}g}&j>*dOykG@H!??Zu$p zJIjW-uTmng=5;jP4-tL{Tl5Q>o^!0LDO5e{EH=BCo)^It=Z4uRyz z>-JyK(R#5x(SgY4P{Xsh!nt|+3Gz0ZbJE~9)IDwu>BON<(@vW>1as!n0;konCLG|t zZdh;SY+`F?2I0P&AIE?)j;b~XL>-$TQ%F4X9#RzVEkNG=2$|&GBW7 zokc%!Rza0n38#;cXszmX<_s^4o0Chgn5P{5D~LHs9)5Vhpu z)z*z1Q_cbQ9Uf2^G{rFm z;*UA#y7nVq*8?%uq$7q12uhwBkWdmp#vqS6^5TpDXksGM@YFxi)J3xZwRO1ghEEd( z_&B|v$3}nfJ)m?Xy{$*~7z2ACzHx8Ze8xnV(SRmfD`!+hSP{!tU>Ef3FkM#Qu5>R1 znP7R!?7=*GE*7Wzr~y_N3%+3+^Vv?7-GJCi5bJ3F)l5-j5oT!!1cp8^nk2IPv@@b% z-3ph*GbX%4ofMRiEJ8=7g)k}EZ7sC;2gVSiR&3{>jcdmjIx{TY11OEsz z0G4s6f%Ddawo)GGE^?30Ula#yf?#qYutWxE%Qm0xF0M6QvT*k-lYYQ_>IgZD!fbQ# zKKjbbg#ao@oAYQCVgtvWHcg~d_P2Kyqqqd^Jv01DA9Han^qy-og9}?!>q`bGxB5@}W0UPTQCz3LJzYrF29VAe=>z%@HG( z9@6e)AKa^#8#%|oQYHV;*i%u+xBAMAmfsoXYhYG3p1`awoO$z$S*mF(33qBos6ay! z9$`$*Xkqr3jXNsu@-;o3C!^%|1xRw2kh*@1Wh|!*$P9 z(&+uH{r0`WKit*onyXh5u{rFm{WM|UHf?(8n~(mctDB%ESHlhy)&Ci@OEaGR`tZg` zzfXR1%Zsl9Ax4$WM-&v)@IaGPjriKy4z|wQ)OTHF8QhD(zCb01lhxDPDC4r$#)-T&I99IKx+ zQ6kUh!+nu!Zs4SX4r54GEc%;ka#h(=Z#956T}dL|_w!giY=#!3>uShkMrj{CO1=lb zGXXm!a|;SwJ^pGhujXt-X!4^)z1+_JJwhK2#wQNlpaw;ixMAP`UPxL~312G2)5V+1_QC@q<`D-S~>y3T$4%ZV~^Q<^r z-a9jmdR{yMeadP9%aD#yvf*$svj`P{vPPsYIggtM% zwS=ndz6LtZ)Dwp1Vyr($X(?xvYFQi7umeDv3z-3)p+LFRudsMRZC+Zsqe)PBAjJW~ zDSQGHhQ#=Gdc-+3S-2TRvx z;EzOwAoaoa(cFx_+Zyp4ro83yWLE}`-v8In1p*2a8RUpUHSJZ4zhcNl;$zQ{QUoa^ zEk;6SL!@b>6dU-48D%%$RVY*$kzLof>C`O@0TM(xN=2&75L2ku#WD06ehrA`!TuUa zqx%0kW+xsIwPo5_Noin#d2k9U2B^y2$phCvPYQHY_RQ)zj6}l;umzBZZE&=)d@&g;5lZ%;#ZG0) z(E(;C8-2tP5&16?V=)@?uAmQt1Syi=j6|qCV$q~JkH>(L)##N8E}19mVsAa96!+&b z!U={ytzZ6$E~p06BzQ!yh}LY_u`~IUrMq8kp9?NQ>RUGg3JW`>ASi4e%d{U|jAtmj zH>$&tHFn!Y%E4B))gB;7mJoI;nLecAZJmmlZ|oXM00M?aFr1U>(27+ z3}t*Z{yIeO@Wc<%e|@dlMUm2%*su4@K1Cxr5)B@rR;6x8Gued-i5T7r(@2LH{G8na zqqvzvQN>AEw~|*}swp44(firB8~>lT7ue;JCGcDtRv8z@v>pWeUWuBN1Q^S(h&G(u z0o6rk3+7_So7pMQcPuwb=pMHyGG%$5Fi??#db z905IY@uH<)dGNJ-#91^EEQ$$3zsD`^z~MwhkQsD!ExUkfVrQXxg9C(q|JS2kU_+_PlnRs}g? zQGg%$Kh-?f$Ne6L^rD|6u`F-j78E;4z^)5Bi0Z9)Zc}VO6Bf#uj3i}K+qlr)b>oEp z4>x{mh89_hl(sqdw_4NXZB}$;B-a85T=`Q>r+x$!(@BMq!r%yRP(5ehbEa#R4fSAk zbov32ENhWt;@eN<9{JkFo>93WP&qc#;_pV+%S_(;Tjm5OCj2#KbOIeC5JP5)?>+MN zIW}S9Jk>^%_tg#+@FBgZOi#(&^ zO(YyHU1ddeq9k`V2n&oCCRZ5n1!EN>G(nfG;T6*&%pxy#E6K zVu(o@+#NUjzwv>-dI4>u_YE}IuKLvr%t8|R;pw3_GjnqTeto4t(7m9wKezMpu`jB;ni=hOgW8^)3g87pM)zZlQe^9V9 z_z$2gL6zq^wc5uGi!IPE8%>xvk~FB%`2Yb$cyy6bjwtkZo{)teNX5edVh9COGM?H_4gj zdsE^R0l0ouG%BHvDtl(_0^r_o9m|ttu7g3s2OPHn8Ex&x5=`MzR8;{p_}6#l*|pWM z0Bj&)9NQ{pl0b=&nYJRhppn1q8Q=Y7duWBR(p-T!Y^sc9ut(7S=4a7ffF35%4fp1I zEE|0&_uV(1-$#XN^ukzw{-F~@Jv47oG~bUIf(b}4#i33ljVfp_=M4~PASjhz%+mRp z+B8Bc@(cODN7L-Q6PFiCop5#2c_xBYt|{i>m{dz0<>dH{B;qJI65=5!*CsBM6;f@W zu+5kwJeh@p+{VJ;gw+>+*1m4=x}G`j9o-ZntPwZpxyF?Z?2wt)T z7=S+IvLJcU%V=~QaSOb04kQJH+F7i!!`ow$0@5&}Me^W9PSP z@dk6{n_am#IX2qM=9>4vxd3Yq!>{c(gdQ}I!^E4($%bs zu@>cr`)<#!Z)Go3?vqWSgy6Lcy$>I7i;<2+%j>jSocPq;tLWEVAfpcfDV2s1vzQDR zS4ll#>rnRq^}{O`Y#PX`d{t zBp(%yiznj=eA;Y;%n~}W8yP7GS&DfbqTo~)5~;n4)8vl%1hZ z1{1R@1#RY;dL$9a^iVcBV)&J=9k{OW!sCOn@y3&a6iYj)`RKEy$Iy>YE0?K~GDBnr zN^IODpdmGIla^>E}sm>8)Zj8O5&+ppbnB9~vR zL?8^T;?tf7dLKT0wY9F3!!<5!MK3bsygg(6?#K4a1P+&J#ufy>YJm5<=kepPToSpyl z)LOVJh28UcjtWi}c4krgxSc1aJahr%#a=cBzZ%2nP-aZy8ohUE;zX3I35V;a77@p+ z*F2Zqz#ZG<@g{^vHT0sZUlBt1%wLur%%1LAt;*Fcz|#7aK6Wa&g5{sMaL3=|Y9E|8 zZ{DJ=nk!cuMrHw}1jk?b=b1;2-%AnBYZj@#dj^}#PhCn$N^Qpq6e#k7VWi#q3n+Q& zWf{u$KknPw_aT8*eU?&Pk@5A6yLXAv$bb=yv$YMKlIQIA^5QIpU(Nj@7}+6M4CqtI z$k}{Wne#8Tg4h(N4=yYDl^M{YLwtc#9U}p+&T^3;t@!J67MI5+wHV+r} zAuEc-BLJWeGd&Peg?_2bvp&&r38@X*_?##{2jN(V>wae-Iw0y;fA*yj6)VR6`0RP#tual*&k@3mc*(3$ z!Fmi_d+F!EQK_31GZcB#HgAXk6}-^#2;JX%B8c&VurDPeckZqRoaTk!v4tJ#G2rqp zsC0$x1SX5;(mgN)EsYM9af%YMpJt;oo`gde6qKnsf`(U6r=7EyRiI8f(YTpUq@gwa z_Cb9LJ18f)*FupqqAdkl6y%e|V!ga}_!!iX%nnDiCnb9w4R?yqL-Oaj0H(pX<>2`; zj`yvhj42-6)D3qL6SM0n#CnsM6B?N%f>kujD$iL z{0_iF8j8$~*Y&N=J-Pr|F7E`2sMjWEI5}ixWD;c8!C&iDGOp!^AQQ!3U+FvSU`B{D zXr3j_SmALY&$v9hj^_qq5IAXHt+XLfeLZUuKpaI~t;j z{HS@SF3yOT+tNLM3UdLhc|UV)@VCkA@?p|Q$b^dhw!C2Kq90;T|6Tx9sWdM&HF+Y7 zNaOskN@i2~;_*T^AaO+V1caV{sr$Daz1M*NgysXkBQ%V?pn05HcKVfHw-AX%#$o^# zz-{#ZZ!5qhd_}mB0e?z=QJg=}_l{f)>cxKR>=eb!95V+110Ls*03$BA>2a)PW#Tt@ zQVq337gj-I`KVfnKyecUc)ZWuto)fJiW(;*}r>-|6zw-HBfpA#u8kG3% zcYJGn#2-0C9GARd@`ON$R^CICh0t?hUKkr>(1%1_)zY7aVRFeQYrH_L2#ZYD>8E-C z_w+({BQhYK4^x%KQuVb=vaOYTd-wP+(4+~4B+NiE`E0)8kt$L7 zHENZb1L@(%C9j+AT^SK;F*xH=jo5tMu$z4B=Mg7R!OA+^iPdA(L-@)QG*VKMLR&&H zp>$>G1K5vNNwZvJxI(fQ@=13EL6PKp`;V2iSm+RziU>HMWkbuN@9OiJ(_PHBL)c*Z z+CR~*yU@ArU#L36vUpga*Qo&bB_SXzaO7qf7C40Df7mAE;p0|(qoNl=ook%QrR@>L zaL0o4_W}3bp8=?v@plYQg^szgQi(Xru*{TS9#$bkto z%3=r-xN!rComx5su;x!dyAsGbrO$t*#_W7{m-PDa*lhdNe>O(8OqDQTMobb=m4O(4 zY+QBwAPEz6f&)NIt6^cYMt1C)b~q3pNYMBYCa3OkRY38>Ej&tNVw#i%^JpsKiJfQx zYR#>piYBoxQvI1U`!J$nJ!G#2?>?#ei)oZO81M)kP$hn(?95(Dd&9@dc?yh#w0$E9 zR6#v_q?)EPZ75{e`sB36_Znt`tQ*_;6qYizH`A9QJgr!G3b|aRrH+V%!)HrXK;zMYnq9#2OS>%ejEg2lA6K- z)@jxmB%e-Hu(ADU$#&NfK~3MEvZZ;4$O)FsKV5Zf-+Z}{a&e?KSY}g5p0if>h?0|E zV<&*azpFW*>w62u3e^bNse1R+YkZoRJ!Tt<%0|#BYkX{4ObI^Hh*o#TwCQ?e;_0?9$V3S%L#czf7Vb61 z)1EkNKwD@b!$E6{l)c;W;G`Cyg>3@uX~KID3sn%+NxVxJXK3}dWfVgZcV{b@Sg^N+A}3x)*jpAx;0&OY-oGQ*(^NSb}UvEhHgxi(f@n zNu&^+&ifA}vZu%RM~)Kcp?x30xek}`+w#l7f>mrmv24;n+x_oa{i=@Ptc8g5 zAke5+P^ z>A`|YkE0OPZYI_2+8q_7khmx86{p}4m#o1OXl=QByL_?&KRR)OImCJwuH|ULi$hWh zE}`|MTYC({IQDDT^2*V&{00Qc%IL)M~2GgjQF{L zxC*P20Z(gy1*cmspXv2b(EP%s>MyFZNWTmF$PP5+FgG#~`PFCJ8h%O(SQVM9OS?5a z&hl%TuD<)4HJd6+b`41s*uS6o{SAelJdUDa0G=k80_Dno4L0#RlXwvvMQMn4u-mic z-6e-kvD^%A*nvN#UNwfpcFZ732M@w|1l&?Rp!7u8%i^=Y606JyuYTa_GSHr|DJ>*} zw-B=DUS5SP$5Q5M`tZrQW+WQM6T>@)cQ5^!<#6jib@IzSe?sK#DJ?m}|Gv3E_!mvC z97Ne1ELs5jKEado;yNCBZBGO$7o;V`Zu`v;;`3$zYMhL$F*mLGP)$&XD(XufQAITF4%CPlx zJj?z!AAQZ48DbCcD|?ia53|jE+Kk+ZdK{2Vz600q4m&@k`9Wq*WmWYLZ2-Ka4Z<_K z1fmrwlyY(x51{^hVG>HHE2;Z0aw>{{rb*Vy%Qzb2cJ-{p2u-*~q+ueJGoY6L#)drE z`J_}Cj7`~rrc=#zPf0-1yZW`6&i!H=dgk}tM(YCG7qBT-pIRcwN*UIi;&Q-rL%JXkS+BYLUi0AG~j^`B)J}s9$np4RNk;=u*;aRT*DOT zOWh+{iJC;=Z#Ki_xhp4NgcK^0TTqT(ivuQ!DjDZu7i&?fUbPnEfJQ_)A)Upy7WXe8 z(e>Cmx%3#!sA%EFt*p{SM@7s)&lf0)DBa%750OU&Y&ndYA~+CQL{bTQ=F{y1eXC#K z1)HY-Km3`n9FK164Kml75L&BleEb~LmQDM2JV%J!)3^v#x(Iqm(5v2)keV!f5 zI}-z5Ws-ys5kDyjv`)DdJ2R(<-Vewn@L#S(Y=FTRw_mf9&i3r&S{N^54G6y=?;BZr z%KFoZODWvR-47jYYH0*Pys#Ax@WxAGTsUsadF82#z6&>5TbbB?;`2xp%6WDkZJf{k zZ`o+c-)2^d{v)TrdP~cNxHZ}o_h`NuVVPInJVvf!$vj~i&KJeSh$qUFo$mfMX;o+{ z=3E39LZz4=ud{s25rG?_s~NjLK{cms?E7WeuxP!$1}FGEL?^g5uPFQdUohs%v@5Sj zX`?KX{D7<*uvedutKn+8BDRqv37=q05?+F_Hy=<;$?g~mM^^*^#_6w$DB75rrwJ@t z2B|^mtizPGV@1R519-v)2?y!2OCi%aHCVMEyulvdTft>ih(5=)>y@7o9;2}e(U##q z#`!}(03&VFfx};=WAOc(NgXWlrnWvz%OSjjKQbEM)NJRJHaQsW2Tm?iPiN|vy5FKf zpN-gVm$_`G+2@(bN3jI?6l4ri5uaJ!Mgd=>pfu!~zJG6TgNWqbI%L4g&osZ0hSEdI zQ6N;zc9lJ+xbUP+d%_0f-3RyvJ7o=|c8qdL>^K(Z|FZH&w;}X%UXJNq)N@mla$=l*Ml6Z~0rQ9UW_2U1N z-Egf-1M+KwwH!5hiAu-%ngM=6MVR5HhREwfo0{Pk@uEscg7iuZlpp5iCOR6e5n8Cb zj!pG%~yDN z$+lMZB+5tuyhPD6gM#%(S*q?AgB-qn+_XJwQ|CW+9PybxE8itN77SzLN1boFbJ0QFu7xExLR^Cf7 z=|&#(nDd+P{&ewkG^gS&oZ0{BxDAyt%B4Py{KMa4_=Z_^!tMSwi@~_cQd~vh|i>Kth5^W;+I6nEYnb@Ba&pDD`*6 zngA-X0u=a^wsWhfi=BTQMi1o2``34zuz1l%6te()Pdyq@WeWE2jx&=hf zK2Uoa$*9^?RIDU6=nuEc3lvVpTudU0bwgeb_vA-41`6_wtC1CPxTvLp-D7)j@{XFN z)v{a4PQ99|+9+u{viPVeW*=#}W9iTkK2?;{%dH*t)(RAg`* zH{7FrgN3qFCmvT?zWP}{cPxC&oD26aph{tN9It=_y77{%jzWmM8k}CP$n9jOyb<;9 zQExq77^DYukaBQaI4AG5)1$UmZjOTtR!x%uy7``2${mU98|L;RFi2u4IB4(uEjVC! zdHJG26&l^3jTc2<7?$PH6U`teCc#vIcI+R6y-)yQN2RK zdjC44WrP9Y@%pX5XPx&m=hZK$4V8mcwpZ8>=9AMYHFD`o`_#lT)RL|qe<`>$;PB@8 zP+GTeALMP!OBvg?Q{SfdWr(`W8!d&bAoG0bY|Ovs9} zhFIBh)*Bg2I)1ce7xoFpLEI7lnbkU}g*coWR8|Y~mhd6iUm}SV!((w*P6$^PB?~N3 zLp9+Zx`_%WUN`G#Yf?Z(B)xeC3lEcdxOZ2wSu{Dj+pPdCy2!k+OQ9XsLn*!*mX@XF zGCb}tAZygnw>9_3_oW@vx&fu*^bU5sj)gtPAw%-q`8h2IXix6l)%RT$Y$^_A$Y@2P z>@*hbfA=MSZ(xF(V{`Q*0NFmZ>Ef4RT9XAG;NbcEDUtlke|{bke?TBth>xK>iYvnb z>%e8~vtdTkq)Oumb1RYxf=oAj8o$fZnn3=_>t}Lzu><-@5v}F)s{}Tl->2SH{fdNS zzAXq0`gY(rg(e@D1A#I+^Y*^8m^6)&{H^`6HdW!gkuiJd&gM{3-LeSdIZ{lJEXF}* zO}E;flnm3RW3M~YlIPgRlFmzigW1=*cTTw(0KRk%#_|GC^(<%1RaftXe{4Rk_;rds z=AmVcKwh0qad*#Ba%sqrHr$U#kjq$TJ*W@}uLrj$+sIJrmN3BQ0Ac9y;M_$tiu{i7 zi(RC6fBRgXl1)~3>D(I)hyG7KgAou4&~gFjGH}wem)=Db5h&v!!YNA0t8>S^apa!x z*1-bkEP!F1@_Y7z{^<4XoGG>cxjF1+coYP{$Id2y!yQ96d6c;^oX&>5`MLp&koM8d z_oyf2UaY_mJ}OVpyt?<7DOVEJ9J*A4x+EE?sqazOwW?9mn{Qy_a&`|D==QCCDnTA_ zGt;5We&_}DSfU4T=K2F)TA!>BL*~Njj-@Bd7C0ae7J>l6|1<5v1Ac)S2FIO(3< zzUJ3L@5fCrYd=@mZgZuE|L9M3SHYG+j1Z-v%%*Nl_RsjzxFC5p5#oA$uPCgg>OO%( zo_NGlMH4BTYV7DL*_v`j5AXfRxAzbpNsRIhQLp$UN}MsHtT0Xo^LBXcCtaQ>8;%m6{&Ap=@>{!xJ+la-3|Hc%B6upNUga-%c!m ziG5WOiav8jBI+~(!*Fl24iN@%3Nv>j53W;=P+52k=%W;?d*xnb@gt2wwIvhx0!Iee z3&!%^cGLJVK7vyTEMVG}a|=gr+e(dsPU47J6_BI%GGdOZjW!C6>TlWV>^?sc0P4D)BFVp3s$BhaulJ&uXNY|p|H`i5fEAZ1p zaYK0crTv5xPT3+oJnl_on#=Jk?1b;x? z==$@tgr+%NkArB(FaCY~1+=qmT~G1=DnuwO292L{bsrLmiJkBsMLUtgqLP0_&V*jn zm4uVCSKcMOBK~ty8q=>mre5CwQ+$KE!DulvYdR!|2@&sRz-7)jKSz9LZQJY(nCK7w zpDAMWm~92=3Cmdql)SwBW*!o<$z|c#KOVvl+Dhr(1nObuF}LA%3($~zBYY0iAbclp zUrpz>Pvpw(U*JwE)F6;6`^eWF_`60LTfKJR2k7UtW-whbiSFEKU=iSeEwqzaRW!}R zsPk*+Q=A09OZhN%ihGpwjgTsm%4Z$MV{c30jWJks>tUktlHu-^l)^k~k>rC#rp;-; ze8ke`{bIRj(QV-*$AN3)?Jx8;O`FUyNiYr;Lv7)J-TMw~o+bFmRuPxfaO5y&6D+YL zT*N*pQs7cPh)w_lt0WYq&^>Mj`p8gaJZ+kVLgEF&1`Q{sKR#kmUB$V%X!aNVC}ovR z1Il7^zM}eVw0JaS-o(IMl}M!dN^+CxZ_}rTy};8+v!C zRZ{^e`;tl@uKG$(u`F1fh7|@B)8-=PsNYEcbe(oG!a2B;JeBUxqY@2?$5C;f^H6124E4I z-A)=kPWECW%Qg^=+Eg+HWmBBpGik1a1ktImLGtM+_6A%V?1UJjX!F*(lr+yhv0(*K z>FX()pzX?GNkmK9Oa4JI5oU&*43Xm9wN-GTh_;K_g`ZE69Zx$d`}g93yfeWZEkx&fg*hcC|;v7O2v(eIRZt zatB`&1;uG~-kP7sjM69cVd+HdT)>3E!DW2h)uv7*CjkOgKYn#nc004No#Qh9INvzF9A^xv4<&;jh7 zz^=!}$b(R=_0;0i2y@)svaf4l5c+gJs6iMwkp)rrX9p*DXOEEc=(xqXgYKbE5!NTY z?fQ;7Y|iQn+oo0i%Rbv^{x^m>OQza$BHh!=S-1kkDd0<;IgW zPQJvG8>=VnG*+zgZ{nb^%xq18`KZqm1mKVOzhXX!We&uE#j%%5!b0yg_3|&8DE*xW zN6Ul9EO9c!fG}g94>gFni}DNsJS}?P7C;Kb$pc|JLZwOcAmWD~R{ySPCbC|GQ-r7924>BPHXS zF0Re&B4#^z9{zL$v~qfo8|@r8p{z}&>>#(3!whFMktCj&NmzGmA4(a)%@1RJN6hK_ z4y~Qd`hr@ptEr(Prr&KX0lvzN5XQ(8CAb(?hpo@?KsMLygOdOQ5XyQdDWKYlU42ky z4V-noW2*Da3*_3=?*TutP6bpqI;l+mf4Ef3x)k8hQ;v_-B&%vXALuKh6*Y`k8y$II zVj?v!w8N{gFu0n)k1+dm^R1N!_IK3IyMD|43-{=ZmG^$uQ+D8=YzEnNhy^r;X~Wv3 zhn$QayrrGB9x9Nk0t?Scq**)~0*TjjeY#}X6x{6Puo~*(jS(2!B+(BcNu1=7VHh~G zqp3HWT^}`W5L=jUXF79_(>H-0eV_C#NqexwN4BI&-FCH3-*#Q~3mqrl?I=KoqT8!h zemV*YoZjYK9$M&-6Wka_FeZ!S+p-2g$F6lkCI7usJtj1@kaumUR*w zFHzg>ZL1uMF}=)EEc*JwcV!@z4MO0<-bdM^P1$wwxoo%9+mD|-)^uR}MkdkR&|;3J z{Um3G{G$3h>Hy~D`TR>|oz<%0BwQ`Ye&0#Ni1Qo-sb9Y(I`T+BNp;#irJ^!f?FGUG zd?Oh=;|q`C_@L%_c&ULgdoGw}&y;| zfT}n}8g^o8sbi#*W8Tw0MpSe-&*DrUjRKCtc^L-96E>8pB_J$BB1jmuzMXh+!_ohY zV9SJK5!S^5a7Dm440&z@qrB2LCz0O}21aC}04*OFHA;|XWQ3)YQXXCbEMh&|cb$1J zTuUhcCu%#;S7y6N;`Nf=@A3owo34iIJDT3TPWKen!=nO7lgz zUHiJh?}!B%>@T|>Z5<4RjD)|U33+M)8*MG3<&0zT0J;v=Q{2bkMU<@Bl|e>E&CaC% zCzS zzFX4<&FbSG02A7vmh&~K;~u5{N9JAl8`1pla$E%L#tDga&JyS(^;NO0Q3$6de+%6i%vZ7QJ7!{bco~k)%{C(z*oFarX&K80&O?)W>Yc>qp3tc&w6)@H{Xn?M3!|SI z-g(Uw;95!3MT7^0vX|bS_i%N0L1WpLn&N2X=eLPu1Ic#AiN93l$+w9QO02$^BujXeT0kQwTOfNWc#UKF!abb%{EL}=o1TAZOL@L= zM9C7ot25R@EW(h)wPa2h=!a0uo81;i?qtfZmymFeD5E|qlT?d8HQV-9bEju_tBsEQ zEH}(l*y;J|Gi1G-d@7@|1+a`Zx?jVbunqlxXtQdlToIP*ReGGHk&)RR4dG$i z)l}Bnj!C_=U6bw)CgxtLryCQ=^#AeGbmM%OVa3~)-VdBkN1#$9p ztIUgoK`eC7+b=Iq$6wjh;_257>Fs;g_M=ztr~}af7@!XZ1aLyt|dQwEEj+bOUGjdJiW)vZ=i~ zoHESjGYKuK`iG4} z5Ja&^n<9DY7DnfG^!8$M@aK56vs#_X=jdWSaA;GHej3Nk-LOh-pHaYgqZiZtG9c1a zu+_vpLWs)eLv_(^T1kPzry}JMd@v$7+pI^KRsCf06~@yIh?TKwmTTgBrFXl!oDFWf|hNC|C|@DKnSNeN3Vx(l}r(fcy>?G zB#u8n6OEyG8l{5HJur^}0n`9(kiUjz6&bdpOSvp%=ru_jR1C{0#3CGYWS)3D&6^OK zOC4ayDhUqK{9@AJdJSmC1fbD^)ERBuchC{e8jaAIq_RQ4ygq=e@{)frxz4Mne0rE5 zPEo<+eTD~LWSC}-ayAg#c8!4 zTuMQg;q~Dxy~8-7DbNn%h>rkRlNIk#K)g@2CV@_(zq&qBH)UE@IX6#Se~@obrVt48 zV1`H&hfS42%Vkz#fRYMGnC)}>d#@ZlAenUP^$`utdAl-4l5jG%V@&#bBs>Kw?Xz54 z*>NsFf9oi|Y#*+jP1{?M!yp)b#srbI$Yf6)eM=%^N~Veo*rcv+)zcF@dQ*E?v6s|% z;ihE+r=&$Eet}z+ann5{>%5Ju+ixUmXRD4(o=C}gme9~;9Ds+lF68e8vF%o@+G-CP zQ=-F2gKLS--5GWLZBDP)JVRxy9>ZppMd6SLYPyJTXmCMWVLa^qT-gGt8n8#1FfVO#Z-GQYS*W-Io{0;lKK{60G#5&Fre7sYF$q}`YQ5) zL-RsCMhiN&v~`V)lwyE5I}u|}ApR6L{8IS+?{e!128gsOZ&4G8pPfw2_apCj&#czN zx#J7i`gb`)RK*E-b)CQKZqdz>pRT3+LmN{>y^*(^7k?yqjY>|B)U1qP=mBl}3&|*j zEsCW%zqAvJruuE5-C&{8H7uzH#P!P{$%xSXKJ3HOtx`{636VDSrxbl+4>QHo3wyHy z`N;*a9$PLQ2u+^N-EGJnFUBHLQ!GmV;tHP^KqR3BOkl8g`6$sZt9ZHQwPEEo68w4% zmMoTLNx#*ZlJ=LpigFhLldu@9iq09g70K$F$&fJOr?X_exAUZ(^^0{PL zVYCMCVQQ7k2)Mf4+GJv3rKE`ih>EWKo^>DU4+T(jbtEbD14o?5KW8W*XfHX_G+*`r z=g`az<$lcP>IW8P)4Qkkh%@iHz{Ws~fM}{sp|-q=L~aTvF{4^A^<+*6FToFz)Y44= z=)8?!h<&Ef5ktX^=olYOM1~L3eiLqVH05}RHiRQC?}Ji;CS1}=X+*!f0;W9g`B`;O<#%PRvRa?E~2(T#aiBi zTo*&$y(Q?s*FWI1zO)0PH`p)ai3)2$0FaQ=b?96}Z6< zSe9_pUZwFDNuDt7p*ANNtfpNvM!HX09Nb@d@S(K##~}DFp$u$|r6ot5gNTgMnWcJN zYG_w8B@<`XaL+`kg=Q~d_5+84iugY5vrS=-T%ab3=PvuJK!o*C{oFhS4VQS2kV|uY_4Tj z_H?jPoqvX)k75XFF|i7`WN0tN6#RH@h4A?Qr%d1o<9EkBRv&oSHs?xMol5s(& zAcT?#PK64gvQO|f@dB|h>SIf3rQoyWuTku@WrV_+qU4(~YTB2STH-LB=$86Gu&~hL z&&8Pn+*g?9K>h|p6KqO7&l8!1k%z(p8Wg1(YpxO!!&6%sRzk))$gWa890G)XA8JR4XzFx!lYs-52lvPjRs@Q`LJY_Jy|X`8wJ997 zl?$wny52_*eQRm178Cj@L6owVO?%Xh1M()FRq3|KlvxS)szYA%R~-S{UV8FfB-tms z=g+olb<(6J4vf;>AJ)AJ?@Ke1I5EPZr&H3rf_qX#YA83r?9Rfxhrx4r$9u(F=}GO7 zDTV*U|G2yNk$+{2hRfF)4hfPEvCz9U*ZPv#99?p$?;p592zPu%`|U5=0H{frPi08J zU<@l(-Ni^A`xHM68X*O#=g*w8@~?Y%Z&S8;J6)mjerC~faY8@-o5Q&mZ(d+x7H*D3 zZ)@79G`>zXcS}#AC9CZ6iWUjLI=_VJL~>aQ^M2db?S#Zmga&cy@?-FS?}wj%TO6fL zElCO!+u|a-h+Rp$9VmN-oh0{1ty@#?Bih$|$w(+!ct*%Je^c^6GVQi;%Ct|-5Ec2D zkyeJVf?*LH&bRZ?XUyF6Ue<>r(m?lM?!-pe$#DiNzmUSf2N{$e>3VgO@&z97tr+Ax z%ajThk0D%#${XRg6Z@?VI4sE2e+x7J#$60$N~K`YP^U3RYL3KIQ0IX$i9|fV4rj~{`9cDR zW;2!Su%8#_$X^B5wId_A0lgKInQ1$rd=8;~uzQg7`+IjDa#5E~!Ah!bM_l2A30+hM zKDKp%PT81D^MRd|gI;sL0JT3fAL8)nsVLykDHR5nOTyYQaVT=wT2HWZxaHce1=f9_$PR^zU$&c7Yoj=La*dOt?Zh0nW(K2#hP^ z;I%^IbzVUAT~LiF5s!#7`*d@<~i7+Ktr#9#PcnN<$a6d-G|@ zMNAxiz(3<}`D5LctG~3d(G07*=bp7J1JCs}VNnR5Q&eYMp_}S=?OUctfKHOORlrfL zcxZ2diB%QJquIk!{G}8{*a1i4#^M}(=9X6rcZ(E_W6?h#cMluMY-&3cz7%R<9o0h7 zzh&l9B7$uqB6;z3zZ0JY2NCkTeIYKEZIcOU+QCw9W4$Y$fJDw4Ov5A5t$Jh9924M6 zr`zZ-j>;vjuLmlIt({e8 zaxt?^w`Rv&D^U|uEzzmqQq(}bx?eduS$%Rx1uNgtD|VQAD|HcJBb%rpgbhQqscQ+J zE{_Fz@cJ})&Gc@eTF4epJI&e#+e&g$2(Ouj6}R_{@9i~=O98zl3wIlIs+MAZ?v9P0 zT&hFE0N9;JOMfHkdR|A}nb+BKnNdH?UcyqVVEd>${sUDsDH*$@ifmuW^s#D_hjkJ3 z)IG}54=I_~GdVW|F70&pmk+%%extYL`i_U1F7-h1_N^ejeX!|t585cPzmU-yPF%S> zN(EV$FoLQ}b&r#%?e%VpH5dv(tHg-%$&1A~n^NeS#9))U zp}MVJk_|Mfu2Sa^BPm(gcY|`DbOhnNY2I)jL3fmsspU`Sxv^VzTnm-0`QW3uYsJE~ zEFwgH>~&}Bl0DEg-c;6-Ykkt0Y|-tcLb+?4+h2m#&b)g5^+B^S`pl;_mVX|OUQIe8 zxU=9y^aU~|B73A$06I)$)tOuPT(b_X4w}}o)a|09kl{gyPgbZ`dv=e6h9)dAO5wKU_AA=;!2CkgKqF>QBNog7 zi_@<=a8a@Xiw_o)c(QyNI$M3u+In)8f9+Ev=?YMe8f2%13J!8*W0G8hr82vcnWSth z$YEW?MJtR}@YafFG$C}Bd~oSQ54@(GuJsgF7cyt;y3^wrZ@;EO6*U0n#Y~>AWh?1v zZy%<^%#?Q5>bp<0pMREEg)YMmp0FnY^7CLn+_PL3ON&S_oFXHw;$KZ-`o+?Mwn#KG zh)^+h-anPMh1194dvP+XrqI)Od3h`)vdcC{hbfn?#DP8?(41e%9s_=B9$L3#2dHJL zr)>#hL7UGmLDoqo^o3nL*K-4;y`uvi-BYd*9Jn=+B;EM1eabzZ?EA;oi6h;Xti9yY zsei!4sr?d^i$bdGzr8ZL=c7FzKL#n@M(m-2V~7TgyEeY4DK>)wt1&8+%iZdh1Q-y8 zS3#mZ`rDp;<Tr+!EMi!}y;Qg1>_&oWPLXd`~E9u<&TBYcZi+WsOH$ z@*1ac?ID8nEHRoa3DsvIcvEUqCz}*s-MW)l+I!JhD7MNqdxN~eTGHr^wFUtZkJ$C< zg@xp>WGU!q^8r1FZs+=;)#_BE!=uIQkkMl0NNcVhFOiAX_-@x0BLAcX*;a_t*nm(T zl)&G+a1RcX90{Yq%WtmW-0Mb}0(R6v!d1?y;jB9kEvX&$pbaJf3kC=xdNNMN@~lX! z2&VaH7fVHDGU3_>^fxDiiOW$ifNfweQW!&Rm*g2n{zUmAxG&Pt@aHEURDH^dvWr!> zNajdjo4ZB45WP5lLcIoF80QI%=;p8E+NHTSN^EqnRcS@}tV0wgs+z`$^E?;kMl^G< zDEH;Qdw^ZL2PXjOp@IY1hGAp&tDV82h3)>!>Z4=!Z0No6HaHJi1UvL4HoNoFDhh`Y z&R$*0xhRTiNWxPR|Kyg3Jnu%gYE%&ZS4Uv#?X>&TH~ch%1;UKSfwfAg8T~-u;buY z@m)D;`%Tu3YDu+giAWtu3Nk{A>cfl#2A~h(Zp)dh%zB|`iLZS_ZPoNQ#@*lC{ng70 zaEJ4cw|9MNNA?U@Vm4iR7E4VgE}orH(eAO1oAaU>V>8=m(tuw`0*#yQ_bQAFn`33u z&(ZdPSa`n%5(YroG8QEP^Lo^{87E}?ELf&6=pJEvL54fg$Y2MLJ%9&tvAa0vW$DYK zNJBQ?xJU$pT$Q6La=uxh2dPqw6~ce96WWS0Bh*h+FN6@KWN0#{+zWu-ucgk0N#q+1d*$RCJ1@XeU#(u{)}DPeX!*67pcn=V~aVCE^%d0?z6GCP!n zIsuSZ*2YjzI`%uZ_%Q7s3-`*MQAazF-h@?(Sb zForfVBo6vny$h*C8bMIdO56{YS3Ms4*eSjsmRQ$1_L^^Y^>$q9r4zk!xWnN}cNuGh z9Ek_6$k#-H7|0|09Gyc?JaDs)?buSTe#tBtM~tH$s=J>}c2nu`$T;j;cMbJofHnoU zrt+a9H4oBSyMj;<%@{|U-36uEngEoRvQ|8aEJ&kQKRxQ`*yiWIhN7>$VR0V<-2M0~ z@9n>sn#x0XZ@IP+fz6EiOS!s*y5c(*pBR60$5lZoTkWz_`wn-!V(j#c02OQsxSo6k zk6ZP%u(V@Sb?;BQX1t!U6h=0IauA++DLX-DTh(8ra&)QqDW^ft!!W55@hkHQ&Tg&B zm1Wyp*k8J*tL*(uMe6dzL*Vr>i^EvvLVo0fDS&Y`ub!*t<9)YFRt`P^9b%*gXkAO@ z^1kn7$7n636F;;t^`vVyX<3jw3!lRF;`|tu8Fi1ZEM3J~6BS|c4qEW>rY z*yn%Np6cw71Dx;UtS@km<=yB1nxW>$u*||rTFk;Ig@IqLK81!zYz0svITDbUb+aH@$3q||7-wWF` z*l!ueghiERQr?WiM!J)QVs)QF`=@uZJh+F{&WI5&KVEmtzfbq3&AA72s>{#Vs)N6C zSuHjeSqS{v_2}ABEQf=+M;2krl(T1*zy){q@~6+&8@KITQVrx7@k2iU6HuUBSk zT)SbAOt%3@zK`Z;NIG#L>N*MRf$Ly$Rz;Af(tX4)RvY#0R@5>dEiVaJZ@3C*;G%3nXkQvYNYM zv_ql5pk_MiFRB)>b03PVWy49r<6tdRg#4u>kpJCW(K4ufINv+da6xKI-XTCNJ~t)O zSih=^H>VZ|#apj`Ghj^tH%vi6cgPAb%F^~jW-30`&?CT`3@0L$3*J!vkCOfngVD8+ z>#r3!{AkC+^PGd^XDkMt4fiUj5?zTA-uaQQQ(DSSU7OODwDu9}v%SByjeK<|=tr&D0ft{B+9e!j! ztu5PqCu0ZVLx;%#V8Mc>R<>EuVobMgnGQpn=bUiC>MH3*4Ru(bV2uT`=nJ872w;{F z1))PcGv{`+h8$aD-b9Y{@NFNiXh*C@6HqM92htkiAwNm8&v6Fo;$7k#W`oa|LgQ*n|ezH1DUOnu$YbG7@10!!&& zHcw2W#<2CWm~w?K8TBP1E}&oWV59+}MSXRr_C$ZQN8c@p%LPBfA1mU03e!X(0Vv}# zKMX=sPshNy*k(-IlDRq-SD{;2d|VN3`QC;#!+7nHR(w2b5jstZtf0+I4zqpY^gAKX z%(U0q4jFJL(lIN@(aX6fV_2ixkXIRli>u(6mau>JfVf4TeGJVpj4pI^=)M1MV`m>- zWqGFijf~9{R^_yXFsENRYtDE?pt3r(j-n>CHHTHLMXLd{1x{hLP}D=Mpal$@)}>CN zHES%GQKUhTfdbV=LQxdb=`5kb2niHX7+_i&6FwpZk`NNIIlt?=pPfXlXU!j*gknT)T!#gniH-p_>{YFmX}R=58Y>>>N36ZTFSh$&6-DrSH=mUIVm z_tX(pW}EKctkuZZn0pruPSjRq4@C(lPly3ks{lw$c|f-#1gO$qdX~U6{KSZoqcSxA ziM!U25dM97=w^9Gg&=+-5|PRZEtIln_XPT4eKJB_*gGFUoev2y4o^ECzq z=}Cb%58&c$@HfOpjGOxZk>K^^%Gc`<0nT{(%^W_$}DoB+?3q%C5 zoP$u2Ir%f{tf$>xL02B2pf!1atr?H$d)Ip86Fq5m77Y#x@O29u(Gt`p>l?>LzcWod zzJyY`QT=7qgw8$3X1!A(xKAWl>03s3MpG^bP>hE#l|C@gwVj6=B{eL{g?GxOTQGST z<(+t3yt=#R`eD|u*G@+u^9+vfkO*cXi->oy1wm1UJsE&ra}D=Zm3%&4qx7r%&R0h$EZpcU%9`gO$pV{HMjv|119 z&N>FA#$j9qlV{*++~XW-A+N6Xy`auK4&M7LnUvguTXYb%2?BX19T>wGr6@#-Le*Kj z@-YNeH;C^BT(9t?)^5kH1Dw!EzK$Mf3k{D1v8&zB52g%hjM#a26X+wr9+)Jcb|%}@ z!Pa3{S&b1}wE&rj-(^xVoErkGFl(73)vC@@hO;hl3(i4 z0XtC|eAgQbY?qvP$C5iLwd-5!Zh*+(SC1;X1*D{+0sHMJQ52qlnY2r^b7S?pjgj}% zOC?@6K%2Ahwm<573v+H?K0xPAfR(vv+8d(!Yk)wa%6;rs!=)xf6`N9Bh1M;z_Z0qC zg8i5{7Nb>0BCVmFuSMcZw3mF%#)sRU=9VV4hhH}g{Ng_ zDbRs`I9emys0E5~1HwQAbe@PL4oxyB)Vrpwspybn=B2`~)Rr+H3ydA97nLRRY2akAuQDuje*9K5YFMG=j z+t)E@Y{b^r>|kx=9%-|Af#sOr($k_EpXaCmlVj6E9;#PyK=719_$*FK8S8Q}4}=Au z&`+j52`Y{uyb3UR(4sO`(bbbLjAm7?c2}ojK9l>TnKjJFP3P5^L%C{sL=M0T^$ffI zp6}g;QNV3@h_9RigV@+Lt{OD7SBlaWe1qYq?{94TUAo@Ur&xaRTP zl5+f=2lP_>hc&V;E|OqHIDk_1UARMYj#Bx_o!Ms&PB~F>>NxE!*m_Csa0ouWeeEZp z$BwDz=e=1kmo9KBbfTWOf-#+QdSgM$kXRw2xXAdKBr~0J3&W|%L%uo?o=lrT_F^+* zH`QV0<6<%5(oeAlxB_7o1IHASf>~t9FpXL<6M@_ek)VZ9MOK0#%+-m=TL5JsvQM0D zw2L0dW#WI&{y8sxa-_1M%JeoTK&?+kQ@DA;B2a}A%L?QudxY%}Qp7U|vGX5(9jUOk ze`~A#6&jkEh@WemH>|%5uKjmeY>^M%cCDMvQkHi+Ra)r2$G#N_JqQU|S6;6)6$aEkO<-;*fkX)bNzr!Z^r&5GZnSSIe-3rJj6G z+Hhc$$Bkw(2vw#FoC%6RiS(viNkAAbVB;*qxgQXz`EwC7a<5>XYI%6GHr1tk-~hEo zaKy;rtFDx@jhuslw~HQ_^afrNPeD!| zQ%Mn-WTt72x>)8$;VsEc21(jh(MNH3F8WAS3auND74LV; zyniJdr-Lo7n{6P@Z7WQm`9bBJPBpmLR-b3)T8yh2wH|0YCRTwt55iGi-?2mblVA6^&WImzbI#8hxO^^onej$V1!q)AC8scW$oq|le3o6j3B~|9B z*XskKd$5w=in9P-7*~*e3)123(W)ZZK>1ECqyQ0TYFcRU3 zfQPP%b_mC!h>F&9*h(siPyRg}+9ytAK-6UGT%NoL;bCmQ!oAA#0TxMDaU8n(r>MCi z)?m8*i6$j#gd6+=k00ZyiwH4iqg8Qy6^Nm!Ls8(6iw5*0n?J3_VD;Wg#&=s!SM$pl8a81I(_ApzQbo?uAvPJF_|#UJ7B}3dkVP#evJ9 zNmGyeARY^!C_gz(o|bU5*k|PqQtO9+VDYgnE*`}tvDd`JC3VI7SUja8X$25#q9-f1V zYh+XbO0nEx-ct{hy${Q5PQ&I&4iog!bbI+iXxcRxh$y2F%COHm9r0ZTayAAM^u$B) zkWRotePN)ru(G`JfhCPwf9JZW;3gQ015=AAYk5ZI~j)_dtcfP7omY-?IL^`!o4 ze(}g$=UlaJ+MF-Qh7DEukW?k;W5s=P=2nZf-eycP4}Ees?gY-02N!!He@xdxZkxj7 zsr+9^1@@_0S}v*7>b#K2H>gDDirpAvBi1GFD0Dguq#(Q*~2Sdq__c zoq9Kms4cE|@Iq!JH{eIWtjSI;q!6HS+m%9$sm6xED$EkO>876?CS-7wgGdeK&lqp5 zHK&i$g{6eq`JtKej4BOb+>bn*j`h{a(S4g|s%U@_(lxg_v*5F)ifRboR6d}>TLDI` zYfRhL7MV=EIZ7X-Xk{mutVGVv(MM=Kn;lDwQ8ZH+m07am|MizY)i4w_wbbj@-~D>G zgVvUibE_BEt8Pct;PpPj)_*KW9+PCaOCaqf-=pn?W|Kg>dB7*vyBw{RHYo|HPIt92 z$RE}!IfMWlQXep-*I3V(%PVylM2(2{N4YC_E+rxCkbUQ}k$EX|Aj$o%kqzf^r&Udt zaJ~GeY&A3`9|3M~LL>Fh=o1ma4O4_B6o#A6(!PYD>OMq9-ZpG=@2~6ipY^63crKdU zJV7Ng6*ji&Iiy+%G;!62^H)CcPP*U(F zKPRNkd%6t$(EH6e`K<_8H3w$l7b=I+$H=5oj7xc)*)WtsAMn!7AJs|_t4kZ~b{ON5 zQyfOBSe*bfUF0H~Np47lgeJ+@rj2);2NIt4_0z0d~DL=u2I$mXDWN7$&2zz4PH0GscsPbh+R21mI zN=?sA0|aEdRP}{0hHZ6M0xHi?%B{hhQ9(6n-oXSRO~8-PW^6rhX4zjYUm(iDT3oT| ziqs@-4js%lku%?)vJ&Nuyq{3TaYhW>z7s}dgA88bD=B@I0wDOf$Kc3lRUr`CnVKI} zz{m(I;&?PewRAp3(pC_bOd);#nsm%k-1F<~uFCz>Wa}M;tEa|vWbk%K9Y6&(=3v(& zyPjeqj0D8Y1W8=Badg);<&RLQzh``P2;P_fx?4crjSqB(8g@42h#R)2WGu7uur=RdDL#QPfF5jV)gtO7yYel{ zYmDf8kV1PlpRJbAU=BPj4|9(BbsF?I!5GMj&(akA$+jzi4K)VvCa(q0Y6+UGgnK&P zF~}40cZw}#Y9e!!1`z{i51%gp&oA{bus*7ISDGqB#p~aHyTWpp3egcGR!jotI&tR= zxY^bz6+U{|<>NnW+XR%8a*MGzhj`ykYQ2DpJyx3d_l}oRA#Zyi~sw+rlBl;=x}4h zd}zsd2A|0<5t)K#T>5WYAY7B5lF3bJf!L?qKlZw?95j#>UQpmpaFrL7uJYIM2=P4% zmGmfiYv5%Rual6LQW)vDD+4ntJE!MnEm;*r6%;4H_LJ;Ln-3L&MQv!mPAC*S|IlqT zvyTI6k!hyb+8kp9`Zhu_bprzVMww#64WNkQ>^R*m`E|jSs6O{VCtR*MG09l>LleP; zbrFabX&voD>8J5FHWc{G)r=EH6|iQEnCq8Bv#JHpge= zhz`#cos^$YZE07R1&HZ1&!9)eirE$Dcwif%$q}DCugr={SXEx>i4z{z5lg7ia{Bkz-ouh~!n0LJ=;OiMWM5QRNFv5b6pD#VF$EG*6x1sF5{n3Q0|t zp{GBd<^a>nfV72UepL_0K-|FIMrmkUNqhSipqK`oXJis*OCG zPJj~G?2e6UKvL}sw=tvR2NFMU6^+SUyl7le)1W;kvk~k z({{z#U$diHNyhsd_6T}M{2%r}AfM_cj^}Gc@+VCr=>GEY+&`6_Knk{RQej_F<%8of z0WZ<#BlY(*R~7p=P>y$|tL+n7wBlu;+m}UmtAVWLlJDtKuu&>}pc;n8uG0fwEID=N zq>Ueir|SLY{yq3x1eG@4Wgn6{J|!^)_$h=}4{AaedeBT%zRZ02@uF{Yzb=!jE^971 zvWNNfP%JAOkW1oPHR!MKDLCT&#Z6xdD9Cq?9n7l6#Uhus?@icPBy<0dpFRGMsn z)=se39fHY%i%VVEFfjA=t{1i)dR*G;NfE^5&D^(Q8NEKFdUNys1AsA}3?}2aQ*~|i zpUpBkOdn{otfSn6SQXvon*OL>WNIy#g=b+^^-B1N%^{GEHci>;yk{}AfRIoar%0Gb zG0QHStavJ;Jhv~Xk47QLTdNNC<}EU#46Z=cq$Ws`pprxcXtTDkvg+vV;8U+qW(eJ) zsy&DA+`;Auqv4E{g*Q3M3c6kl&rJAr9lF^j`!E(cftZ+nCqlP=`J;S(T%XXKeOjc7 zXcZg*iv)?hhU)klI5E3tvF^HJg9l9JH^Ku#FqIj~t@kt!IN84>lwHs(%ubS{0t{t3bBAgy?`fQ@Tdlyrk}F|F zljrxlT-5*eC-mR^nUq=()s)3p)OS4frZdmtt4)&~HjiT;J4hF7OI ziOOvuNJEJ{cK%4A4qR7XR~2Bt%N9IDOjh@cA;gag!53js3z;*gh8swnFkrUe~3!?3`??79k z`{YxHCZO0P;-neLc`tRjM`7*JrPx*MWTV7o#;I^afM z1DDHDrxIQqhbGFedUG1rg6l|$cBRI~U0qnbHG8C^ZQF@f4uO7i$@3e_A3d>misldv zSUnspHEvnHtwhp8s3Bkv6~N zOj!GFFDEOX${DR3QV?EpPg(?h2YG@6i8}9ikdSv;4|Q0Tn|pMb1}>_wO;4$j%BAM# zJr|MRdQgGEVGN&yWtDCP>yOi%@D>q7@()hY+Y1+hnWXFdqUVRnv&FK7gmBJM?#Z^F zAQIjC7KYnGl{6wf9L$b7t`M8qodc9* zA@X(^Q_siQr&1Fm8XcyA_w)vUJ&XSrl!v}33cR`(8H$`GU(cFTz{S{HVt-#sWY2ys z?5M{^mI7o-ykcA}XWFW4cvaVPP9k*cOslJB9EJLp~U z`FC|Q*=G~V*K7xS*!781m`z=|D@_)#H063%eg-ZilQmb@sjTXFfEBWHCic+OMb3Yz z=*aGRYc&~5%7=j8fwaJ-^1xh1T3YlU^&w#Pqeyaoy)|I!NGH`wYLpqqSu(8im>Tv; z*(*EhZZqUTyKY=w#B&NXSA2$AcW2*B!o>5s {A6#N|(fy|O2GFE9EBVnQ)mG5;v zG5KSL>3!|>!J~86pBORCB*(#qf>D72%J8Ega34$C&!goSu|l5|MI!m09x)dJJR?Qh zcZ-3lDH=QOVemRlQ7>}j9x^V+79>h0CW)|wAb2Mq7#yoaL@@narVs|l=5T{TS+zdvj~x@WtCXhWB=Ouk*bdSr!ChZX;jeOz%8)Mh(>?WxX7qw2!&H1U#Plq8BEF} zCGCnt#uJyV8qqRc6ST3tWm%3oPwg3nvm8eEQ}G}LaYfzQfwfpi25;_2c1igFjEa7*sY7YwN>x_eV6 z1K|5U?YMJlbA=S$NUW?!%*R!r45O2(xRhj(yDqK9I*}17xi{!m1noBEBPXKXbd2kpsGub zHqDRz2tdWnis?XV957;ChBu{_8S@5t4?)zOtrLkPOJ|zVa7O#9F}a#_s{-m9aQp1a zu8toNmy1ai8d%}5ka{>va3;nbDF84L73?j+^uF^yU_6Kn8CJVH7Y@1MQv&!~m9qp$ zRAr1jst~ydH58q4nFH5OzdT!4+t~Ja^rHIF)IQhC&s4iXQD`#^jRKT9a1HiANCf<1 zps3(QQMD|D#_bxvR1eg<$bT9hXc&HCS@Ff0wE)w9$_OzCF}kFd6~|o$xET4{M^oH zP{9iqU_cc+k`(dhHt^vOpr;Jrz(uE*%snQ9>Q?0~Ipxh}!L*eSk!IvskbqN?RjG)W z2|4#z2x3Pg4z06(f+cu-pD2Awyc06e_R0vl21B%~qh3 z=I^)n*8*UhK4Ugh7kj``d>V}e`P=0pg5up2cK_{3n5@sR(q|WgZiABm7CO2}WmYQx zc1O|JA<(`$b{r4)M)8i^qhG9xgg|YZDr10k#16kZnb;U=FFO;*8KC^1#x7-;g z7AhiM=(8g__y(cDO?g!#PyPH|hja5}k1Gyy=z^S?#A*d9kSaacSEXG-r&GCpy}g%r zFO`$wQ6Rfo%FXx2u}w&d&WkYlJ#G2vg|!6Aq_>Hfs%KSg$Ud{13iINtHtf9b#~dK4 zgC{g3jiMYALHKrAs-l_Bjw*DPkwYBt5-X8S3Q@3(U#vs$@S-_oPmvROx=2FV!lWuk z);?s;Nz4#&g>z8#7GFIFf>14Rx{j*y{hC__GWIe?UWB`w-`_`|Ums?Rp=QO~a*Ov> ztyHdj6)pB&Ia(^7oHb*~$(A{(Zdj>JjvR$o3=byxL z!O;cz1OqW*Qd8MC;9XP}DKqR_;{N5uP%`+~+!8>;w)3;O=FVwyimX}rAK+o8V}A)y z1*zaj*G-smqP9uSmvpxC`8OWlwY#Eaz=qwTTIYXk7ejPvu?7Ak5HRa{kl<-2lPBIj zwC>g%nLSY9LS|*O9OIGe2li>9Rfr-2u-2;N+&PZp5L5(W>T5KYfuB!7pSa{iR+EOfuFY!2p&}v)rm=!+X%5>x%#B z2T8Ervm{n@761E%%&yl>E@0=mny{nns+{1bOPbxRl(O3I@-w=sjWnNtmf&;an}`R< znU$PszLy3ae3kM-=$B->68><{w2>cn4_L|xy=#r~zK5AiJqwrr;Rk#j1Zf>kOo0F& z#%HR?+mn-YTb2NpK>OPb9`ZNruia~etU4l|{}B~67S42e84aQD$$nAvN($!&7wXRZntUQih{Ona?ZHnKweeM zJuO> zs)*5_4`<>XOE$6O^(zD2!236dGyWmQFc>vD~r zr?8#bt}$apr5reZa6D}fx7jc3+3;CDy6SL88^0vh-FPPf_d~fQZ6to;*jgSOTq56N zJdP#CXv^Re#laJU5i~+1-5^xcud~Zrj>r_mF*A^d7cVU7C?_nWg3`6;CQ0&eU#AY~ z$T#Q1*rh&dGVb@d%fh0=lKt02B9-jF+&KVFUpi<%2nuY(3D=My<4IS}YCNhQt%W-1 z{+FEe{2bXTEnn?c55$Z)OqYBwt!Q=Gai2TDC<&nhT!2-Br0`)69wEj0x)L?*U@!o2ckm;CLn-E;Re z)l^2pMnEe_nic9qWf66LnfgZ1=XH0MKeB5<%;FDv2~0KOYsEEoo-6)oR@*1{m(2W7C9VdGqNLD87o;1ilgZT->j?etVl}gmU zSmy#u94Z9H7hhl@=eTITWHl4Ux(4LFIZYZCHVUa=UMB345SRv#T|f6f>Mb}@bKVxg z&Vcjnhn|W86}sf2iejqrvTAuvRLhE=@kB$1^gWs%Zc}us6Z8Ljk>8)XV?S4QU`-!a z*di3HAeUuIvp)z+f}{zrhmVD=2PnH2B0Mq>?5G-%VJWD@G83Wkl+B55RWw3C`N@*@ z)63T$geX!f%N(4H*(RX%AZF|ce(7!dc{?VBV(mfD%cuX`RoT2xhOt2P+KgdsDXvKY zZ5**YFyl}$xIhfED+O-pqizDlp+RgR-L&|3U0Xk@pzwY&BK11=lPW6mOR3O2?zNX; zH!ctsitVHGtU0nPJX!o;(B z3z_uazm|y(SO8T_^jc$o?|Y0TOiSS9?Ec=^%fUuUVyfRsKUA7ao`|YTnV0;t@O3!) z&i?iMMEl5DD3MwlksQCD390L;-c?ryT$Na??l@OBGq|b+T;6LNTo@$xDSs(@g-wdI zp{i0H-ldA*AOZQ?xa7gcj?eO}a^jH#%3rlxSfY5$vY=o*Rm^GeczMo+b3FPCFN!w@ z8+YlMD4M9CPAX1j*3&2nN&Fy0YS>TlAG@z|305xF$2JU+8&TtmT1Q1O%19;ny^xMf zss#5hyhs3Wrs-dB`X7=Z^2V2V5+nH|ctxd+@}SS0bHUZoL#&5ne!Od}JR}<)S7%PD zak9E}oJzFlxFY)xRT+1}6ua4XsTV*^OgdV<^5f>izqH~62c=xdbd+h9>#hSOb<H@wEQ&<57rg*{erTDhMI^?gX)vE&ZrucwLhSbb))8ZIu`7I(@5e8rvfDU+}*gZ(dEtV zZ0wsWUyW0IxwGJ{X|C~zLD;S)1M~$va2E<=|L^8ZQmW(8MGzKMv(V6-D?d;I!*ylD zTx98qqH-iInV@8{LYdIz^NaskH-{K~$s9TVQT>ya#n^^BL*ZRcf_`u&ULL>)z&WTY z4gt>6Okq`1)zr>e3VcNf$`nM+gv=iG8ACb;&pq&or0p@g!RzboNSSVI$76TDvnJ3E zi5yDz`tv11B;0|%T|3C5cbQYCK*i_8*)wsh69`r{?%^Y=_9%~%EP}W^;eIE`Z5$EM zZtviVU|D3=2a{WmnUwsxG|wl9cfXULtC0O&R$Kpfznh|USy)TG-^2Lj7@M8v6{7&n zMi-%=(=)zZ9;fHAG$S+I3vD~G z9pphhzr^}K8TD3$1XxhS9lg&+jr#A^xtg8_cCrzfkSNF~Mtv@;X9d$PgQikK+GHe8 zN+b(D33%sQO1G!AP7=ieT3oe|ZX9F|4l4ol@OwcW8tldjh;4Z2=}4CROnDVd)SxMv z?T4|Ij~>n~P|3}Af*h>4kO_P)I?&L;n)c=-dN50G{IY-*RQe+A1Q0nEhH@woC7geprFAgp_(kz}#1Az;o~Gt`Gb!~Vpjy<}VK&(a41LHwyxs@ zl$_3-E^S~%wB|_}lQ2t#6d;}{oOXozE~aI^?vRRLKH};Z@3z~V7}2x`wkjrD^8v$ zInyQimk2)-)q~nSD#O}83KwaD>Mh*PNAhE!B@II_iBng68`UmV5;JaUu>Tkit4Jv; z0?k!Oj?*~}gBL+{kTF2Ebbu3cWUBRKFf!XXPG#ZC#cj7s{9|~Uj*YFJD$vJ~uyxo4 zNdm>fE4n^JVV-=YinL9&unREyr-8y}kIIZ#d)t8rm}^IH|9Ujv6(E^7msH zIHOtSk^rJ2kstr2D!Z;t#Dl||B<&C>*~_2bu` zINq4dfYP@dvPJbju?A+88MmAfc5kXoJkmVr%$cz&A3U(#vF!Mow|DK#4OO=PLgr5+ zsVG=Fe)*Y4epK`a@K_hA9U8b*^~HibaCSOa#Y-n&Obk;^zTnta)jR8^m8)5C=9A5L z{Nm$htLE_6!ubmL3^2c1^W)Et`e@mLihrN6;XBV_EB;5rBYmNJ{G>L0EZS%O`6{+kM!j%`&zybs7h7(e zTi4gfQp-qNem^nOaZ`R!8Hvj)@BLrYSvi5AU9)MQ@%YbMPRazeQmQ0dBvRXGulpxN zn6mLCdh1hKf#K8oW2N0~O(bA63>%WaE8c4G?;`|~2@+W8E2=I&saNFeSXyZ9<<1Qj zoFVH02MD@Ldug=--BGa_1x{jckS(mA-ICyjwyY;DE2AhiK}f%xJNZr>YYg!*brg$p z*BrT0yyBk*w{yLO&<=0U72P-jC$(@pkB_I?(F}UjQh1!;nzM`axKrH>A;9NJEv{yZ znCECa?GmMfE_RbmhsX zGLP%?9?OmUa|d9#nsV|6{cFktf7Z0NAb0(9nD-Vf_Y_q|kNd?vSH4_Qb8gEWyWW`p zw+Hb4SfoQ7r4<&Ua;O>_*^oQx?dc9*dVN!$y-{qm+@~9PAdkB8CS{v^pD%dl z)C=y~)Mw9?Gi?Hp7B|Dk%^yZ9(5jJ~4CkuNoiF_cH2*``2L!3rGWgBauW09_KNB)h zUokv|(zq=NyAu>=7fH2Qe${s9dshb4ZdFg*hj~{~>p9MU-Jt~DO03bXdN++^*u3$p zpIv;Uu;J#<%j#?sB<8AD92z%Wpg>f!&CxcwP3{fGaXc0j7f|u@=`oUB{DUqRvg?h} zIf|$rxJEK_^Xc0T=dNq&^ZTTsq-_LL^nM}OC9jQ{lX+y9PDLr4$)63Hv~kJ#i8Y}R z(2}B}ht-ccQ?>ueU~Kuz8;V}ux;=E+FpPZGdSQteDW)@TpV<9(uV2T(;+yY#`VYDZ zxNDt!pj_>jg*4Q^%~8Y-u=i=k>yzL8Sosr&7i|1y;}<6`C^$d&8`fXa{iCzP+RN3u z0DGZ|IeSSt1}9l=Kfj@KEEzMa`S(|y+tvJ4v}MMy4HUwu!s+Het=jx$8MaQA;n35H zpU^IdMjK^QW9s1xnc+E|PdPZ-)~w9JOQ|T722?D#%OM7cTR-JS)Ev>mCM{CU2^LtD zrsv0f^nS1ki$aZvs^fa*)pO_P%2#{UE{a}?t8CK0aA^ZW9L}xT>4(Z!b^RayVAqY6xo_3Botu!6+z-6% zd}G_c;K6A$pr@<-qot*dB^yROQ7fy#qE@vulR?({$VkYgWe;wAy!QSlw*U6PEiaw< zzqeg>eAI`dsF*h>(zo5f_+Pev(=C5hhME5%mw^p!J%?Ves9k)%81<8MYda@qku)t0 zcwgLhX5ejC{PC-MPRM97jHsJu!4Hqvx^U#>XTQH?$jA@f`t@^P+4%al*LN>p*4Q8F zyz11+F)Pd)bvmNE1|*i!SC| zQQPpx=l0)F`{u|Qo!L3_|CUm^Q>|^ss%6EOa5UED@hdm)X})_v$KT)cEE--ZXK3K_ zue{+d0D80m!;dT+sau!PCy-`9qE0y5+)(_^xL0qo;a5QyJqVcnW#x}Q`|?lc?cI}` zw>pgSV-QY5$9%4K!@W=Ov7eifTY69FSEl~YPcC_E`n8f21$nl-mpyeMv;TpDFI*-V z$3$8BAGVJedc#k@z5AViSn$}6uHpk-m%WYmgPVZ`aDhNYH1h9xk}% H!dw3j4N9GW literal 0 HcmV?d00001 diff --git a/tests/ut/data/dataset/testAlbum/imagefolder/apple_expect_flipped_horizontal.jpg b/tests/ut/data/dataset/testAlbum/imagefolder/apple_expect_flipped_horizontal.jpg new file mode 100644 index 0000000000000000000000000000000000000000..144f6a02e098d25671e1ef239b4bc46991572cfc GIT binary patch literal 439581 zcmbTf3w+h(mF~ZjG{%Ou)R`o-)|zSObS4c!+KyJGN|2tEQ+tuj5H2FYhR%RDyZ}|O zVWXBh6UNDz7D}f|fl!1YXiNbS1ViniOmPYcl#3vur7__q5<-AncFy-%>;3I)(3yYF zUq729`}TV;>wVUt=PwJ{o2>R7E4Qu#nSjc)_XAa)mVC({?otd`tL&- z`Zx1KAIkX9Wto>e>hwJ@!LoGWq(9}=u7;g>6sZBnb}#F zW!YQ)pZ@9H61(E^%irj~B_r*#vGgm_GOkGL{XemM+L>w1qp|*9T6)HZGB3-b8U1*} zx{uTRj0_r`NsD=RC9lUmbVcT$eddd|T=wUs-^u#ygE?P%>|d8&{+Gk+f0g^UP4E4W zzo~fW@egO`4fw<-ul(F!{dK|T3$MNI%h%s<7M?LQp(WAi&Ltw)cYZadR{w&UFS z3!Um$S}eo6#lIieuPf+Ry5=BD{Yp!Jm`BDHAIkj97ccv>TS~LO^WdL<_Dhdlo-_Pk zm)8I4!+-fVfBRl;#Y0WmdH>_uw?B7UU9+D3X&rm~|5wkF9qa4Yf!Ief(wMxAD`Lg5 zo>*_}wyp^|@qG)AjOec1_w%%lul2@O++BRMt9{kkNj=$fvWpw9Xicipw6W{qgU2?H zU6PTwVIw3 zH?1TmzHRvz^zGRN%lL}e8v1{rcz18Cd3#lHWATw!=k$N^(Zu|}?%q^7f>zVC_9gc` zpexa|>Wu~0kI(9^Y%0mW_gqiGjj!s8JYP5XH`V96mZVo*(;GXo`@Nc7XA;j$iK+3m zx#g>MabBZI^R^F(AKa4@i@&xYZ$|m*k!dCQr;nDza!N<^%ueK_O*#>8-TX*r*(~0w zw}(f+9G;2Q=1=O4E&fc|v8}QE)={PUR2i-Am{U~R8~b+gb2*i34!1AKc(`Y1>DNwm zexD8&H+Re%!ap{cMfR<$ADnqlSu5XMxHXnDuIa9sy|Mh(MZ|VY&w?pr0=q)Vx?*& z?Q7+0i)YU-o5d{6$sSaEsm+Os{Pkbt;luOW&YevONOL8uK z4BAq-bxdvU#{D#W^B0Qr4u+7SWd!*KJ#Jxkp04ifHfy=#_ygGmU6UJ*mP{Wsr=X@P z_5qKU@pDQ`avNWa9c1{Xm#$Adqj9lOf1XdvKvEyIG^*@G>ym!nO1o&k2WqN{kL-G4 z>AiY#@Y<1gt{=FEuQTb_S(ZRstKzj4i}f|_|lKDu{nb$w-ZZ){IlYj5mZ%j_LlH&#^lT)_zUQ!{t0 zAFt8clG?+<-(Uc%+LMEm$*8d^taF8%lT9!KRdbdlCQkA1X42He+C5bY}ng!V?|ZL^K)O|b+NHbiW;ki*A2^KI;$@{6s_6?C?hR$L0D>_! z^lDjRQ^S>Eoy^-Fqj$>^Lt8P8o58nl)jY9iDvit0Crj0IcK6s*dxh6Iyye{0FJL%-&dyky_eXMjcpW8iUNnUeRLCq60UI0x#6+gP8 z`o;M2=`F|0TE8&kN=xta2`s4T` zeDScX&fL7k-_eMxPs1`pV`|!u95>eh+?_O+qnAe)WVCeFFRt&6E!h+Ae6Z*4ZJqJg z2kuO7I5>M-Z|rz&{@Jec4Et?|m*@RtdRqC;#51e5UN!DOll$co#$#A!erw)>iXBz4 zI@wvr;qI(XmVDjl>*Kqd$1s-p7andGk~RVz4c!}iW82Q*po`4jzd;rm=HWoi} z;1AUo&eiu6yu7J5Rsx70D@fnUTtB)!X7*KrTkOfMDtv`;UOF|i3eUmU74IEh)e7*r zpJ93@b@WUz!)!_15#ME(cSAaV)__M?CGYm>es}shKHpwL_lu8iJL|}o-`336wD)wD zeuHtE*g9%{tn@D09e--@{u7nQX2t3fPeYl2_jDMME(O4b#b;})GV&J7dSm}Hg5LP< z>f`R%M2Y0yK%245^D)6NoMBaOjOoau!NY0i-8}2fVG(*RoWv)O+jBC*GmQyQgSw z3K2pfWV$94bx1liRGi?s& z4hE$>3?H#hW@~G1c~z{wthF{LtsLxVjn(2$e4R7y0OEBUDyr+dX0I)sd<#xXGh1^i zadUCD{Nf!eYvvU3c4I+}>Skzngy$vw9_SCO%JuLnkT7iyem=0(x{~IctKK&Bvc)R` zDOQCy8nbk28dC(N5%@z$Bmc`+FtC@7tHxqrQ*ut}7bwXrh9Trj@i=D^#Z%Pf@c1$a zx5ymLHKU5VUuG%SmE`V78(jo_Hcv4uOI}1vfKC^+TF-4b80bTSJa^kSU*rP$@`*2w z?P^RfO*|b&oF<#|G?rEaA7KPW4~z{o2)8z`YA+rBMn5K^enP)H<(%!o+%9P`T5%6V zO3Ydu6f?oGGP^K^$UECK2frD)!1w>eRPD*EA#X5^ADjCECUMo<8cG@g|AAS=#;$5F zlWir9PK0%V5lMLP&{mntYQhtnfV}B_GXc^B`*O$K_lk^SRz2$>R#w^2-a@CcwW!T} zlYU=Ie`Fe8qNOw^TbIv1D`K6;uUhNCO0HQw04IzYyGG=J5D0`geQrcIOnvtbF$)}a zZn0y+STqTv2c|Y%PtjAko5rwDKKTQcHF{ym`%%evI$L zySE)I-nDVs*;VbaQs$sJhWk79c8S5a(a;(OCqFTvm8Dm^Xli==S%|~Q!Lp08c|%&q zr-$Y?-!XqGerkKcl2-H1nL>BLH_WML=Im=R9i}==l6}2~t^q=xId8j}JZ#8D z=;f}DV!c4SEs#aFP_?)UG5ZPeBijTX$F>TO55(TOSikBP_bDp|y|Z<&=g6(lP= zIjv)2`;d>#t)G-Ak01SmTxwwh(dVv>XW9iqH@><$zOAKurb2C7l(?AEj^g|Gt{#~g zKi>_m6b7ziG2(~;KUo`#KfC9t!|fOFpEmAry?A}~FDqZsw8q9bkI-mqHEP6b=48>w zSspfNooMwb3})r#O)vf0>}`oBh*r!zpG_OlnLDPII5egxrYx>7I9*Xk&0~qV9kVpR zj5jt3JGEadhE3llz^Q}zE%*-1s)}23% zQ>;%s{q@9RQ0Dh*N2VzftbwpTi$Mga>Ng~b&w?26jf;%kxbyY+ZU+*eOjk=|l`cMh z`>h2QQ^p9k4Sfpeh!NQl2WP_}kJm75vsf?Hx7G_6Flv4dUN^C)_{4kz&j8_Jm76ib zDhX$EMy6R{A}5vlSUqxmj2W|Na`oSV9_F9&V1HUK3~N0jXP1>R10aku(nJ+Oa2Zk5 z4^kWs>*&8i+G)>i;j*TpjE*T9!$O3n29`92>Py~Rw($Jhb1Po3xno45Ae~@142q(#Dj(qVHwvUVRJ9Dr`_<^daQ;C% z9;nSfeU2`MxWD-LYp02B9->Wuo+uBq89AUL*nr1gV$2o_$@^UVaI944_+pe1MI*2{ z(f;Q}WfUgVf~muDkC}BBZieBAmoA$6VywnkvR1>h(h3GX(~5dTT$jRBN-)`2d3fp* zgI?3rTz?BAw%(YF2HQqvb*x~GY#GT)nv(evkiV(`B2fZ++qfTv4$4;l&2$@RCL4%f zeER5l$q9_`NB91UZV5nu8tCwIutr&<7R}$1Un%grtd(y_wYiVUV~5WaH?Fkb z3{!kPKt7{kl;D9u%A2`_0PA(K6Gp*#gv>Jf2*`4AFkUiXek|VFdCB}`vxMM4F@n+R z+g?TE5aIPuSI|@2)UmVu`Dc3W=8GBN@aiqM?wtnTx#L!!lbMAcTD+J%b-x}upM z5NBparp;j8N4=H@GumH26LbM`Bla^aGq*f2(i&#ihaOt5P8-Q%owFEaoc$hvIzeM$ zdKFIy5d^Rs_P(_qCv1Lx%<%oK<8u?v{PWw7bncng(Ei%<3(DtwRL%mQgqF!n%j<|& z*!Sne%-2q=-`};Xlvwq?oKl)72}5$7MUY$uSF*{50s}%%4hpS$UepSDbq@aa(6t!e zE8BWw$LvKcN|BR7gppZ0+vUL9nnw@ZX}nugE3TZ45@Dh>UcbCIcIH&czWXnnTQ2Bs zKd|^j{PgL0$Dn$*$M-!o#zLK;TRV34#>&R7U9+TR{M-TaaZn8Qx$}+F2UT_aB;LBN z;#6W24q-+9-s%f8yZRr!Wrsr1CWUt!=+a}v32TS?00o6-y25XaQbA_4E||-9i^?wy z{+Zi``p&CKyzPxOu2S1=KmmAEP6e4*fY5=sK~A$qVQje~&GPH7WyT!^8A^2!-&Eax zy!d>ZE7~AW^P5$L+dA)4kcL_=+7-jxFp6vHJL=lE+X}TQ6U~f3A6z!fg_gFC!gtBR zrEf)fHm2mT-LW&iOR=>UIWwYOY6dU_Q4QKf%b~0ywxA6^nuldc1Su^c#3%#A@rW3Y zW}CR_#Ke2?c&xbr^IJICbK>iBhXa%e@Pj+(JjDeN8+9-zWR) z54q72A|=66?PPwgo_~99Y-zieR_GyZK7@(PFVknT+XF95w*^NR3bzK=#9CWxIAG}K zXl5V8rjfkWG`qAo5wfEoYrPm}eaW2+As$w`XzMW0*9HW+Fd0$u#2CprBLC2O(^B4t zRscp(tDy!fl2tDsWqz6q>*9*exK7_A7s#|`X%&N_TgS+SSF%t?7d9~Z3o4K`fyHuTjLz3(A5yC!<4|14_tm?-ot z9TVrenELCDMAr=x#$vdBUTzZiFE!q>Pa?U6cX$@zZhkcwSiUv&rk>C%D@7US- zK+lvlrZlD*1cl~UJnHdVd?_PfuqZsx>k=t#(oL4azF4r*sgq8IQBXYVJ}l~;Df5`& zoQ#fB7b?51%ZV?EZ|k8E@DKZX8z5UHbu&kk=Ggt75_ZI4lG+VK0I1zlxEhc5A_ zoo9i(>RUfHV}pX_lVfC)#3psO>>%L)$V)EwM(o;QmtWP6qwbCU@i(l@;g&rZ+`T#e zEMf{)ER3!ZbXd5vwfQ+kUv3l~JI|l{-BXx=a7t^Rc$m>MRDi^Y-}YA+q`)Mvs8s`} zfI`;4DFK!&V2ZX&g{f&BI^g~l$ zIrJzA9!z`5ggqA1kJ}R`Zf+?md$i&F*{&I5HK5rAYu5-c-FE?*48dJ9OYZDTJ4GP? zkSvlY1i58Lask+-jgGm;5ytxrc!KClFQvTiCrk&$oP?W{-8BojdR_J+~N7N z^H&Ij+XeKBO>F=redVN8?5I-gIbXK1q@RP=ith98D3ghEVHD?WXLUS&~IiM2Ijf>##3z(<7SQ^sR>$DTVS7--&mW$3Ofki>q-Zh*P+g*KmGY7d_`qG2I2t zgE1J_#6R&R6Zk>;eC?W3kIKcim(44_z!y)P0=QUX9B{1x_~gPUxOP@#?bHOari7F@ zW)QOr_pbmzeDVOAgKW5PvuRI7dNW{>!|H1>?i8i4%b6r_Y%ycL`A#CUyxlBSBF; zAmQY-lc2}K19c_n*UlTTA=j0C+De*j!jxrA%vGLM{7l_$Oi~2XDr9;Y;2-m6>I_bo zFA}w4mdhA7d`X$GfQbixfM(5Oz4%9+3J}yP%P|zx=A!<|HM9;h4VFORB#EH_%Jws) z6^tR=^ME`+mk z`_**gL~DN4v)XKm0p$|+r|eG%f3B|u>`)CwHj)M8W9Zu;hWrjc$%Np8>Lce z)~!|rE{Kj`9EYs!?kA6rmRXpQdn`XH?d^^@t_8qsKA8A<;kNyRY_m?c3nRPAAIUQh zUvsLb|8w*(yMAWb$>PRD)6WWG`9~WjFFVO6zx{jCCQXUO9UEiy!}y~3g>z)dND^ee z)NpTp`g3J%d%ETWn2pNrHXWlc{vJ+-k7MO9B5{rjN z9JDyGxNVXzDy1L$ zr?g3=9U!PpqS?*j=lwn9WA#U=DIU8j)@eTn#E&kT;^CCIoix-)j*#M(OK2$5RcM%7 zu4vIBd4|yO|6;tk2ozmKwMogF9^6%k{3L~i>tiWoiEL8Dr&N`rffewX2vzggbL#)0 zOy;Rk?V0-$3a=D@7lB8cn2v@h_1{Z{Hs);+s#|eX4S%SO(O@_Gb>;SCm)0!IS+cfa zU}#m;y1t*9RLf|Y&DlMPCwB!jpL=qC`R+wV*q?u3pqx$K z+qQlpug9!gWuX0tMquHA8`o?!I5gq~Q{xo@u~hCf43?r0tcO|4dy9NK)@be?$LmBR ztT-+I%lfIjuTVL`%UV&JjFd~2FWE|8%TkNn z7xf9R9mVa>Zz76MPxZH!9*P_gjPx+5*x8DO4_`*nM>N?gS+ykme3rt*3$q}kzo6!3 z=gGxwM@kkkr&^%-lxlvLPhk(>IhMOdQjETPLAkjs={d%3B}tgZg}&QfncsGt@xBQG zin)iDn%?EIV@dH z9m3|QYt*%7R)JgC(!GCCeJd)Pn35psrPV5xXegRLN|~Ffu}YM^?ypI}uUA z5(DO@&N3y;8kR>s2YggA()Vg*} z*QXB=i5Q#MvgUVno%f-rBH#8iz22d|sIo4x`0oBPx8c8_dc-}rtf(zyxMXTZ^WNoO zq&*gpQ7lm{>jjPuP_rNkt*{D2^Z6! zi?(0&wj~mOz}JXBBvcS-zg=nyrDkY4Nr$USOHA4z2hk!G0AW0#LUDA# z_{R`z2@*9@wRgKAn<$q0Uv(LI-*ArtRUzw*teKkz~*6tUS&11$QBkt*~T?Nhdk#>XttqD)L0UZ z)n!Q3uo7Lh@TdHGF=qSUE@VWXk=rg(`wW> zJOT0D(c<0lmhE!}m_e0eFjvo)Jd-o#MueF?RmJaJU47wuV9uu72yR}P<9K@^QRH9z zH`De(Q~E^aOZ45Vaa;7+zA|D6u*~`rClNUesec&?VN!z~93-}YT7$Dg=}{5tD(1IjJm z)wbG@f<6i&AosW%_!QJTH#YX2wqus<<-6~BhGl631=7OM z(P>5KOj{;d^I1;0NWXZ;$WnfaMq;U8GN{u3#;BOWRFS=#h&M z&v>ce&w|*rw@5vn?jR$%U18GfPaW#rM;s+C#4s#MO<+``$Xmi0PYG?7w^K z;+`9A{v9f)v%MVg#?nugGsF?d-Z2o2$6Q(2qdPk*B_j;W1^Uk=Ca&7rJZ24%AuC=! z6M`qw=KY1$0%epCnRILwZxYcpcdVpMqEJ)~M3_kwPuxgxowPM6tfVL5aE}f_6`2Ndv*3f)P|h#Y30PnH%og-Wj?F^J;feI`QdYKVRObIO_Z zhdDeO?Z{cu<~yS?O8J>}L#fHzr4njZofRMn7Lt#U8ti)lf3%Htsxl2iIg_x1v(Pe# z3b{j`T5sJWC28>5G~51=JINfTP!#Ut6S(I2Y1*oz{)_yGac(l&$uG&D87y8J#S{<- zYUfgirT`(z);Xqp_I!#N56PXwfYfDRO?D$w;O7Bt{}}o;!XN3&;Iw06~ z#0?b#4dP@rPf%=UaBW{KJq&*^4ouALInw zee9}^=?qW^(nTmoW16Xu;#T|qWqQ{23(G2M-z>xS>FThq&q*j+Ga& zQL|>pyqAa2)>Z`xBYJK~tR>e**44II#tPJ;+g`1{FoU00V|y0Fe(fve`y$%Y?z$89B4Vw3B=b-;H+xxQdVLI5qE>W||NkNB*wu&3mnG|IkBN zQl`wXcJcyfue3sq;|)-W|_ zQtrjuOf5T9Er-b4ldI8PS*9nCFDCbJ z8SQCMUQIPKWk=_XvkeQm6|OG1#2q5%6ue~&Q)vE`PSU15zLBH%EJDD6V=3)F|f5XKd54cY-aI6zEKF#o6F=+wt)UWB&& zhR8@{gzgcf2(kw)x`cij=pYHDK3R5gr~NyEWP7j55pjon4u-8#y$w`~tik142dEl= zxSM&7f9CBVo2h>0(7kdFxsPoW!uJ=OrQ9X8A^y7toU&QwUKOZSkhTEbz=auvwX^Iv z+ErCS&Bl*kN+$R_#Ep&7CqMXSh*5a)BEi!=m~FNsyc4fsmzdbl`yPrRk}eE8kj^M~ zZ&AGSU<$O6xsu>WN*0BQ=ULe)E61L6 zu@i1g5?6@yt|+{}sd)F@%7DTwSq~wg^9)ann#Z%q=!+X}t%m@!?=n<8WX>8~K%KQv zW$@Z*GiQw0pV&6T;)z|BdR#->ZWOr#fpA?236GluYc(^ZjYqD(ac?ubB-j9~8M}3F@D| zZ_w$=GSn$k9lSM;4Pf*`A7`qbgPM@eRk%C2nr*joq?lom5=|NS1sZ8y8eL@wurlrtj<=(4E-DFlqbz7KKDa zmbR`SA<%o<@-9&<>_adoQPfI)%od)UdSS?Z*gycMH1vi|>uM;AQuNbqbps}HXeCUT zI&>?Y%VFl&tx%%+6hkUJF-bM;i(!tbrl9EI{29jiK#>1V#LxUuUbMB$;A_k}?em3x z`k4rR8770ESgXmR2>5|4i+$2>frZV;H|m#k9*gZ)wKq(vKZl5J^tn){Ah74L?Do`b zdXDtZDI}u5d0Z^SLWhpx&UlX*_5ng(Rpzih6_9$(dEbs>YEBZ>HR0JJs~W0VEzwXG zXND;Tf^zSrFc#Mr;O=f~-+5s++fqJu|1~RZFrb!lC&3sTEwnnAZT_$*dQkE4&WOqP zGq9KIFoGFY573g%45=_qFMYsc5bM-JE#_y2a(e*Lv!Nu7I)7#g*Fh>N?DT8KDJCH& zu-iT)meVn>DTB7iOCCA7m4ST#p-dP>%VS}7@w96f0pF>5+FAARB&XMl%-1S7Gs_$) z=VWd^8f()hsHv4D;mR-{j!B?kg4OVXUD8iYxRd<^?4nE-+-N}|68JdEpOGq4W0XFl z@!BU4rM@Io`;*8U0WR&0wwWMR#H0~o8<%OX5TW@N- zo#$ZTsZ14*=kh}`Y7x&%@hKuof6~LH-%yOI)tobOnq_5Oe6frF!uHCD=4&;(&QF@| z!b1wg%+pwCFAW&dfdkK-BB?*?{TEY?OD=pr;(tEjkf z_MXJkwk@lnqowRP?hX$-k#wWli%3~ZxdkN@(rKxF#|t#~d!XMC+YFRD5vie~DTbowJ1*yBB%a1~mK;VaT;!bFmN`WWWXJ19Uz0c_V=+}v z=-y^DLdhTPXTy;p9^(7)m%fJZRmi(TTbE?$&>Zu*zMe_3=d)9xs5Rr)r@~$~;D&0@ zt0)ci`kU&(7iPPrLxlqF#9Fu{PRzQ3sLb~3?;E~91FsTkTynv%4-_(h!*EDJPDam{ z_MJTZ9i?Km=wn?2_H+7A8Pl+e%pK*4u!SdR^Ysb@Keco1SR!Fr5ZJ&2y}r18mULLW zm!30_!f-gE2PKg;JE7-?sw}ay5t3dr!gSg##SY7d?UU0(%MpWQq^dW@R5)x7fGwP| z3X!x3B0UNH&GahO6>wc8{NUWahB6uGy7t1`z|hjg1*|r%Cz>(B7GY9 z9+LbPC&tfr)oQuYdoLzZX^SXZpH!8EMK*=P)wM3n!+{y9DJ&%9(9oQ%FxN9Cb(?>3 zhK1I&L{3nnEAIGymt2d6V<{3^O(Y;AZWjNcIY!rL*!Z7F>}PFjQ(a|~&>{dgurgqA zd$UyNv#N_BHc&zf=ds$6FpL`TJ5uLf?QM2lmBtRPCQQ26;DwZhZ#k$`1tj=7AU;T-%aN|+&QkWRA zAwvk>fE$HnnhGi5sU9D0i!j8VMrW8J%WkSrS4w|!XJoWsaGsWF_P+a+Xw!B#yBX6DkU?5*TZr7cPiQ6P-fi?7%2 zu_ADOsC**DGHt!{Ha7ejE=BySCakNF&aQnkzI#DTPDdGmkjOK2%bC*}!hlx}kp&X> ziSIs#dPXt-xJD!$wbsL5B1et`5q4RIE3cuolKBy$Q1ocD*W(nvf05gy`|qs4t?8~P z4yrD~tcjX3Cv&#Gi)4&v%jz}oR*cE|qLC%B57R6$nK9;B6<}0vtSeEX3>Zz!B}CGg z8RZu#X==CKx2xzcR6(8Nc0Y}my(0ho)GLxpVISiOTl_QvGg{{Hn zR8X64dCdIjv?h0=9Y3Ru_4CypQJB$I{E~gvDie#2>EHxJSj~}*lWnI;a&%mVSg=nL+cm-o zb`_d*)YD}7=crE%dUf^f@udgID*^0>1VmQjs`75ZmerB(eVgx^`l5({zSLwzyt*ua zo8-dZAM!N0vX$l9|uQrCfErZ1} z-EbTRQntM($pkcFAyBuR(1CgY7V{}?-sc=KNz!(yA470%1D%pF2Pl_j$y2Vi-M zl=)7{6*7x-lx%fCxgpB~*pn<)*{|G{&v0(V4xSU>rFbX39LdyHG;FF98bDQ@W@=86 zggvV_WO7LCjAEr5UC|`TR0AWi=*gyw4Yo`pGIazVimA+OJV4N*Xb!Tvvk{lxIhX3l z&tMe9E=1)PaA_p#oh`%&L??HS`8GNf8N9?lYOL(IMy`qdwRR2;imhJLE>)#_xEGg?*?wmV=CH6in$}4}S(OWxKFaVOkvoJGBQS z>M@C-o}IMfN+_H-81V9O61An_dcuO zf$Rnnxy~Q0z7;rwE?RD)xEG`OQyrHtlK%YKzYn`;sCG)7O-ToA#o zI+JHV0nMoIGVgBrk43q2g7OhEDSiRWO|%30W*{0{qlvSJoXt)|LSM*+3*iu=fu<6` z2-pY_AdKC{8j~ES1REdYtr~VPDM4canPC<4b@;-AN)(l~-uUXf zZGJ!@DuHBW+?6p)LUr3c83LldFXjSxAsnX%DltW)+OrnU3xn!^(|R6UTf|J^?2y>- za3aK>0mWH%I+REk{uM}071)3!9J}lCCf?>X zIhIwmmgw5$Utt10RqV_Cnsm;F)7|JXi&sov|ndHq-N)IQk8;Nvp0E0-f#6e z>W4cp%5Z>d&jct0;aCu5YYf88;;+GIPo2gid&s61YEGmsY>!HYZNk>nAa5vZz(Y=R zjh2g{Yit>Zh5m!S0*C}S=3y||3gbt6xoLA)48m@Qs1iR#Yf#Y%ZK5^D4Cq)8`8vTR zr{aV{Le}*4i28{>*1_yTg zxTcv1lc$ezWQ}h?_Wjb#ZY#&P6d`qKB+6zF9)yqah{?UDC1Yu(9bxj(sm5URoE_oF znaMhgm~HskibKF`lrp6|O@=*nnr|+Bm@g*FsJ*B1VmeL2vNpa0O6S@r4pJB=9j2q9 zlXMf7T@UuizS(oQLioWmUW{N{qZ!7pN-Jei8C$@Z)`+S}Xi%^13{Jq-$c0LsLMqTA z12Z13ASV~E74Il;l$FE?lT0MKT)*gvnlB;IbE$N z6nQJ6%(?;J16WUTnjc+OurIkssH-^2x5qONOqbJf-su6B!hWu#DN2dBJgXOhwtofI zLAZtDk+3o@Wxi|t^f@|g!_GmPvJ5;oq^~Kd!)m~jgv$ePO;S_RXBtK1`yxl`>m$+- zDP31SMMp8n5n~`WGAYg<1CY9F*6vbRTB^F@7!ct9;e&R9BR*uiJp6=nZc zeCO6}Tve83-r2%_Nm%9|83{nojwI26uQ#ov(_Y#@`_#sqaZQ1|`&;dUK$(ABa7HUE zn{x6^hB`|nZaFa{NS~km`lM4|_fK2Gq+UHib5hI0shXyUK*JQqLIE4%NB$6H^$2~? zERU%US`-W~Y2(CxxK_z>2n@@0AR#lRypc*_s$ZP=s7N1YDdLk994W&YrFW4T(TU1A zd0hUDa}vfrKVt)XUXXZ+(K5%pv*TEG`-%9{v65yW&y@KDQ;=@>(}|pGVNPj(pH8zyI)hrhlR?|@^Ev_NZkBftHpQ`yVXm1J~Vc!L8GjV86yqd`B|-s z&)Wg&KUU@^T`HHTQwoeK5@yY;tH5xi{Bv-IM09MvarVCZ7oR_e*E`2&IQHh*COeNz z&69^xKu)$kd|d!nYbh18Gb$v(D24!HBZm}z`dJ%UT9rmJ(YjS?dFWn)K;@hmYBo_w z^!llx2%WWqt#d_YAA9t9UD5A2d9br+%-Ue3?4 z`CS}Er4T>|OB%hSJvRO0H(rc_fIu&>_d$dHT-)E!9$UZ>)CdTP;k9Qvi?z$XqF9eNIfpN+Va3QJH zn24Eb*w-#4!AilCt}HeF((l|qz~S5m#K6))5*QcYVWVLab09e`TJI4-aBmUft7t_* z8Ja;^kgtsRK|Uy54$_!V1@sR`R9+@f&$XBN8sfHsaQhK+a3Qi3TQm#s_+AB~k-$rQ zW`RV7T45h&{sO1FBxJ-NdU~)gmW(1dDbnr1pMlXKh|NBWxY&f3vy-Qp;^fg+Mxv~n z^5@VwOJ$j};#7Py2(X8Yl@hx;?|6YqHwAYnoWS02(J&+iqsxj#JeDM19jajR09OvM zG+*WO=2cH_VSe7126mL3n@3G{d=nVlKurc?jLK?3#cJiit<*--mn<;zs<2>3!PQy@ z!-QVNf5uKY&_4fkGb8-_BzwUy+#s;AyAn@x#dQ`_H7a1*!_qQUk{bySp8_mksO>`& z^fBC(ik+Q*n>XPq3Qb6LBV?X+vke z#sF17hP`sCS5-mAuYCgp0l}78bvhUA)akmLh*c)N(y^?Vy?(b?K>_nZ5oJj7kWW(% z&Q6U|np-h2RI$W8E&9dZ$y1fGeDTxD{Dq4Ap(!#o@^Vd~AG2pri1@w~f zmUn%+dh=)*A=}rT(lES4rYT1VO`B^I=J*{1NbP6zl_DS$L6KOB@K9AZb6B1k80(?g z2o%3yK!Qh$>FB`X^|RMC&Jn*<0@B*>Id^HwEt+_5qGU{wMpIcU;$5KILSZh=W-e9i zOvtf{QnZ=MrUv zaS+Vh>m<^%GXMAQWH!hX7VCyhfJ$xCSmr|dk-QGB-Oe_qSAG(hqofH`nA;_ePk;A0 zzkVn~48>r$lu{`m$uaa<9T&ppb6aJit+#3*`n+_J7MUG{VBK*tVmOv;XkQX9avIZ~ z> zSXd>X$)1D?d|g9~-%qXr!9sNA;kHkT0UMQ8vN`b(%a4U!c?yK=kR=(esi={7s5OQE z;hY2!qDz=W^f#~cDFNm_fh?LN9mC>vIbxik&ru}Ff{RKg;o~D_>7p)me|<#F$Itks zJkH(W95fnINQf4l_|R@bC@24uE+>_yLC#e{A9pj;SU9t2w%t!UcG|^cIwk{&b{GV| z?^$1*w`KXGKuA`tp^z`|El~G;j@F)wb8c}HC!6~jlx}`@X3ZCgNLC8j0_mWWEey|%zm)9|lc00wi`*@)xCp0UzhKbC~l$~>Xbif@2tZILz z@g;F4%IlPKDQ=e?FotBRIrB-!@Pz}QKYe+@QaJCMj83TCv;|SBM4OmFC|>8TDDfqg zKVlAbn>NL(iG9k3n9m-&>JMkGpLlfb7R+lFwx-XeZ?;0y?SrP-Imm@u2Z)1Mm}=EC z@{2&9C+VEOs3a;NpdI~cwQR(9#Exv2`|N5Oc1_t1c%YiUhCKvXMti_Y?JT7+8ePV6 z-jL7FQD{RHNdnLiB=8byzq5L7m~`UzR_`z0p!oJ|#qJZ=>i&p#84)q@OzcRJ2JwJS ztT%S7IPMmzSY6E0@9@t0?MMj}`j~%Xg7ikzr)=ZoiOl7Rw*e*}wado{d3{ zVavsZY|3}g>uUG>^Y=E_J7+`Y*FF8XC!OR@YTZ>cj-Y&G*H3YS7Aa)ut{PqpA|jT; zmyu#j$`t<$DFpMV;Na|`2emf@!b3n8R`=j!QDd;q?0O9}`Ve?)Kyw0Z0hhba@Bt?<2Nq)ht-fu%T5BJh@sbkXeTO?cDp_?RaV=V$RwuRQz`hlpi$NeTxIV-w zv{5F&cVsO|x>vysLlk6rg{RTU#z|aozhcq?It)XJ$F#u&-K?+CUjc!2Q3yrhRgzKnesBt~#A8uO>LNdOiCLVZe1(YFUJ zCi!m>k`r@@G$eBzmi$x&89~47;t*fb{~Ok3kd@HtHeQak2VL5!eND~nT|jQ=sIpgsv03!c8Q-FUm$wM1g#0MP7OHU)H6 zc70T{;Wx)2$By|}HH-8HX@P|dst1cjbe?npr$~88_`fGC#T;%shFe*l6VpCEmCOjY zbc(0z1_%l`Y}ZEx*Oal0(LFD#>SrS8)VXMDG0>gV&piDa+r+P9$HQ>*T>vZ_ZD>mI zcQMilulKCsxDhcXTJQIrvZG-cDGToO0tva%#cdO>b7n*~_}qdDbI~@rGQjD7ag)Sd zj!eG+1%mCnvVIXTGRV5 zO}{oaiFfa*s6B9w`)i0r+@WDASoQO ziF45UcZBY@eNepl&ByR8Uv8h?8(Y81*SeL-jN8NUXFWe5{ zUZYcXzK0Az8zMad2_@L#Eiy8C9TgUIwu-A{pSup~cPzu)TWF$Iy-u^; zW^zA=Y%fc&+k#sli;-DJWC$)^7*186@4>E)Sx!2Qz3p7a#1UF0J|ZkWcTg(>=@0A&H38p}4xH zXS%Kzx8nCi61WC~qH-C_8Y%M8MlV4HsYdn*q8nN}0EPHl(N>6P;8c(ym`|3)n9xe1 z)&|)l(WF0MM4>5;Kf)j$d*4Ry9do1YpJp}YP`nG!){!&D8{&g}Rl0$JXEqmswaAZA z;~^OZJn%FB(8i%{#+3@5^&r4JnY>wFO)rBT6(Z>?!h_{*e~=lp%&me&Ke?EV;DXDL zmu~Z98)sD3gu_|xcuN@#4FeNJ^J600-oGr4c$au7cuQ`E2t&YoanGz_Xd10B2g6rJ z+h3~Q--C;LWU%1!f-YbWt36j>Un7oO+o0}HR1APP;_m;qEl8w(r>#8iNZXyCn-ZB znEh=_G3KXa&;u2H@-Z7UzCkr=pt{8!EH8MZ)*!3e!fxoPJrf;7Ay0y3h6Az`pOQgi zL-egjckWe!kGq1h6!IzL)(YqRh@NX9AlL_|BS?)%b0(^(?iMlS!mw7GSZEgX$yC`I=3L{!bPKyjGB!f);P1FVsz4_-THixuZQ zJh}4t-$4vHO6uewJJww0DkBXX7p|tw-WNuuGYo2I*MzYzYiF~nb91z3Vd(3LdAqBh z%dOB^#Y#45k zRqB{dXU%LDo*bwL;C1^F7hEM~`TjHk_fFl~2Y0|Adj(V20~{Npqwfh5N22Z7C|Jx1 zR0_GDAYsTxiJ4`s*N+zkv%|_tXDTmYXhzGB;6x#IyFg{p1iEoJk?AIBh~LD z)JeEycO##p=*LQv0({t_ln6|QSyfBo%l&N|Y0NTeT1;I8aQ*^agOykSp?`R*lu9Gt zS{oKqXz8ZLVz!W=A$mjDM_knE6QRt6qly6pj6pa5)b=F>#7FPc7x7xe9{dm<&S!qHVjWR_8(D#%2=>;}l(Yu&!fY1vO6-{yVvJ=8S^VurzP|yK)$QjvpKO0^9Vl!}) z(82tk;_JJjEH!BJ(Lm~_=H2^jp_GStNk5$IHK+M)4+KiSNbA==6{3$L3PuMC7;4+6RmA0{x zLDFD`C>nwqZLO1=CrQD97t!{NB3mf1VlDe5RScAFj-?1Ih;ujC zk8brLRtchfr$A*+wl!YoDP3nVHK9D;U;u34b23chY(MVsB5~fb_3TvY8CX|*E381f z#krv!wUOM=&Im2bP#)N7V_XNak_}POSlPUW?#qp2$Y%ZIbwm$&J2RTY-Ia-JHERk3 zq&cF6vg3{HOYL5}Srx|;qQPykb4sa7s!SZ5gbkWyk<(GOfo;?TC$Ie4CLLU6*EX?B z;buFGpqV*s^TVlQHJl?o$z<1HzsZX2;F#PzuPVR2 ziP6w4_)HHCQMN}&0v@kqkTqzARn4j5_?~L4!7@x#v#!PBzL}A@2~A#mx%i=bVFxLK z#j;_G2)=FED&)u`Kl7X$OdR#}2jt~zqbR%qrf1c$xRmxUL#!F58| zPg2Zi>_CN~!irUAzay0!D}>k_UF+NVgGDBths+ewL3_thhO;dCmCv(NRV^DnmOGJu ze_F|yP0Dz({h|C?XfT0`%1%XpBEqDA%A6Kn-8Y3#o104~Xq1BXCAX7lHx#n6(P+b# z>U>Ei8f_I^2yw;s7?1FvL>P4F%hLtl3Wd;GgWvH+UX^UgDMBxbJILElVF3c~uCynae4e~JBrwZu% zSxfY>pM_aAObSaIn8D?T~2isXhht{4ER!?Vni`&DjA#L#V`5!U%rs~0kc>0zJVP1 z@8(4$d8s2BJeGh{Ai!XC3uDw`PIBO!A_o6HL_O1O)p-79%a>gL%Q3}9g9a(y}& z6tf6zh}shU)GUUejCVLNWCI&guhuFNG8xINY?72ZI^i-AhcX=99(Ek1ZwB&5+mxU} zZj*?a?K%_Vh}wW?Oi^c1a4wuHI}#icpkn*F{Y2)7tB$D*{5sDmo~975a7aLwVivtHO?>KMnFsT5Og z+bF$O?d~ZVTdfeVw0suZ9Kw0+(T&ASmhQlX z^4rc}Iq+N==-8JjA-z?lT6v*r%nIb(2hmU$k*vbJ;N>9Z6)~HG*Vz%c1__2^*f?j` z02&~#ZD}!A6!vMVDcG~(K^222X0dD_!{C+0A`TLHqzNh+lz6@Jq{Bh_8#p86)ha1K zVe2h3X_K7(2EXCUlvcMpA!XFxNP>STT;}xd!_?-z^usY%NBYlTH2Hz#v!IgRD z#~cYUxd@|5Jl&iCb42~dcg z{NX_!71hb7`u6>LxsFS?ynHc(ljsL6)|}ZsRTIm6J_8l^sf(tOFu;IgC#mZLJT?ifSeJT426%~aWK#Zs5=OhQQ2kd6nHZ5FD%!ZSM44b=yz8C@} z7dgw&l>)B?(uQSQC16*mWJz)z)1}u5##~<4eqge($85WV-@h|d=L9uF#;Dmz-#lPK zL(7};oK}EB;5OeQjLBh8?9wGa{?4Dm>ch=w^*L+S+Q_;VDy)9 zO%}dLXbyGQ+CN8Psa_pwE>ZNPn@tNQn{q4smMKUIl+|>7>o!%5+a~_y&F7YnU9+VB z0AiF+?Y!_{&8GJvs!z}&k0O9*JHysufxjGlhD1eqA$l``Fmg8n1_lFMb1wux?Zhlt z=Lrc%x9LF5w5g^sljtv-6#$z2j>;RK(3S!XBo*}6oRTn|S*3#wjdaZ09##&;CE^xU z^oC$C2@csfr8X)x#Nid+nC53%KHv&5hfZ*aQh%lH72@_C@=hJ5?KA@h1VvgjCR3%5 zvgFbMdPFyr>e|;GpT_)6vV+d_ls2EdzO+-}YY3h=OUEnTLcX0G?abNN>4X^U zhcL?aBMg@#TccxhUhev+?z|Bdnb`rOK#@GK5ebF%G$|C2Dg=TvZl}Po>jBne_%A9a{1Y5#3llueVk`obiR-uZPqu5g)U^2pT!ZuAQZU~jcPNdMX zw}}ebk`%F*LHH9YK5qOqkS9ZN7ay?&clEMAZ;TF3Qt|Ai&Od7HpZ-$-Vu6Wdo!~M; zm*4j#MzA8_Pobb5K;cv@)`V0jhw7ikkz3&dgQ9vBCfvnI*Otxmny{Z6WtT2qilJ#* z0`Gev@?1XRe7Oao#H98@e+ElyhBd+oz{yDjyKsl{gI1nat-EJczdMDsceAq~PlhPP z<-4jHad?zWEOhVZJ4r3@P2+~Cwpwl{IGe7u8|(VcnvGpEc#_rDtl^NeG~xg!#LYBD zXR@kb1WU1Gm~m+hG01db$0 zIIDsLI;k$89J*pgVMi=2>nT3LasTu1H5B2qBu5Y{9pA~y;b7jGhbn*2B*w^TAW|hw zlk0wEFIgdtv}zp4AfMeKS(1AtwF^g;F|qmroCkE?x)Q5%CC>ubdTfN^J`QeB3~EW# zP_I3BEjwR1H^a`{kQkWUS?kA!+VPR0!stGNlOYVgR39fdR>i}}OMTbKWBPQq-1e`H zsZqIJ1nA(Y+#ReN0gCJnjDW3K$W5mbHy3B$X`F@Sq@NO1nv8UihSg@qaQCyp2Tr}R z=j&ys7*_90>PjX;q-$D`B4;{Q)WQHwujlr;gA7Uii5!S2?-^7^)N73TZiQqL4kk+{ z!o1c58)G0%ZA9w;5ivbnJl{tDoGF{urGhoapS^Bu$<;9m3c~)JlO3fOvg^?KhZI5U zoZbw#0x^t4j-2+SDogO6N&s*{-M3sMq2Amvc4K`-PNkZRrEUYoZf;qA>MprKL7dxS zTB#v|btUSF|7;N$aqBO9OE}xN-cSSE42oCMIhx`1Ark&gG5+q0&x&V8O#?`N%87+& z|I(?7SfsXbYQkt$X{bYCi`$(Jba)P4gshmi9si^X3%8v!EA%^UqbjD<$g7-3r5+h# ziFwn(i7nDDtZOJ91ZU#Q5fj(KzzOzs0kZ;GNTd^&D4M46!6ZwexEZ-`9PmW$|7ftNutYS-uaLr6N8~Zp2a2YDil`0#uO4*_@?A$iCmcJ1zMxV z11=6n>5BT3xv~M7L|3KRRfeaohl=niO5HjfDlnt zo<=KGQOxcsFzwFn-q`DHGx5yI8n0U69o$|f zv0(RNwooxfI5^#$M#&DaR9sh~kNGb1t&AM#Vk{Gv6XRx|A;p5v-xpG~C6(eJ{mw1Ot*Kmlh@aykNaDEA27w`3?L;gw-FbIJ)Kz|s(|guS{o;lmb~TjQB< zfJ1%xYSujauhc5)^woGj_Tzwz0J+d;EuU}JmxL^c8=$ixjL>iSLNiwDOi=epNLPq7 zEO2RxOvK6D$GMrN(kwb=8oT$U3Kpou%F-{EUUyZ@J7NEd)1*9 zv6^{1br>gJq23k=bjxIY=XMxUn0&kM4|PxG7)Ejj`{4F7dWyXj!Ssc8dCgY(!CYGO zpxwmUh!Qfzx0h4tLUgO}MD;5c+337jZCO-{EuzFb0}=$fPQ%Q}SijsuBB#m&nMByC z!|x%gq$w={k=z4Cpd@+}A+Ub>HsowfL^Zy$0LxB1_S|d5_i~d21=iYcVH1C;s{)G} zp%TWiKM+b4tE;dK^r9yoGkN`D=1YvEoD`(yPMbztmofI}MBu$|tCrsPUmL{w>#q0Be2z*!Vl+tAqg^<{aZv03^%`bHFT4mxol6v}ZO`EEy#Bvx}6q~Z@ zo-4`@b3m(q>u(-?eInZ)Dt$FzBeubp8j!|Jq3}8@497)2$bKqgPMl><6zACJ2jdhU zJ$Lf=QE8$Mqd6Hsx=_br!LYG!Dq!Vy83fv5#b>79SdqH+ovz^kTuUeH2^IAF?BJ}# z(>e@WdDQrc4fi)xi*P zA8q-*1)y4}v{ZCuD&8!qui7$X50%Z%chV_-y)ytYRGSUFq%j?}6L-uP$VSQiUt>B^A zmT|dG*^W+)F&1Fut>K%YLEL;~1Kxr>y3x&C-^C7*oc0qJMzwwgOXz~)^kSDp0EYzV zUUFhUX|!G(Bx(geJ$iEC&d$xm9RonR<@Nj1`O(3MeTO{3l2pE@b!K53_C-^|>b2W| zsrwh*x)}%-WJNP~F}VE7&UCzGW3<&sCr)yNFJV0ELuRz*1%6okbDKn`)JY!|S#X?=gT zDa`hzr%MG}u_TihwX2Q9LPm4?iXCbqOpJaS;ggJ!;a?&xAXWOMf3Q-=-<~qM#}c}uKIdr=m8MAK|B&cHX7)m|Qd_V8#I z5U9B6ue0e@z)ObBoV|tB$$F@o9ex&xZ2?$mNpmY|3iS=m+AW6qU}4y%C7CSz>e}k% zpW|G7Z2q3azwKf}lWoTeb;U@{BEk!D{a6NdSXZ@l2r&R*op}h=2y@Ua?xpXZA5mep5_R3tM40@H@LX@O$F=7Q<|SpJkEK=3RKdLw3n95 z3=S?7OMA!}TLL`&ZmXgsVeHVJAKC{zkRWoRoF6~+tyQN$+IEA{m>n)LZwTQer6)Q+ z6BA3|qJUt*0}b?zENCJ0eImE9;j?8GjAh%zYuyR!{yVZ;i5Y?BN7|mq3J(*f=*K3j z**03>`5_jV?k}s%h@W9j@M0w72efbcnn)RAbwWo<9kg8)WIC4y#YzC?KrOGrasNYT@FbY zb4aWOy=1!@a2^s-tX6^H)uduabnZJW=uVVV2oS&<$~U2KHXKfyhraC?M=oM=1#F;(RbTpH9I8*0^qQU{{F*>)q7 zGR1=$-ehnSBsM$25vVZz>W+7g-^uUj)3P>GSoqn;4HRW1Z%i2idX+@bo=+{|gM245 zgXz5Qfhc04AdRAkX5J8JMRZ=1kecyiU0R8YCyB^5hIS~0$#^RT6!u&86>x>T+RX~f z1dyt5wFZb8ged3zqL@(&CWP^_5QW)Yhu5P?R&1f$zx18pcxhLXx^rMzra3WMe)1A@ zfrNp|i&&19fKpDi!zpVhPe|vbLsEM4h4Sj^?i-6!MdUY_5HQ*=%@u9SDSAs4V$b z&oSX2yZk#8Srt+Dsk(2I9(BUaA-f9~_+%OgOr<}=eO5t|pQ?-TN)T~P%;he%j+c!w z+iet$5vXpy_{eLCi7O=ps$iSp3T~EOV)c?M;gx4*&PcgkHCH>_cAV3lB>~^}a>i5AwOf%Pwynu3)CSjlgFUHcT zGQxaupA+DlOv2?Y6_79`bC zVH7dlh^Az%5h&y9)9bZ^aqC!3PrSu8)$GuzRrn`uv!VcC;*?EGr;3~wZvJ%qq>@=W zB_`Yek-T(LhT~>~gs}1Tl|R!ADVlKeL3CH8y@C;?OEkhE|3r3AE##s7C=nq<>f`3F zLtB|6R2Hf$h?AA9V!8RvpEX2bcVAA~V&z1?wd(hBJRyJ0%l@tUW`F3jT6q~dS3b$Y z{N#|iCLr3@J#aKQFq>dCn_7DvJ?HHV;F#O zV@)@{h6D}KQ~A8fe#Csyt?B{%a4eQZJIv{w4rRgC7^=V>k=16rwD)bLHzp~J0F87E zlw@JcT%`0=yr-||S>Q_x^Q_`gw(|amrMRs^*UzNVS$+(htvP(MC8r3_B%nkGAi@Sdql^Q z@BKquS+#+~Pj&SlQGE23!rk{Z9DjGs+54;FS>M0DH};vjs}7blgVM?eX(%^&2P|tGjjLxL2=!{IxG`uD&y|?G-WO z!s1^JzWMmCw{_?I-B_;?vEQ_ucm(iP#zSzev29pF6m?mAuBB>G3IVXD0sr zot9l@Wr_dTbgr)d+MW$JQ2>?Jy70QCcXi!+&rf?}Kbm&mwBFeL^Lcy!;;w#I{e0UC zmH*;3&+Go!v3UJoJvn*jKP5i)MsMtmzdCdBy2eKy?$~nvE2|%Tbm{R~5BF5QzHj)} z&s2QrKP$eHJ+S!Khd(v+W9NVU^x&)FzliTXv}i`&3LAABr;N|9`1h8|vSn9Sq0{^T zpVT;eRQ%QJDtlu?FI#`hKm7CE11BXue!Mq!;XS}*|GwVXnIqpU+TK38+Q(^G;)>c+ z7v8(}`N#ir2m55>uikhHbgn-^(fW~&1oM#b`9oQIzS%kOtBKD~yimFC+QZ!+Svhpy zTh*Ocw(soz$nvxEi&uSV9W_*ccH5I*j-R=1%Hs2%ELrhSv#%l;Sy;7}cP~3$FmcpB zRE~YF-#2zg}RbP*QnA^@0WmA?5Dhu+>f5!=Viba3Q!WJ#pR_ z_&(cyLn>BjJ0y4`^7Y|<2Vz4P9$MZ4Sl?yLFjUO<=Z2p?>0X4watp;oMq8_j+bexZ zEBIB7C)|$Qt(H8tC;~Au?O*o2c&*N98G56w3GNZc{&^##qvek!AA9aj%o!>L%g}d< zQjiOu+f?r~j< zM+*pJ`53;1^E#y59S@t}Ou);un9C>wc5Y{9FYo#UZT}fZwBAf&(f7rS*1e-F({M3f z*!gZ2ubc8?GyH!!d-J#^&-8nk)@hxQD$}aqNF_6^Q#*pven+%bAZ>juwH>9-lzpo* z3J5|iBAel9$0`*{^X;_CRErU_P+v0h6$X5E2r9=eiTDotfX~ z^SbdLGqJBiNC3>bI_Ptz2Udvpf_ zx1awJ;AlABDW z3HJ?CwgX!(r3L5q$GPvAW)&Lj4W36X2sJma9N;9KSRVm4mVe|?@ZFnE@?5T3l)vhG z-ceXVxW1@OyI?qH6s(!w8D2XZ1#sffpp-TwEt`Gzc0rbKMqip6m9uku_Ikh1s(Eku z#zmJKn#E?0WoiU_r2nNEA8n+ zq1xW>{ITDh?;eQe!p;qSaOd}_Rid+Y4u%~5o}gd)LU%a(UUQ`DEhK_YPH@)T_4iz@ z1Y#xsDQ9~_-u@uV`)#7UjMLv~6mpg0)Bc{^r%BFSaHk=;clu*>_i})@eA(H`54@t4 zLj3L)*VTb|%S`W~lN*eGy3_WhwqG+o?uo4gh;XkLUp*WawCaA($~LD|agoAzurt&M zQ2C~K)8emrTaNZR{7z~2JHxlm`?CZ51*!hU%{dBTOU25#%M)#e4YkXU--2DjG(Z% zB0s_2UKo6)rD8pI-d~XCkL&;bCbS_(zSGF^1*xhH>$n^9$L`g)san0-_*gv`OT@8; zBO&3M6*il(!@9J&$hPi$#!8i{zuA!A`f^MZBCLC5CgaBflVcRmmv_7VoN=+GVpFyc z3ZjoKAM8k~)bR9)xqKC;)^Ewg%NO!|HyCfZ?fiE3ef7X6^2YBKhZr*q#eUYJ6|){8 z!kKt6ue6mvtb?mm?@JFX_6Tz<^_EY{F02!EE^am~?<+}rUh~OlVq%bOXWX>WKdIY~ zH+EX?)2Yrq{`DVaRb!t&t#)l_N_xp#!J9A`zX*8q%YI8mUC_*;wo09QRVr`BsCSC7 zv(sMB^$FlI;~;bYTJ(p1jJ-W1$P62QzIHi(HaMvlw%ODxg3%|u&}&ZTOjPXUy(EZ=h{}G>W>*82)`J$;qq5ih0A8LebWJI74>xQ zhk*2hlAaIew7&`BcWb}AQLz$>qP#-yQafufT*XAXF~2XX&QGX)({;6<|Bfl?9~=FD ze^@DaEOxx#pGb;)>0U2uR5s6ePjgFIJE67>OYcr6&C~XVgx6N>TaSx(I3euFsTKr2U5Vg$~B`;iA15gB7<9aQo~cSAFyw z-OK0W(!V$2LH(~|N`99}e<;UF`=+>}E@gJQDakqXV8TUROxmr4`8e~#tdv}y+=mPE zPycgBHdyC_)x-vRo=|7WR{xwGZ}|99XYc1+SyId@#n369tS z_uo!XsNjDJ%4c=7nCrWk2_GSqM~$#dSchRKltZ%0F4%QawBHmp`HxWUfEnY@{eaaU zu%HXjD;2D2tsV_tv-lL!;A*_Yi$P_=F6253LO?15U@lTb%vm1LTgVL@P{&+4g!GtZ zWX1|AkyqU^fN(v45aUaa1NAJR>P}OaMvw%`hEOWOlpi~%OqcKe?mkpOK8>CZLx_pM z0>o5vh|}9xjjN^WFw+{dCI^;N3?bs5tXodtTsJOLwY@1w#9l%ACleWfsvegD`o;_d zSx_IMg?WhSoUMmc?g?`#)FXHZSIT6D%(Iyl>JVUL+|f)Gs5TCP*D@-1W3ghA{h6i{ zupR6+AW|glBMUb0##K=;@Qy3bSVD#hfpJe*i=dp?L1hZCp(5o6vO;oQh%Znk3Ft2{ z`=UTfEL?nkQufU|^!9+SKEQy=MRGm|2nZWHl zL@*IOC2bsopB|LE0mLOKC5ZwFOx8oD3^HJrE&_?amd{oW!-Yg?AIY{{F|D2>*I10e zoVI#oVP~^KLacV&7OveKF=ufF4}eH=tbkA&?@VSgOzbna{t3L7fs8_ATYHjN8Z})d zQ4(v_`ZG*T>miUP1+JQLmbnU_6fveN04%}-Sq+geEM%1z4>KAMuM>b7gT)e2<~7R} z6bmYgx4dLdQJKXJRO-O2`>-!yxJ>3urlwic(g$x4{ShS*Oe-28PBT|AZ9uGm`jOZJ zO?Z=D4-=F*_C7MZ@mS5=&vaQ7Nyb=FmXT5r>^b+m-7QgNnkvT3ex}QBMbQ{czFjcfbo1B+ijus|4aNV-!Sd^a&XN|#tF zp07Qkk2v(f_@Hj#Xi8)xf4He^`X9YkAqy+xe%zsZxsP{@x9AX0Bh+r4-G8CWaI%c+ zvzcAOyE}RqNBAL|%sYypsp)vl-dXheE8E&He5PAaTBRG*xfIQocqZlay<^MQ9Ryx1 zxfg%ZHF@E6fh^bA2TbCZ+UkBQHSdY`uFzK8$FT9sK(0@=V)ni9fugR8-_Gv3T_Hp0 zG4}YRVXy|4qNi)>46BUg_Pi>7SJKK71n+@+q_aA&c|U%w+IcPfmkBjF{Z>bk5J(l; zkE*g#24*+w-vDBIU%IC`2a$oMYkK~CoN)Z8k;q6;*-!YYLv5>CcPM+W-mzOUIJt6{=48=T7z_-GSUn{^-TJl8UB=t|v_U(|K_ z2!zJ+A;MNONjYi0#ro1Z`;BhrlHVRkEZr!M)r`&<5tM1>25;tbpc(by-`v0Y@_jM` zf|4LDGyr5|DqAlt>OiW*YV%XMyzp|M;ZEwX%nKSf_`rUCSnHAKUc}X<>zqC`>^?1Q zNlWunuIJ#-`bCt7Vv{Pp=e87C7kiOn)t@WFCBdd%aAaGsg}UANk9zN0ph*`Ezi0a+ z@GYj5V*x^o^4#BUk0L@?{n@6MU;){>T5D#Ha|T)xJ6>!CGV zLl)1J2^0vg7_RMzzIgvab@#y7Jq=z{z?2PWZ2|~dcid)!?y}`Jd_oXHvBH#w{P9(g zcIO68)azC6Ap&gVk8i1+dM|eQwn4GZ>40?+4ll7I#V0)RQHk-!!<~h?N@wME--P7Z zl}Rl@#>I6CzVTnVcFox9wp`W~N(2y%JZ}`bD;(|_E8V{JJwzO(oonJ$BiX)j6S|4A z(@m2q(S*U%(X~7NfyHnA^e0`fR_d*uOgJL{%rSZPS|gCX66$-+-f_fw6`~2f;5+W_ zZm-g^3Y(zeqP7jzYQgSgjomM&t+c)(M5)qw`Hk`YGVRti+;Cr6AiX`}ADx8I-Z#8_%BVtUgoaJ^D5RNjttwHtq-7^uAB2gB)7OLx1VK(2OF zw=E7I;VA9b1_@4=rmgbUugHt!1}Z^0OUZe!LBcVooE?^MYU7$g7dwYMCrDB8}jj8 zgx4P6cD;~Leetscn-C#FsllKz<7^Q3uYvgT3z9=zLsUfg8_m@Sun)Ltz8>85;ktyH zFZ&mMAP}i8l%%by;SC23ONW>C>)bzcOu~nE-d}f0HUovL@Kl2OO2-!yU!QnW7~15N zwqVlmq(*nIGvC>z-|AVdtHStVIdTcB?CaDSyJ~J0x_01+to0b`mAKvPiwmFMuA6s* zE3B<{U&Ni!iZ&Y?i-;IcS^XlZows0iE6DXy7exDucvCZ)lUlh+FxtfrMj_)!vL;3I z0Y1A!_{-jx{di~X>{?x))lRgzT8?iksJ2P6)$(O(Mfmo#0Otu9NWAp6vpTo~44E>0P^wHqC6@}`|K>NRZXF+~5H2IJ4(SJgGdrBJP zaAekc00<;Zn~OZLAlOd^KV~dFCoN!SDEVcfrp|OGqhe$3_I}ok^Z(ku87D9Ed|)T9 zDXeLd`XT(sGRJ1b4pc{#RpatPG#HT>T+BRNqCv}G=>})!jvIJ z2n?-I7?0rHAsrL$5>?tixvc#fM2CGMLZO7?>HUZfP#jse&teHrtGR}oWEXFCGdcsO z2f=9sz+o^$i1~FaeMO^*4*+&EBEfK)N!SN@h=jy(q%DN(5Yg(#`qUJwhFN-G-28Ur zWPhMXi<8G3*Y%)s4Is12YCihuAU{A*k`WxbN6HJBPB0VAF=hQ}&Wc}KQ^#VvW-k_` zg-juG3u^5;C&Zhxo}Ytw4?!EA3u7T?1V?3t`Tq^f6;bmgh;17RftpFziRHplHH`C^ zj5*NZk8vwAJ7!RQ9)rE+#fkZK@Eh;OIr7MKgAai$3)|AHr>Q~h0v~m|qT;5^9mF=- zoDDH|F9x<|&gx9)V5X+@NNHBn7b!a+mN5B~)3d}N0)K)nU~SSzm_D5n@sFWLGxSJ| zRDuID=_B@+As!Av&-K`tGR(`6CMx5&Kgq*c#Q~dMq$D9`4xt`C0TgC#BYGSLvy1|e zz8Fz?lq#F}gXu-kcdV(yay3<+E~3U7X2OLxAWvSvd~{kN0SWdR2Y@Q17*L9}an9A~ zVXT|wFDSTgDn|PEEsf?AJv{U{MHvy&41wGLbl5O1Llq6A7__jEkE2U%AX|DUs8Ow& zOdQeVrNA73pu|GKR0j5N^5QyB#6y;y>fL@A)ku*_wW(Lh$&jR+O*fyM731HggV zSE~vei@j@TYmbVl+voN!LcH6~+kfl4erez5eNFW`9CA0a_sYjo5sy^y0~|Kg>ZkI%VvB;;rwd%lyom%m+!sv`=bJrqMSYA@riqJ5sxceN8ueb z8xP@e5O=OkGQ&rufe0S~G@NmXHW{BmH%S@6^xUZI(|@o0y(CD# z3wJzUg{XBFuY;gK;Rf7AW?0CsxT{&#qMdoK=wqVqydGvu4%gT^-pCddDLEse+wnUQ zH*XO=u=rK-ku7cQN!^-LDXv%fSu38nAko}(Z4+pGYx7%OI}nG2>bXt1K^lh7%;0-2 z)(1}ZoY4mvTw7AN+3A(ip`blkHa>bs-EN-G6Gva&=Vf%2KK(8g&P?jU-%xUO(_Qz` zW4xQ^7=-v?7t`Rju$T3#a*%AImlT3BuxaeD9VL%LC>pS#HUcs5C5~T|vI7{yg1z-$U z9%`-b9-NgffU~!4-Q`NfJil)Y-5J2JxraUV3p3i}=BnupRD7 zP@>BlFUFD^UiN|uy@Vs?r8S=I5Rdol$#g`)+FRc zWJg))mQL&33>!TX-{Gr$#W4H^wzdlzYFm1{(9*v_zhVj<>)Jg(&Z~y~RY<1~V_VYv z@XbnYs60;R^}FV&U_0nzVtvZ90Gx3`wq#2Oba|>MhoqL*1m5j7>20FnwtV04d2Zl- z`h^9x9bA8X(@2)#s$N-$o=qb4C_Iy0v%SWxjycJeh0D7{V=2QWYos`4`kS|ASsx%R z@HxYiG9y2!`E`6AjpJ7y(HcuxUi(l9&-g@-hfpHdab4$sfrbMa8PlDYsHtK@K0@DH z&~2C7N83&Vu_tl($u+LsTe?NQf*^`QC<#E|qmbLzwHziah;m7@HJ+k_ zK3lNL_?q=pu;I!2HX}c)-8;j|=6+^u2w`wM{3YCtt%8sk<0(*{eR?dE+$=+pN@OhI zd|y577k$atuEz@AmSdGHUFy95uFGFkJ^2678`GeQ3j?Gu{Ba530tY*|L*^0>r7F)-Y z>FH0aoZ7=&oh5j1>GuiIkA0o~i{1p@8Qxf$A?jXa0EZ2DIL!#1t4hrG@uiZ+g&xFb88ji}(3`7b?E==mtpCR6%{!F5*V zXO3Qxz}ozSe~YZof%gBaW`A(Nr>tjtcypu8o+k=h&l8LC*|G7^Q7!~$@e3>8@6 zI~*#{#xgDe z>u9r;KB>20n$@p4KoL^{@9fFK<=UhJg6y-Jq^us%ABA9pcY+->$jI4`7CElj%Y1mK z@i5&dFy~yp1U_wG5a-QMnTQ_0A6xZast0!+g1*K?>bga`H72AhrQ&bG6D*5BatI=Q z115t8ri{Y{&#C5djgQ}I%qSjrVN0| zM~{{pUWM9CtSQ%jCPG$&kAz_>-Q6QAjs2Zr$%p-0K!7nNI=?4^h5-KTe0mMt8BTEM zs8~%5S>|9Y;gSLCAVzd3T+O5#@4^<3xzMr|f#rmvcDi@WF1Txx4*+|qMy}P?P68Lx z)FVWW-!^}-fOQ}?YobskH^5QetE zc{*3ZjmE-6pmw~0d1rcjMBXraLEubkWC@k|qccXz`?yLIdteX99B5~XE9!)hch=+v zYPx7Za+>k%7J$t$+gWnXG+}T_n5I9~e*6sRBV$1!2}aO>?V)7zzhokCOH4rn(_l~# zo>&j7_y5DkyYLCty;P53%JrD3$Li|y8MY{F@%3qiI%a1`xS=@Fm_`BUn0ttuOCzif zSIk47F%+Usz0tTnF>@sG-s_{R*7OH@0f(5Ha;!<6s0#KIPw5tlqIT0?Eo_0|RY3n~to_ zN`bWh9zbE`o@mW-L;{7I{J1OF5^==I&PSh02A+e^=C7saR3{Lzn3#;#To!BU6gOS3 zqQaNjTd~ay_AR*~$2MBmAaE-R&X=|C%i^n~NcK;)2qA7dDG58L_xfh?cR4=-IblaF zL^rnUo4V@6Mcg^hWK4Z}%R${gTMHe0;}+FcCIroLe*SlGI)`*4P~IJLjx89Q*slbC zWe%S9e3fW8;8k5qVp*WoO#EfdNtZOy62V_bD|e<$$HJsKTac$p=Y~9>oD4suSe19Q z@;B4RO){9!fDY|@LmO=5UFrNTWKED@-z#fMs*K0uwOf~)=>1n~(eR1#)#Y;C3h~?R zcOmS!iMzm3y3iwkH^$z75PR}S772H93)iZ-y)pvz=*n_t=uJWQfR~9BNQ@ax6E$7R zDUoIqFJaLq!QBM1r@JoYmo<#8MNuUEY9Uv9e7bX=Av8Y^hugJ=1J6mj90ZEZMn0cp z61O}?MiHSg@Qh`{uJHW@RL;}9jOS|S)ds!0^3`i#NT}Dij3|EuMP)%&q{HhTw=C!B zca0hrNJf;#+896Ulhwl6I8lC=M`9^|Ch5$O8$UMQcpNWo*_FQ*A%N|2?R##S6WznEhp7ob!(Rbk_$c_ zM}6o^x|_}wBa;Su<0{`3^%+|L&tJVNeyubQLZ(Fp!T!=u5AkwCHPAdnM22omx?1}6 z1B+dHF!jr6FVyEhDI^igsbHv0Vbr;?2=<$_UHXO^TNt)@EHikkKDJZZugmB6*EgTK zFKfb~aM82+B0Nc#Jo}F{P(HO@vs@f@YLj>-KKE~+9HOD;kcn`%$`_>63>wQ{eD!aw z6MAKVu+NsSzKr#?y-~$AM?~&Z1S>^d*2sVjt$X}V?}!pBS(nP!eVAqakszHnslwPC zU{zM=rybZdp_8e2!}9S-KkIa91eoP2l7nUN@*(I+XqT;LeH7L#Y9FwW^@Vs}ueop&~|CtQyQ=1qn3Vaq+`R9rXpUMP1F zt0G;u&C6~jOw=pvk?YZ~AhPevn^;z&w?d-m1zyKLPk-FYPbre`)z9zBYNF&op|2qs zwCb%Qu?Q?Z9%8rU?M;Vv8jlB9E)K5#@#>ZommAHMv7Ih06~$x1YOjrJGcowbRXi&$ z(hyzgW_&uT(-J$;wZbn$g8ryYtj*w>v#vEc@6^cEJnc-8_??Ac^D+hxt&_M|U0atC z{QU!qXV%#YqObc76rN-1rNB|QioCvx4f%N$W8L&Ab@-I^^eH$Z0>{UqHBw2LwQ;Fd z5p2k~#MKsQie^g#yz-7#-I{yp*N z4@n9Fg@**N#q7z6`gwM2qr%!$3}gnYKjpQVgu$orC#B6#zF-9J%#FmzgA^Z45_>xK zO{sbobTcV$9{CS4cG{A$WFdiPhP#y3wwKO6+ zW_E0ZNjz?bSVRf)-k``}wgqT~W5G`Bdp%h_r~;UCORz>x1M=P%QT$CMdtg*}Doi26 zKsujjoZXDI?oq!cP?Hr3+k$Pglu*FH;JBpx z2pQ%8(m^4^A6M;%FJ=clJ0`;+3OJ8M;@}^~LoadxiP*;wIX@89BH~89Cr!qTB#mM1 zY`|4my)zX65N)qzOM`CqaX|U&k)SASD|j@>VoJ@Cvy!QJ9eDuLR`nPSz`w^aKxA5Q zd$)t zE-Vsndio!ydu)?=vsHGzdN*Us@L7TLBeQy?L2lx$f!|Pk8 zL|KAlH%u>VET>W^UO6D{s;CS+`1~`16MiRpU%E8p+OczYc>Xhm-pqKeF#B{pFW7Zd$sadpc=;`9tKLK!CCPBIGfR849(f5IHR-7* zht@cZO0QN_?h4g4wgw~XFBfDP*4O0oC8JUNeswFed129}sdMRy7r5Ey53Y|n?d+8C z=YD0-uv3Gtpk(5HPfe{-KI{=xW3&Zec5Urf70I!4PfokFe*W3|mHngZIJKWpeP3p* z(ewB_bN@D~+>q|w7*)}{Mc!Uf#|*Q;pEMtI<{{%E=BpD}RejcGJi?;w!$9o2ydZ>PKA_m(A`m2bIy$A=-lS>+ukA?%fg|<$Rz8q3HyK7gzk*WdH;GF_35=PM3_P-?t5_i**en&flc{yw3F{9&GY8fOw&UGa); zKYmB7apn3oT>#VhWt!r1Mm~v>-j9p&Uk_ZLbSoOALoXX^9QlgAtZr?j-e9NG)O40C z^wUSX!Io=idZ`Eri;!>}ed`{R+~p`Uz1PoBWO$+PUiRJ+mt!sbE5`19niQEf@qxuJ zUG9~WYGg`gNDy2V{u*lg!lW1VHh;_CgcV9Tx%88Rk6&+8^9%gBGi^J*RaoHfC8u)AZPBU?OLsNN zln1YsqqJ0ga?m+q`@Q522kGEV)GIF>O0sR>nq#^|Dk~jYWT*Q|PT5}9@C++28-Lp< z7S1L$`~A&X>d9}{P3t0yqxVNeGnQ}6fMS8S^!DCP*W~!~n4Ci0mJd<=s@dq$ay_R} z{Ypq?&{l-Xg+kRSo?18)os(M1*B%GOO_de}3XnmeP{@$=Jr^Q5OK*oEmQhzktIW5y zMcOb@QK?xze|RLH-)q~pE-6CV6i#y=m|f=- z5$FC^1)kT{HrHnV!L(xsUsgGT*)4Y0ey6k#Hq2L+#9gf@L?$N35)}0V+;2*4OEx02 z(*By0cdqa9k!e#aIi$yg;C4J+Ou~^Kp+=S`50u02PCha0&#$ttv%QtjGi?koVv^dO zI9}rN+4@Vhlm%=I3(;(pXzE6_M*TwXQhn@fMSI&DPfL3FGvnZQ;ctvypKeXKILiq{ z#tKC;`-IpoXt_|j;_|s9{8CV$9OcJ#qR!!*A6>IQM?Q`C*tklA36afx8c0j-?``@I zZ%R~-r~+bv6gW2(y>{iFe-t%E&;{{P=bTuxafL|t#GvN5mwttC_WCviVuTQA>0J)4 z%B)zmK=PJaXcf*Ik`8&vho)`|@;0}w*#}0xcOr4y`yHpqCw1Yfs?)3UsM~IM)k!vATc@}3Z5TPANKrbkg78wTw!hy# zzHmTzjNd@R!A=yLbZ z?1NP&Tz}t0ipc(+vY^V&>gi6*rTZhUcG^m?<_*w?k*up`kI(P!9QwofiqY>ysr)g? zq5oOy-kEK@6~8A^aUba+J7=r$!^WVToGIjiO2y#0lF-fxSJp`W4;5ZC}nlYshiBW!GkWI_Q{u{N*Ctc*(To zJGZQAHU@Q-y`1ak@7D&Nc5d>=+BMY!QPcNamhKvnftm%DxZBKGto$SbGQltc5A+4B zVm4G}MxXen`3H}_fFR8O_b;*W6m!QRlm!hIDKJ==m^>p*^s!Q`Wbx^i?h#x1Uf z<^|lBY{1xfXZ!A=r0gXO7rBm@)J77zB(tlV9y|v|V(&7t2IZUo-6eP|%r%l`9#$(H z->eWyHzlaZUr>nRVbXb-R6c8=^VYR6k1|1?(Ld`O=EDdk!xTrRf&o_Qf?K1k62tja zOO8OFa5)~rM+?>}&9em|TCha&8hs5vTmT;a5VZquf8>{JoY#oZyjdV*T45k5tm_EM z5ln72EbBpSM1`Ah_7u7o7<1OCo#_V8qV9l2E67V_5b6?W9_g+rH)Rqc##8nWJb`JW z82QI-^IFN^W+{o^Xl_!9i3J*Vu{loj&*Zbto;Wc`ll+X5N5so%B$*`eA>o_bNbnOz zc!*81WkrQ8h|;92uB8?w=nF#P$476BvVCxImoiH?Q@u6D2T>Rpwy-ue6qnfYK5;c_ z?1nvtX*}tYiK#vEHMksk-;{-|*UjF_wDL);LO-StH$N9l zEz~5zwg{v5>P=8xXhiY0c-|#1Ql`!N1UFOwi5`(UK`Mj>mmhn=S~{Guy842U^ENtY zzZ)C;{@}INg1V@D4#^8c*b4in#^0>fxJ=`&`mCOb^SZ9X{_dXZ7uGt3fF)_Clw7D} zYYH!H9 zyZ}RVL2FM2;Eqb3zTx}sySe?7e0$N0PPL3j220Wd60-Q|0_M%B(M z7izl}4zzV$q8-#*KK6I-1X2B^y*a>vf(@V#byIdOue@$xi znT}0wOZV8-0P)0=WwURtRC_L4=PBr=_$)WjhxJ+8{eIN0nw&&I9NILO zpU(*O^?`GIi(r*oMOh2KIVvXyjl^d3=MOwt^JR!J)z3Oc7?wSBkQ@Kk+Q_}NU5;Y= z>Z-oi+SwV_gUyS7(eRvPWb{SxCdFpIQ@jYA%g=lA3;Ff-Kp4scUaz(mu5<0s$ka+y z(-6mVP<*ps`b2EI{sYNnottJLz0*;A2peDI@m(XV4glfqwbG&YEToiwM-Qki15U;A zqxOii9xR~L&>O4^39%Y%Dff!p3XZnpUi;;M!PuHF1RS_LeS!;yoK476C!TarD*0C; z9kXI2Jw*;-{GEC)U6_7bs31@Oyihxn%T-RFc;S8t94o(9`dP0vR-WS25bw1@HKcQF z54USp@*%16dwdeJHs5a79a$-m+}r28u1Z?hBvmGGs-}EO)B(p2`JFl^%pdDc%YdLN zjd{KX9O=#o>BgRbH6V0fc8$6HtU}J%}|K;c{lx@bnx$`nOUJ4Qf6<{Z|Jn&+^Z7lJgS9$ z2}-A^#}IndZFy|!aNUSnut;WnO&V5s4Lm}7T99SjEINO{Zo^m)%FGX+ZFM<+R{_Gu zTVgvB%d?7dSeF={YAaW#rnKibDwA4n;lxT#>NW~Bjhe)Sp|zu~9WYLa%KGMR)y{&A ziw0~p;Wxum!lVC69D)zKs+@%{Oky*uz{Y{H0c1{Q4|ya?i+t!U&a^6a zW;}6z5M}sT!CPA$jV%u>mQ<$|b=upF$PCtU#0vK0p4*z&ppJfE@%*-e=+j9nC0T-s z2@+M}iCvUk^$@&JUsI1x5;yMM%NpM3k~XL#iln(bG=H`fNj12Yd%Y^lL`005o%(ZW zYEGRkNH#oI{`d$KuSWmK$h}exUsIh^ePAa=BqsC}tw}>Kj3D*s$suq?XEL})u%kL2Toq{BFuUP0SCwvf-gp|;f};gOtj{Zk z3q74he!p5aeIlV3zh^(1w<9Z#kAj}A^t1OmQ5JJ*lLF$kR$pFc((%9%L^+I`gTe|g z99Z)iRFCav^9>$b%gkBVxz)6(`V){&NlyD~QLeWcAdt-P|{d+#J z*qI}QgYSVsg}=-Bd!?Gdb(>C|Y;WQ>Zw&JC4LS8`<@4Ve+wwHu89M|Tzr_0k##&9C zuS9bznm^>}RMh&LL4)@YH-IGQ-$zhouaq{~i;V3R#@qTu1*kM)n`Qvd$4c3M1xanH zU%!A+t3F_7muIZSBt3pS-uOkf;Pdj$m2IM~;vDUrf-g4ZfM|*&F@{RF< zoboAPxnoCi;wy4cAa3{ty@}VDMiQ(M&rDy-#Uq@#^QPY?F3+pdTe+#xfaFT~XEr^y zw~Z)l8O`(OWEUyjgMZke-#SF7*387@l zC(lX!O2iMd!cD!-fquEqf)##<0ve?W&54UO>skj8g!ejg z&(vNFJi!cWOo}`pE7TqndYR)=CK&z*HhdIL{lEA-GXOPX9pul5l9+NlERYRg1&=HQ z*-b!l4yzH2N7XkA^b1 zZ!mv1_Za;5GWh>etN>RZ6t6r4>SbrmbCyyy)NF*&4!~M-A4o>bP>AkQeG`9A)y>GMLY%8rO|A}$r@oX>qYuy0=A$c{iEY> zin8F3J(nVJW0?R8R@Xw04_(Yy!Vx1PpVi)@zaIzHV$d5S4!aIt>7qq0wz7wH{THn-W&kD2N8iVaNgj zw{)?()1CbXSwVex#*(ynTNJhEALnL@Dxb09QKIWC)KHAGn5ht~vblaeau1V6=Sy=w zh&U!$!&&Af)jY*jJRaSpFkp!cEWv- zrx!T9X(ci3nwbNcN>Lb21clv(eCAC`4eg+`>n_Us3|Qf>4Cp_cXAg;?iI$qCEe;iK z9NCRO?P5=WRfk~ISat_}2%8~0WYH9IHG?Gqi^0&mfjW%XyNNhcKRx1p${=i!Q@9AM zG0M)G(`feQoEYO{`2R=rw8XPal~U#a#%eq`I54qpHSn-HOT|3o5eA{$f4D>8+P|VZ zeh#88!D_K5kwao$ zKdJhD#?rvpkpIFSMB^hd?*z05nPWceyJ>fU7b_~W9#~Kn4kj%nESW`BSV$tYxrl$& z*Zd$T(HmV9!f|P>aGo~GFmFqRehUm){;X?q+itsH+-)Zy3yuj@G|1TXyWC^LTx5BL zSY&*`*nDzeu#?}1dhr?5$Jo>?jILQOZjTV>hSyZxb8FmvXTwy6EKV!YN5o}SrE~2` zmNW?oh5#xmP22$Tv zK}x9eh7t`#LP~h{G{cmL=Q;jD@;O{vy3XAoXu6|rJ+&0hI#%OeREdc*`x%uhOGRza zElR~Bqs$5Th)10^uQt6%G_L3Txla$pHybt<8}kqB9FzNOL~fwc;HX^hQL9a^LjA&o z!M}V^=qMey`M^TiBI}8(RHb_@gM-vDmv7cv*S;Q`y+>i>UoP*FjQUG8)rJA<^y5eB z)!8wSXH5+(FBsH$fg?0cKr*$5-vAUFedP1MA?kQ2kWOBhD9z3+7&LYmF#up(5<%;PXk{0HA?H)Z}Q+jAMhI&h{?o0g7DOuU;C z#T!i%`lchrjZ?w9z|RgI7iHmE!#fQ}^FZRprf^4HM?f{)79MpFt|LV3|fif2$Z?4Hca|PeifhrL+L2CQu4PzODU2BV_xg0D8F&QpWfrmcC+UxcT zp4TSWaL!C1QOnnP>Gnk%7aQ*YLA;iGqe*61Eniw6f8oe^RMF&(?nX|G>#sEm_SSr< z;ax@0y4s@XkC13VXETjn-F=1pemH+Ez}6NFVpXS_zw1>&pv)K~-V!~1aLarI9i-oO2zkHNw(2LFO_1>k?a}O(qK^?T%*ReG|Bdv5G_yXYFnU>ZAbnWL5t%+nE%@o z7HiWc8fuYIwUDy!CJN9CQIUd7)F)oAHxwbWWxP|?R?)5DsS_|XF`pMYHJI8~drXCI z;=OJ4`qB}lGN~>qSg4)KvgXtXOICo#-#^=IEDL?SR<}?zEA>2r5X=uDhWSpxy`vhQ zW~wH+e_ON?9GV+C@a*7@Z#&3})zYxMaEt{}7gbB19eh13o(iyD+7!;^c?h)_@>-#0up=wI zDatWl@E7FCL4WCa*GBtc5tcS4O>h^_(8% zhef=7(DvEc87ukY{Q?ce7mOV-8r!bri;^x{h3#1=wHDnlnmmc78(zqTpRgykUkiDma1^+Fz=}X;mik4gPMZ}`MVJX zfbkMUm?bAxPUbmmXx*W*`(2*2v;YFyTuybu=0us`LQh|zt6dNF%y3a*Bb-W9?%Hwj z>$}|3!u|fIHd*{c#sQl{zJ^j`QkGD^Zz?0l8_d=rYGL)K<;8^;r@jOR$O z-NkjjG}vx)Ft!zmxU5+h?RS~8c=hX8SnD%=Z4$OjWyqJR96V(IeA(OUlZ3B9tHc48 zfBEX)s2$^Atl)}-UuLjHVRQU|f#LteqCZt=nb8%Mb}+4a_+;sx7-osWC7IhatR*gtaC@5~Q}$#h*b81hAi_&TwlivCDqL z0yS8$NB+P$iT-q8U82F6OuVvb(y%)9=Fn^4luUiOv>tv5)PIy=ZAv|X<02DkZU|Jc zi}68d2o38=^!YP|t$CE-41meXD&g??pF5~R*_>seU64SEIUvUsw6~gyi>Z>+G*>4~ z;Rb*fgq;<#CRVHiGUuHK4gy0m9G%iE)b7(%!L?F;w6;?^*qKT~OPc8H_py_JGqLH7 z2S%K*h4QQ_u0cng9$4~lHKR|M_&r3iOl#q?o>E)X@`Of$ zMx+U|&|DtG{3Ql%QX@~yOO^qr2Vv@SP8C2{#UM{ed{NYfA~txkre;_4I+ns>xi-Ww z63Gsm-5_k};M2E*q{g@+;oG1`Bq@9(S^@*0 zv?*u!Zw%|9j+mstWE1?b1+{5dY0{X>Bv~7|9e99r84s;dOtR-8(<1z7ry-3a=rp~W zRY9`GsWig)&!2sH=R>^)H2ellI&@1%zzy~bCxl}Jb1k{K@N+3M`k z_s-?o7O}Ja-Is~qbZ$IyNV94QDH5Nn$cfJdueccQB@Zb=)bDL^SUgIUkN8VxUjPbd zsf8Y87s>{BvwAP55yA3Gzu+?f{ky$O$AAC15Sg)He;4^@#DlzHAtlr8o7TBJHnpez zaFwgGv<$&(;r;&EEzrgU6?yUcWGN>gS$g8Q0f!P8pCCTm|AB>%%d_i!585HY)`fH) zc;2z-7=L1NZkp<}jJ7Uu_^DsZ!gl5~46U6K%_NmqWP_~$meuexLWcQ~t-vg>hRxJ$ zLR{Ncp65CL&z=rIZt84(X?^Q^DB7AR{@#=ZgYE73RVPyv#=;xb^2Zsc%O+Vs?n}si z!Re4l9`H#6U)9ILy9co6AdD5^8^|H8aS^kB9`A1>0*J9o%&L&kT77Z#Fs; zNy+L~*iDIIF+@C@DyEj?hULS40)OzgVAI8W=j#_9hqAOA<0M9|NW&}gJaVy14B3&< z<^GAmb0CeQ-jXsPlk~l{C1n+Pg@{d~!KEe@-ZVv*rxR**tJ=deBM$UVTT0b+SqYMF zxeEs*7+IB2jeo_)xweo4t?>0G5_Y1Xr;586h_udNm;J5=Z$r9KFf{MX*p`8`d*xBP zo{%BWv=jSEEVOua+Jwo?d3v|mSr1_$rj{&O;Xmlu@-?`wZa?Z)hcTF=EfAi6P-Q|zTuwxviDR&+$WaM#u*Xq zMt*K+jgYgH1R40)MvBOk@69G?BR>w&yqwyFrI-@rv$5jpX9yT@g#nG=bCk=&o(d9m zR+ydKxd8e&{oxV}Y*;>UQ6 zurM_$|1VbAciPKZ9irAsk$LHp%x*^hR&)PKOm&1k+qg}*nm%?Y8oW;O~+T`Qofq!e)c>czg_v?I4yLsxJR9mnL z5bSP1_;4_7QD;Nx_z|IS9q&pH@lPC929aTtYRQ`VZ(M6%$Ami}-RTtA4DTinb*cZJc@z-3df`b938Tyo%n!pw^!2d61|{l| z!P#qu3q2C$D1SiP-u61gZ4WGV--_N9cu|zH(Mr2n88HPb5brx(er>O3`N+8&FB?zc z^R1F3qeQ-soCv_)asbouPHmZhdg0>vvJ8K}w;F7>XYS z+kS2w<7(4*KFh10#kR!Z0|uu3bF2=hi`sRIoU%YjvAB*Z(f6Jgb}c7um74@_C&#wm zcAn?6tE$f<%}Q-A(@x+WNxxp*BNS+I!#CeJ$OR)&9|4>0?SpB6_W7+&bx_xg>9@gR z5&6TT#`$F}Yo@yFZBHc%3%&MJWXbhCq{d}wssn<*^7B7~IpZ7YynPj<1=W|^)l3>T z-b4%o*~_TbG~fCfL`5#?erUB1knu|!DIC{O(QHihM_{j;Hm~Uw32w9Hg1>2MFvrgI z-7kM??Iqo_>sL5_B)#+mbL7(5I z)U=?#VCVPC0vNDsc~hngn$W{|eCS|#=&R$tXQ5XR^p*w{VZPu67fagpyFZ89O4k}I zgB|jpueR<>s>Cl&NwPwxRm_Qlr|#18IQI1(eZ}3gEF{=IA4WCAZ$IS z{dM@IrU<3r#aAV-oLXjd7?W#i4o3+R_)Ye=P=9;w{t3-0V_;Sfm4#wp>IzUyp!LY? z3G=d%PmI4VJNjdR^J#>Q#!cq3t;w^`bNll3R_&3IYhNsFtcFxz-~7C(tP|jNB*HA8 zO4!mpsHqlrWoO;O3JXUiV0KXWdg)HRRmX^ZmX&0(`19pT8EiTr=qSiq?!2$Je6d@J z@y=MaN`{KO@kK(-tv~naRr0WFh1hnj1f{B=$?<@9K72Y;BYT#s)ucix>F@QrMv7%v zYTeqlR0NUjRcMLz%IT$n&^E`EBrNOO(Eytd6yuKfLpPqGk%UJHsOeEH5PRlSZsm=H z+O9QRzp><{_Ajfedbc0$9j!|Hu>O8eEk;2I8zk{1*Tu6DlJO(r+2<-&Rp?zo3)jZR zj7vX_5R3WE`F_YY`nJ}7QWh`R!|&HC+pJsEK9PS-imf`d%5Wi9c+F{_olbT>%_}$P z?uME=-m);RJ#dxjy>lx)vHQF&zFfI0GU@r0y8Us^0TR@;U7l)+|B>~d0IH+{E0$DZ zn3{522*8_a!tf`Q)M~zVW?zL==aq z3CTmcgH$LO2Tr#G1T@K?nDxh|6e#P;=XGY#VF!Fa@QP(dk6lPht(LMHDB9*>xI$ES zZ*>Ma=;>`{69;FoMqu$wIoQ!ADn4YQQFX3qG zK&*yzgJ_gQOMtabol{Vvs*;VtoYLBl+#N8bFrnXnbS0T<&M;CmF+O6;ki|>bK(Ikz z>y_1L;#wz0*~0`p?q<&dVgfu8L=33=3SA1mOxb$VET-r6=n4Z}2N^UGGA8~;=qZbe zRzV0~FKMU9!fG>>Hv2>(oIw4;q3gu}hGMiQ6mT;$kCW*Wk(qS@>dnELPI8UF02 zxF&MYqqpvytASlzyFT_kMy|nTO|Co?kXrNsl01)g3pIx@*cM?9fJtatnYsqElHY$L zlCK@S@V|4TbQ~YazG@Y} zd2K~^PLThdJDjW=rJ@?X;Sjk?yU!TJHn6>D*?F}RG{RV0a818-KNOIw5^Bdogs!!n z*9?ADBH*3C*jC}$%H0=V@8d1w&ff9v2#P_s6H!D7&`K~057xug=X~_(dfTTWOvZDv z7RZ8-!N>R1{xIWVG7*Rm1{hG2IBe1JBe(Cbo!yzK<-empSbzo?(ShzDjp;ovx4?ss zFah#`H8wpjwE>J7?D9sYQW507((!KMD-kaTdzMugdyv{(Gu@GcwLD=WQep8LG;M)o ziwkUfwUswK9V6)-#afTpD;occ+`p@t6|>8+mmA{(|wqoC{^=uI-W6HxOP1fDo5j?G#)_j#ZFminMD1q ze)n`ibcSWH8|f$|qv))y^=?UZFYKIaWVb&y-a}pLVfhm)au+-Ck4w+4^Pt=8++)oryD4+{+TF z<@r0pLINyP zHNxKy2}4KqUf0@S!sehJ-@na@U-a;Cw2zStpDphmMX4QV{H`ndE!DShUTts1Y$WP8 zlcbMrUdR1l*V=CkbZr4$JFuqy6KhdqUJ0_KU&SS+`lXJ}=q~rkU@8sGGW;}-uzD&< zSbS>G6(Z>Zqf@5;-2-HY*RQ$Xo?n(5*z86RvhRM0l@<5HTB_h1wr>0Cu}q|?u8apf zDGtdT($M7qYrmDP-$*pBFDy!s=s&oAWqZCpsw2|1+XuY4>&3i0=)p+$h$Q5V4KWx# z{`lbQ9|5;th@mYV`N6I%l(Zl+NkM=YH+Y{2ZZw|qW7~LlDrDf%1&92=6^DDdR6t~tE*GFtF zMnXo?*$Db1NLN0W9UT%wJ|h)%`;CWRzjyS@UTI+fc+D-iEdvE0KqbM|jr~2yr`sdd z{h1)@Rsyh^i=q$>U$-sfJ_>kt1LmeBJaG}Jk-~MQf)DHbF(eo3+~L0wRGofWo-^Om z_L+ETsxxmV(oa*uUjF=ccvrk8-E-|4UYY(*iP~}#xngJQxA957oAK>7 zuGRf)<*w~X&3E~;+g!U1#rvZ6Q@ASqc(CV~-0(^(O!y;4D!O-L4oJ>*4(kSvKd|Ub zx)pEyI_&@O_3Z&o-RZWrPHS5R=}YK=kxU;mdPQV91*s^c>Y=n+wVsx@Kqa-HAe174 zK-_JuQbEjgEHYIwA}Xx`LgXa~q>7L;QeGl&Ar%IBBoUAhAdlUDN zy?>AWTi;sWT9~6A#SeG^ZzhF+`V-(3)ZA8_bWg{~g#YMiOG6*z^P`zI9e}`--Wr$m z#c5eKRx;D^b~u{U*GmEDa=alxP%~kLRWgZ9K<#uFwiChL&8pv9T0(jW2Tf~bc)=lv zG}RBPYm)FFUZUvsSB6jfOiztK8bAK`@YU1pArljEJ35aXn8|9eSP!LM84i1js>lFg zidsHW*4{o*R|GPHcWrt`q>oUE ziR$cl>pkfOFsTdGt|i&Kf1b<>KwDx{Q4fdP_1k*#3JUnmOp7K$@D5tj!obT9O|tHI zWG&Kpq>w`(4*bE)$N6c24tDhg@=wYx>(1E0<6578JbKj+XRe-kC%_+PtfndJlSJ-h zLtX}(MFSiBhM!~8>ib6!7S-Uy^7E@(yr=f^`p`%F%Yw}q^|Vu+tO8&4y^*n|j|Vi1 zP@TKsaIP5j5odWJ$q=+jkZwx#?ro6)5%$`pPoF_d7^d1qUp_fA3b6fbk-i0YMHOt# z^BB8fWn_VI^=sO~t6AIlZFv(BI+cU16kVZ8?2o=X<3GWY8w9!)UFf10$%!5Iao8xn zHr_kk4M(41=PVvbIk?V_Jg7hw2Ne@Lr=q}Se%^O@uj-bP5VWn7mf-_e3_GGxupUnf zm*6qYH1s9xi9p*e8R>9>Y(CJLUXKBdL}g553sRBM%Bh+GA9AtYG9yLr?t87B-x9;7 z$Hv4_ZPFwDg`;j3oB7;Q3hV7MrK%vMbm>e;QoEj7%pdG8M_WU8#L3JMET-=8@}Zrw z)u`g#^mrBzqKiLXF>LF4Ah93wlC@}}_@#MKFw=)i8S`$+`mF*ygM%J-C{u3t{cU~zXT9+m*D5CMPH4VpQ`YfY11;Z579}Cb;qP1kYAT~zra4fWs%yg!lFrc> zWbr{dK<*!ePX9f{`_bv2I1qSX%yU}A$dL`huXfwNhB(gQNh=~!9{v8WqzU(qbGQfz zJ<~G;JMgJ+44(V$+*fm@*?buO4dk4|l=wc0DgnRHuE)Kl8J$)V*@z{m!;u+1UvqZ{ z_AWj+1an=AXuf;=A&E<=HxTcNlfP3jz$2Ym$%uvU3J0JF0^qr|tr%sw+*=%|CUyjn ziS9T>$DZf1Va+6~?*?@4kx2|bm^nCoB?nSis>hpuglHv_6ykbO=ig%v#GZpa^8rWz z%LpzfY+6LJNsh2v@FF@ZaR(U%Omy*=g=}(U`inte&K5y>dH{%(j8G`79Y7a*!h5hp zJV?7br1QdXkXIZHif%`!O@YI?l+;^j;05xA^~x}P%lrY~e}bVaz;(cYm%uS7#%IuL z7Kcp`+FHPhG1D9QOIJO3L%?x{Ry-;cI00I~fd(KSB-IggULwhq$=}WKlcYul@!-^{ z7LF`eQ06&}A6^c;7IC#?>Fsa^c&7rv0-cVV0UvJ=@Dk-4({A4rs)Y_4-0U}T__b~4BWuZayUw|Gx7nE_Rl=>?<$uXzx=1h_( zp`I9V>g0QoVoZ>Gi}xjla*j7OWA;a=Q-%cw01suwNB;xU$mJ|x70jo~VzxmYjHFV! z)zl+QgUzjx`P~Xz8fSQkYvl}9fMscPIg))2TqEcC5z0ny6wOD)%;|{_Zpfc<=9ML! zg>qzwU8g<0Z8q_|&UVXb+p?q1{6!=2w)aa40t0bMsUPZA-Di%8OWrOYu#lu%)z~Pz zy7|T=<~$sg+G45eZ$t4ovcXADxOiN!_9QS#Pk*zfPVS6G*l zwvL`wfAAHLc4T(fDtn~ia5CA|q2KF!PJ1wqd|XsKcB4&P=ostq%L1G@Znf z^ewKe)=F%c5ph^;ctAL`u!bE!nEnn!xeab3}pkCaP;`tzRb6p5;8GBx4Viv;x2s}B_$misen(* zg7GMhoyPVWvGLt@_aE|`)(0!r@79gFMBD=Oy;79Jf!C-~+q(eqPRO>|t@4RtK?np- zh2DM$gS#dWl-5I+jtK)Me_8NYtqt?g!VqU$f<84ek~vzJX*%M7QF;YQO2m`;;*zAJ zVB~RhfKt^$ut5Z~n^8a@KhtG!>tythmw%~?NBOFNT7{T8(-aCO=miT0;+d}YufVa*80{{>i=;m1k({juTVZ~(X0)Wm% zU3)msod1S5JWy4Bn%iP%SD(A4y2ZyVxm~2=75UU=;W#_;;Q|T@J}C!(vCdtmu~(rR za?Iy-5S*jUxc>Jl3^6bMVJD`nJ>J1{&<{2rPQf13aMgGJ&=ciNXr^_6;UP7_*^AUb zl9=3WviLkAI?vrniUMA zyD0dj&TCY!-g+MWQ*}-9ZpYCo2nJm?r>UHZd!S}}IHapse9o|R+F0%_dUja9tP8?s zW&m3S%B5q{<;CND*TUkUARQ}i`LrAkf;cI6M(}QgSgPYhF9xQy1f>AcTn;pqA_OQJ zVZmhrlYQE0U>hJAy9U@=)Rmh89@b6Py7EE`1`Mz0!IS6_Cefa_G)vmU)*vE5oR=Qz zFN)X&dlZ=5*!IH)9(NmbiU#F`Hqq{QHUn;uXDdWA#tJ;D4KCibU&p&yE>nBFhSSAS zRXm(%h`rhuTgj}zs5+HE=*`rBjWw3Icvrc2H+!0HA^#JJ z?bdIQ?8}>tmsp?N?|KFfpil+9zc2LlKOZ@fV>%8=6G~!vL0zPbzEunxdNh5(J;t9t z$hpZ>-AQalHSrxM&j>HcrlOyZ$UM6oZ^A3;f$G`ap?RGr1nznHmv? z9vEE;{JU`=@9h&F2Xq2S9fXP)+M##uFey7D9~5xHX50-!!B>fXL5(3CXN0{4b(CEF z-biX88pcyT=qbIkpj&uIT`W9~mdB9%%eCCzLTAr{!?NZ>L_^Thk&N(n7FAsB@TiH^ ziS$NI2ixKQX1$3_`%y{@StW0A6R8h-1mpD@jWZP+2LHU+}HNHyVxh=9% zgZooI>s#N$#$p{l-DlUvZ$8{)TGNJYgL0tP8(b2P z5!bc8-C3?rD9tl$zBRo5XpQh=^*Vp$#AW@quScs-BthV#dqL>0Lk>wr!B^@7N^h?V z3ENP>wFB>6|8mGX3EOU#^5OjnQSMlg%DU~LJf$R!r&ZmyD9SzbC9Vqg@Z*;&rIG> zJ`bs|#zw8gBU-$v`C1FB2u^QWaw(ORc9;$1!bl-P&9U4WEA}wRgU6z9Y((Iy7IxcQ z^V6X-IC+`Pw#^=+b6nB?aY*A1rRMJ~((nDvzy1dR%86akYDH#?K$Z~oapwLcDTV6D z?cex|c^e51J9&|91;?q)pPm07U+2-|b>tes#`OKDFX!cn9Du}mGj%zqdL;)^dg=G! z0+))6mFt-18TeekI+Jp`S`Kw^1R=q0u7}Nh=ERAf;D!QI(J`$XK0Ndrr0*pc zm1oRc*m7LzUQ9u^)wuOpbKLD{9V9|9SjQHCV=eK21YzJ*BGOu2!F|EqZhq`KIr5qV zZMhFi)*V16CZ6}O`KYk~TFEAz*+(80FoT@^B!?mJc|@CVz$d;6uLNQg<|T|GOS#&a zfB_8@k%QEnh@Y&$bfCxnx5RxSGX-a&xjr}Ih9RY-lL+P0psDhg1%H0sj9L7OsO1dw zAu(Y?u%^X&2A%X3@C=+>fp`dQ0*1OTcY7AX!^Z}+WR3$*6wr7$I7)ykC?DV%jrfOhGrZ8>;)b=DjarpAFt~8@#$^-Q zL_5jD3C}F5mUvUKfkPiq^&+yDKo^v^aO;6Z4B{Io!uoLqdSOw=Gg;+R%marHX+;rw zn+t;A*u-Zz)vuPrM#RHQ^*o49Isa1vL3m?2;O5$*>@nA)h#u2TX2{Ahix#)$WVNZ> z41*Iawdg+bZQ*Wtw$Gztj~Q;_y990N&Mx54A>izCs&G(CELRxdicW7?b38spYU3uL zo6`)jsUl!rA^2s^uoX{`mkQBgH%&Ey^MH?-TIfnNzZsCTnd5gV4yH9`?%1JN4WyoN zUO7atJ_2)GWd|dMc}HE4ew@LM=Zio7<^zy6miu`sN?Fjmz?q7`JS}EKiTu{7imM!W za~Ep9$`A|ioC|u;MQo^TT67bU_o@?)WSr9znnCn{&<5I5%V*X z((%iJ@H_m$hhwa8w{Vx0C8V*k3DH1f%6LJm+HINfz2x-(koNmCz%`~DcS12UP#f$l zXIt7G#1mE`r@Y2&p>C}BE-JN(!QKS?5WB*n2fb>ev+AwO4J*r5uC@9)WQmh>71~fL zmA(^4I1DGg23)w-$0HvH{E)RrZ;E#xso#=!p8-$b=(JJ1Wb|nk!$&VabNtiSEh2V68@&9`UnI`6Ww3RcsJoj7vEl zvH~2&m?$t)nWu14hhcDklRa}|r&L(R|D+7xAiEZk7C{Nj?BqFZ`}N&9%HTcXk0j`O z?36c0eqY?qmuU|6V*9fRnHchhTc`pw!a)D$E6G!)e#)=^VLP}~QE8B{0`k*7<~*#y zxD5rzZj@xh5OEq=0b>QO_mT4Em~7;gVVp&Kz^ZL1h|P|)&iqzbCYpVU$!SZv)}%#{ z6-NTD7$R8-O2{Kw;pZ%%K*ex+XX8Ym9TfU_bkPE;dosmn=mgG$U1bTA>%+&XZ#RDg zrE$MdpdcbzBSE+;2X>gU=4j-Jn6+o_D_GHGEyx#^`99Bbyw;4kbWPZ-9#V^>d*150 zlpd>Y_WB&|A5mytgfDyF;qtyRi)*)J_f%`4r@E$T6CKFw?As%TZznlMmQ=X4Uza?0 zAi^IBv&ucV#<5~=VW3~N1X9&5mra7e(|DR_Vszco3#JN?%%-bMPXkA`%IgkgFPVxi zG!>-3s^)i#qprA!cZUh%FYjP3;ry5t8H5YI;fi}&rUMd1GiDv9M?@cGWy?zKHHdR( zua*}TFuv^`UG(<7(c!jtJ7IE&-{Y0FLF0_`icoIYg03OmQYe6>68^hIvPxJg6$kRP zC^Xfk?SR4QVg%a4oy6B^cTQUUHAY74O61Iyg2;F5a1osvX5bb?&q#UbEMUwo{07Yu zv;azt2@^XOtr5&MlE%S^6c(CDgThl4XCU;u5^1vnQ*tZpd7_pt zGS)rHBjaGJnzu({lgKx2(#&4giKEU9Ij@?a{?kz@)tB@9+e9*@|LLUBC(B-kw9ix( z<-`nj=fPN`*5{O1zZB-8Yq|p%V*DVFOfe?=xc3gG54MN9dNMQq_6|h8Y0?|O<)D56 zi+|YnRO7Fsm#0D30V|1Y7Z-;I4fp@l)A}#qG!95eZn(cOw%9=rdX49typf<(uON{e(bDs5*cJ#bzN3`9VgPH zZAQQDQ~`27Ufs;rRO9Z5_P~-;y)OyZX?JR|XSry%&QLE&vvU~JrUi7iCBJ;&f<@BO zu#H#C&Z?R__!577c;Q*ae^Pr^(8J__iQKQ9YtGMnDo8hkY! zrTpQkY$y+BKP$K>tN(gsQ2(tCn=+^t(eP}-g?4xwH|kWkWY{S@)?C_zmS(=5F*$5# zq}-FRhHyAJRi3MPHXJLsqcegsA9T~5=w%i8>3xmDgMYq;H?+FaqqMbJ`@$8AD&9tl zVj1hIZozYHJbgxjL*>IK&@LK{*UyZTT^Mesg#{c$qQG*))QPqs@X?p&>dG&>AuflZ4e*LR{%in3>i zrlPeFmX8BdnHHmRuP+)7JTnto)tfaPxPoXkP5KAyI%cr3!a}ZR^MnF+gZS{iSG#XK z2s0h)L)`=g37xIGltk~^u_6RVG{Z*-xMoZnUF^HTJ>%$`P^2qoKf}#Wlm&np7UWIj z`I#i@)m|F@9zb7<=ix4qa0M~jV;Uu@PndMg`gX>65Gf}69Mb|av|^-{Yr@Ssnr+W? z+sR)LrukJG_+63uPp+)<3h* z3gJb{3Ag1S?4_F~o3ef?RP4NnLTqCX7-*3s*sm{O(g(_o1dm>wt*qbf^}KLv9sl8k z&dqx0SaFyWX7tt%xwAONTGrh&9DgH^`sV`kMe8Gx%81|ZIG2#vXz(Aes?ubjA*4aj zJHm@>_2@~;p6nZy8a>d9vL?>sYuFAAU3{bA=Cw%wYvanqd&Z6TuED;^g!SMh4W~}d z2n>60oTz^(PaUSVDdmS399ygZ;fUQ|i#!+r>$+7A_LJ-~>#K|4h4gCe*@ZCSxk}H; zq*DWUu_7IIK?;R1eQa2{YMIK5e{XG*Y&1hmfgw^YqG{*~dc6i)HXQxK)IE9^(TQECc)vJFM_E4Wg8?ldhS}&Z>g*un)-e;k>&pDR6L$ zr$y|yL_YaS%q^*Nv8_-<;ZP1D#TUPY((rw*Q3~o_4!r6+4e0?o96Q`K*NjPNp3QnDbbqs{u7+n5HJJRKuRLpc>qBV8*fhCXnsA; z>Hase+TYNxkDCb*Xpqf+;l2XA&@XbV9jTFj3tFLg#i!Z#>Ld4j%BNTZzgG@w`Eg=X z0&aB5{>+@ZMX1x<9D2ahY;}&p!2|A~ZiSBuPHEnRh+K2_zPJF9s*{Fy>0t*hXn~ad zf?mO!P-dkWh~oM_48C^RRwR9b_+KflHG!@xGp#hL^n_YRaB`&kAphCvtl{P zh*7B+e#nj3z|58)d)-y=kg1Cx-~;0bR2fLf~(Xo$eNf>@^zy6|j(wmuH# zQVapG`uynNtCz9TlmfpRx|isJ!;6S;f#;_N5ZA{Q>$X!E>)gG@J)lTNz!QAKJY+<$ z4&z_=MSLzZ*~;j2Q>iH@D`fOwcuD@`q-FGjofAic{tbWhpZ7J_4?f)(ZKY;3INnj)>3bw!cqdbg14w zRN=&`+>HT1eT0UbsxzoC&|)>OUfeQdb7OUr9NhdPxYsa4+D@r*65PV;QQS;6Ymfi( z8(_Ha{jnk~aoN zhEM^W3Di^(Ec>W=0j$iPknZz}n*C+G#RqvI@GAdFfNftK;CE(Q8`-buTe4xF)Oym5 z-|g{-dN~?qgB0wB4}=Jd8S=!(jO#}4mtsk2M=`*aXqCTUGeLi)6M@2Wa+~JFC(7k z3Jspy!sCa_D>|ELdIC=as|s;`wD)ll_b-J-3YEe|dnknf@}q(KWr2(9>P`$;EVb=0 z#H#tb@9?!JNfiutQ(s%nm@#=rizL`5p;B(}HCEpJ!0qHqvU_V^KX6%ijSYfc2kte- zt>3$T`4jfP62WM9Eu?<3^>-q7`--t)Z8aC>fIhe;b|=UjY~nLSprbA_3Y?z+JniLC zRYVeCh|p~Pxs@53QYac6O~#~-%t(KFpcfv5Q9Y>Fdn{kL5ojl5q^ymaS$E~f>dcYE zVohdWTcbs6HsYBhHHkiP(2m{}u1M(tuf%Hv;H48TG7#JlyJg`fHObf$e!5e&CfZhW zt=jQ4h-4x|geU$3ClqAc?oU#!!Ty}*3ayEbVb?`$MAF}WP;fY= zMOIf92(VVa>Yls_+d>#EczlCXmDB!{aYGkl8*YYf^D1tKzpQ>q z58q3-!l{osXR*8K6#B)h_L!9*o#Ru3qWnc0?n-yxt6JQs(kOS9=ZiGr(&@|78NrA0 zBT*L@)U-1r`*Gl*ImKN*lG|>hNcCXSCWL0kDe4{0fK{%6eC=)Yp4Q(X791VHI5q6U zrtJfPTZyQc$z?8p!3GpG*#7{1adxENZMfF?z=L(VyIgizcgSSJpBRhr%*t-DVN&q;YFoIj!kyL8cdLqTj@VZt`{ z`TS5^N)+f1uM~MRxm6AHg8<0s9L5f1(w&k~kwTIbLHWIi&IfZpIirjl}->v%;ikMpPN%URncu*M%=#;hv?VTW~SB+t*MVzr7;@#8O8w^Pr%zk96 z<5Q+Odej_<(;0=QB%?JZ8;l(G((I_X{A0pp*QYz6vBVXePQ+e{wp{TjxT5q}zuRn0 zl7NJPe4s<2-1MrZr1=Osa)#u4Q z0(B5ILRS2znW!rU_bE&a(dCHPAN-p8CpS~Gr<&KfT8nbjDc?<~Ka>XT!w>8k4YZGX zZP;0)UW)|K9xDSvx4*m%=i!|NA@5S=kN)X`8?9c|SpKs2CS+^BqtXH*&3mFqM=%Vi z3h+RE#c(6$jHzxNI-{7uoc^P{42SFe*5aL2_$(;!%X?hv8(=zAYTaQy)fp^2(u5m2 z%%6j=Z{PFNO97%^DRp#7*Kb%p;+A4GwFag~#j^NK(e|l4Q$VYM zw}I_8ph_SDb$+tsfzj)pXy!jXYLmiZb!tY@zP96asGw`#g?EqL=y0HYowd5Tm=`(W zr5oyDZhQPwhiyOXd_puHj~m+AmhT~naIk}OY+G{z1E$|sH>eSHFSlRzLZ<=@3MR2g zi7Q;^4wJ1!6r1#(wLZ=3)|~HE9jU)r(ecgBSUf{Ey6>rze3I$Sw<8X=KfQkd zoObW=+?u}Euna(Z8evH(Q>LQFfg0Scq-1IjXP4Q<z4IzR&A2iX7$RKh~iJXe_*m;-G|r2 zyeYWj=o~%s9(A9$^J`QMu{DXhY@Ji_-4XpRIBadaZu4D9S;38V*!zYXmK-iA>t^8B zc~&B~JnCWP?A!c+a#4~l_9*?@6=}{wtNynE+!5OhDNMja!{+)iOD5N!xe$1_W9L2o zOD55WWjLlKozkyd{o2fXTZ2Nu-bb9Z!x8H%Up#SjL-E1)0HnQ)ua%bS?Wx;{_K-o= z6$*DpJSuMQeSmUbx$sr+)mCl%!YMg%>Ea4)VW(KaTl3w=k~h5Wa5t|i%=p)tZv_3D zLkg84WK{+};OV=@=_Q8~-u9)fX2kQw4+76*BYbT`=i_PH5N*Xlf}z0Ap;b=hKNN8P z13Z~x%y^ZQkd-;CSiWf$;01h1(I&Zo1AgSyf<8}(nWqSKJ+d(2&a!l#{eS(T6YjtL zPcu@PJMhl~4T584)`*y=es?FbEiP!me}wdW;K^;J_@nN^~pA04%-d%{0L(Q4$2@Z*XGbd0Ssn z6Vo*ZZ>PRpGLAw=BThUvRz8n1LIB-XPOy=>47~7q|X6TAXlVxU}h+9 zq*6C1E`V}K9z}O>a)7z-aWjq_z8(ST`Dw^~1kSt}0U=naWY5Kgf{-SfJ`J71A2kZV zQ?U!Y1NT8N@J_{#-{my-s8j)cW5RRY(A<)sc4ZFNQll%F`p-}?$irbM_tpYVHEM2< zCapWo7y1L(kL#mL!KWrA^>5Z8Hf(-AQN7&9fo_@K{5@0%L-Vybj&pZ5%S*Tyn}5-N z`}&CeU5?-7?(6dNRJAcHC!lvNE|R*=Oio-;+^{aQUou`hZG3Nz>J5E;wY{C?)FlDi z8dAIrRtpn**m-0P0?qNz)XH^vUVI6Jf{PI=gHYFwrD)oyy3~9%B|q z3WNHDS11kRTKmemcF6-n=f5$#>xA_MhX^m{fgfhSi--~EMxn$PUNPJ}b+J0-o=BTI z3ilA%IEv}fW;g)={gnM7!Xz(STn3ZyJWO>#q<)cKZ!13vB2Qm3Wc+2pGdK@Wy4oNz z-%oOa+zoeopUltm>$6C*TOkFxe7qf5kWI4(a{yKOs4yRvjpV0Je{0ty>%J>YV}rv{ z0?T!pnZ}|(gUXFDFclYl0i9q5nkJuRYmvQfIzCED`*WljLBZg5zB*z9^}#OkDyfbA zLL3LH$IHz{z!6Q_%NQwpG(d*=;q&Qg+whb(>vODVieW|-SA;2c#y1u_EwpIvvbDR}iVLFQ;(%2!5LUBi?8FhZ`b_i;YW zzXE1OrafJ+f|=C{kffTV9g2%b0!|V9CSL=NwB8t_VsTwoJD=d{3khw^ ztPsc;x!56q-SHti`MVL%zCJ8R8DSGL%>N``B*UCgh^x^! zoB-SJGf=9gZGm*4sn>L*7*bK5W}CL#vZXnIxwJdp+EqR=fg{JhfMCjA$o2PuFcVZA z+H4Vi(HU`OIPi&!gHwa;(b3f{0nmK%y1!F&$(vOkq$9WfM`hca)#n!XkNHibO^ghT z0rFuK>)-+7U=3bSbZYX)VQ2C4!cs^HB#4M}Jy{jOd)Ht&7-zc;9iD^wLuViy*lG6m zy}ivSe?|%o;#x^+=WY2w$GA)56UREcXpDUhb8?!C*g_)>Z(UXjs&&$pLo4Afdqo7l zs6AAK%n;z|3u{>7?vc|8ujNu_>5SgEWzp%cJ3Fze->Xms*vdPBamTaHri8__r@cus ze7m}aJ+WixOc_nvt^m_jZ>B_y)Jmgj1J);6)9~8-nWwQ>JLg&BFUApi!U=(Q;jm3E?6I|GB)*@tVo zg#M!}i_oe|-v|;8_IiqspmuQ zEq*v&woM;%2~1KsMyuO(%88-C{NralYST8NXSz^W(Om@V-RM;XD`8_yU4zg#ORHNk zO-O%&X7d*j&U=y(6kmnC#fsWoeGpXqVKa&2@bPAvs-Q41iKKd$+C>4Kz?S^`F1USK zCQ^6|*cTb8tUCikq86W3Y%elvamRb!kI5k39>+=?j1?kykz@A6^G0x&gKtl=$v@w3MS8RqK5I#WFqO*R*6Nl5$ zKDEww{k^g0g+(6%&ly~xE4M2oLo<;rK%MH8JE9W9tzZ59ej5gj;W$!ZSEN$_?0ZKr zv4#Wxv4m9sn>dx!qaR^!g#q0h>^Ffn@!}Ehrry2~|C#f5O(~dwyFtFbw6`9Om3Zx? zdkpF}u_WQdMj*Gv>F0&H>{->hiCsgT)Ww)t(V~v#w<6FKTxZzt!}s+4iSt;4h}v|j zQS@wg@QCfjs*EA9*E+W*@cEHY-RMBTAHXf*_4q1xU*h@P8HD3*zXX=i{=(DhX>TND z7It+rQFQgS*o0+8ZT$Yt0^J@gMgz(VN^JzSJCtI5pw_rY+{sK%wOIxxM4mm1!t8ah$wDYGwuNt!I=gpkJ-PU|sy=we#6Hn$map~_&OTnTyrSHyUhTV&V zliA%_`?ewWe&se9G{XL&cqYM*kco(^K0PHsbmvyhy4du3}tF zvWuuO=YOi-DngJh;)}Fm?JaG)qGhM-&kEx~T+yX=p;8|f;7nw+qa(irn)w(sCPnaz zTbB=eyrAL<+B~Rf-JUIh`}YY0??a5xR_U_hO3>j_l)`{`Ppmh@ZT%0i>vubk=e%2K z2+s7Lik>;qOS&XQRU1YQI}y^VrG~(Ms*Xm5;i!8%FrHVzq3ibe!M-2pKxIEQ zvQxIKF~O#L9-`f1Z1xK(xWhuKSi^6y+;$+do> z0dT|JjKb}ll(H=F5BE4HMv^LCk9rI-XTfKLbH#5N@&?xsG6&sC8q0FC)hY$TqeB)1 zFplMjL4kV^7Yal4Bf;-q|NLt8^P6vSKY+$T;2e{*)5|al_a7QsDK0b z=gnL-ox$g2+u+;*SHYbTe>VSzJMaH`5C?(4xvM_ILDVIe4mIe%hKxlKsT3xPEB>j=Mj29zy#AI(UJi=iY9q@)FJvaQ%mkAf!! z%zvExbuR6jd$#MHZb4f)fF}Am1iNslUnSTvz>Ac_oT?iEUvL0`N0Io;V(*m#* zfOVl=_EyCeT4N?dJpUtRGREhac^o;Ejy3BwcN@%Q2QI#o z+ZyJvE571tx@jGdjcUkcu;Em6pgTkB4RxZ?T5edVO`UqZaf#HsPqUq5_S~h*5Myt? zZi@D$Hv2jGHhf~e&Hmn8d&J-Z5;SbYJKD|_K&Ub?_h+62hyBz7vwj=0a_YICTjdKm zLu2xvs0g}4=cAX7(hdm+FR?wrSw*4(UYOqo& ztQDbLKUkhYuJwQc3ru&w^syNreH}{b3woKg0glXIpM-D;;s??B2bY0`d=~yC{gQ4VVae$+EjSH? zy;@HD%ItpXxMQcsR94V5y`f-lOGm{mDrzO0IDj?GCRT_TjUZz=^ zFK-E7ux&O0J$b`SSG-MG#bB8!(MNa@VHx3uKV=$c-?#XzDpxnA%k7M~?JU+hgcNo- zLHMkysRg8uv-LG5vQJjteHT5zl#Yqz4TmQ^3u|PtZX$U3NxT12bgShbF`oq|;2s~jPFVdb60s$Wu!W7E&)Wxdw*e{xQgYS*o z%xGJDG*odQocGFprT*8Yt#Fs;fUzZLXp zKoE`q)q}k|6mM?-Wx?NPCA?4q%$M^5D7nV=Y5jl0Uo~=_H`=XnvMeK%BrfrVX>aGS zP9BZvkakL8KypZ{oQXEWC!l3aUczsdx2Oc~KHk{=bA%6nY6#?OT+2BecKMlS14>}8{BIZIH^DHp_hiG5aZ*eKVQ|bne+V!l>Re1|!@RO#$ z;$b&^u z@nZbys?{IOCzJft3~cn!0R5=8hk(Pu^0cyzSiM{dItkWQS;!M zh1?SIi#J1rXAjlg+?i)vD$`w)7KWcsUk3ehM-R@4{r#{`dc|=4mjz*l=kl$rxNR0vCco^FhiGmI_y^i`y?CBLi;WcPyy}cmVN6QD1Pw(;14R?1513*Bd-#71IYf#Ep+Vog98T*e*&1GTQe{wp3&DY#U zoyD4ZevzqEU-8`iPy9~X^2XDZ!Mhpl#NM~R*+r-mM&ZWaK@FmQoy|(pokmVC_LLHl zR%5GB1w942`?Ud9f$gj*y`lE;@nD>Z6{4xrp+3r>u(0+Spk(G{}Y_vE+*ilHt!#52&1pboSMJkRoV$!{iOP`bWbF_q##9b{{Z5I7HEFd(eP(hM`NEz+R=Gz z&6}q~zvNHnU=avSl%1&xM@JE#&qig6fIQURfle_-oMbaNYmk>7;&phNs9Z_3Dv6f1ME@V zHMgnpYrL?N{aZtPP&$|Dm%KZ!sa(1 zsR&TQkxN|ReEM~a6&wKaMKR&WOo8NAP0_F~F@kQajQWPE18hEOO-^mu_f)qQM{c!K zy)!M++PSY%2BHxtV8Hm?z`}4xwuk*D8p&P-{SR&_)R{J=@hJ8Y&c4zAsnYus4`~jP zyK?oO`yfT%N$CQ1%2~YX+o;o-{@QUdtarexqU?SZX!U>HyjOii^gfwcuf4255gjAl z9DzKrqbnt+wi+2x6N6G+@88hf5`mCwu_AN&B=qvtIJzCdmdX=rr-F|M9et3z?74r5 zC!b4k-;fITM?wDA+qz?_3R z9I8103b`yo&P?)vAy_2?%&D>;3nh5RNen4r6b-{5rp$Pg= zWu76VzXS{csB%KCm$lTkj?X|QQY5>5yb9efAf=^jiW|bX01mz{<~obw&|PadH3839&BYIqLpMbr0g-bf zu)qv2IoD@`ge&;t_MboFM0gMFA`s=Ho4SYJk{E-YX;Ns6P2ff0nvHmzxgjLddzJFx z6gSK^VlA}ioHKaliaB=MiU(^f7)-yo-G|@oxO@Q?@78^s#vXm*oKYXx{}s40&ZgyC z?p`4%lv<3<2}i)QZ&|zH;ge4T$+5+r;!Zgc+Y5>SYs`8Epub1&jK9Ggn(BQ`8UDB` zPWe7h-J=BcD02%E{COgn%V0Gd8L!;VxiWLu7L^eM;|!m(K%+8HwRr}EBUE`>P&+Xk zyD*jzgq8Xd}7*t%Kj)f<#5{?E@jzvTzE2;p#_^3G=|IZ6d|F@jC7nTjpPUz6j zEi|uZZtxzd@bSZbkD)i-Y(qWQSb)X^t`aq83OJWfOd1{P50G&yWp2Gc`a8GQxYl#? zOFbI1^Qp}^cfznn8g8*e8lUGk^TPV89WFVnC?ezq_Kw`%sd~kp(&-NQa68PWMuJi_ zE`|*u36tO-zVfZHkxwiG*2|R7>ZO_NpJz_2&T^>zDQ2a9U8ZBl2cnsj0Sk@b!>c%a zx^oX3K{zv@&r3+T`CUZ;|E3#A{&GLSe}BEbjm1YWd!9Ym@Y!@?txguFFebGLL;SFX zi|ke?%)O*`D~8$T-Iflx>a@b$k=bZRS@9giFbYd?J_r<+ttDq)%gL0 zRQ^;ujr7p_21vZf`bpn`(k&k|0g8W4*?NNQBvTS`lP@Imu*@=up z?>rXTs_Eu;P1X1Cibsgab-|B9@RFCHyqU<;1&i*5{GR!5j~>%94Jf8KY!Z=%u;^lV zWgvc<>bUSVu@oVw0w=HA5#NI;603dbe1=VsvrR6@F}XE%xMuYCFE*~e4{j1IzNnrs z7xxXRyfsc}U&u8CiMl1#!iDz^o9)Ides=VbIz!m;OQUb zg#)HnM5SO6`%$UcVD3)vLT$mfGjeY1a0i@9AmRVstnj#1>U_GX)Gl#=MO(fP$`GBJ zy<^|ISu1wNCp8%jkQR3*WUAvsCI(_-b$gm_mxxNcQE)Bjv$zOL z+}^i}&%8h0sIUt%LR;1RJ!OcS3hw zs!+CRGb)W;5>aub&3>ebg5NC2sm(m;dMZS}1a-|0^w4*TfSpd=)fWEq4M9zv0h04x z!e-5A8ga5zbe7ad7c;FFF^OwMT~W{{Db(0nCp2uPpMKlXm1Jsto;RCJqF_Jusxf}Q zuWoE+Cg19JSYkKz1Hh1(%xvss_62;r5m4)Vz4xLOcVQ=?rD~LSlEewt^As)wNBycd z;Soq%--vv~xqT?0Fd*4g){Z`bU2tREfsN%t$9i}m?M`HPBRtfm4(0=_*7i~#t}d>& zWIxX;dSx{KUoMz3-gipZ$Y8VL<&*~#WT)9UbUYb?E`~#n!bJNVInks?ANCJtlzwX% zHKRqZgee8LK4~-9Q<$p^RVx_EvucjQfGoGO{7;H-6DzBmKt{WEO9bq5 z4OVJ2+xR1yE_=9LKAbqD_gr)aMVrPS*wX=c`1|R%I`LcO52Gq9nr7qAp%3JY3AIxU zSd!3>bwqk}M?Mmp(WhJiHvvs%$j&SI=uzK*wiX*>^85RMnLs96Vi*hhQAl9jBgD7& z#iGB0*@d+l{6+s#D33a9|7dXSdI0H7i^@5zv zyFkW}b98D85E_QCEa^KFmRacJnoe{_Ui7w=c?Vm9mK7boxo-{9g2XTOCgf}zFPT2f zv^?f+w{%E2j5w_?^8#ON%iWxMJTGs%RGA7Z+x{@kNXFWGEo!1EJ)pZw3O>l z`%g@sx0gkv?LJVz>|wH@bKpBPJD;w%gL_IhY%x`~L2{iVFg{juqhz!l*gNRQs{EG8 z&NE)^#c;H{DIW2)<_|-^0zbFG=`IEph=v`g>j@+y{#Ou`07nzQYU`4U&I#E|Xrv#U z8tQh~c;l4k@dA%p%oW%@>r@SiN_6_52r3)z?daSWXdAZrj~!RWow}l8laiwx-flH5 zR+%g`E=?nD4HJCSoT#aat%sQC++>U$LaI@+){#t-3io(XU^NQ$_%zzK{7ff)S@4a6eOh3X?5?R|+;qOL zynSd*(~Po)!81U#;vsz1@EV=^+#cMIzN#BzYKXT6VN80@3IeDX@aW~aG?#j$IR11n z??9FE>pL@{KX(OB#@?<})=i5eI|?Hsx?MsZw|u2lz2O#$UbUzAAUAT3cr>lNnfn8# z~mYE8oRysv8Bi&^lsntV#NkX(~g~$p% zfue^6htMfxOy&*ZaWgTp{`M(ZbHX<3__z^!`*x37lnZ8N%POQ!whUV^hz)28)rSbgnv`X7Vx$!DQTdN>VjKu=x; z#aO9DFETfL&tsNXbr=uE?BKicQ(Y_<>lAFI>LbRLV4s5L&@35#ssY=Htfnv_UrTp84c%g&jxD&+=RW_(|U~Dp!B2@mURxt0eHeWjU|9#~^&U^^f|CMZtPlLt- zZ**So`y-V|5LQLOQY)Yvh}{Q(=bbqGmf7))>o-8??0Mo3K|}*FL6o1$`LJMtxZOo@ zfzuH0dlgN5^!XQmR+Z()X$v@uW8Ty(EAlZ{4G4hZM;aw2vTL(g5`5graSyPe88dSd zYSQRGDtr*4$C}6m#jT^G=i{ir0C-}~S7GPgj8TLV|1^4s;j#^n<>knxHSvXKQ)Lu! zg=AT11i;>cSO)Gz!Ny=j-MRezCz3YeJa20TjB_Z`?Ac8|XVV6>-_Oa~smeiR2GULd z=z3B?9~F8n0N@<^4xoq2FpoWsEn`UP!bd?ledal;J^-dt+xFw;=4OCGPH>*X3fWrE zT%(XtOrhZ(4eUPff=+-HJy;Ghn(la$3Lpo*!`x zJr`8$R7}^38ORkx=8&g<%OeO%3J$0)Z~$NRA<9&7*uWm(W}Cn9n1F-7=F$Y&MF69q zB88u!GC0)&^1LC6nTvFqFC1`Abd9Lxaggq#$GpejBqQm2|4D{wa|HaKS&ecyH4bns z1@jAO?kiz^l(@{+hvDOv29F+yaJD#*2ZvDk=$2QEn*om45ayp&0qyL&uG@r zeez;O64-8xd}tZP=?Ki@VRw#xxY9gF!yhHbffz|$j$#`1rJ$rxL{ss9`on!p(Esnx zjobV_K}rp{B+N-Kuxh!s2f$ikGl>c2YbL*bT>&EoOlNb0K35dvv;>@dEOT^$qnA-0 z+|dNcrqsNwdn0CCij;XgD<)>+Q9{^kc*oD^&>ZFLXv1#6xf9q8;qy$7Q%PE2dMvV# zJnM4V6Twl-UJ!KGf7Nyk!DLP4K80f-!g|Ot~u~ zx4vemEQ8RpTTy}lDkO76SuGKG({l|c+dSbFK!wVI(eat2C;DVNah%?3z%ige(hGiF zs$Ly3;*t#lD;&zuiS^BC)-c_k7!0sJLmRgC0x*g2VOOYQXJPrr36w&Mq*8jxN?uIES zccFBb8<6}W;X${a7^FZ0WYmf(B_4g)31{9?3d3=m}=t=wQ*DkeJ z+5Ty|d+%sne%J_`?uT|)0i+U4lsU9CwHeno%=`G9_5H(_L>zvz+aqHSnZ*tmHN@

K7mFtII<{a?yn{XGbTGt^l&j zK(6k}eMte$N#X0 z4C~Ol1q;tSfGoLNi|{1AA)XpSWZgb>us61lmf1dRS_fhG6NCA+>0_qymJ1QQhXbb^ z(VLAL@RHCbRvdB0!X_hZ6D~jG$M!gAHe53rV@{K^C%S+3VML)7p>CJx8C1Qtd`Gfv zX~58$dQbrCNUB+vw!)Zlu0!ViD=ST>u5 z&19JxLTKgeL&@q)u>dkKUmWJQed~BV%2#4t4w`=#HdX;zy-n70bt0bLHyGVl4`2lx z5_JWERGZ<30o5Q)DC_$cu{`e`)b%^k(qq zk#M2H2SB54&3-p`5;}WzqdbTjp+^%^j}>*f2ZG+t&kGM;SkbEG&&*sczqBvZ5|unP z%-nERlvpFLZH3%=f5}I|ejuC`{P3&r&b}n<`;knL>Wt>e<%d>EZa`Gp8&6wD{KR4rQl zwM)d>u}zKGAyC^}F&Y=K)^2TJ40;c|L!eb;u!%wb!1%w8AZ8OrQq%>fOT-RO!OgDB z_fye%&>1SV(iA-pySwI z!lp;RH|l_rV~8-5JqS1Q=fZRii^#gefm+`G^}nz^DTj3>oyqR){C-c}0JgI|L+gIJ zIf_lu<6zAWIF{^78q=s^cH#V&(Hx|Dvav&hxWJ-1n~qbt#&;vb*Qx(no_#@hbHvwzvPWQ*DUdv1wOFOO5VkPZ#vBM&r zsc2mwE*Gs@OT8`ok}9L1Y(+#iLu#o~r8Kv#N~>ZNv|7RzSt1~$6(Ow@!csy2feM2x zNx(=5n2^)|=RLtr=idALp6CBO^(i7GIp=)O_q?C?{VYRlu^7=&CXj@{D?V%KO|tBm z)WsnaI|mzYyT%XMaj2H=udtClf|!TV8(B+lI5=iqx*ck+>e1-K%yt8ECS*Lqv}D?_ zupt8r@|nbnJs>^<3N_pn;o9@1!umz0*}!T z7}tAzoR-9%T7}{CSdhr;)WEx&-5@hds|>#c!i!2Z=4Us}yFVQO*0^DNs4y;^&5 z=J24nDM(}Re7Qw75GokFe+9R5OCYBr@HW2!)`qyDj(3`NHg+Ru7|&1Opea?-+xHvp zEBoZy{`(I?V=s0IepojmGr*vKZ0*wIqCUPzi)iSGstYmCoL%iPzYbw>(J8kvVQ56! zU(~)EGuRDxTW9 ziuM$M1!JyiZa1&M5UhoI?w3DN>EfPudmR~Qfv`-#E$%s8et{7shQ{4Rd7*m%GM&#< z~`Wo`d$82Tpi`6ceMuI{mCHC6W3Q>P}@~F466if@T#STMShUVoMGlLo{_B;8GZac zGp}!CJlwr2`cyr^eq5O3!87AVwNSq#Y@4*xC1aNF%l!m*B_vjXb|XaJraKpzS$@dr zbgfU=Da`IV7YqBmnbUqde?dh2K|FBEi^yuElnejDl>5e8V^tsYSf0d1aW4*+M7+L26Bi4cGpZrBrUzUs~P3hq|dBhp5O*2KR7aFI2o8*N02jx+$d|ARc_` za%cEce96HvV11!UN(a*_; zH^cR(pNr}8s#~P6^u<&(_zIA6YW3&X{pn4*2T+aS+ARbI?OLE&*v4E=j)Z5korQLz!5FE65I?5HU@D zYpma3VZFqwcL-qnH;A@~xB?x(yNl3i%kG{#rR_6|4&YyuT2w0|4!vVw*MVAbMhmc0jm?riuh0N=2qJDMre^?`RDu)d|x4QCnn z=o?uOA;1(XAE8zb{@jOH+637GT_Y?ZkzPm6@bqGIF6rabWF1M1Dqz19ot~9q0L71P zBNm|NdsZjNxDI}XJ6NnqBX%m~;J&M;V_||LEL|a%Xo5*HKx)Tv7Q2yHfR+C`y6p|m zESBPUkoja`65u2`k5k7)f`jw{2LK44054Yf#2uU#0x#=UX2mXe4C!Q}>}(&_fLwj(bv|BZZahIv z2uqK^pCy98hL0KSpcU4G@B@(QGDi!Haf5W_a9mcEmsJ#QvZzHDug7i zun=W1CMvfO1#nw!Ed~VvXpHP(uY5J z9KUT@KR76vrJ?jG-oQq%UK-a{N6 zZ}5Hd<+aIiI)_Ha%WPv_yx9~IzbkI3@8X3AQOw${-NGN_JR9t<_Ul}Um7<$0@Dt5$ z(7B#GiCP?RT%Jj6$#UDrx^U7jh6pZpeu;m689GtJlB8>Aw-^HUJTqkJBwRuRewL3X zgV1i*{qbvpCw{g1#XU)t(>k5>-lur0NKROJKI3?=ck1q`gvM6GKFQvTiy_C1^l1_N z5C(J`GU_V)PkC8uvyklS_+n23@@9&~VfCl$g@%xt&yG6_tCie}WuPWw+cy0CT}zv1 zFTm`_;MQVb0ZTAw`5}~oHt@&uk^3Pi^W#ENkQoRvD<%cY{5stgh-e7DJAx@DpAm*Z zOx_nY4vYG-Wt#e&BCjcUHzCERD!tuz`5V{EVV^WldRi|EA%Ns( zZmg`lR(?L^S=QMHD+XpdzDEn2U`-Gi#=IoxM&n^m<&yLX1eLmAZx4!9$j*kEC+ZmE2mIqOzci;2cZMycq&e;4<_b1-7d@SP!PyJih{m> za&p}wEtCq8%$fBoJEc6;L0{l0d{!Z-5{-**Ezj8lJtE9>bIoV8c0WHN!g#Jf$y05z zrR_^7=9YCJIJHVLD0n9F+}ciA#gxnxUsjeovDvf)OcCrAl+y8*8t*@K0oSU3r}@>D zL5no%jdgvCJJ*sum787&r9w_bQfh<4@KIq;nKyM}P~dg0wa&*1Mxhc2{Y1~aJ`-mB ziUAWDN6?J=|e^hXB!(`|2;{SWzPZ(PbM=p^9!)rKn?w96G#mB!YjnksDLPjmYz5!{G3b zLb2dd#J-=GB<-rXtb#t2qP2O!TK_KM+>4h==d0N=y=OXLcGz*XQOlmox)J3^o z;crqNg|>gar4<$tfGOexD9@tx;}V9TH&(v`sW)C$f6|E=ZD?!tmMe< z4|-k7@vFw_xA|Z>w*_~X7~h9Gf_c6MH{)JBm^fiFYPq$>!)Uq&UkEf?c@?}AP8oln zU}hGrlI`C}=ZmhJ61#wib>*^K6`0YppEf=S<>Q)X-?D`E`S6Gf<|QTjG)Z^o%ZeBO ztchp-g4^`tx=`t_R#$c(;QRw6gcFEXFxbVxpiYr?@B-|W=ulCQE_0rj6I$gVD)+}W^M9hGg`&al?F1^u{u=F! z`Gb0jpso@=p|-9Qgttrx+V+06Ya>W+Q@>o3m=g96-aO}2YVycJf5rgybkQKgT~-V1 zy9_+L>0sGS=QSMA!@f}?Jt8>sgE|NMgGgh;@s7Y^AA!>5*TF>5BK^_p1IAJ5-W0}g zaedWS+7g74QGsrd`8;G>*qKEb@20^00=6k6zeKWMB5d_oHv zf#1B?vP=_ueft9K(8(sL;eEL=WIg;n>I@Sl;Brl3-`-yIas1c)Pb@3`ub+}~$G;2q zj%)FI67-I7Ov_gc=a}a=D<`&CfI z;T9`Q1hLQ1fTj5$v9c&_Vsq_v{A_#Vm3yGHYH`Q6pnTaCSQ0W~T$0m)58`|Db6D+_ zJubYUcHs3CN5QQZ_gE_i+aAa29UPxR<1t0EOjFwxX&4*QpTpUo2-);yCbujAvqS#l zi^GQFTvb>nYy+3I$+zD8+Oj57t(=3lwJi`CcX}r0zX5ce;C-n#bK6;mZ5w(u@96djcPFhX zUP{m6a&LV*$RytnJh?kLp%a=J*+6v8XP&*`A#c1p^7pqkZ*082WkB6O@AkUO|GDzn zC1P&f`I*dt=WA97!lm;WkL`($FO*$XF6UMdRB`_6>C#>HMIpoY8q#6Jn)F!fjI4bS2|R%5!OjxaT>~ zDKkFSFc?czkSvMlOnq`pd))nDC$k;Q(Dc^Z=!P>H`3Xh2?WRO64-%@0Pyw7w5K?Gp zsF|-7MBS6Au%9eD;Hu!fc3ZC7^SPn1I&*IG2m#zVsKzt$U znW#;tRxCerYaEZUJ1@0+-W0Sgb8gbQIhWi!CxH-QqVDzPpFS}BTns7B!roy2yDTLz z5UZ0uvcu3x2X7;%+p`{}*IScuf8Di1?f8f5C8b4q-#aK~Uxb-Ktl*!Vgis#B0%DcO zr%-6m1RcUctuIfd0J@1ajka%MhhO3cwh`*U<%*I%`qx7L;{3XOY2n9sK0C?p?-LSG zTLcI`6HfI*ud!Hrz6g7OqKU9P3Tn1k0zAM_lyE#8_z-(VZHK$}p-Uw5a2$I#c;IBn z#*QZgTHp%?wsBbb$t;VC8$^b=&@qnD^WmhB9etg*VrMigIG;mNCU(|afbapR;3-*Q2Ndtbw1?Nf9VN42 zyd|d6gmu0n1TTJ?J6qj?1acI3a@9mle$8czF9LoCbr3RqL4BW7ZLnC5b)^4rEm#uy zH5~n*5&LZ;+<)UgJVDJcIM)&8wF0N1{oW;fDrjr*z(xSNB@>OT>5Pl)4!rY7GTXkO z5j7b)Jk65jEfAO}@(97(k?a+Jm|a}{7OYHS6{{84-pL~HUz~OtQ(LSXuoJxJ`W;>F z)-B8#V36`XBq;D1`35LgW=>-)Mh}#s{eK{K`X14h=Nbw4>`o7V%88QXmRpkQ=oP&d zeYhYZpaX#tC>OZ;iFJ0)p7#qd>0}YEpJhc7;{`{--W%-q>A37q{Km?m?hq#X0PnDQ zn4?4B{sNtWEIwi?Ap1ZQtpqC$8GmdH9re^~u@JG^Tt;ZgQEECq?wr3N>uW&~av)Uv zb{O>Uv`1Jn^2i_r2b0F2aB%*Ud$yjW=ZSqTCi{_TUZtt>Vp4)-2q z!u`$@a2+Bnp7vW+MfQT!wfK|W5GC0y&aMBq>-4*8W@!-NoU%HGHL98WWBrExWC&O*xH+gfIcXjm z5+2opIcvYMQYvl7sM~N?k{_IuZ!l>cjx<5`&FLb$G{b|?WiJ^EmMz|x-En;dS3RQ7MYnvWzdB@FP3AZ~uC zslz%~YWsxxMU8C454t%aTIxk=gouel$@cGP70DI+Fi8iH9dW9G6P;_g`KpQGPWmpm z3;AorJa8dl0CZ|))nx;xr%+bL9c~lPq%)yTwO<-+))Z?MB|X?Z(ngT{)lo|_^)R4e z<xO;LXi8Bvz~6#}@-?ZD;?UYXxQsEYfz$_P}Vu@ddIr=u{p0TNrvO-q{I1A6j~^ z_cLL>ASS8J#CS%bmeMYp@N6>V%c*4%DT3;8-93ZwPDnU3B5U^+2>`4#%7Y^C|JOO# zr`g`)&c0B^|19%PP-OH5wRy2Ah`|umAiURqByI7`xxqWl$Ve|3qe=K%)z~e!m|WS} zLAIll&TFcvz82MvfainmfmUC!HWkoxDjrR3VPOVc8i@HPAr{(F_~-hWG{`|(Ckpg6VH+i#o=tJz;whmUMJlc_@5VXu z3OX5g0aLM2DQ5)UM2QK_39g@}%4WQx^snNH_#$Ub)&u{I;7CDUDLy{C)A*D3EF z=>=;wsTMfZVoKTq8zKGT)H@g~=F6*L-HbcVX&6C>)c9pLQJ(hbeYN&ydMtwyA;^WF z+GYkqYg062fY4%^*SX#@N_+5%Es>u0D`*c>@`oTBLLuVNLiUOn{c$w{oet++Y^_-- z8hHbJAOAL7m)021dSC9Kf?12rTLmA%&8qQA&WL_-;qfU^VmRa;lV*d1;=GN+o7mKz z1m{+yQj1C{vJR5>DYd1YFoXwHF2#TPBBztdQ9sr4m;yiEO#}?`5?WA+Fn4@eLjbhh z7xKIniSNPPr<(N5EYn9Q(5v?q`#OJ$`=6dgP$-c41O6s(?2wHHE)rRK>gec5$gcLX zZw5T7;M$i0tqbKxm)uP+T%`5vEx?#{g^xrGOG>rIod*Fugs*Y@<5+%O>qo#>2toOu zP%u%hGUt~im6jL972cYnOg?o_K5Fxj1vxpwu1L|J-+sQOEee8|=2$iY<}EZP@{7Y! z40V3?LGNih7&;inBzR|lEYyi2&nckAqC7+u;G|~T>>FP7x5dvI&Mn4Y)J||>+Y|c+ zRw=Tyg<%GGHi8OKYU0c$`#t-U)OnNxC%HmfI^_L$0(<^yP;9i7Q#nsC1z{!BeEAkas6-${VuIusm^LTD=$4T#o zP%u0m84~Ep3_*FGS=~6X6)A^G2c?oBVg%iP8$Z1M{0jYXVvX$|`%=@&C72CcI617ge?{1#G!Cyeftg$N0Qz2JS5E9MorL+^KLxUnW~_m$Uz&c$9-+pPw<3cH`1UFXR# z({uc6?bI&vZJhF!grltrJH}G;f3&2i{|qq~SDTJ~fRV%4R%(hhdUwf&$IF)IAUI)d z=gn7B(11e;B!0^Ei4JiArA!QWcM`CLxr@Eh-Y7C?;537fEG!}-<^y5O~Cel2P4kq+Q zA~=BbA=&2iM9`9FqPB?cOz0b7K}o@iln{P2>lHv(ApnX~D1f(P#g<@oL4_p4&5 zE7sjtiY~4^fyoIAawyN45*ZUtqHBi77Z0?z3sG(bSLGS9fw%LO0akR>_~w zghpOzi;ar4HAX)ZJ+)S|sp04sF1Y*G6=tqhw8;k5I(I1MD1dO}gC56aud*$nq1m_r z?t?hst}R6$Rf1c%QJXD@L4R%JW>ox6sVy1PVM9Fjate64DQS!D{%ZB7vRH%@v>IMC zh{`%I1`a~zcXHUZ`A}W_hUPL0Nohaxf-d9mtfQ;q*i>Wm8ZB>fnwVbxYCBm>YiwZ= z8W*E&RZkB3sbsNx|FP`v3;waM0$YE2Q2%}HdofyW>nGsRp~w%ni|-CC4vYLY%>6=s zpiNDzW-}A&wLURV)dyD1VqJfL&inA%zgZu@?iv;skBGsv!1>tj%D+Fhy6k3Yeh0Rq z*2o_j7r1fkJp`Fiu@CtA8+G<~v`4G?I}LlsvtQ|KyRIIn7pa)mkZ0cQ`Jo>wJTAfs zm`3!Q!e=r!n|7BsR2tN=Wz;nVTBymt9U!=|#{9X#Q|e!ziYCFfvXjR>pGQ}-z+SG! zR3KRJ2`CZNjJqk)o6g5ifLyi3;V+&-?)T>Q7p*W;KksoDBNQcqptH=QfL z5%1g?_t&$(McfeU&yn>o@Ne@n2oqxV?wp9f{8UEeZI*}zI^Hj>>WvI{Ir^VIsWN?I zX!*#w^^!ZpiiiSW0<{j3+hs96MU}t~}a2)@S^5#f$IO{Z#VfIr;@n z$|dwr%zitQkwFiyX=Tm)8-CQIgCb!-S=;D(Q}nscF=%Q}1c1mWXGI0!;R`YGM=Y^p zDEG6wM}n*bBgR*W&*#k6*(~Z~0SDRLi&;)JOKCnJ7-5M3q$i-?q^BYA(KBWG<}$t| zMak+_Kwc|u`}jyTT+}R{d3XjRyawKV~Rb-hPhb~*Z&VL0;?7HO=SR^0wSUh!BXCmYE9YzP@vHxa|`u6 z0bc^AdMl{LA2tcV@<-p{QI_~jfdlWv7v8{ZlrpUGfsjHlkA<@sVWQo6ih3UI&~uY9 zn}tw;A1BJXTRFM7J&o!PQGRD>=1h)+?7S9M5D)gFgd!F`KFkins8*<>8tXBI1!w%B zPa{vFGPPOsvf@6RG0+_Wuvy8(I%|5s5X+!KkgFe5!_j~hk;4K4JZ%y2_5?m$=4McDm9bay36Cg7lh?j2>VSJJ~dxh;N_+;KHw)7X!w4uxYs+9$`J8h+C z39A4w9oX&&j`GWA0M~E;JlPS%uL+6%vkSHhbVQaH29CtcoU|>#n|LuN39YsHbtAc% zwbb}oa9$wY$HIARDzpHRdzK+P8cx3h#b`eL{%^m1O~zUeI~Ihz7wwV;!41Kw z9uf!Y0-!xeX?Qp*^Zp%h6Lo_=8GE0>LK1-quVUnc&%r9xSt`FRDs(FRKeU}8OACrQ z^rz@a%kk41BK=4X+sLqf1Ir42lOPeJjbm9{|Lgx!GJYudS8zx0pv7>Ingese9~CaB z_(=#sM?ds$r@(U1cF+sahlONgIs9xqPu`Phck0p^yW)D=k`ZaY&JDix&9&#v-_$zU z9ue3~*=X%d8spK+OjbD0`mXVo5{gQ>`9&sJ-F$PvJJPid=~Bo)dXH>heYrP#UzobIljlZedOu9H0&4_#Lft^ak$4#)eX{NWg?FgMfN-CSrN9Ikpf@ zd?RLUms2K%7wVq-)hg4?1LiLbsV#(y$^PAXk?5C9XP6!Q%x#HDQvhw6&KpV^GCmOu zY(L*)mV#>^nL&2Vm|!7#$T_T`?YIl__@i8d|@g z$gdPl4mj0gnma4we=|^d)1S6-NLL;+O5yXXFxmHT%|5&CXRgMucRL%e)%dH`Kt72U zB?Ce|^!ver_Ws$_9fn3zk4)d=KrZ=Rof9J}&Ybs76$lu^T`ua4 zHxjdrD{As^#6kRXokYVPRaTk<74@Mnbu3%!lY8b&tVS_2Tb<9m(5m-v?DQ^?bS{hG zV(GWVzB>OJ4pg`FW~5-I$o0BcYy$R+Q=*Z6SZR-S!@kJVz9%a(xU#)ai|S76>MUYZ zk}?x_)+yxiVXZ?rrYHs}&{B(o^r+&Pr~N47#Pl-tplxp-x~yW_*ExD$ocU6256Uo8 z*Dh#{Vwwz9a+705@}(Y?xoX(7Q7hVGdO0Ggw84R6FxAOq1<* zg7D4*7)_3GObI~t9;#ZBlM1xmm8Jruq5r+gR${(7)~`iuH6f4T8I))>Me*%HcHjU( zDZm|S$gIAF>K_@W@berrbC4+~md#d^@$$1xSwdzWqzPG>p)^YJg&hp#G-waUh3)pG zY(hJcZgj@ig9MoCX!|WrG&~1afYVlpM-7kmbs3@d{hY$#5#~&g-PaIo6UAX!xrl8a zvnqQlOZ1v);Ys_0$r#RU0DD5D$JEb<>B4h?Q)tBr15{1JupH_Uo-wh&$YRal;Z2M8 z%NZe_{bL}$%-k*5YoD$3l|p+LJbd0C&_1Vw{e(bw-y4b_JScq3idFC(cizy}jN=74 z8|D`DbQ&@gg!(;8#+MkPPvtfv*r38E>lMsk&_s)=JzuAO83vjF`ovTqZT%OhA8PCl zQ2kFB{O$JwT2btL3jPCBxG8(7HE>(dUg*aem4%D#L@|`TJ~5lx${aC>j7jH5yvXpj z`~v)!Jv&mj-9ope0cHo>Ht~kG`!kcfaY7-pfFk5njkD1_^3NL0B2CN z`P@A#EeUAFqoNF@-{m{7;aN7qQO<+5A%c`2=!}cc_Ja0-Jd0ng9#MPV7n_UmZ>yE^ z9l%7$TkSm;Q`;L15iCd(@L@2oHm3hKdp&%KX9sJK```5nF7P^cB!>TL^>hu2!P%At z{FmVSdD-aEkpQ^@2F^u}?HG`nK_lWUY6%RDJn_uH;~%y7!;1n!W)WgFV0W6#PmVO) z3BrsNVw|3cRdH*U(8D=01rKeg(r}K}1H1^GD7#_xL^R{| zq1lZEJVE2_x?=3nKU)a3AruS|%Mc;XTZwM?-3^z0ewx9)xqpVLX{BUHs z7}oBB3x04h2*Bkz-vUz&!z;-*|gR~c(OO1j1rv3 zT_!*2_=58FODzXnwg?T4Fb~i>6;t(&-_Ec7GF*_HXjtqd*-Jrh>E|WML3Y>R1h6|E zrAIB@_|VsQc3!&8$Ft$pptI_DjwTFrZU6*#(Z#O*0OgTO=Mud2-6I=rmX?&3N|fj( zG@y4bxR=%2A$XinLjP%1X;Hc4zUu5c)^T=U{ZP(SP9X+F9elK`@3pVmZu`dp4gcs* zN)P0Zj2F2{eAjZBjJ7h!bFhj!E8J^2ahLbR8M42E17v}TXdWd2u&O+0pW-3xuI)W8 zNBN&qn6ebH=qH&U(XxZBoB|K6^Bt+%!x#Z*L7dsUA05Xr{c43Si;$ztM6{ZAn?U08 zDW#%_5qQ}@xZ(ET8XI;$473Y}V@yA^UeFNH%OD0-2uJ(SkJcXA0;ZCH?TnQ|-i?2) zSb$iLd?ci))Ccsdnrv;+E4V#ikZUoso%ahsPbBg)==BAubK4>xpn&XKW}!nHL!UW z7kq4==(BOtk0G7so_ zSn<9=<^|LiQGw>F>a4#^Gw&zfsDz8%=riTlp7Yr$jCHvwsBq*!q7`X)3j25m309xV za%1>KPO{A9A118r;In>}NYqT-7N+--2*H_ukj5k^db>Jz4sc3fG%M(VHPShdd8ZJ} znvM-;L$ieG3d8jxU1TJ79u1na*4S^DOx*rCboCx|axeA?6nBc!>(Ia}Qx0bxTh0G_ z^H0HoOP{6sa>dU4G9SGijN8|6wyr>T=8B5E8rR^6s%qxfb&6ER zDHatA5y=r&C50R*WfK)OmwplC`IVj;Nbq!9(-BXZdQ89zZApRMRS-A z?lvKRfYO}2z39Hh^bKgthx`CPwAWc^Kt0W z*)f^$K{5j7)-A9*03>~3$|C0nN^jI+9tT)M{$+E&{p$H)3y6o462OV2Cjs3qP{6|p z<^Kyevz}l_SQr04;G@uZ)G|IL7|F_)y{rI)gY$uMKOj4B%j*JBNa*J_+{UO~+zOOP z>}bMxg!ouPz!k@xY5=u?qu2yuJ#ZI`_ymYs}m+pf*|?xkRgflpF|=Fd$@jY)Masas=U<0eYgiktTS> zn`WSm7F(zYB;wCK&TSE6^v}+&G4U_@7>o*xhJjm<#Yf?FGz14Ox8NeS)j}Y0p-ex_ z0wxX{Zhm_Q80CO#u(QMRpvfyc;U=ZQW?Cb22>_do(X1FG^A>xAR!+2(X$b6W1}+B9 zI&P)I)c+O@Q)XYr)eKF!J)c=SUV9s-!2`duohXAOWq%i#pViDmrp=?SG<}yi=x{u@ zP(bjpaREYsocEzhYNhH1r`i4lz<=v+Q>!B{PO<}V zAkT1%{r*FRK8_kOjPTeeUL&y+T)zVj%!QAp{mEU&i{Vm8%=ZGEM+^M4D9Op`;CG;# z{hBRbfDy40eL!ZnxG>B~T%Jbv^zv%hL6WW=XX|)x6lGJ=Al>;aCf)4xnj$;q{26c) zST60qao1U*2M{spB{W6obXw(R(N&!5w@^0+ug$Xbai%H!kM7t~y0_V@YJ+1yV+;2L z#38w4!}c(glA1Gfhr}Km?ssUVvLD{dgyiwAPzTUgtjH%<+)7Ichr|#C6kKv*iDg=9r z(wTL}1u{dYal3r>P@i8mM1agw{m<@$g|^9Erc&46jNzXO%4li1XC9v2I%WG@s7Rk0 zJ2P2qKa&A{{#Z@ez@nk)^jWvqUr+_!!!s0BR)<`9U9(BvIwW_42~f?u=09{`Am7jq z{-fPj%+IDoW~;9tYXThQuMDS#MPt)woKDpUXU&O4bp!UF3hpev$C-WJ=sMjszD%5b zZ+dpil()9=mR$F!gg>zTUG2|~AHA-@uvgf#`gRBQA<7xz+~!AkEGWTw(B^Qcp?R@y zF}G~-Eii^IZXBvfBeh!!d;hk`vyuDj`y>~-!0Y7cbl?!)UArUJ5@|$n3nTUk z7cQDJ;GtVngC)azZ~#xx{Dn$6>H^0@n_3~?lRI=^H`q1~AZ-4Uh-e=11Dm@s22g)Uxvysu|>N|zw-9rsEEy%@-2F%)&;qsR>e z0h=f0!W=n#Hm!4&@*~Kzoy1t(-z31coBRte3Zfx0oULEx5_M(ulUTJP^DAIm2*QAW zFC%wP-i>ohv*!+(+K9Tc6m{ihi}z~&^g+SoAV2sQ`x9v1q5R?qvar*Imbtor+8fl0 z8ohKYOWWP2Ko8?B9HnWYm2|a@cS&EoSAUPJ)P3RpTXO9RB(OcePE?Mf0v$%lz>=_# zd;Q(Z*SN+rFHidlW)?xy_BzvJP@qs;#M7p01+f5#oJdUKw?}{$n&=bSTm}k``plEp zTyLRdti6u(gbdR%m{7o|kv>~8Zl>j~pM4Aw63Oi)9T0{$pA4MyC@a9&NnnZgYB=fd8d^~2&NnmylH0JA={YBgi^^XsCbAQ3Ph zqUoFC+4jCX zvd4jniVg%xL&CTPr@wpnS@fCR_S`cy#H1s zOy5w0D9V+JI`1R$I|r*hEAM;>P-EWTs3lgoC@8-?Bde${ce4P-b46#(5oK3RyMY%4 zc^&9w02eTsaqpzQ5yCoBvU!nUg2&{xVX3C0LFDR&2N&ykfLZO5Ygwx^6 z#mxiy798`1#t-spX{L1xDm=_Fi<^E`~iySqR_w|)r`pO zFwn#Bo&WT<{A`MK26>SoM^7P{DiHk9T&+P@D-TLs(+Zbj@z*gsr~VNY2(eSuHGA18 zimtFC9yhAPvlpGHf7-|KQv;0NDlTX>;Fw=neA;q_oS?hsV;k5!`?8;GO1s%;jWT;L zq{j*CmTZO@K^Rg4`$JWo946iTt>L-O^@)S9jx6Z}jN+WWOo3mA?=u&Y%@MmkA6GZlkA#n z!W3#Xd3;whx35!EEzkEVLfNq)F>YzQ9D$`>x2!#q zqm=;$Pj#-p$Uk~tDaEAw+|djb({-tXkG7aUXJn(Me`?y-vop*PC^y-wkZIua(e2?i zwvq^*e#z5;0oVfk+`@yLI^C)q|h5SgP$syE0dYpT$&=^|QEuiM_%|39wk(#~`yZCG>r3J{oz*^A`$7;Y8e( zQ0F;NrHjderi1x75`G`1-Mt4uy#G^p?+ErMoQ3sBKJ6FmHdh(S+@p8NP5*_YP3H?y zDcg3{;|RQ@RLHP~FoG=bSTLk<83{U@u|AUe4A|c*0y}}xK}kZpX9!&)h2+BC0)N3p z9_PV9v4gh|29qtZkWz!U#ewYQ1OlMsNB%g*@!h?<$aeNpJv0f@3$WfhJ_jq7z#6kdzH@t-mW(C47`=%_OK(`!Q5vGHd7XgAp10wSlAqVL4 z2uGmqV7nNWxxuAwdrXg!IJp8!=m(r? z2$EyW?gzw~GK`bpIl_W+F)#~s3dbRD$KVur#TK{-0E>lfmM}N!XAIk-Si!g1!3T~A zW>6<~fRhM;dMjXx1z*>qKZ2`%0`LYupx*(jdKt}TsUs+4)rkS=7im74P2nqhDtV764rhe+}(H*ZrsQeNbLKW z)bK^u%}#RTL1eOk>^s-n`0-FH0cS$uWXW%T0)}irLwcL7#lC+P45`g4uHXAJOc>GG zh=zncfXxfrS2INI3%?Ckh@Z^7Wzj1vdv6DeoC)vJ(!=Twj=q?npX>}N+qpn@;4 z*@a>6Lkb8u04;M_ewg&K2g#Dr*m*j71=h2vL9Og^%+m-)YLbXRLOb|Yq;|8jd5|_D z9!|V5p7x>U8#xwO`UQlob3j_iT#efd-==^xHqMcZ^tkFtvm;U%SOComWYB^54eO)5 zGdOO!f^_ZCZA0GY@2Ktpt(1?6IK3x+2f*pI=Dz(fWsfeGr2~YCK^xu<_mjo#!%8B; zNk8A}17R&~6m){2!AJYVGJq$^{O^YA)Mc4>+Xc_w4X|GzNk8lEm}llvyUx;3SOn&b z#R-5`2=$)gkC4%WZj?+n=H8qR9cP1`Qy#g7ONp)sr5V0E53YnmXnL}o5>}}1g?D5< z8rbkoxOM1rf9o5d{|A1}!`c16nh{uR*xsxF2ZoAFefc?cDwFn?NA|{>adhwxes(dd?|b9XMzEHnAY79 zlyvbgYwcHx_LP?88@Hd!xYOjOS>emosbP?vjt3uz;l1&O%zZ7o=m~>oHYWJC;u~jW zHc_?ec3b=$&R*GKpS%;&H2_@~l4Z)lzlKngU8uP2=38P`|# z8BH4f5`>_1U<@g1k!hRBLk>$&h9`_{!n@Ld)gj@c|mQRVGOkCG48--gSV6~HUH%rxQJhv{Hsg{ z1#PdIc7vPVpXyK4jPwKkD0yV<&KtKVU0lYCz(9!MwlGahHZtu_d7cQtgdn)MRFSqtD~$L`dX-^VQNajVIH9Ai_A z#&z9>hBAul431cBR}XF#h);410JL0n0|elu40j*eJJW@a+Qbv9H#^m+!cgEXbueVM z3CPIM1)-_}-UAs_e0Ds@;c3JlR+zIn=uLp_DymZ5%TAxT{=T54yfHGiRG{eH@7fU7 zA;9h5IHe_T$Bs1I8XV5<{ON4W4T>(6IIxZ8ddtKe6UgpLcrBI|J_b&Em>RR}N%L{w z@)As)JLE=NqlbSRsp?Z$)w)t(wC+})T^13-PdE9QSBSd(oS8nC`G2@Rk zYU1#A7TOY1Q_&4STwS5f>Pg+2vzOOf+90u`Jl)jH>3>2Zsabv+;CvNqvv-=78WU#^ z`5r+Mq-VQ^vzta9@c75rZQAVYS*pd#@uApQEN0oL&q(Qc8~a4GR{DZ`Qd+RA3S(AX zh)MAea~z*sA=`;YUG@n}u@(lh_$7&iIRmiRt>#kEO^+Ku?0BE4_gC9i@UM1~cLiQG zzyPxR#f1w4e~K_z*zKvsP^m9mx*5KT(tM-Dw7ak-Yr^PNQ|{N6iuqHG^{SS@q!>A_ zh0twgridKscMephQr#69wgp?kR;-GYx<4n&d{+xALWfzt_+})G{~2lJriGZyM%Cg{ zl9l}EjNfZ~f3i?ago7j3%+;JE&cgGL{n3S(`ylMGse;yTkgoxfDp&GGd|C2prU5;} z#>jmXVz{~dT76b|J)Y1Ze3_Y}Ee|t=nH|I+!5IB|3W||PR_j*VVOZ>JTZ&R-#o9H= z`~rS1GtGEt#pl}&9O^b#O*0{4m=fGa-}^0zX?s+N+EL$p=PTWkO+ zu5w3n{CE^B2i`2lQ66`7Pe;6E01q97{tAI{492r~IfGnx-m?H#kaNS5Etd>IR5eM_@;X(G@iq~hY&!0> znwr|8OOBl>kFl%rF;n~1wCGx1My#W)As-VoT@ED6)1<+sP=8r~N%-Ffoo zByY{iqXTkS$a3}KOc;Wkc2R_@m%4$>(DYyMwwS1i0RKW^UJFOf@a-`N%>Pi74Z>I^ zlzK^avInj{{!FC=W(NF=EFh=->E6a7qxTn&DMjQF$$wG8GO* z7$q6X9c+(nYtcu|mHegVzmE+V;V7bzjGdY@#?XIoWm2+9-s5(pSQS;<6lfpk$2G^*Dtmv3wOd+L%sjh_u%2?i*QCeqNhP^xxy&37pw) zoM3Xa>Ufp0v+6zFxHvCqaQ)Hyt)Gcjm6<$xzg36&?9Dsl@n2-gs8+GgAzO>xItny` z@@3q?yGO?l4CSk~EkTCC$#Y-F3PP+iWbLkzBGc~AT$!5){_K`xUn{{#fiaCfF6VBwhPbq+2LLOwIOiz~mp?m|o|N zNEPAkE1=|}z;{yKKtuE+!9N4eXv?GmdZr2<3$-l|j>srSvm4L4!CQw5pXJr{kEEFE zKNAd&p>6Yt=ZiJ1*WDLPtJ2zI1cUedo;1{j=Lhxs<&7zUlV9=hvi|zGRN4ANk~idb zFBfA&5~H!e{FumU*=IJ*?lzFeLh(C05F1)Lt2cx;?9mj;ZNs2eWSFTzy*nh(P3=Qv3?_z zAY_ge=fU5AJBLVS3aHQlyiK&K76@W#YKf5QPp zeS(T_n&z`(S>uDt7CVzK8tnK0E^Wyc4DZhp`5r3X-0>T5AlmHHD1qoR{NH~Fe^J;(*}v4>DB5=f6_TPoU;iEH{hjLL z3%|o(u=~3ZQJqBpPBr)*=;zldY=D@?c=GzYerFLR)32kONjL&O?Tp%=EZhOfAsEIa zTTAm*FPbrnfz*dsH8IX1G~2C|w&qk17`O^!OAFE?_;k3i4%2}hc-KeOI2g(LW-wlg z!|`{bRjW}D27W+o z$AOx+gFQ~<2)YC~aEmJ+IgaDP!iNBt=sH~|f|+c}nMLrhRp z(AhDH-$Vm_U{=!n1ky!)3~%9$cX6W2!d?QLAq;`qs^X{c^Q32X?ne2f&qaO8z9ro9 zT3Bi`j+vSXNazkH>OF?ijvVZHKB)c_S)(WlrUre~XIU`{sUL{hboma3_x&KngOiPJ z1eobAX!^hjCgGa3alu!p2~W=rD_B>4>VUAV4vS41X0oAJwrq=nASH7Id=c3zS|^xr>G7tU!UdCwLpWk-e-yieS;Z<(TQX;iZ>@4Nr!A z;WKa`iDtfp3>r^cDfcFFmNdUw&F|vj@i=tI8+7-{(4V>Okt*&!p8c;@H?|BmZJ6O< zp)@6#`_aWp_RBoxDPE>hP?{dDuutPmzkluZ!mJ?1BgRdvm<=dc4DwayU-ka~z*vp( zNUQUQD|>1AHJ}IB?^hY{IA;Cuz&egU_t`T;<5R+!)4rYsf}xp?Oc~kuub)Nq9$p(y zz^%673o9GPI(U?-;l8gpcsO@am}!?uBimVEq+2y813Pi%>S9A3D613DINQla_|V}r z9O)RIWkmWE5br*9E91K{FIBAgd0+Q;*E>Axc6(hC9d1&lJ>P}f5>=bx(j}`ngQ(}+ zOV9P+z8!?8%B6_KjqtX$_QGTF-m(hM7m_A#2A~c#xto$|V;dbjiJ6* z{YD8jnq{W7EB!#k-nFndD06jls|r=}FOv}*KBimwkDsVUHXX^a7sOmIM|)t(9B3!d z4~52oMn1|K;)_nVkRMuc57L?+ssI)Gz~T8^1ZK#wVp3%fn0mVdk6ZvQ^K342QH!3s zwwRs6b;8u{3{UFkhFSGD2VeyauebO!cCcnLsMkQLg0DG*X z`~vi2_ALH^#8hWkdHKaDK-5qBQj3VppD`R+rM z@1EC@=H?oKN1fRIgsWoXpW^A>8OYprOI6*M>ITEm+*mPuG;w zZ2cRMllMP2lnT?bUV);1ZlVbs5edKxD*37=8}EhcY|*H4Qvo=@5U@yUmhXj`*w^U3 z(N!K>rqKvWqNi@*OlePXb%VQovCf9%@S3>TCAisEMo)>()whSyEE$zjN8kxHm&SXS zQ;L7dgmt_4c(GV;0dtKlWQSUcDby_mkmN>C>XC1Es{C*RvIN`eZb}j4rAl16NHk!> zo!L007Ce|1JNRWz$>!N)_Wy|&#L=%-RM)`#@LLoQk=#8OVs>U!hHcR;vCIA#OZppC zE4;KEyj4*Ee~}52N{Kk&t`d65V8ou0XoWe6go7`GVH;u}@L-01}`h*c#3B2%_T+XI2-v z!N{Fh1noiRQf{2K4}GbnzSy{E+UOO5(8Wn?{_WD4W zY3*@yZodJKm2)eWt}9czE3^|OT@k+hBORl5+2mEn+^%UEN4s#itxYoPanV0hoXM%^ z3A&x?d$_u`v;yKH4E#q*Zpkx0fzO_^`6`=dh|AJJiELc`bCg0lb6OxrcB)@nY;3W+ zx5VMmGAvdh0$VtHu2mKG^go`&`!4Md9kbqsi860t=^5EofxLXbUsmd1@l!n`=N#T> z*Z0rNRu@bh|JCZrGEV+4vS)}t z8Bu$+h+9vcw05SRkIz&tx!7k$7w<^?9iiAi@H0km-z!2y9CbXb&4&DN#Yf9H#AUnE691gzDx>B(Ibf-C$Wl1+VFxH=e&Y zZCE0jy{61SUoEMN=jWCPoOn7fr7{^8_Rl;U{<)f$k`#}pJo;twE|0aw z=8ouC*O=NM=PzUXA1#4?DgI5<`WoEvnWvR)<-BLl1ZXCThQ0HHcmVl2$>RhwG-u6{?vvH~))&{Ktep zxJ75o8xiCw)jM~N1oVC-Kzd2XzG%0tE#u`fO<1bo*mq5rT(y_7`YLzke%@|=&yl?=%|UNMhmN31>%3C~-dwn`j2BLb;Z~qc5P>|e56W~XzM}^J#uRN`UWt!_ zJ+~CnNG)s~{S?2m$OxE7LFKP}-ABC@Ndi~Ai&Aj?k0^RcIf)wqc z9e3V~c)U1Knet0*7m&bvf%3FJZ{0dV-Y+1wkM-;PRJA`c$DoYEG!KgNQzeg}#7ODF zuKM2#>gV4)v>8sm{841(I!g+YcHQng8`9P3eea-QCir%QwC2jH$rtC}T&hU}W)V6} z3|6n)7MuJJ$}1KeOL?;#jCXyZh*Auc{lc<$H!pidf!x|MJkz}jZK}EYKv?ZMZ>Txs zEYYb>#eu8JG9_L5$tT*?iW-5!maEGj4F8mmeDd+avX~JaE=vW zKq_DX4%@0G^?49LjbOXt%`gLGS~W)u=98=!;NZ8MYd~~hB3RI|Rc`TS7Sl?SY0zYG zyF=n{HyfD-hy6wivzF>%kmp9IiefdU05h0`vsl~`j5P<99zrG>y*vF2VpM*|A_;=w zfMSoKB|t!pAeWy2gihsmIKEcG#w9F3gBdo4?!Z~}FYv4P2OXWb#K0kzaW*jftCr~~ zN?i6h03oKk|L8V8 zA#A!pqsBR{TTIQH>BI>oNR;88{B7>fLU9&9vK1GwZtEr_ z2C(993pQu<(d0D8b|U67>=%7>d!BGMyXk2Qyu!J#h z7F2~{_N(*12Y8+_6|bGkZO7Cyt&?vEESy^$1fQ`c=+qXoY;pi$`aN82=$U}{SVDl+ zqA(2)Ps1tH+xsbQ5#1YD^o?0QET+TY*rTOFWC`rpSjm^>P9Jga@#(1}L*Ni{0i2{@ ztO1boSh8M63WhTPrKt4nnNV5Fw=N@~VF2HivhWXGxN(|EQ;Bv1@0 zK7t)wM(F5RCxK90YuLowB5^cV+#3E6>az>mvMZpK^?M(Aw!wwF!l{Nt10S;Y%`kN* zTOLICwG)4-_Q5tuTd! zni5eik&{a$B;t;*NlGhCwzJp$9x){SXB^|Agj7`l!kyyTefFJjHjNB&e4zuYW?)9i z{a7;j_!!xDT z(t*-(rY`(EzY3N6Ma)tR9IZN)V?-_S7UjoK_Ic!4eM$9M1`&1Ng9|c+C@~7X3^T9P z!pLy5O2b#5HQ*v6U}InZ&Z|`>+a6h)#|Ue<;4?k}PJ6O}3u`L#Uq~C-?wWRcftB?2 zspv4!v+tj~m*25`1< z(!2)k^^I7RZ?$PPv9b9wL;VzNWQ8h0>&RbY{HpQ5@xD>Ox)}^X=oj9_bVsn^Y8{ze zBcGb$l8(&c2TA86<2QggzJsI^KzDE9_naQTRDzniV8>p_?UeXO!m>9Y$h``5T*z9` zn-&1@oeu%;G2+yLScD7O~yJ`RgslCQDndl zI3wx}G_V~BqH@6nm|KvU1!zkKEd6b_tR$ewAi_9pajotsOrQSY%wf3P!CM7; zJ;{SEJI;U+WfeVtGM5WxHjdZcaP}1UhVqiW7#lEQzuW@cXMX)sOvQq0fzX0pHacb# z?Txc2luSblh!}FelpzwfP$ImFr%5^1?C5)dkg!xorNyZ*=TbHuI-Q-2DE5lOZSs^` z;txi}5XA7K1OFdoZywj=m4*#t9qUY~Wm*eZsAQ&_j(|*O1Ssa=!LM;d= zwTNtnSRF;7n072u7mR>g*TG>l>XM8HHUGsa+ENjCF(^_eESW#@6A z+MT6=R?21V(i@Fr>IAh%9F9MC8)Lhof_-e(eBqh090J{WSM+7kbL>OYj@H1*q>7%2 zxf!;{GzP4`ol`NEFJxGy_Xcvw>v_1@|LW!p@PREK5hH?`7cTJnL-)YThn870l>7~_ zwNHh%lSEgb;FV@5%+g+B^nH%->w5|6X~b_v*Jnum0r_B$JC^b$`tay*Xw32~4Kzza z%f)olL-`;!RNP&9gHA^ZtK4Jhy2hrnfRM|daWzC!W)^jKT z{a-VXS{au8N6mout}%>gM&g_f=pJ@1qEh53!B+x^{SYMY<=M#j!lklaEY=GNJqCB8 zkHZjKsVv!vPBFOIRc2~z$MOs1)j&wKvtzU5uJ6;X)m~x%eW~cA)=H}G9!@x7Ki5931hj4Q7u53k) zMfpYWf3v8+gYze8xl@AuM9Q+kvValEch0=A7|?Rp>d|TTVRS2|!#4pcL)XS*+%f2Z zwAuPkUh&%MLjircV?!mmX?3SQ_9i$i3(pnY)S13v591^_9d2z!$#_`UfG63W+3w?p zOB2al!SMspeY3gcKf0@ta_Jy~yQU?DZYU=dK|?SSk!Wo2W9mb)V>{Q9nVz`yNGyn6 zVfw4%&e(1A&+_#0^&u3%ZD^K8;ui7*qMb#^p?3_>`|?p`PJXY0 z8OxvI;`;^$q}uDF5AxK@QY3IQ8@k`Qn|JM($qG*dOq+PO1$A#~o++^p8>?;^`#djX z;X_WAM~CJdShC6B5iGoOWwt{Q+oX&i+rWQSdgukI+NgS;5hIdvs~mc!JV>?l8!fgp)dDnN z=Z)TZ>A?BcA!|RY4L?~6{gq#HMJd7UKAejm4b+SQ#K7{)xM4b$%=Tk_o+c3;w{@&r zmQ-6+mLGG-OaA_zstcj(x&PkeVrh*=s;Wa~*yPbV)SiFaI)<&dpe(4)9O3Fu9Q%gd zV{)(Hy1{l&z|i4#6FCP~=8_ao%YVM;QDxKuFHFnFtnFT9 zIXRZ$7Ve{$ByY8T+G2CG;riJ(rcYS(&!~4F>G0w^mQ)lBe(PZp&}O&e&fnx`hW>n9 zZ5cc|vOc@KEi15~lUvjL*|1GCYk{kej0UIsg0)Gdf9lLi#hTwH;Q9EcE<0Xk6V8k4OkB;9bX5Fi#jp6vt--?y%bN!VGC;akGtkGTuI+)MH zW^HdZX6^r@F@HG1h&?^+W?2`q$D~^A5$utl^Wh6pEhqS7NiJLyy*1fgvb*Zv4SOv= zcl1zi0@#digpa#5ez6{lXn3Fi!j}7o!udBFGCvNVfm&8}DWAKgIeDk#52bmlgWskp z6nMzx_(nLbeN%?L`H#bQm^|0x4bgUZ*_QN`L|KnPHlFSz1?$3=pUm!?YsY9exa$8rS6xo zsZQTO&vHn7>LtQz(GJp4x^*036rRh_D|baL`g&MExZ~>&2{qVB)zG?EzMx%{ibOV( zd_^?}381OepnoSbH=e>^4yT@_)Qv&+C80tkC()`N+6vTO&4HEVWiUyZqf{vf*3NPG z&S+}DF$Z>NofIeaEwPG0!1*s3r##Sg$IQSRB-w#M$*hEdxcP`D~KJR zNygOl&nKBqBU3d~9mRkZC>Gps(`f85H4LQ2q~#~b0)^kA3!Eq276BBn*ELS^r;aI1 z-NKnH^nR45)c#{Q=Zq^B%++#mgD9w|2?!Q1{>OkQivt6i08>MvBH02W+3XzC!H1*U zMA=pNW*AK^N)I53{SqIP z(5PW?+}C^nJ_L=tTN<5F1d4TflOw2cVJ#6pcoK=hCYW9*Tj{wM^5ru}6=+-}w_8CB zw2YJvL1l~Gb;?7=X9Q==G7j-ujCAukg$6Q(soj2qZ=|`235=jK9Pd)eE?+la8>sLl zK@m<3?&^cQGQD?Qi^Xl+rgc@!_v-rB7%N6Gols(nF=YDw$?Mn(4zZf4nd+ok_MgHY zed7;M%hPpJshIEtUWvF8)F1R&V2zWjI@7_;`2vnknDB^}HuNF7ev%);3gPKaPyzn- z6l!+{6(K0=d1?Z~u1>HW+CuVo0AgrVGj0GxQlUREpY9y$jR-0u5a=8)CtD|)_F{j4F-L+4opB}9M$@!8i0?wnqLR`=nynUW1@?c zgLwty(wO&7Fm}MH4m*mT5VJ5O~2{%C8S<;rchqM!%le6tj1^Z?M_gCg>>pq_Sk!jY<5>ZntiPv*mHq+a+7W zd-n?4JAGXZGJ`Q8D=i+8oUK9C92?Ao{Y-yzRbb@VX#yM!6G5Egro*j^ zEv%1m+s|TLAmB}a2L;$CC#(UST1;a;PT}lU-9&)_2!oLn3bR2u*1opO_g{;tx;cXNSpbAatId55_UE+TMR#3u5|6 zt!}0)X17(0j^r+E?=yI5u5wq?SV%>$_tD$#m@&5o7fwhFdRP02km&+nHPr296K*C| zEqMIEjq3xO^LdKBigjXGW)G^umq$|=7D1_n;C!X%9MPcTYOhb`BpJ`h|`kh!~baI132M4ry#5aOUAUSYz?nw3YGwH&+9+I|f8QIR^t4AKh zh!u#-#%O@2!gS|ZXnL^0?w4)Qk+zqDqfkU0t^((H{uuapIbqrQNCJ`%;)b1l8tXvc z9KAN=$Se4q3<#?eo&cCMbE%k_(am2Joj%h@=wt^mrlP_q;<_<>0_=6bCa~K^1Z*+X zwrP`^_}L(LmxIql)VI19KKY;FmvaI2(fcpVgj}8j%H#uW$$}uEzHsWg)L? zoeFBI1LFWj^gzAq2^8dF;dS$Bm9V(Ap7r|JiH&`x2KvD^w#f@0RrHUSFwA~Cp343x z#_a7?DNH*X!zj#pkH4=Gkj8|N0;HRZHlcLZc8x#~Ml9v^ z8;EouvDcii2S zZ^Q2SGy5|*K;u>1UQs{l&;)~VmRzHZ2VFxe0%Etj{KgKhlHCD8HsuB?c~v2l!q`$^ z&0P@~U7bu>wk+24>{eelY=fZW2UYna*n%DOU{Lc`!Q#m!EM+Tl2Fe2Ig8iqJBd`2L zM|P|SuNi7dBx2a&MOu$NFopW*qGs3TnFLdDZfjd0S2;^=j>O)BdcwsD-GHKx!_z9r zT?eji2yOo%RMjHFdNvSZ^@!&oB=IGN%z58dZhtpcK!{sgJA&*qo6wxp*g`sfZt0pcmrOv=KL>^%c z9Ov9!w^hFV_;Jr=AWy7(^hn_B&^pSR(!i=trpy(1wzhF=YL6Kmk7$W-T_p&9Dcis5 zaSufeyxSEL1V*t|?0?XCsJWtFsNF-&c5CTBCaEF^l>rI5&XINxycz7`)wgSMw+wI5`NxeKobc!QEf zX~YtwvxW^Gb^Dgx;}yuq%#?rP*&O=niOppQy3}6>}GPg|4p6r95MU^Cb<}#!Dv0r@({h_I;DvbT>O?=?>xKVB16}x4*sap52=C z(c~-H)kQI2!+ATM3uR-7VT@P&=hnk}zTUB>>nY+@e^l}jRv2^3>a?ZZ8vAeZ3LbpC z$uMCuxu^TP%+Q?3k)f?&crlw@`e)PlnBc^_KOjk*J#LvjKA$%uNUJ;$l5^s^`Rl?j zkB)id%FWIz{W%8@9){j*WWlsIRBx_$$-!ew%Es5nS^wNV=j2)1IBt>8S8!!kiqi?U zQiuS#Zy5O{HjG@g@!3lQeI3>{-gEnVjg?X* zTlweda*@P(ROK6IoxSEC6}e>&kq-tMM}&E`<{$gY_RCqW1t&fEiftb}6qu^5|0-Mi z`6Z`iy{4N-cdw5zf4D-+?rXsV=-oA1lbJATX&^JgA{FeAyW9!yU9C5F&5d&$}G!w@1fZ&!Q$cdu4Zj{e$&eu!cN62DeplbN}{lyY1=H$Pms0UuT{y zM!zPspzWm*yX~J@%t#AAXTp1b+OK)(v_N0mh}w&MR=fI>QbHi!H?6k7>(^U`vKvR^U=J84JjlWE`f<&rLW@Hi|2e`Rjsy0K6AP5*Ib4P) zcC39MIpTJk{;u0U<%Bgp&b#^TFF;`)^FP5>GNFjcfJJFFug{!YU;;g0@6iafYMcs6 z)z;zN_PJ(Xl&g(6!>N9S!sbZnm~R{RFv8@P{=M0u<8wCnCeD^KJXz?{T4kk@mkZiV zv(;~)QEHKD914dbrLN_MDV%1hvh3EwQp2Z*j;9SYT^ZQGn~xNhYZO zVtm6Yhd<`p+JoHyDaO`k31VRvr#N#CW<{-SF6=mggDBjm+Vk8~d9KCAmdOBwxQHhm z2Rk;DR}339r}{>mdAbm{66^vCV#jiBhoRykRx|yF#r7R2>?rwBYLF2;`mt5g)<9Ub z>$u_;m=BcdO-KfBC-8|Zyj~?UpkbmWq_f}?B4{nem^A3<(P-;3q+fu>s3}n405Pdw z##C(R)RQ_{kKjaXLbD)R_HQu)qZ@}Uyn?uiC_@1YJTH*`|Na_w5_kv7$N&DvSj-ba zfbI%4+7jx~>$Y!-WhF z;Kw6m_XAFB#%vNC!1ZtfK~Ya*%hXqZJP4J55}$IzsAmjb|GWk~0t9_8jy-zf%sG-O zYx)mt1*^5`>=3}P)PL>*BMHOn@UtTl=?E!V{IVM_)G-X@#cQPN1b%cM=mAuFcvNH> zk5@hpU<^vc^g^K6Cf_+duzv2KYIUy%P5_sD@R^AVSptPY(Jb-Gs0t@$y&M^q7wtvA`&;q z3$R^?MNs_<@yk)>xxYj+AsGh3>eU2Fe3nUK69MGg7C2F!06&+E+Qja%f~8P}EGHEd|9?yZ&Tt*B$Osl9Q5W{<3bOx>K7(*EcL)fJW*~FH%w;m*4wfj4 zCh@t<4)98Nk)LkIs)NoN@FVmW@Ju+>$#DKncE30z=XRL7!GWk@!TcZxbI5v{tE`Sg ztgbItoE;q}i|AV#C3pH51p?n!5o{~xp7VJ}vVp7LfdlRlPcO|K&0D3NI|dv2AB2gt ze~G*(OSU~T6A7ymZ_WeBJ~W{(SWWdoM;``yJcr~E%f~z=DTs>Fo`?JS9~|CVpQyj9hDF{OR%}}0$h!KFsgO^_ z;~ieFg4?*b;Nurz^As@Btn*_az4K}HdIW6aY7oh%$x+Kbp*Iq_#r0L6v;ZPg)A2;0 z{A(0WWvpiOunZO=QM=V!BSBj^T8C$hf&$;*VFb&8Kq_$5PB`==2LVF3xC)roZah_r zlzRk%!&dx$w860}MGvN<01@^oH%|ols$+rs?-aT#K)$h4;h8^!J8@|m+^n+U>&occ z$hDkx4E#Qgxz?d#r6(JeV{$u=7$B2nC+w<|^4I9>^YGuVP|rRXFqu$SvKPa^UpTP7Kh46%TY6+AYn>hg)JgoV2jX> zm89fpUd0Y;P=F^@&P@p+$+SeHuQ3shRNd;|^M@7sFjI>F)!V5f7M0#8T1o7wK3u8w zIU~6GmEy{tkP+X*XkmcQ=kG{g4y5$~p|--IVtP=QhgQEEb7k21mqHlNMGv-5tM+NH zI*-wZXKF5Je}U(L_h@R!i)WbKkS(jgJ8uR&iiK0!E)F6pt{9yjJ6>_XkU_Sgl(c}h@U_GJMEyN{ifj82yUtAi zT!YdwOl_qOqh!uo{mb{5z80uE72!;XT0PT&cIfZSE|XQ_{gG}u8i{UIv}B=-yed$1 zj5y%+dGc31j~5HgVT=GjzUJPYaYSYzwzCui;`+3i+>P;{%d7H`(CSaH>UeJA|i ze$@+tjPv_4Ogi2~UZ+=!eYcPkMV1JOA(3xu^v7lfuMO_YPfX*ifONc-TbL3eP~d`k@bgA-J}rSH+Vh zhN`O*YTy`nKhx+EXCm-7`Q|`>eY}7~L&mKX6gr=#ru3WGHO4$mkZ*?mqO92`&(0mO z-9xsqaRh^a0nsoa*Z^b*e-L$;hYLkQK9b-MhAT&{<`;5KJ>|l%!8M=M51uA}H@=kj z%ljYvtDJ1U+BFZbY2I|P2ftAhCu@G{!-Xwki*V+!8I(;5`IZC1cOl}Ko6AsMe??eY zJSekHg`2z2A=sY{Neokaq;vBuB#g6Nx6WMTmrQmkdA0ssL$fm@T8jq7ZvSnCEs9{N zN}ZRp6r;KeKnKgk%yi`sBJ#jiJE>M2!?llbdsY;}VuTmNn?C__{b81zT__Z_TaIUf zN3h~ja;U!&{b7f!h~D>Y-_#}raH7Jsr@fcAYDIWs!6xHoWawD9pqKSRxBJ|ub7=}R z2E(h4X_(ngZx)zT;wuo0S@9O4d^vp0U$Q)cSK;u;mdm$Q21O`r z4St843&t9L_+&hk-8a%V>@|C816Se6`4Tn;qc`LSVcr)jO>@mXQ|UIp;G}Nw9Scz7 zk?{IrYC6QKG{>i-zYu>3$F=uCfCTSxza6;jR$C@-@{COWKNRA^AQ zRE1q@JxTGbl7+}-weKr|{K>C_X45JY;39jM((&BXJC=pc#ZHNxCQsmX^oJ;oEcnif zvmwvy`YN@B1?;2{??Ba{imkLvWJER{nRgp4D*7LDJ5r)yVopi!!$MP zNU&#>ZAPbMd102DXD8?Tmc8*ZtI|yZPKiu9G>Z&14@5dFyu%;bn18CFRiU_a$2+)3 ze?V@1wy<|I9@^HR53&tbdV!DUneBO~#pweZ$;RUqi6+j?NnN#NGZNqXoa>i5Dvxh0 zUvp3H*sok~8HdAC`IW!VXZ_84yzW;$w^EaW!UG2na!^8YMg2!vubr^5KJ&o%QtXzQ z9~6RD-K@n*$=R7K@ejw;mbb&amtE(jS!Wq7e~cY>)_#3O|Gy>9m;U4?pKOf!Y&`P9 z#har(Z{|NJvvFk&b*gD%;!5=s+BrE_ywy2-nW zd#$;1JM%dEweRF-K4X8a8c6=aQN~xE0e?b+s#GG z4-b~E8G9`GV{lerv_n~Fv{g>~mR`=oSZ#SOUeZ2Mbo_{#h41bKd2xie47%=h!;FBPY-}!N_{pK;( ziuR4h6^ExB>X&_K_SZaNdCSCAcEE!sCy$Z!Igqz3Q?mOTi?}9Uts}+#WdmW_I8TLL z$hx4jjfx$MdBzk!oR z`&1dt%x?@uHG^`W0L=wsu~7TnxWz}I?>y-?EH?T3kJ+ouH7mVOYhDzN zLi$v$_SO&M|Ng?Gk;%k8(lHJK_29(w`B5)6!RdY7DrMQy5A%$UHy2G6U3e7K{`b;? z>fm#EWopA)aH@4p^A7UAfrFEmWO>BrY#2BU4;25o__34{Ri&3^fd5`1dpc?w*dUE0-5d}3oUq>e%L9f)( z+_IeUmZvT<75kZ{uy=w90aZ;eQG}{4GkJ|DY_-(U$+$uXN%6`6l2GP~9)q zieTbX=Rb}7*HOgKWsszm3P+-nH^-B7f$DCig4qWb0@P;kR!0@XNCxJa+E^LE2!V$Z_ls+KrJ96Jm46x zdZ5-4R08mMBAfvB`4GL}h_ICDmO>%|vcHkeF^(C{L%JkGoXD}05B}u(!EzFp?;{Td zFfF_?Qvp8_NU;7BlyxbjCZM)|6e@L!pCj)>-F4t|nfSg%!b|`$XS82wc^cq8756X+ zm|_qB7FGKsIsiNWI*Jk;F@QxdVR6hS0}|;coND=|9wk?yCPd3IEM*BJ5^;P3Ma>H8 zD$>uxr96MU1bZ!W!y)FQ;l%;7b=`#R|WqDxG*#jN(i#c7^hrJ(?tgwp^}OQ;)A(`J}tmJ z#<~z(1w4ifIEDZQ@n?pNR*DW!8zgIpF+ z;!%b9_$E?7MZ@#;1oH{zqjVyF9ch9Fn_5n58QTgbFM?brbdqr7SU>@xg9mW!fCrg4 z2l!estRVt2XcW5KNJKT3@lzVD#Sx=SM%|SW&H_?SJjgyEb?ERE+!%h& zEhfy8WDn465cmXWMEB$cF7(B~Mh0IZZrECrPU0%=B`R}c8*_BkBuglE20zw=HCFnEFz4CVF*JU&$|-u2X4JU^40F~gHzm>kkBeQ+bvzmJ<>T+v}>*Pvh*BH=w(wlN8{p^+V; zUj$yhfqePIVQnIGZM}Q*=cXT<{Yf36vM*Fwr$n;Pg~BgjeeYw}gmay0yWCfz*pJP{ zvV=^e5pfON(E}y*xCy%{1Xg)iq3yx`Bh6XIj3X{%m2Asl!;>%BcQ~ouJ7IJ)*;I#j zXk*R4`M2kpI~^g}NB#;7A$&kV7JB}>_IB%sM9mpxg9|0_ zaQBZ2=ivp{(FM3ewORnjsuQl8#w_C%&Iw}Gml>p*2P?G@eO8VKDy43S#uN+YXoQn> zRsFF)U_tUK)Wd!Ne=J-fC*HQkBf_`5pypP%KgtuF<(I7J8v0|kP zxjS~pC^RBcj<~2-&*r<+mO^Wq)B<%`E*vcIZ8wa1Fc8=%llg_QdHZNTkgK=Wp8}Ce z2!E5Mq=cZ?09wW4VDIOIzJXLekK3H*w#*w3ET zij~s&=qxosigngSes$#Qe}7c=bYH~8uX>>f96W`DI1>KUb1q26-a2VP#cXXwYX)3F z@pGE6&%rmJJN2txj3adC4&N9`Y%mf>uLrK;YaR0qS};O=mA?MK3qpAJHssFN4 zqg1>_x8?!QIy}lo8~F3GP=5~-VL5xq=93j~iO4mBf{_IQj?z|#8n1yNVn8S|#-R7_ z>n;VXP>SrcX{~Q!RO_|(hb%8Vf>}-`P56(7-Mh_jQTt~y+2rGQa?sXbPJts!2W!}x zudINy2Ih6V>G-81KPgtM5yAiA0Uk=U2o@0k0(EC-N0GXG1M->9GB4ghGt*ZhP?mJq zA_%%wa}joWIozu?zVk}pISbL99P?|+hE!nMxF*L;!)h>2pi%M9Uhy7*16mY~>z(T( zqo?zSZ>+$4D)8ulb9x(W5sCor!?JH%liO_07q-0oR?Q`R`GgAL5AP&PgRyeC9zu9T z-*Vkl7ry!!OErTgTLtD10G?a@iT&A(_d@Xvqy9usS!V;I1k(M0xDJa@%V@yw2}Z|V zrPCTM&qb%JG}(PAE-Dzza%GTM9f}93fx|>Zz>bRnoGfo@7Ay0~$OH9oT}z+F_9U7I zj0SRu@}Loc+2b1P*MExs6_J8+cv0PCF<&W99bqzqZ^ z&`G;d)DqtE@Ro?cy*Rd(m*&+b{i@e?H^cKJl$S8>bY_+?x#d!lhGgc*g=;od+}~?X zO>4Kbu;EeXIyw6_=nCbPF&w8>QJaI*ZT5I%v!xb~+^o4ZQT%Zuw~5{KgPnU{cF@Fj z2jhq8C97(`U>Te^0K<5|3^3EEmwzsVX#?mTm|G#)Eh7F*sPcZJ6rm}r`X!&YW{Qa?h!W-v2w-|f; zp`>gixTDhhx{>*-ua}tZm&ezrKlMy7e^qIKS(D~1dH>mk^%X*gF73BIiSn18)VNCi zpvD~A+QdzXs?O}~NuRsa_gHz_?o$c*Jj6I8SRQ^>m%dg0`^F*+IpF6O0u@Q$&Sx4qnvo;U6NyTqQPy6K+XS5Mf@xurjb z@b=^W%p$Xup0&2GMg<(%yRgt(+V?2dw!zBsmBK83E*EW=^lpzXrQjcEC@ZT3*0KZH zpPg{=H$D?Fy{_<|&4-2Y4`H?6cGzB)a_*G2)AfeWTP;@S$0lNbiLLrtvSZE5S2j(y z-uZ?12g!JNzHk1h$-CIMKB|i1J+?YKTafG<+F+G)Y?P-uCWw%Ko|qkZwn?M^Lhl3V z*Cv+Z+0UlAuwsl!+;W)9oNNEHb&9ihMM z3+NeLxh(E{mZ(jBXt@3UhMJmDPtU}_nWiIJ@p10pK>MYFT-&DI({KM`&GGe+8`plm zp54f8sQp3a<#aB#GAct8r?oGy@yR@#&Nql`ANZ-Yx3i0z=9&^2=Oom2KLvBq*FN59 z$ld2X)!S^e4cbSyjWewjA9L2@jr_^PVeXI(`%t$0mi``#v+M?k2Cu=Q@UXX<@Ic{W z3NK>B4x~S}-H4H-j18Vpx_Y^Be%LqsaF8#1cywC=7g=@l`k zoAXD@a{d|By8FpYh0;?x#xCI2y!3CzJJnlx%IVw&)_wlYSH9k#f5GWP^%wG_GEA}L zt4|c1m`X{sGjb`j7S4;enTr;-2FKhSZJF)UIxMM5i8+{@FuoAnV;WQQL7{c8gaBSJHLNj8*LEvqJW13h!E#`QxGW(X8xGBkyr80FN$R<$XvRoMl>4V_Iva zc}cCc=k`sN+*6tTEkekDM#s4zFE`0Oys*HiDl8WmBfnGBn-DXiWc?1TY*9;j<~xtQ z<{d051AQ4CN8O|NwWTCJfUI4YAV8L3;C4gSfMi&#ZLv8&&f(Rh4f1Lyzi#=Zy_y`KvM;`q2I<%DVxhuy+osCJb1%brpdUsh+@fPUW8}W*6@VW= zV+Zhp`b%*B{|cMxl993$0}LUrNArNGxrp#^V=2|ayOg^o z?l*FWRH%yqVCWGO`U!i=cY_X7Au$i(7wFtq6r$LpFiYrA>>T2m!EhDWk@eAcd4wlW)#LUnW zhDX860sU`=jnm(OT?h1@OpHyE9yxyCD;6`C=%`JRj#SeoFFa`G%}rxnuVdsB47-5F zY|#aiq#Mk~!5j>}4u^@1n*)e~844O>JYozuRtYWwB|GM3AF(swL4e!*GA*->VONu; z1LHvXpYSbM6l&nlJL$KC50Nf}E~qAna0So@Xi>WM*0sdL6M}Ruc*>M{!0-^@NKhBP z=z4&~!wRo2AWp&Bv_%6F9Ri`74Qu}lCllxB6%BsaBbZM>XJeYNFC6;h<0UktfFNkO z0PNHGSTaBn`on1qYXBT73_O@K*ir~I(h+9KArt!AOu!D#LpfhqI7eIJ94W4%LMV_X z-MYoV5!#vbnFR#&4&Y@!8HNCA8T!P%nVVmz*~|bmf+%8vSo;%?6#I>`#A+V$ay*|d z;?yAC3{C&xxqVbZWGIs_gPV%S47`LPxe!|hs~SQdxQ7sMI2FH^7*YEO;UKvV1A9E@ zeK3v>LyRCq$e6hj;6PakKz#20kCFs?9}}-aUkPVDSXrmF2m(mghH^#=%6Bojm4Bm!8I5iF{3>h)YzysZw-{ar7J(@rOGK}N?FU;50$=C zGOyN)nJ=HH$P8xrflnHy{Z{!z4z~x_IGL;AsF?+za4S`4TX};ItBIsxd1VlZ{*fP(yZAy6{5X0E6$Wn{ZDuHIF1Sg_Vi22QukM7!A+vU2n0|$ndQQ#Wb zpJ6n33nBEnc`m#;_b;_mKJHyVD1gC1tbB*_VE2TdDsw9-HAdSRdE($`E#I@s0p1&P zQ$URi-KQPHuEx=n2AzZE47t**tHK-BF@;Pav14#9z!+{bYZzg!3A;A4@;Svt;@u&E zLG86#AqKqyhgL`0KH`y608rQgGIcP@f#Sl67RGzs*w28Q>i7+5nQ-2FVFdM{Ge08A zW&H#~bpP-Y;`m0l)tw}3LBa$$XLa_&MJ)hy@DXO;0mQf+4hKlTI>KSY36US%)Y20G zb(L-sVRB;oWETGVv1ydA_%0Nmur(ig3sDRP^H}qZPXPHrp(3Y`p($=1dLZ2W^7x}a&=A3%91i)2Hz5bVFw+1^L#QvTDMRI&$ zN$1@a0!NHDiXDMwaWZDuq~N>V*wy{9Yx6fARC3(eM>`f898lEE-f-E9BLcdu>VsjY zI$J4q!sYBm?bUFliI~hIrWS(vD~0u!;tuqT){NG!&jQ;tYm7UnDliwBk5qwc1;^)o zRsCI}*H`+1KOzI61ZJli+b8%WJSpTpeU_IK_QQchNIfo$B-m0p$nz~kdI2>|R>N|X#;oY!$OisBq&k!VuWz61@p9VJo&KT*F6<9N__NWr6 zaSpA0Y$Xg6emQ!{&YnO**?==?PS1iHJoz$d}~&@E)BSl3@jD&j{uVj~$&VKK!@?ayJcf8}`1Qy(Tme}=L9>E$h>;@8Rd{w{4wDEwC z=3&dj+Aa^1J@?r?D_X2;WGiJx%4X3>yF-!ES60(8k>{EgHOFyJjYyXwqH{9~f|Dq& zGW1?Ivoo=q?%M7)FhF`iOWD#MB~MwBb=AVv>RAk1mXM*~{l;B`u!->K4>+)SIBFr5 zmtD@q7;iJgt3uQjZf1dr{-u&X6yg0RXyN}_Ur*~s(lL5(^7^;HLuPnA-Ocsf5KTfE z@-@Vdd#!EO3#=+8O$ZkbX`I6?#P@64{G&X~Sn|~uz70DX6Vup%5ch^kzkdZ>Jzwc< zR?{PUC(@;^a&HlrZS;Mu$-B3Cy{mU{^S-{)Yf=8kuE>%~wtva4tA5dv^tl%Ysynt{ z3sh~Kkd;JBS`G-`f6QMQzyDE{H0(+2pS#_MjsH4SZmPExr`waR*Sx>Ish?op@n&Zb zcS!nKZA^g7KEU$!N$F%invHS5hAvNs4L z(nIV=J05k;zu8!8x%+I0JUBu*bE+FZ$8&>Byx5o6^3ru-Rm8o9%T2 z6|5v5Umr02B;QiZ-UPj@`!`kx zo!IurhQkK?eU2(V2)>>5rE#+Bb@Qj6ZuDmz95np5g9X~@Gosc^>D3+DpBo2jznLC= ze8L^@_~S+?>f!1t)}e2EUu#Mgw{>%C5@oJ=Empi3Us&)f$>NHqOXMDcxaj@i$4wUjM zALA4tN{n_wC}wPJs5WKm(7@(ZaGnU!w!sz@jM(GS<>Q5c`_(1`vF~Z(G%W4);O)=) z`-ipl!d#p?r;pn9=tDaxcX&|cI~lO>H6{um0o9Ry_%hDGn;m&K6JVATkO(>@o_Ft;tE=KyG)je7v7ebetE

sE2p0ip82vvw!NdXoo3=MN6kOv?UFsj|Z$NfDJ9Qv_^7Lp=~sbaB`CiM+i z|5J1@%~cIke?~?fj!Myu>C-qoz93&LRpR~rgyAkl$Y1^^2kc8}7$Ywzqh^fmsK&BE zw21J~0m-zHuC>3SzYTRfAlc2(Q_OH8EtnW9ljKvAOc2`_rWeh~6?Z&uqvC~>-DEU} zU1*ip6Kb(`(iKno$O{W!GL3^W5jIzgGupN7D8!y|>?_&`koR}Swa`Tpk46(>GcxN~ zj3K&S+Jms7znPfib$3+L+M+c;^NDTHqzWKA1w{SwI8jZu$Ah%GX=I2ll*>DxzU{^; z-7j?3qezKsV%*X8=$tRquvNd94OdHC1qwd!p-e$ZHC@@z-xxSEE(`k%&Hy%hsRoV6 zfiWye!A{<-lNB4hB*rlhAyk>bTui-+i?-v&hCGp~(!tRVtQHFEzre;!Piz*Iw}4*0 z1>roh20b@cAQ;JXCbeeR2d6g;25KAoJGLSF^Q2Hbw z4mszBE`!^9ap%gYOBuY^qDd-&m{Jbi_0!UU3I4+Dc-$zA*lfIz9ETpfWhW`R@s@u& zej0@-6S02^9$PkcMr#z2xp~8RtII%1!9BedFg8H!<@#vWqJ4}ROw0SGU!@g}z4to_dT-wT9O2ja7%9$v(SQb8FrRWi$G*u+aVua;%HOnA^Ff4ot9_Q zup+d%V%${Tj(X~!z$pdogk!Z-v+j#KS!q_LMQ5JN0H~w{1tOwlF++bikhH#)bSK_5_8k*my><(B`T!b>E=lXsdmmXmH;1Lq0B7WY+3W?TzV##@ENp7P#SvB7jgL)ga@ipg6Lr891 zex}iBdicToXB&SCb8*ntOXjZGwD*O@-jkgJYYuhDUg1;&CkspE2IH*^uTrN}N%scr zyiw%np$JG)NIeg{*FWG``S62c0Ij1bft#@KHd8xhej8d#27U^I3zGlb_9g7p+ zke)Z_?tzIEmxZHI{mJDIWmhkj<)n?J6*Wc%KOgReuTvwRjY*l@S%>zz-oNA*-c(!{ ze|D36BFI%ClyVx~3%`nkn|)Lqwk<_@OY;rK$ZKt(@@_9`?x%a0P^=CVc~$pex}vN7 zGI2G4W^k#|sww!{aDB%M!HC$q?-?V0LP(|t~M~Bf!!(lu~>i{2fYM&Ejg+p^@R>JEb zZ?xB1{M+NsGsP$6bH#8%9h&cV^M%FDifGZ`P1kpJ^l(FeKK#)m<>0%wURa=O!-l)f zDxFKcyD6?J%BQN-{Jl6>tsi`0k?=I9Whd9rxsQ=i9|eOr77yN9vkB8KGNmfCS==J7 z&CU)>9s2B)<5|hIHY|wmx6iZ3?(%Vb-8?SHXrukzvv$)07(2C0BbnxhitBpwGj}}Y zhR!I%4PrD+3fzLCv2FYQrSGmp-Y?j_`CY8YMUvNU`QZxQtn&s9_zjF|@ zkJVf`ym}BZ8n?ct{WNf`Pv0_exVpLdA=s3T@e|@fG+uKt?u)N)c5E=%RBYYc!QW}` zSy=~)2}vC|2L9DAvCgC4il6Z(qaR4c{^CsT$wKc^vyux15YZ>ovI;okl5-Hbmbt+o z+uKS~PH|Vye;s1{EWXxTK5VQ1;Ro(XP*WCk|Jv7w=KJ7$C~H{GVLZ>*!h_gr*)oTM zI9R9R7$9mtV-Wu@5<-hfJlgl9I zotTABY-zPxHFi9u1LTE7&u8%S)(*Djs6dN9IH>$$%^|ZQ3_bB47Vn9|w(Lp`!BnTo zN$^Bp@HCKAx21=|@WSi%X90(AWydK`o)!<8&51fJ#T@VzCi`q(l&_(!`xnFja3(O7(?W6lc+pLFF260RLgS|Ts zKv%U7RQ5MM8WFbcbtvuu+kLnVe9B+2|L5+F@>(GWD-`36?w&Z%O7gPo5_KnJVIA}5 zgDXf2lVj?RO?XjXc1*~1)!`Aq)-@$b?g_CX6W7+*f*uO<~1d@4r z^M>XY^LxDiV#75q_&gW#bXE*&a#vrB-;GGAIvDqaPek+>t%0FDcc)ZbAU%2SIlN!r zs_o;3E%k`lB4Zd6+G4Tx@<%+6SKljH^-Qz)mZ_O z0Dr&s0DT8_VHo{YO}Iv03hm*O)p=f}=!K1jIKncQXf+F%Z(KK>tk zoq=5gep&hZdq;4znLb8`fM{7_`|Rjt8~O8Nk)xr#*5Crw1(L(-h9o*@b)2C(|JYES zo)spVeOUCcP#Sfm7rfjKdaegMeu6yht%N-pA$U+-(6JJ0Gv9TJzgM z_nkutb;`!xvW@@j7IZ|KoLkF9gK zzwFp;(G-(CLY0YG66rlu2#vRcMFX@>n z_+-HYkqv^N+dxJ9c1jn6&|qcjf`lEvBM85(Agg9y`LD4B=aEN+5l5J19{d(j3{0p5 zYUO%O;5m|cHo@Bah$Zu*8Aq$^nV&OQfz!4|5OZMj+sb+2&msJBn2BH&=;q9_0pfyOY0 zs5qvKQ>7YQ09R{fog^oYokk^mGqY7AYGAD2MZNDu$8f%cT5% z8Vv$uNS9!BDjGHFdMy^OJcQKwdSS7pE&+WNyL~!`51?u!P z*N*PrXc>SxdIW`qrO)qu<i8p@{6 zWbM}B{E_V+q(#p-8Z*@FL+>Q$QS6w%JeensbezG4LdqV6R7Q{$f(7s|hiK=q2Ha#i zD!4aK19~LFlaM1q(O@VO(zg>)35$wYi>2MS@F3XU1ZLD2iJ>y@r(SW4bZ|ah5qBS^ zsSC39QJZq1Cwv9`+Va+7MxKpbau;JzHmxcT!E|lj@~rhIr>&x0%f;k z#JGv+FQx>XqE3jeb+DIzm+B-~nZm3HIO1e`@Chi1Ck+VPP@yB4z4q#;9IQ^KS1>h+ zr>%wpZ4!~OF`wp*5kz}F^bi*?eubd=nNBPOEK=wqNI;Q=YKSB6L6Nb2U)Vk+QD>f1 z)x|RYvkMgg!>nEhbE6r8K*HIRR0E0HAwxk86Ac?mIkb_U13>HLDp=uNoUD+ULo<|E zfd=Qt7+^3HcqnA&LPn=V?RBe7hmria1-^%x_>q_nbRVb=P%VckmomvBW40siUkg)1 zOndmTFj6W#JjhH?8iX9zU3Sob2bSllS&#FHaTV8n%vV7gwz3PH=88Ugpo!%Hkgy1E zI8}5&ugF?pPUYi+@w`gNfi86k<2e{a$J)f?+&*0jpPk(Ud|NQv^oYCj&YG@vl5~G0 zl@*!daC(5d$0{ZR53uqaQ>Z}$9K^IQv{J0el?93 zA?jX&su|fRzL-U~b)kzWCU)_d$f9~Gi~%2u9bC~KfUBAtNIUl=xg&HUc=@1(W}6sx zGtB9)P;UXnAvY4l(6hr_q5XBi^EjHFNMm4onY%li#48tmjj$e?KP&>M?~15W4vjj% zvB-4}Fa^27B!Pu$Q)#`F`4xKCH8PNhi+c_Y_~NI0lN9@zN-HzC*u<|f?Rk6GqEme|_Xm=&5I&Dt1HE+DA*i zu<}fYqIT>e@HV^~(0FBHcm6}Rx@-kW)Vqtx zQk*{*T0s^32E_yO<-i_WF|2GGA%NIWNaqndZHV2@=)G;e>_uU@o>U_Hy;YkbN zs!FW;dnMYni4_fdb<1^LOdSvR%sO0L@x zv<%ta+0z?k~f zaebJ_5HaO)S=O4jT+-6Uo3qvz&_G6gvgF6=u}W9hyZ$0C>+HW_b%;h`gs}pGEaO7l zxw7MqOaUVZRf@e;{Z~eJy#XUjnEN+PaLz>Pdc#28EM-eLH~31nzsadQb8MINQ@coN zaDau%EJ^PS{1{Ugh&hC77@U(GW&Wf?AWo9VspKLyEAL$~PmCJpypW8>?IIue@R3_$ zR%`codssz^tVwf|dNI@}YlG`&)W|_tgej$gmE|6OxIvSz@C**wUl~WW(f5`PgMr)6OCoeX?HC!;6 zL45BVv|DH~svEfW7@K}=8%!8O`_?Rawl(SlfN-@g8L13R^~wkP3|qU*uJsj}jj`?0 z%gskzrrm9SWhqG4TDJ#<=(X?_Ja_cNpS%Im;P>fjK#=EJ3UzT_CD zbuGm8kLA2+rGIijXZAF*rOChGS_dXA1$;1?FVMk2%pJIc1Mbg&-PQ-);z)t0r zU`G5Z@t!^Uz1c5ZNjHjOKJ9KzpHq>pVIG<%!f{YJWag6t+(Vu!PuW)FwG`(4Pdhcs z`!Te!V30c{?i}w(G?z%PA949Zzj{NpAuM)JkvVn;R2N^lE@*1yD%?Ag&dLq@bH)7J zGFdakHP7<2^JRwpx4;4)*RABxvOUrmcMGmcs{e6a8O)8miLQxU%Oysz9K4Dym_;Ra zm-g!NP!RI0_BKB^zsnbt6}X;&l=Sc0Obg*_Y1;I`r7MrQ+SZv-5s3~@2lSh7Z3^3) z-ex|#t*AKn*d5Ml)WuGtZLUjxSh$zeBvroXy1+WLApQiS+h-EB{Ze_x*kU8v2Qx5jtSN%6pq@wOG^eDfo;qpIco zv#QXF67TN5Ar8X0%C{g_(vEYvw`_mokmCK!lS`%7jm=r)6BEkneSaho0kpp4Y(r{c z9L;6`I>hs(qCyYEA1n#A8J-X<=lrkm2esV>&}uFSyUXiNA{B@#i|A_r&}d?ggOU>e zNKG=O1Z2JFMgYnGWDc9zLgEbyR#E|!jno?&0mj`YD@8)1fN7*2#R z!JoLU%9<+#NK{%)*)CwO>QTgSco$UIJA{b&Ut$hEL7)b3IWm(Kfp>~Rh@^mB1WLXI zP|I2b0=X!;y?qk6%}HHJlBv$WzlfT4iDS+Ob`kOc@ZPb12| zHQCsTXrJ6Kqf+pTEI%q}4bVuDMjD8)!kdXU1lxPM5{x~X(^pfO!VxMKCu9=_X~{GI zVl=M8$kQJ{ZW^7*sWHGSudpy5NJ|R8N1QxpVFXn6@7n~EOHyiJ6%@YwtE%ax@`N=` zBk}?CRAruEIR5X^W40PIzy^?58bei)mV*ZKAP1PDVFe-rLI5K5R z=mwV%QI#5bjsK2AXcmLLWd%y(yCCovymhFihKW|FBxRd8XTgqUTnqZ?)M_DWyD+u3 zZv|=8#gJ0ar;!`G7zhq$aCOAuoEdpkAJo~{yWT18~QfD7WHVrIccbh zmN3%=0BA7?gu$8wfN-D(%(+DO|DA>JIJSvc*xt)~{8t%JV-F5PHfmjnSE-T!Mkl4W zoX#~-^9LNJ+1uH~Q1j(b;BZ6ZdL+L05gi8Qk^(m)*#!zon#8+|n66wL=)--JE5!uT zZTN0$y{?EwrE`3D;EO0#tBi!TGTkIb0%jisKydz$E26{V=MQ`F^e=STqYK#tUoojw z$b1cF-VCNLB9*z{O+V^M zq68AuzzIiU7Mn!|57G>*U}XT3zTMP{`Ks{Uvw+(ti4j8J!QNC;11(B0R>#)K;y*S- zAW!3;G` zNQ9|T=tlyp4EnXCQfF{FJPlyNoT-o1>;r;5zyp}Edj-0lM1{U$Qs@=@2tGP&xz>8$ z{N32@Y--JiEwo7EzeVuT!59-{UVm~|l$b5pPcXHXRMA^V5z{qF9t+rmM074ewE6E> ze%>zfeUC1q`v0Tr-Q%h}ue5&-HFl&4(@z1@9Hi5JreRQH`bL|h?8FS?Thml+1D+v; zb}*V~5;R5>w~1q&A2wmyYV0H$6l=7CAq3HQ#3n_i+6ZXye5#g!3KA6%fxYv7uXR5g z$+W*eNCY-}KhOQ#_gdGw*0t6$ahF&bcQzw0{aY>uiFdWzh_`p_+x9M#5W43z+%PTJ z&Ba_ZV~EDE>ur+GT)8$8Li}jbW;>lDt%OSVY>##I-~Jf8njM&0ZJU5Fu%e-p=7O3G zzIffL@V2!+H2ceXelWi4cETx)cGqzGoec|2DX_tuMeT3eu>16m=vF?5qKP|85Bm-+ zem(2rnXi!j#K!KLLlBiIK4$6N4?g$FJHDa?`;hK;&rK&_Yj);A-N<^!DrN&yjIl|FCPpaAaiNZ~es|mci#A ztEf!>Iqfl41jFlKb^hIVCwiA(!L6UYJNGwP1jvEQQ>i!#evnc9db2O90tZaH=ZChf z4Suub+`5MN&lgQTapXAD<9*ZEwxks|cimbLlRITe=bx4)#Z&Mw?@9_~1~U=R7j$J8 zwzSXd_$9ggozdCt{?2P9+ZPu?ZrgqjhJg`0f*rm3})WF+Rg~VUIJm{f__l^0eaHemxi6`B zmVeWd&u)C48~7DAxpX7FM0Vr%?+Dz!Bd{=d*B4=b?mzj{ie}QG+j1whM;`H=ulo3o z&)&xRrO(Qpu*!GoyZ_h`|AQ~W{&fePFs5KNuhpmj%>VJOU}ER96&>FkAN;4lFM^53 zj$ivr`sPU(#8c~hMfb1m%D!bjR`{z8Cwxrmg4yFM!igFDxOY#fmKFPc=7hU zyoSr&@#i+t)$hBsV`tp+>{f+J+hK?P@A=k(j+GM`eO|YCd|megQtt=ib}ZeM zQDr&qBa8iI6&Efa6nk{zYS*yQb#KhvbpQU#-F??|e02C+SMv9d^pC%Ku!&k0ZMt8N)Lfr2@E5hBPG&E$#aL8UpJfwEk}!S@%Xk z^EduV58FWgE_5z414Dvo(=SB-V>c)u8P4PNxxr0mhi8u@NAq8#?k_ECz3=F7E>KbTLQdYhxI(ovH+@e6g&TYe@ zi|$^`upM8GzxF+=nZ8`naJsa4=!^NN8Q!cb{oS#SaY0|n{1=~}`FM3(=7>Xod`pFj zM^B6!`@7SPbGB4mId3A970atmy^X%(#vPm9o3{HmXNN}z1}yCS{e`%K8O`U43Yefe z|Gh;`Opt{AsNJ>=aJKJS?md5?z~*Wn^}c+?SDF6V)wfP26~D|`AGeZHsDasR`I*b6 zoF0FnKMU|5smIR*cef-RN@*?lMQ3xv0|+SS9Z~fk!Pt&*BhOuLI)m?Bzx~q4q$R%J z1$K;Qom$HLN7l`zlgPXPmUsK5_dhRKf;40M*3#PN790}K&=Mal8yx$}uMd;HnAr9G zRqL?;r=D+ndG3cR_xD}==zv4i+qD-g)B7O}jH1yz0`=LS=6>+ez9Vrd2j8zca4vgC6U@igDzKWos_Mk< zj{|ErFrnkr%l&hzI)@y6uE>X%e(&x0N3T59Hi+3AAJFFQ?c_-G^)a_Bp7$_==s;RE zB)f8%H;UR3(*(XW&rjyBq}y+*=J2_a=aN8e5$IN*1eI1_tU$Pmtr|tPeXARirVN8u z)ch!X!)cPGU|&kVITBY}OhSN|me7OZa4X4hqGKuX4IeXafLyax4k&*_${o6aSS*%V z6e#**h&E#G`l%xeTr5uvL(&)C@UBx0E;)-2?#SH^)4~tepUhe#f9jF&%6UIu&q{g!#w>EOVHwJK^Ov-$Jo|3Q)F} zpuot6UcZ!}Knr>>6eGLNmeL=nuU#nAHQj*|C!qTfdM9r9;U%)jis*<#4A7Q7egKAS z>|6s0h$*zpO3eU8)mV3#3AqvTD-UR6>|jl2ljpKN8Q#+g$vGsBnCL(9x{~yzSdW#c zi+%J*tA{6|2%V)X9?LT-Py2185<~<<>$`W-+#+J>K1F7UyR6WG&Ou=bWnutInSORr zQUQn$7z4}13dUI4KOt-VI22jufdF)o4NmfPlosMsOE(g1}==ei+q5qX1nsn@(; zl!@MvFkUb^fi){y6D9#5&&-iK-UyM$Vt|Bvr09qSwZK#p0p^V8wH^oKxESWKN#>%y z9Nu!%D^oj5Vf!=s9Azi9eHdD%st!JYhNM;SU^$JC%};lHQ)zHj*7Udcl>`^3C*{yJ zV?jWJJ1?{^gRDuV8C`6AYdz_9(?u+Ovbys{LMF@1r;3*2r5a9;P>-sanhmyy50;g? z9YH({_qN5wEYwF-zkv~=dt>mN_128qH^}0m2+JeG(Ct+|84ks@&(tu)5GAHRBW_P= zCU493?}G-2GWuz|hA_1L#*dX&x1J<6PeF^fFj|cnS?K+{2Tg_38w3y_?YOj}6DyOS zH@Gi~#tjh_J`8TbTct)I#9wV@ScN5s!z8&N$fKJ8@kLz$8P9`kqXEnAj>>T35ot2` z_<`}|Sdk=)keF^l2|0*gxpra}q-p!Fcri^Y?ILn%?KV_TM$zZ$$ck>if2}%O{3!id z+BopH<FY&j(OaF=HUoiR4-_c#( z^$$84$uAAwlcb0sIrdX(o;1@%p-JrjD6JfG_^jBMc&vP;lc~{Ede& zsKnq>{};kg$M^>KuRdaTLY z4sW@i-EGPK8&`SOR(Er85}hVIK~?Pkyfpm_W?+X=Oys=^@#l+WzaGPC_sCqMGtSFFt*T9v

R5eiF|_-$&@)cw-rC08;=36no54pt5!BgHN)|ml zJsQw7{;zlNE}>>?WV~Il#p=OFFYi8SM9!3HdrU=w1Y@(2Br$ucHA7H-H}R-~ykutS z9jkpdYoZJ86Wsg6t{LerPNE5&xYQ+~yREs!haoU&`&7MwcZ=B86~}8VY}6~mJTgPO z#x^m_VjGsf5$hXX5L|`C_@u2BBkWu+lZ9?Xf;}h}-aWF-;RWuWN4wk;D%H0=(u_@d zwOItza=YKp421USbbHisZ_fH!6`_8R<5S6~zLxGAny_|#lUd$Kvdm4h?S(WDUA5OvH*7;_K;CCzPYUukvRYN3F%e2={c z<${{jx8}J%{w9Xi@y{ppz+mGX)25Y6P2|d=USL|DSdZ~1a0jz97&kR;vzpgQzs23$ zRWTu2W(He`^tmpt-!^2HG*8=H23{qlGBI~iQ~8?5CL>+7_e_|y_@g6%=yf1`v;`$^ zY3tQaDqQeE924#-+H6*z#L^!oF%JIeDHD#KMi;;2`4?4RV#{KmoA}xAGsN1b{eRAX zUIFv+;B6hhJ`#Aqs-;UtT~2s!-alj%WdETXZ26T-9ociyt4zsms{1VXPhW(+HnaQb z+RMRHXDec#-A-<~bIOsUrXuv;n6vic`qgERPPp+!*tz(HXJQ|XpVJ=`T2WvyW8~2xTij)Ap5|!-Nmzt8@tbJ z{`~ItQWeRdcyBGFLuSLV90;%7ms*xq9IH6nTz{lHPfcbDuQHuKn!J)M;$M`tFNR|Rs944Kmv z-Z4N8R;fCxdt>>kvTJAd(j<$enlkCk7hyj=dEz87OZIht+3jTKUuIh1+c|-+UWsn% zdv5LY*ZWtd9*I*^&^cL?bI&xzU%BJJrRFoA=lRdRdv)9H_o|<`^_|JxEp@viyZ^2B zaz#z?g4&rAuBSy`sVwNcm4}pGc*oYJ6<2;tFNfWCNswJJFsY?|CcnPp$Va=L44(2G zoZ7kLVgHs|)|W;qbYjU~b9hPjzo;n-142Gg#_(&`^KbmBZ3oi}YH=FBn72Eg2y=8W zyteJ-_!Cn*{6FaMohzbE2ty2ZUm9`fY{4IAY)SooUHjsOKluKWeh-~Vyvb;PW+A_N z{aSvL5AS$ll~E|q$KQE;e%~vV4ab^h(uT2dGc?kjbSUWPC?O#GhwgvFWDHsI@bap_ zHeJ<_U#7Qy=VE$C-zP6N(J^4;A5m3Q=bYXC2|_7fU_mO4Fs@@LAA97ju5TY|&+K5& zQ1CLOpF|8NR?Uhlc;8?76m30y$5YSHO{VUs|M|Z>QqceSf=|e zfUO%&P9%0J=p2?a@;_RqH)jTa=5K1D!l^`cC*SD4o0T}<-uc=2;Htp`4?cR&=Ot%b z7B!=mKb!kNbA0RdtoFD3HoUS|vlzyGT=0wbiYv*lq#i$ho{>V= z0gv&+N{iBRKe$*_JLR4}C+O-=`0B~ri8G68AKyD3tV{%QG!(Z z6!KqBg)WUa0h7xs8~4mFlw`K-^B@)TlZPuNkAB;IL`=-=TdT|tZ>*~)cVW}AEXiaE z_F5YBJNU9ZjVOIZ4yF1MA^)?Iu$|CWo+q`zOjwyl3cQ>{d=U>Wbz5@_%lnDM-V=_WIwgf78VkUNxF^t-YNoIyS00t zo;ZP};k68wxOr(99PfkWRdeW}}-xjv^fb4m^+lV4_l7(zqj3$n6R5RQ>|#aowcXfN5paNS^h9 z2O%5i@`rz#PrmP=!}U3mTA;^>0lqr0%v!Ba@w-l_WR${6*nTsIs9ulGp1Qz= zGeN4;MzXwjU$k@{d@cTP@W&3{jb-QiHpPsaNJ|D*6!FbAel+0aipG?Ev?k5JF`H`# z6|H+8VaMd{ucqZ$qX5n$15c@2u{q)fue=79zm2_22v*;j_t4JmZ222=8L&xD#xdbi z3lMFGCG5aujk41XIi}{LW-@tTI*ObX)%g%QU(qN(!+nPH_(p$Ig!|yYh1r@Cx(;f@ zRA~R&Iptc<+<1|+a0Y!1!qoRwfn{Y#avy@%%Kg64b-dU|Es6j9XXiAV!pl)(7*_vv z5E~{J8!rP96ArhSTyI?spqmoN;bY)p#@y%jH@iA~jbY>13^td+d)ORek$@=y4B?A5 zakqkT=1?*Koe$X5j}5S}m0))Ah9X}hn%9)wjv;XkRcR3dR)&T^7;68JaNULyMv_nU zFtB~1vfZ{w@`jN~#ks$#xNw{p+S?%dv&74)Q8N|bfD@9)D4t3#N ze==(cgBPK?czTw%+D%%(^Rtbily<^j z9oxV?T1QL*CLYppyIN+j^va})jjrbWO{*Ml5t{S9DP&BFMjR-fa*QA?6tB6MNjF;K zLe?P|3?ekCEi9bzRa|LFo*JOq_1BjVxCo3uXy)OY>Fp2qR4cnF#(_iwcJ}G`l2EwGx@^6f2@gfh6Q>62=wz9i^haOA zzRwokPb{sO6PwF;Df9nakD14ACv4>MT&QUm8L{MO6-@}*zlUoYM!a|-{Ys!(sq^MH zSpxNN|Isk}8(SI4vd2^^5B7gKUP684$w z+@HS{f5P{COA1K=MdbYtKV=yRFzk2h$>*#ZzZx-?DR;mZF<&bU2{8Hpi;E@D;Rcwy za{wERJ4lY9TU3-yktr-@94*>WLATc94K&}Ce_efPQlA(sE|3?m9m1X%)ChAaf;jT# z(htUgc!a~?`ssmgqBPp{KuF2u^P9O2qJb69e59VwRuL-D?XZ*p7=xMj<_!T=y5`uC zbTzHlpUS_1I^df>#_WpA2VP*%5$+-U3T2W?I^#!ynz=MgG}Fpy)s3sA2V$r91V!>P zScv68^QA{fZyJjI>~4gK1Ow~`deugpi`NRB*U1~J+6su$Oy*4B+ksNBFY{j!0ltHC zx8-2B{|x8o8k=iH&}8u}q*l*$fcql(&%pkXMib zQ^VNp!8`CB=C5x1F9N7T&3c47K{#42oEgwuziOeY0_MqdgX|nW$|zG_tx$rmwHn>! zR$G6uhVo=%^K6v?^e%L=@L;V$cxAX+wK#e?=?r>ZDMobFdLeZo&(c1P7kHr{Hz4gA zv4#>_dM5hgE7vi>F1&~|H&Ix00uDw(>T#~i*w2TM;SPwC^<<~ui?#kZ^ zwdOmNRQp!CXx-IcId|YN;%T#Nygi3Vht|BcS{RQ2CnS!N@{O(EP&HJ+m4ZZCm!~u} z?kFoQ`>RV#St3QNUaD&)t>fRC`>yRJNW50cUPS#-e7SK%dg&bcHM;w zzvW%{JHNcXYqvdy+g>p8JZ-a_)`TPe#&I}|_P6A1;%6~hvov7Fgf|!=aN0Gedq?M3 z(DBw~VWbt-oSF^hBUr8@_6AC-gcOpy{&W%S`Eu|+E%RNB=;{=2fi{-f!}x)( z6$OaZ?s*yS?;;q93%%$EJ04G^%)Ea`Y1Fd z+YX*oQs1^JMnz9fB2pR@{@$m;>f&0i&_v3t7}nK=_6D7G(s}1oGzdt^DN_@`q^FnZ zV^D({NQ|*UcnTiXcF~(D7c+)Q>v@_T$ou`~_SN;1pc*K`B@%xX0D#eSC^sJwxf33S z?ywPPAHK#(CDm*!TQPQc0Vs(n0{&~Atn#=`O6t~4BX1Ebs2Ek+{$%_I zwU9EdcbGjCp&N};(!EX#wO(g)py?rxhf0vq1r|@}_AEVNsU8QSbmr|{gv(s3HaoJJ z_W4QzVA*Xk6a=wsT6PCD9duv7p|dvtCbCALr4Kosd*u`Cdi1yU%0A3s;uDCJLW=@; z9vJj(b5d19^^{GbJax9cl$9j>v69THqZQ}R$L}l7wJ6NqF6t$>-geC{a9J z*H}=sj8lzWTht5pTg2v4la<^ng3j(6{Yq~xOi0rI<%6i*w1{?kG{$?6CV=_ZtQ0okSf59^OuXq zghrmEV#-nPN}r;bAAkwLDX6I7f?Sw!ZhVNKFY!D$5)t>h_c?Z{3Acwn`lk}bQ0523 z!r*&zGjPGoOCf?kkoSy%O<``qtAQ&Wrak-zhouaLBCHW+P#9_Co$RHh7%{I(0}HB% zXFo*NA4FED9F%7B#1&k4bC)5Ff0ZGdqe}CoP5$6UV;#-v?qJFT7&xITt8>VYk~UjA z2&$xvLUg(l%H17lBrK{+Ofb^+wig^9y3-1{42udW#%2H)z?0H!!RqR)NoAxWuU@Ya z_IABrY?Zi4FlQ(NvHq34CQ$4OK)NuE*`lJ z#u7`%iVM=|OQIgT7~y{b3x0>09;c8aHe>6gyXt8Q^IJs??#t~~lf6I`QbdCwLcK0L zt{W-S4xB8aA0u-j4J9h92(vnpopQrWw`3#8hkKB*kOLa}^1!0Q+cg{P$JE{2d;veP zM{h6!HxATHpQh+--QbLy`>fJ>uDQ;9wOX*sPdD4#)w~dIxN)tgwsHA2hJX1n1>WU zYr;V+_MsKWHiWb1IaxB=85G?XSw|ezY0+zUBPa7ZLmSG%mHWz50u=lrVscc@^E2cI@ZJy2ar zqhI@*v)C(KCzb)}i*FNu*WBKsQKS2}GGtiAU?d=T!Q%(E4m^&+VituaoJ#otq=&S3 z2|UOMpZHc@U~s_PxJfOweoWx9h7x8kdDF(c8n%=O-uo{HKPJI*Ex5{S<9u)MW3k)9 zDjst`r@iII70#VJ8BH{`Q}`05q@&FBp;KOFl|d`(Pi$bMkcDSN7PMwY^^M(gSFVOV z)-aRP|&ZCe=|G%H%xHzKkKhi1C;?o3CMBOXMFpxzBIr4 zJ`6n3r5IhcgaHRrS7bV$n|lmkM4r&Q|A2?U{?U`>`pp8J8LM&A$gM6UsBm zl}fLSSfUX%l^*jmZ;YY)uWV!AhcXqTw`HJBp-lb!X+<;(+tMx`=xW-`Mr27}l4&EZ ztU8u-#w-EDUG@0K3g?STSdd%byV~r`t3^`+kazo zfjF-un5MuMkhst5o5*P!Uudm@!6&sNPX3$?H%<+j+~ZOs>|F>oB*KU_(MCw^=dd^4 zlD!5tupYDC$2;r*m@$w-2mqgHYAqVaLdzN7=!$HjMoY@43wrFEdKZ(tG|dnELM%3B zg38SjrRwnMsTNjQkcov7&Z>}4xk=FKoy|IV!HZOAl0PlBV+)G+q-U<}3>x8Y_L8W` zvUtx%`VDuD?t<__?1udr+j zpVIOy&henJ4#!}I%TaD5Jbg#0Csx*DEG$k%6m+c4s33==rEG?O0#Z(Y3C)nauu^mb z3Kmbrif6yQDX%R9f&qMnxLVDv!=yrWzCOh^c$qy!%0m@J8-u~5E=K*?o?j)G2J0K? zSQuFdx$;tgjU}u5#2`EpMOm1~={8stn?YPj8kypE=99sE^1Iva6PPTf^nxs|1`ELT zDDo}bI>}Py_w~bgN`r3naN)MSTfauUe+%AUtS8s2bT-}{`>Gr-L3eG4E=)Lg_3%PE zpW`9?rH}8GXIuKP*@qa7zB7YEseA$I%UWk-6cVi!>(&LsiHr331mHH1$@pXV$39e} z&mzF`15-&j0rz;8I;BF-4;@xJK5zla7dvtZ%sNQiHG$_<0+!jN?srDuZfu2d1X)72 zApYlkI$~G+tbPuENwseEuLMBBHZD;O)XPH&3XLG%*eMAdKPjN^RzS z0}MBEObvV%BtbR1DbhHxp_@oDu!g zjiv)pyZ^hrrN7m`blD=>`A-(lDkl6&fK9vN_+T2k6pwo&r_a27?2+QJ%z+r#jJJN6 z;6Qv*vtE_hI4MlJ`R!!LIg5Q|EM$J7ML;3K(F~>hdMe1O;I0~2%rC-DA{{|^;%)`f&RQNSXFy;IJqg$>PvbZevc=Myk)P`s z>vx2{d+ghsu}VT0qvqd8@N|6{7x}hR6~LCFArCu*swLD;1U^gMaNz#`9vy(Oq&2_o|gTrz+&`h(mzE(LQpY;1Kh_%s@e z9{zQ4$^rlmn;?JxkA@h~8Pv#;G=8c0xF4+kd;0BI{SEySM1C5zshNgmD#FIGp;*#V z?FoMs9hbm%6*D`~ivC|k)Z#AEQhUv&%ODn`%whruKp!87virA*fPm3M8 z8A{;bFx_CyCNO%)ZnI_!qisuBpyiFSS=gE!wt~!oF>EnvleB2%Xr*9ZUT0qFj5;Wi zuUAOGrxB9W5#v>#ph45B88cohk}k)*9IS@>oYaJO&5@U97=ofQijcx1=BB3P+*+fH zEnKRe4b~!}2K%@h9U7q!dpm|+g)XZ}hxU#s3)rf8?8g<|AHu8>G>JLy=;q=R#q0cm z$(Drr>@8_d)W=c!v8abbey3+^aG-@ED4#tlY7L1^!KKuYkSv(j+~@jw&s$3_d-r~Zb6^D(3ndK$IZ@Ix-YC6Kx^y8 z{cGXrLReHYdvIQ>t_!V#_S~z>G2#M%22AzDGo&s+YMiPx#}ARE*mrs@I_So(yJ>PC zV)^t#c?Oe1fUJPgDqOPLMgc&;zR^B^kA0N=GY#g-p?5UrRwbrahw|=rVbQu4nl9Ki z;U6r&ZtZ1V>XtFSTO4Jr6{7-QnRSQTz$CF$7U^d7033?;(3?l81G?mI+#B@pn>TXT z%y_9sB^2sFz~e2YtGgFbc)DzFA4Co`th$-w4D>v27wvAG!&Fc^QhS3QMWv9{F?UZK zPr5>g9?Tb&{a?<|@t5|qs-wYU_)fKzb<(L~)T(O`qc|L|Wh+;ZGQO}FUqZk?I|CO9 zhbd>jx*?ilYg8%rXk9^P3R=nuFsfwvZDpswP-f^rV_;w!fP#1ZC0SL2&P$Jf?*(qe zgiF zFqi=7>!sECl@jTz1dYId#5*}DlrA|?L@u3+Bb)})8V1v;JpWOi(yL>JG0Ro;a^oH% z#~~u_r}~h`t7)?{ZuR*=f=t+Ub+^S!s8>m$CJoUdRwa_q9>$@`E=tuZ(p$nJhztPC zNWWahNMs1w5;oPh5ICwiPFP(^?u10x*YOn52`EqiQ*|o7?MCod$qj*r^b;{Y6!jU+ zrClfd5gP2}0828$s$Pm-U4`SQ@z*`XB@i+MOhjC^0--|`ANt=K$V-=rbqRF7(Oru- zvh7l zf}pq=P4S{~RBfo-;_B+@LREEQIDs>UP104l-qjH*ZY6ey9wP~)5FSp`UIm~eOg5Dk zEGeWWl&Vp!s13ju+`!U(*jl)NtMFg|4$+;$??#WjuVSO5MmWwyl~A4)R&xm2*j1p7 zT%U;G3M82OrNyLHSAxvlwSu5wg>THfcLSuP2x#~^aX?ySHIP<~Dm8MUGvWU4b{$a! ze7sxL(CWJXx6qJ2-5Ale!<$NYXM2&aCh1RFQh7md_L}yM_dq=&IIdu3*J$JbBGM>g zxbv-&P=QMbqXJQg2cm&*!cj3)UHmCx@y*&4~RuT@VL1Ag@& zWZ+-&eNp>(SKqM;;S~c&r(+8 z29)E9`L!Y?D(lHdJew6|D({RFA7!f@SpReJwljn*JWJ0Gv!Z=H1fEs5#G@NX**xu9=-bQvFp#(J=stRM~8c{soQxsg+?) zlHGq#zBpZ(eg%mSj1craSvNm|q&)YD&K&95!0Ie-E z5ZR=#r|*NPyb3PVrCp^WSw|d&)06EqZTTW>*lv4Cc-UB4{E*p$XQPV!!i zp65)j4MYYX<#YEaKA+3UksC>uo|kqzCsaqyOtVDvipE>upw-h2Zv?5NZfMh)uP<8fgywaCI^$x+`S=RxPe{@imwj-L$0=w)fv=YF{rmKWr2skQAf<*Si&wz8>D z(!^sjp5RPeWREaN+DDUYKsHWF2i}Jn>FPCh-SQ(k5MhnwSD2s{IE@$3aY$g-z z^)k2^H1J+z<*vjN4rDK{>Yg{r6;NBnn<5PJ8G;5JcrPb$_%DKjd5KlQuYj%FWE{>D zE~8(1>=ud@a6LfZejd7(pA;oK3s7ngA|;Oz#YkYM-bdSMb$)I3*K8danwHv(9_Yjo?vro0(_vPXJH$!FLwdyP1uL*%YCDJv43KM zOxPp<$-YD{WB{%w*UstEWv2w4WSNyECbbv@7)CQIyC>>IV?9ZQ%$ChKL_yrn1|a_o z(_&!40eB`K{EkNmR7|{OYg|zs>1!?F;%%S8u(|}3F3eRa+P~J&(A`q{w)@Kqe9CO2#!pw!8Ecj2+f#;thVgROj;svNb}6=*MhAJo34 z_@Qa0o08$A0%6dflrOr^H;PUF^a3*%eE?JqdUEi&2F;vwS`uaN1@_@DbgxiownVjA zsR`sa&EbfnRp@i-IScWKlBZ6nwHNEMKB<|&gVo;?25I5r8Xh!F>eJAgjgulyGOXfx zfTD%bHdI1%SZv~U5ul3qUKr%%|8JsU@rGWt%&R6ux+s41@&5rINIVt}MWC4xgHTm%KZ>76 zwj@{sz;jJT_!u*x!WNNuN^S*P$unFdnO9yzm~zKZdgdl1 zGMGp5>GSU>f#{(TT9EP$CVU*jU%3XMdMf87W+S#1n1j}UZ?%^C3X^p@_OP*8*dT!gvoA0j;E8;E@jJ{M6xEkhZmp8wW$)m5b!c`f{@u!y(RN@$n? zB&qsauvJAE=ao#_^ag_;3iFt z+v@22%S{+hm9&u`*qu8g2P` z)BCER#c2-&ABb#0JU~R#fAIf~ubb)^=PPSq2GOdNXoYF|3%PO^ee&0U;dKpx2WyW6 z(<{zjBpXt7yrMx3iDiRyRJaXnCP4fS4eMQgs9=+Voa%PoJHPw4m7GYkO(6j6WtJ_o zX{$f`0Fem9EJqDNYdu|&Y>W&<9L}epD=YP0E7M{D<(Lu$##TqXF-Q$FiNx~zm1+^f z(^#7>w8l;slJgC606cSbcsRHL{>hZArdSarT+=59$w_z4k6{X!3KN!UTLVbsOaX{Z z$6R-)WIH18&gc94XYBk8sZr=T2Uq2{f)@$hXzhY^d6bK6SMWBnv+pqiy9ES_m$drm znjFx^>+x(CdLn{@)&0|F(7u%Brpnx)QojP$O_!>PLdJV5>d*0E1`s74Tspe7TD3|% z%n&Rdy3BOxCKsUut$@?(YrD&@7|`CU96CF6c_e44IQh7Nd{vuSjunYIo3~b z8k#{8YsDC0RR6ckOFk3uk7YDX<3k?@A`X&TZp2b*G6tZ9jREdkKEBN2mCb|qxcR%4 z-dveuz6<-d$9+6$iLf7pm&1-pHMrk9gi!zbA?({IFjnpjsLJtQ|qiH$ZA0IQ!$myz6` zLjPPFzH@yT7_rbQ2fT~`nI^;}w8~bC*c|++Y}LTa98WS8M<}xMuMfNzq9qlA4Pbe@ zp0d7FU7YTrPU)?VHB!Et{cvM*EV8h4`@GyGO`ih0EcG$hOuu>}>CV?uEbnbK3$F2^ z$ePl2(p5_H|P(SzBioN0{1DE8{jXdX;!Lm~Vd}fS}AN(Abm_LQvB@MJgth zs=bz7PLOJ?FSrIdxjCM)Qpa7&V9XaSk`27-;O-)u%h+go0b2+}p7m>8v|!K!ES^xy zXZ=aO%ZcJb_)#x6`$TT7FT&*_M(#Fyn1*m5s}3yVphZvdlu^eRkDmOt;FK$FyGeW2 zIloMT!VZsY-4NXA@Io47rs(!>G6l0DOh?rku!m`y5$^cvFQv$r4p!wis`*5Ghd z_AJ|7hOHATXF%hMv;p>rPTWl&GQuQfhZm_XgvN*yqhj8y-!sI$_^%3}|NBN8AihR>J11*h?P^U;(6VcjA) zDO_t`$HsXot7xGNIcW4~g_I%6#{!40c1Z$dsD|q#XGt;%_f_B1C=W2(r5u)(lfX<0 z8_AOB)~8a*V}&a{s}x$TUjO&VN{?Z7Y#R4tj-J2Ps#4jUa58(lKZcw)e z{}pyBABr2d2W-zYkw1x60ZZZ1(Ocac;exMrX_1i0tzb{r{GOw@gxiuF)GEW=A+f?Y zv{YyjRkOtM7@8JRwXaB>=Fo!zg1+q@Sqx?95KdNH{WqtL(vh_W2+KfkxAcJf(4 z`50*)2vHz}t&2dS_vixndB~Wh^c62@JucdzU#ptdO>;!F{Ds;8lnl8sSHd?;n}IoJ z-H1CBN!&|RC62oxtPK7Xe#EU!g!V(&+P5Fs7~Q?oUtK8~iVdr5=?;oP&LkB06izSV zV@^FYzIm%b5^CU*Eo2V!zgz2$LY%LrqJDociGhYh$tx1|RDT3(MniuH849wVt#~`5R zuIRX)RIXAs)%44{M(wEXd~su|QJ|EVsUxLkz%Od9%aApQup`Mbve=XwaR=1yN0+AO zt_VJQ?*bd#ZjB6ieNdCoU(hm zcaDjALHok$l>|uLHyRs<_?n!My-?aFh5Ce+*hP-Q$j&~ecs1)9n{i+@pUURpEtZKz z>Y~SPG`S*?qSk&yVyu$XXwt@ra~?>}NgyE?p3eTap7lCPUcgRvL*i?tiG{97Q2HA7 z5=w^(;-G zW)q|rnN}44v(l~JL31&qsR!YD5&ediLFr%itpj64I3p05R*4brH z|4KYd=Zxp;%XK=8LsIVy-b)`QIfJ_%M?Q)3)ro$pP^EiFmE7G(XXpO8sb4~*Ppv07 zymE?QL$;g(f4XRt+7r69)@@v>Lr9;y^W!d7#;z#l_QCsHA4A7^q95KLoILdb^d%fZ zb@5QnbYbh@Ju2Q7v50!1Ft2pHrAqrg$S=W?Vmc7%7!S5KnEtFXJ=9Kq18zshU}QJ+ z$C@-e&tTvAL!Z;2^Xg~b6>F#VHm#NzO01mqEt0HMqplP}-b+hB~%;tWZ zC?ZEb+&bYX_XZKrLkg&{ML~C_6vC~y&cIhDJTM8b)+!3dkaHz*oU2FRAI<7-f!}s% z;jjsH^Y(GIYC~fDk%2oU1BZ*%+gBK(9hh(@ZXteYoKnl#D{{X zH#o2A_&Zw&ErwTGq1|X#AdOQl&2mJ1q)(;(75HI?S1(rkOH?noK@{pYA069DH=~3& zYC(i+N?O9EsDTnHE+A0tTw~yiV*4VwU-RQJeaw>c@I)ARnP{h?jBw5oyfk2W1|O+J z{L+1*dlR|_FhBOJ{BHYRBK~f+anZQFJ0C` z`c;=Tc9|fvG46S4J>0j--hf@SFeI35CAc)Lvy7^vkXQlTwgMQ(7k(LdYJs3Xv`oD4 z-CEaLh`_*pfJBg=N^<2PT1)qqvI%F4P87RP1E6u+f23kUhry`0r#c?|!b`0zBTlQ& zA(`)=Yma05z=p#OtEtK1+PD9sE0U!&h%7g}w+IPyydhL3+y_V?#|Sg@=9?X*umHSb z1j(@kuh4odcH+wk|2!4~dW1*BA|t7l2p-nU5TB(=Xx1~~9CZdnY&!6(NO>Z+lvF~- zUq2K7DR$B&rmp0K@3=v{Y5(PF z6$APT9gww4A4%VuyPFeDdt!)=M30SLx-aR3n8PWdj0cc>eRob^d+6DyL}cl{Q`GBQ z%K*t4o^exoa?3bscJN&F=t$Xel*aj+2=!$FLK-bRsb^k$>zaO|#XxAqnHf?}Hbo~H zG)3ru=+Atbw{^xTtH!B9lHaDR8=H4&HR{AjDG@-L(n92UOfG1{Wv)OBxW0c)nOEH@ zDrxctbo&VMv?HTlD3g8NsnS8u9GHh$+xXdmgJ<5N6NESDLz_RDiX)NXCJ~hPBt>d{ zrHtC`y1nb(q*Ko)8)b^IR9_~Na|jPvk_D%@81mnZ5(0wi_zobBHTLq|npU1e-mU)RLH7fvS~4 zFpv^a2S>Ejw=`ntd)B>I=}DUF;9Q=mSA`&%b&O)}8wSYU0s)Gbg0&B4f^kAR zmzEQGT3vB;15mar3jw0_$MZ4Gy8Z`37G8}&+j*jYmp_*kha9N%?Eg*fCj^+Zn`Ofw zoNT4J@?x+J`#|Y(^u|GuB5KD+c(-|=l zA=ickzQNtBZrH_v7D{ZiXq4%H4hP0J`lfBIWC1`Kt_Z?qBb@=N)jK+B-lnB1zQG6x zCFneJP9JBAfQy@`0zg}2&=NTAERha^XYOs`?daMq+`P8lKXdozQk6s#p_|eFrDb8o z>|!8E`-tojK9N^dq~<`)?Dt7L;)qFnnF&!T+T7%rJ2$-sAfmI8au`Q~r>5d@oap^| zMDL;~K&eQ!5ieQQf*c74T)&7xml|PWcz6quBFe2o=_%xN`D}ac)>(mlB`KnMx-iARG>wYebZ=phd(+v1wogUj(;pjMcLVwn{-yR5q20do*T z>n6hW++V1AMp6=91=LXC2nb4QK;;AVU!{KM=NGh=w4P|$JeH9v9aY!xZ#%PVX#Zc) zdZ0W_JhVjHvT;?@2j0Qhxh*(ie^t4O7WF1!6KahJYfKrgXjLAag}{QcNU}KMLcNpe zrAMuX2_J01I2%T$ihU9@=6mf${!*>25wSeYQ0El`LO>o#_$Ytq8yxPPGSbh*O4(@|4V?BQL9x)TXt=b7bxktB6Ck|7q7B_z3cAlA!HDUWqz=r`X2YL+eMsfAnbJo%e|-CU2nd$^^F-jalXq5Do0q5mm~Tt;Br; zc%Ex$LYZJZ7muG3G61ID01ed9r2Tq|;)c2G>asJO;{Nnh55@5+G00>=OWrEQ1MT27 z=^ey{97kg9G3szps%@zTriL9Ay1G*(V@f1ABq1VN4S;@sU3qYOUk;FnfEfR3AZ58V z16swR^*BkxjKm*YEB9iDH~zxbH$Qdf$3olIu?lL;53wn&yh=wIb1Z{S+r2=^;B{LX8}DUs?Deeqb=6gt;)8z|(}JVl>uk{{+_d$RRiC}c8>?+FjNWL{66 zQrl<{@x@n@j^ws)e22eu3mJ5uhIR9(9OwCuo06tqtKDa*`L&w_&sOQDLViN{5~!kJ z1MJmc({vgqAtIO2a+x2Aacn7dCT;44BoQiZ7&B}eE2g)%Yrd2EMtclyy^am4XxPWP zSR09(ucD0zh0>DVx$Ck8$P4Bw|LFZtq8=ui~W#8q5ah4d#j>IKra)^*0n z79>F`a)-~r%Tl~`)(9VYU0Hkg=rfxt8fR0&vac;euw6kWu%BVhmH5c*&W@mPLBz~Q zu@a##LTf51Vb976c{eN9A>xMUu)s^Qb5thzD(;b0Ab+3UnzW3p7%_Qo$BA_ax%*Q2 zEcYd+Hf$R&Tr)$meJ{C6RE?*-PF4dlTcXQr_EAPA|Ul4C7XTuQBxZ^5!&7aY29 z{_i@O$lXAEwQDFX!B?npo7=7Xyg|zSIQxn{Hc~;>8m~P;i7TszR!EVu+Jb1CU41BQ`;H(A{Q+yW}=wnRk#*yis`}sKDh8!@mLHC zva@6@hTVp7&)PvBZ|G7VXI?x35-XI{NcklyR}ReTDYeu?#`#-aB1%W7S!?fNAvg|1 zYo;`jU+Z})p{Ha2R{-gDlnL3q+tjVz{4zwx!=PD#_qQSgjYH{qQ@j+}pc=y;)KJ=H z==(p)kvT4ZY}!zT!RV0B;>NkuU!MSi0&hw~INK!flMN zW_LA6%r>CTkq|RCqkxxp?1vGByAG{39>c?AUQkS*Iss2dScwA}jpAvlZB8th^NTn4 z55a)5qqeEIfs0k0x$AI~f0|!|@U#@{Wk+3SK$gXvh3T)KB<<@Qh$Ew&8)rp=WaaKn zzd*c5+T6U74L7a`M%J1;=$Xwbv5Qv(MAwLO>^N|l@+bn3lJ)0=jVe*%maB01nO&~v z6*T}d2!Y8WlTA3tOLrAyI(bwR10$(%(gxRhS(*Ym2DOjrjx|4-EYD87m3LFgPTDU1 z)nJ-hMR;sx|VX@lY z5-X5}Iq?XEVC5Evym39~2QVZMS0&hfopvVD4r#TO_E5f)sxrw|`XzHNyFK{;|t5dFl*DDt7 z!+%|~0a@>T3nR(p9jl_2WFA@PMzsJ4U$KOH!SRVD)`yYHcUfQ7-##)lY~Xe-ZnlGP zf*jawMPgyJvNEi5pF447I)Yq-QbcSw>-0E6^ej&?1SbAogRLk9Nwun(PH z#%Ad!g?U*IimxFV>foYe5MOe;^02i=yepMq=RI1Z7|fU}=GU@&I#)MD?gmfIS(jcy zYyl|3UDnlUc6ml@cBMK9cSY0b%qZ>Z!2qjw(+^kiANg@?vn-=~AsU9q`ZygrcMQv; zU+7#ZxigywIN*@Rai?=HCB#-7K3H9QwvSaumIf+<36-(W-$~B zlO@oi+jF}a6S$S&d+!F_AVN>mZAxy{C6(O4xrwMEAv=#rNhKPF*ci5%o^iQvbgD#Z z*Luq1hf(TNn0J4Bb?Y7KE2R9L&aI#S&mEc8D_nY((9+6 z3FbDk4%-Rpg_w>6eH(m@{>FF)A;w9rENBXDQ50mlLl^?Js2~bu%Gl4vokhjXLIlm^ zy5#jolaCx#bSwpui-V>Yl>6UxM{{{-_l2@AZhI$kz5GkY`UHWh-;24I4tPp}0iZ0F6Iu=5^o~T_L55Xk)bPLHgc;`XCwJ)PSyr*H{qQ}Yb z$C#ZEM3d_QFnPZHDdZyLTWXl{sN3&(H~V+7agtJ9fYYsF#T+t5ZZM9<6oZB|;2$I) zkk^(z=OZ3DVU+L&lX}Y5nm2daZ~Xs}V1d83(Mv_>&&`b}F;!;D8u~J+di(HS3NSdU zYpoc6gI;Ga;?^1lwCH@>wlZ!>Hdnv{nv;$VA$V$DdwX0Jb(xY4`5QUGB5@ttMUD3~ z*e)dblipOwuN~IMqPP(4PxR{96brgR2}%o9%HA>S4P&kj`4Xaoj|A~=oBbi?&;dQ` zb4(Na*0hzm;;f;wX8>3C$=OP{sGZq!=>}f$?`Rpxu4liR(GuLL0MctYB&((`#haEc zCc(kyD2vPZ8#!kmg^t9#i@)(XOv_HI9rCS%W6xZrd2QAq_P_9X*?g=QuEwbqhGFlH#ZDC^y zginkd(CIccrwi+1&5z>v-G2VDrd<5SBZfT^x)*xj$@YX}5L0N!H9wXnP0~lo3}XH1 zNPR{H$$98`3F{@+!MYpSj|_JR&1@oUw~xV{cRoK_4pf_eBtR(OjyXF)COXg3hPnPH10k z#m%p312MqL;`0zh7IXVIRsyTxoFeI9*$<=$y$lH=rrzH*SE$1;TO=m_#+6+|z0GFV z_E?|3!THY>Pc2-SG}3MLlDk@kara5|D>%XB4e(zfbb4SC%6?cd$9b5+pnIhXP|{k* zs%1~8s9V_4sbTuE;~CyfUo$JNfS!|kfVt`*IV%E7bdEj?d4X*w4s0~-O%yrnd)hpd zdUAN#{7mP_^PD6`;^1Y~8E|>>4RUQBpdIoPf!fvZccAO1AP9 zk(19*FcN&L2l}iHM5rlm*Uzt)nwX(3TXhqef_hlICL`$B0hD{NP#%l-2N)A3J!YI_Dn8-_qVvHnx3#%QWo^NLUF+ z1~sb9Bp-~$jcq5)hS`Q~_$S+13Sq^LQ|m*ECrXkUF0G4EXDR!!&Q^YF%LjoSL&GW> z86otOfY|>B&iuUVR<<{Fe%Z^ZBlgh+RsJCG!9|z-M!jHdmUg3>Bt%st5cLo2C!1o7 z-mxVeyT%p>Aa9oh}%VCyuW{t9M!U0s7SOiyqG$e6X zUXkD+X%t3=Il@8sY z#YNa!p+NB%4~ckNr6plGWmRo|t20fngo2?IBQ2~kso=+$8L#1jX`#awGP;L$i?1K9 zyqO;xU-&-m#JO<8o-~)W#sozD8owNWa?DorC{~Ur&qapK7TO?+iJ)jcIBP;}6(xe7 zsX4OO+E%=QZ3n4e7;bGO@zQkbziZ>Pu9I#|b%3=LaW;;x@iyNv&@=_!2mZ_JGt6gL zalz_SxEE@v1ctBC8+hR$<19@G$@Z;odxi_vB3s5?X$MzUqdGTu0CcB{Dczhtk}7zo z|8)($ugA6uwNNA&4Lp6aXLCrt)$G8{62mc;*btE#jAu8S)z31V_h8 z46$*-iDD((qIkV6P4UjUJEQIa=GCYF{Yapn9UhB~aS#MD-t%SG(n;oGW#5Mwz)NmR zKyS?qcHvU++5RgQG@g1cTkXsgq%h6&+rMS~bmH|Lam^qH<_=9|LG?(Zlr5UvK;l5L zfE=qyJVu-}sRebyh&=vfA*<_3YI%WD6ONl9ZLh|N;vu(=c%%!fXL z@BZei{^m=(Z394}{Q#%5=iRyI?eK`My}aJA;EKy1xC@`3Z*`CchFh+b+|@W~N&GZ? z%UjO8pnzLs*o_X;(r+0Ea%n3nHK~QDtGKXlX~ott2=v?cQe9(RJP`D`XJl#t5d&B@ zg@|mA^w#5^30veklv==3s7&#~3AOGqNN9?b#dG9*CJdZ3L04swB`}%wEnr7`-k?kH zc}~6N%326>)YCvyN|E{9=z^TQ5#32Q@+HR5v`usIS}c#cwC+0iJ+dF_yuf(t~jZxeg8&hYXFO^p{*#3$`uJU2BAP zQc1bO#oH_eK(56ZL>HbBawBeIcB-Qul%yb?ud+vY5XVY!xm$Cn{2Y~rjKL~4tVjh32RmXa}On2H0-Kthn8MHxT1{^n?NWmXv z%$X2lCUMek5x9_bma5&8U0)@642n^!Hx~=?7;C~6>Ec4Plwn?j16UHb!okIQiA`={ z0JF&VA9~U{8Um2Zp~aosWW-@-KGFlRB~z)9qcx1j z=KU0HdQ1Ie`+ho8b$`{&NF+oUK*iH++}x&)+~|2T2MnCu7qhQ3<){!3g3_t--C1Z4 z#c0@~@R5?W84E)F%E8{ML#&qd`ntImdWH#Wmp(EPI=`j)%$4)02|K!GuR22uyDJyc z9;%7IaE5tkcaV6oWHGHnNcj`8vK?q|fGkH3F^!6PRhdGL^f~q4W#{PStwwpoDdMz2 zp|KfPL7T=OSkU0;iq-t2{j>(+MBKazE(grg6>Ml!VSm?OagD;A($1fwl4t;VQb) zwve1(N(Q>mDGxv3Mi9fzNQ}krvq88auP0CXmhoLReWtmVMYc;^G$?||()*6Iv7n8; z>^dMzV;SIjFB+UGQh0z{;#0}o^~G7sElR7wj2UAPKpZfAjYRp)Y^qkPsmwHrSHy+* zSRT`b&47!(4^F5R&mn!U>q^?M6^^q1@=VRRuqq1ck27yH4SL77cSRqcO0bcozgOsA zi)D^7DZq{{|FLO`nl;#%C}gQkQXTr{EcMqOb?UjO7u*6u$*XI{PmEvTn)!>TR1~S{ zJrKR}g)$@w zv4@>;&HHo(biI=*GRQc3d0j^)+Sa;#@c!5|Ly5OEChXMQ111!S=9r?=kE(T%^io(b zF+fDC2|88zv6=8rYR863Rn8mD8eqxy`L`F^c<$EdR;qzJZ>`@%-T`^2wh-v#)z}Q> zFL1^dyp)AZj_1-O*_2}!O)TC(ODiA5Luwz={Z&;|AiXr#FE^!a=o-2d#dmxi>}A%P z-Qh|(@(g8~J_bqkMryt{1}v@FsC>0zF#P=7ck@`($!Sm4~_9q(J>th13PMl zedvNw*TUv-CEG$|4%n^N1t_(zEo141FXIH6F)vGOzOp!X>la}WN#$wup<;KLO~r^YYrg!F_XX(TNg zCM|JoKLO=7yxnAYyg5%mK{?G89zNO74+?=yR%P5u@IwBv4Q$opY*o4|iL+9kps2^S0P+LQ0mpQrdc{fEp*bxsO?75L+}u?ZsD0qKu}XMdNXcs|E|~r zQN;BLlU7ANBUdT`A|2aTa$a$2?|!LR^H~eA&7Yh25^aEmvKkRo?S0-a(&CXxC>CHNLpa(v0J_@Qn_#(sZ>qKz_h{^ zc>q!kcv&A@lrUcJ zmffHMmk6U+V%LLei_){lLVL~b-^d$XFI?#XWr-0i{nd8ZYtRJ$KW?1BIV(&Rd3#XmgtG{==eJ^2?4TB1o|G*vq9I8$+tHq2M8OW_BfyxCf4CzZ0_-t1D!?I1-dJ9+)P6|U(^K8igbe#eoPC8k6MpR>X z+U2WSb!>AfO_T7Syx#P8P{bEDr|?#)(%QMPOHIlYEXN|}$C7F_&wRwc{5|bM^SBg_ zm70J|d5DF}B1-$`{9{uQ0#Q_<@&O(_P}LF13N7HspdXyfNz@cE33GIz6%%iJ0r9~T z+ci>}eajlFdqPHKy^-MRwh}ix18YG)gxZl?k1jaG^akI5ha`9MN}ugMJji_#omqOapi*|84YadZ9`x z@`F#T(Ub?rWzax$v5g&}-6UP5IN8=I7Z;t(q@34^PUUxW>?YXV8m8$AK%UwsbS|+@ zb_Cbvm-&y>J>wK9p)IF_>|BprMvON!u3ULMbK_pCi&7|yLPgBjd-iBtL0iC1uY+sc zx_S#*=Y~+Q?cy1^2!$F-yFrH!3I#dbtowT)w5#@UJ72tB$liGrz3Dr(AKWqiP(=5&D=FFE@q8G-&<{R|8!<|@S?g=E92%#jU)VvER zD#jI=Pck>@m<^7_-G8u9QQYwIbF(d*KCBBpANHVqt0+5t#2rJKJaJ^CAr9fmqoc z1W8=})%!uOb8*w@=&a~U?N4DBp&pd%{}|$+#~FG|$C(owHbQ|O_Fq-4Ck79jG(sK8 z8R2w+-6^omPkx z{Xk8{>dVGN<%CYRdmwr0A&|O?Q_eaVL)~c@w2+Y$iLzY zzX7(Y$}=X9p+W899SqIRI%3@P%h|tib0}4I9elm5N|a1UWW#iKDoh+tvotXs9^Cgc&1Q7#{`bi1`W1Zalz7TL z9W0ISC>Knu;%(wLXx)@Nj>3NR%p0k&D@^;TPFw{258y8V$uQig-8glt`Y#L9=%kI6 z-Z)f@DYKJXHzrnCV=&i~zAz8BeecuZ1m8-w8DN#zPeLHNFvT}CIVg+v7uc-qGiT3Q z&}pIJg%(DQb}hu&d?uMs8cJk8_+wXLBZ@%M39W<*9&S_eOOIN_K)qZz-FzSWL72@X3n>|PB3QFTi;D5^ zBmy0L$qD5Z$QBTkl*yuaYJu@jT&X8VL9YZs=u*$>BBSfK^W{v7ugRmhHeXgd$^ zvfE5h2X-AOZ|#EAzfgwrho`s`a`&~5o2jK$KFJ#N2*ql^FSI9XbXVHM8su**`ygpQ zk})5*BhE@#eM(2cSsJ^@o7+ER-}R}R&E$q?N0AakHt*rU8!anNW4z<{K^DSQ5HJMiv^0?YOZYPt-C=z449ijZ`J2r=Uc(8#d!81Q`mI>`U-ApNrFL7>TNkTU)) zo^+qT<9LxxSstuJRytKr%xl_(%W6HY0dSR=VK@XQ()a9w(=IqzBm)o{Mq#WS*+;dg zjYHKel%N%tpG>txT{I7;RbWWU~ z<)m5x)i`zPgU)omM*zul--;G`Idwasp&{giI2zGl_hc|d&&|Od10kU&{pJ6=k&C@t zKR0?)lU>idRn~7J^U?lfd^rzG9tpsObRN3XS=)p#EWrR@W2-D0(eqY>*}1irdUH+7 zOq_lhpwq|Gs9>CAXRh;BJOsUGY;)4bY~YG#&A639c3J*o-Kud!K(=yl$OrRn4#2c9 zx2b8qq({NiuDEi(GR9FY+OAj(abjMrE4?|eBsAmdzf6z9Jx3EtU4_{)aid`&xnzA^ z<<<7D2B3q83d0b>7KAsFQFxfHUhC*ux+vu-Jm4}&_zQt0_sBeYlkB%>P2fOi0*v52 zQ4YNYYsmB{J-ghceHEt5HGmIt9T#EvMHr^XX=`H;q1vJFO2It0Zm(j%N2`0}?wvxn zW+^ben@fnfoy0A6O(ElA9lArE0}L{FP)pzKBOsZiXwThEDTA&Q}xsvUS zTKj)}y?=bw<(2QdfuYR6ZF$WM;2GPRb5D;`CImBcs%@o}pq&g;TWe_z0xGdEGaxFq z6-q(lrh0A9z}Vc*NV%=jKq%r6G^P+yei%Ewm{T|ck)jY3sx%V7$N(XbU)%HkeAcs* z*mmZR@FUsT`};i4de-`^&-z7Ps)m>9CYg~XMe@57_v7FyiT~>E;+~$S=r9d>(+qi; zdqlouzH-!-?+O&;L&tR&!?dzFRhXcyMc0r*rzta+-PGr!dI6{hIShcoPmHkLTW}UfLpnxOL1+r*6lWckV{z)_f>{ zqev|s5bpjUDR#n&0cW6?%qnNMe}dQqoa_fq%|Z){#-y{PlHadVUSs5*V5~SfF3!;B zG_-|0FXxtkowzvxh;`yV*1jB;0{S5!E>~bcJa!`;xZAE&6~t&a zvlNTar*%UaFw^1Kv4UM*@6hMUpo6oJFNe^@8lXG4~{opHLK=W z{R)y3XtFLp?XNY{9R((1Ebc+^5bT|Y9jgvNK_a{KDaslRQ9>C6&w(?2mS$JQ2j8hU zxujO$(d0oS;7P#^6&u=i^rvz?K9hO-sz4wv%xauu*EJiX= zxXvhcC=Cl+XKyIM5ipMkm!<09>h+5x-b zBdxNV>$GSz7`;d(IB%bFTuyX?gmSdA)9dJa!1F*WXmd_Yt8WeqBs zuo;!6*{}mg-H~UumZTjXOMY@$R>tJRPd%hOI)!u;;nCam^W5pZu^JoN!CP)8NLC*k z7rA}fp}ZAH1}Cx)j6tMGQF27l{SW;o?+o{UW8e?dzxj=XhS;RvdS$STd=`BYR63+o z_<$XR3R96^;S8?t4I3$^s)`BJcsrw+vpXnSkfF#l%Zy|I-F|kn<`7i|1f%NB`N8$N zx2&>{*k3rR@I9@J`Bmw4QM_zrbg_KsaWESAY>gVhu79iO|*b^RU(ChM1nYmr0zmjjoQfi$m6b4G|(76U-iP+8NapM@TkB&T`8g$wE{8?#T|seW4&7DTr4}4*I{a zmMQzbYCkxCh~hRvJP9D$D1Kf8l1nykMg`~;tFf-g{ugh{@hBoqP}oEDA12yw?pHp_?!Pg6S$^t*Qe1apK+C59uCVU}BQ&gOg?fe!R z`+aUvS*t|=5mMbg)^+s0rLms8+v##Pvt4%hgppeExm2=laH7haMuW88G6p3#bp$mY zqJ?XYuma`(+6y0zcY6T%c6_8F*KvSGxu)E?9;Qld4RuFd-7B+LQ*iF|?7y#2q6TKV zXK{A-Blv=owrM*f84>c$&pIB8zn#)S8`BtBNX2VO&#)SLN|CeK*d?b&cJ0h*Wu2VE zZ5O_g1qlIRa|&Kv*|zBDyq^ME1yA8cRbZ0x8}d1HHa{8~;4R1HbYE(HE@_3WN88V6 z!WT=m(v~$9N7ePfvl}`Xd-*AvZy5>}Y;L%#?!r&y&*PRU0`UcIW%-HCRa%!^jWegK zsPu^$c{dHZP1!oKS>CZ&Gil@m5m2Dc?L!6tyAGM_N@*g%z>o^Et6Rad{Wgfgzn7UJ zcwSdQ$-)78LaWz9e9)(^6+uLTN_qR9N6p$7Dmw>DHeJCr7OKVartc5a=ZC#Hu%Q<{u&BFU=i}tPCbVh0hAKV(kh@^IGhYVqgA9JNtS^zMOhj zXORlAR0^J?H3B5$vp-BNR}7qxIpXsbkHgDOje^##veqXMat%=ik!kEJ2@+Cxd;<`= zGo?^&#}?b%D(WO9L%*Ayis$hp6hjWgIr_IqvH_NOlh|(CPFJKDXx(gJugM&GApn@C zD;=e&z(r^>Oy2ESwCGx7WiNA=tQlKUu)vM(uyz*jhlNM95qd>mt-3UR`)OiAO*V}V zR%Bx8V0Gl2vK1cy5wMw@mflrKk|TCuD+(4mdq74n(ld=*(UP!ms@f!|Ous*E)YAlm zPp1$|J{!b&RwP-dG(aiRXnpTIXe^9>*e_+4s#lSUIfNkP@AO$ky*JXUo}+jm4JT5q zl%Qc~YxI0&yk6BUh+`;g5)Y-a^Ua);t%M)Xk5QsR5X0LM1=0HlDtrDgoTezB8M~od zBbJEDh-<7P^)J$2f~qt9D9V_9g0zG|mQJW+gWNO14uTSsl`2@W3AbHNbW$FTVqI;1ZWCLA>HJFT?No1y`s$DF!=uWWgg|b%G!85@2 zxP6B$%`oPJjorSM2L7K74Fubx-s%0$07L?7j(O0p1c1i3S|MJ}srFNz$&k>F4vgWtG=Uwu{F-j&Zfywxwss zEeEHUu`DtM9|ob(_8>N?jnWG0w4GnpUO~(fCQYhkz4A1qUTj86SKUqkAyF`8t%)v` zuPuK_GWnXK!lCGga2jYYp`II5(x_()t*em?JH?BsL1`%&Risq#A+zC1iO5c)SB(sn zlSVgH`!%Cpw67^`=7K8cEyK>wmdbOq*ZyxR*%dCKJ%Xb4leP&et?w%6X#n9ute^PB zdD0j$@I18|&Tn5mwELr#J1#%k`|T#KX#Lj|RpT!FTED&Wo@a88iW5r`9X^T~1c^u? zq7osxkMyq0Jur7W;$0|0Ix-%1)6dZysJ)g;s@hvpWJu~(L2UGvo>XD_5D(z8YBT^# z3EWEi`x8cGY*5os=-eN0A1+ye6%N`$rKK&qqFOiFpsli_thw>V;l>u4!?+aEl{>tO zTn_~@I)|rAU^)O4L#D?n{E~c@02|J|hhlB=zvvv8Jp>;)3U|NB)Np#ia8Mjn8Ppyw zw%$_xoAM}VY&8SGpUU+2re@<@#`hiW!K?hHb{h@}C8(JNSQlC-0jRyoO@Gxa~M76X93b*?{egem89 z0NGASJwGdfgldtU{c3d?EelZ37+9q%>fX1S=zI>-=onq?M$J1z)oXqmhZa zxq*;QJrRs_4PQiev`Xvo9DvkgujJvM^q^yuPV?a0O@UCKz!nP?pb;$vq=4Tn%?t9& zNB?9D@tQzKLa?x5@|^c=*jbzHyeI%4frA8UWEj)J1ugo%giDgp*bOzz^v^E|5bu>? zXB>m%^O$V~R@Ik&HEoi$&M*}oPnk^iDWFr-pOZvf^RQ<#@R>Uz_6w}R-!C*#wO|vb z{v0vtygyB@bQL^dCh~u6`29qtQive!-cfLuGpZETxZD-8w7j0TvxRI;ga`k?$9-(tMsy>!-=#g+^0Bd30O(7H%vfkv_yF`dY*&}|!+I;b|^6W*n*5wJj zj*lm>kfW{dpv7+b;?mko!Notl`{mdt%b4=qbef<#?hI$8$fN3E*0!QcD>n6xIU@Xs zWEew9Musvg1PxNhHqXbZP^2+N*8o7YzW4rHQu^lQz8@o_C~?fMX>@+}7%KHwq(6WO z>fg(Jkq3E$C+V8ZJ-k21xT7teAQ?)P+O?t8B+x^Je<4c$?fV- zEy3b-8U(QPyrx0yhulcpIjn26W=2eIlI=WpF8A3D=;@|wG2z$qD~~OqQ7|^O3$R~< zkOm)F{1V5wg5v?H;q{K5NmQKoR;MqxXA24HuuLIhw0F4mVlnDx#b_K~I(h%$)Y-mJ zRu8OAElF0{Y6;NVrG9A6W3@~?t}R_|Gipl8{Tl6gk*^A$G2y(VDd6ZX~qK-I|iuV?C(7Q^VxG$5ob3_5QX4 zAX~|^?AKCLX?M-y85Q^%mu_>#&$^p!?tu)MqE;Rhq=pm16#A)S5&X}cD82W=*B!u| zu9_K|OtqS4mgcpmr2;Uy1ATh%egEzY9i`TE*fggtT38~HrEe8-_~zK{6X*cW2suEl zyT1i~=Q4LGfVt2ogpbRYy>n9BR)*L;)nNuOfb4*RP5SrMo!s z)LA#;F!`P3?}3Sj`M&Vc<$p@)?FLuiP`98;4XmZoQ_9Xhb5sEMO^9_$txIt{+U|Z} zFFnB7^JX)xXT?Pde4M?CZb5sL>!T#Gc{~47i5Tvkj=a{#0@MkGWHqJxWEvVmVa=)?EIuR%PRj%gCWZdHP0j0sRwBLch#^-Jwd7 z8AVp4)ow9*Ab9t4=`R^9cfbjwe(%GsTO$mt>;X;kk~`(pbj8K6yxjDtKJ`Zl2 z%tARj5JMPkV((VC1?82b3PZUNd5PsByPOM|@*1Pbg|pupBiRjz8H|7)CPNIG@k#UO z2;4Op+Eg=J@FpJR4{5r-%b7%}fL3_eemz${1d@hpI`v#JB*Ke>#|~*(!jv6~2`m(k zDEH=Q53no*Rt^a5-D#?qQBL6BRqyQ``A(-S>1s^w6TPC*t2O8wmaKh*?XWuSkM#rV z@kK~0D9085oB#l>ACwv%|4B;82CM9(c(bI!eh&}$KqEQlTzNc|16dJ7a_-rMq`dXm z`1AlK6t^5P3A$`P>PvB4#6dJ?qzmvp9#008bd8H(2+XY4uTa1~yyr1im@?E!?l^osl6{mzabSa^8aE3~oLuU}OeLl#(!$hUY{)ysO zQf_;5$^P2;FF@qmwWBPF{B!^?X5$Xgaj*n(U7LBHfB<8vz{Y=D{e1Y)I673dsZW4y z!$9qXadYF5yfRP2wLPS)-U*bCSE>#IgFony@#x4hkD zP@n_XjlSZh>CvidMA&~iR915C)0^CcWi!87Rp{$Djeo8$1mTpwgDVc4X5asYCN--7 zrVnHrN#80uvycg{Bh<6_Cl-WpII}_IfDOE}a`Y>nbOUrnQEcU%9;@0mkc&x}C$IpD z;iuS&QiZKAB65-V^3?fCzAh@D@gQj9LffyngeZg5E+k{pBV^>vg{k9Yqy4PvaHXmk60<5Xqoi7Nm9Zzhi4L{@|m|3F?z#atp z>lT9-)dkS3BH@afQ6G$8!q$WM&2EU}&n}h{HO0nh@AsigFggDSc}954cIH z7+O|5Pti;{Yo{n6CjutG^dLx&-o8$nPaHK#h_XN%+R1j~w#vDtG21s|rmqre&BC1? zJNP9vUXu|XpWQ!!k0eqIB~Bzj3ig)opjP3~u~>PX=DkbpjP>34dS6sRy`Q9uZV>xU z<0`0^=zGgCI|Vz10i!OViLnjExmxq1>vi*|*HW0Pg?eJ3K-<%yAByeQW>uWWK3d~m? z_=S5oCl4OF`WJ|+?Tt&K64!w`7JILBA#y+1NTdOEOtk{_ttd3fxVS{}86cPPeuf@( zTqe}$V0DZYZnoXro>#hnhHKTu{>H0HoI}&np<6Ads9-@DYMVpF0$6S$D-lj$YLwb} zYlO|kV}r`2@(@}cvG0qfmBZ!Zpb<09t%ieOkC!Dlpr;NYkhc|6^ftshxE?6ln`YK< zGml7~6tp+OvvE`1j$Ciu!FW_SD;apMmR&r_qmpF!vmBka%y&HKnA0-zTIjO z%&~e1i9lS5{xOa%{-KllEhi6=E%2HILtHu!kn+`B+<&UR9`J%s!47NNL+9ik>L_o2 zou2$1*NgvI1G35mX!Z>eI6q>Q)VWHn0t;vYjo z)4?%wvw@1Q*Wl$7EWuWb1P}r~UcG&*tR!Wl6oiRWNkwh^*qaMN^7J3Nu*<}xnK$aU zGs99-=-y{3JiwLxQu)_--WN5wT+yrY6hbdn_Hwqa@@EwAgjE(N@MlmgAGF7|07WmFMV0N$?*otI8Bj;8;Ix&G zXX+PQEWRb~lddc5mvfESD?vtCJHi0bbTSMS@E2BBV>Im1JGQc}s<#A8*ZIxvg~nMj zin4cVlK+gG5k(T=lrKuIA{@g zSlxi4Q&HQN)ak4Nym|j;xCc`0r1858PFz?((DI)Zy4bz)@CF|ngoz%L5!erFDq`v& ziQmj-+`c7QRfT%hm^`TmOEmwHaTE)H;_c53w+#=XQt)C*S*vY+nR&svk!PnuBpVcH z6YOqLtMe;{$r|B(zclfe9bF}7%dezq1&F`?fGdL!JQ)?v- z$TeB;Uq?(}|C9|OJCH|6Nh9-vdWBg& z(;>XXqt`;SUU3r~CD2aYZEB^3R%|^ure)cuH*I_F?PcSJP#2}-CB0UlOj3<* z-3!%E>@j}yJrC4DBw89$D1T8p!@^!^rXB=a8F zr0YXggVU&{@Xe#^%m)?M8yUM_V8QD=xRJ8nUL!M@t^6*I-;+|~JVr_+@UJ8vmn&&l z(k?nngs_1Tk;*UD6B4dyviB&4DKshR40PV%?vbmg10L41_3%MZgJ6sZ-WQbEp~Vt| z5@!)fVG3OET!IDH_`X3m#$G2y#e^w%HGa#5((1>JSB}l19e{jxe>p@H!Zr}4jj|J& zXa}8g9)hdaL|;!PKnsB*y-NDFVBGolGFxVZ{h@GCddR8jB2VaqU{b@t53ECRr#Jn! z-mN@IKwMdPo)|a?sqUh9Ne)4qJ|`3lMXOCt67_o;Bem;_O@@R!zCao8?#D) zI@mIDP860mP)AFyGS{_gR0M9}>55dQ!9hR}m0prP*Q@w~-kW14fq68=v)7aiC$pD5 z7ru^S-MGvSm`h zmD+Hw>AXhWo7{Kq(5R?=%&<|#8-lZ&fe-Dt^on!e@jxS8`tfK)N=qxjUQi;6W4Frr zrSb>JJOY#)kU_)AVR8Jr>#3#1G$f9LO-FZ-ib|qDVN$QL()s0 zd4Kbs*=rj6y&acAZ%`o_y+RtA`ZKdb7!TKPo7JF3nf>+6pZO&7ruOcAF%22W!JkU#WzNad_<%K+$0RL zM$yQ+u9OlAow&V0ADG=oP);hd`BRTY6ttudM3K)&G>vn5;+thsV7Z6v*>P&#NaA@9kuAKQ9*3 zEy{K=>C64o&d?BCPv8GV(WZSRfeL}KeAgi~ zngi=;JMnedPErsR{wCB9;8)Lq+zyvqr`ILEuv}Bb8Y1*($gU`&Q9f2YkZ*AH|E^Q?%>Rb^{rlZyR2*Y*A`(GDk5Y! z>lHye8;y#qxi9peIGL zh@9)NPov0y9YN%i(j=vdI&QtmQBgD$@olb6lHRD6PyKCGF&o(mgw7PSF!z12JAZf_ z09b+K0akyVY(8e8djC5G^U_`5i6^^T5{DrPfu|052`xF>cXlAR=ZST|DT ze1px8@h>db+>|3PBQqR>O6pM`%HaJHvQO|sV!#?S(^;vL2~&t?0{sZ>^Enf3wCi*v z6cYm|Z0l`FmD#|Cs{mn%^eXATTbkYaXCz5elK(81h_wATtf%8C~n*! zItu#W4QpOqOD#|iFgA@9-LUpC^_w*@N^zS?ZB(Gd$F>|%Uefb7e;SyM4^M7p0)TU9 z=|m2fnFhTX?1*}XM9ScxvRq@#tgSwa87!FnLPww}w}va*WA-bt{nEtahCc7mE0$vk z>3!6-6=_iguhzUO2UQA04o!X#E8M#izlPqot8nMFxrbG_GU%G%zjXEem4)&k%!9ID zWm7_At)aC7wXLZ{!sa%B`Nor|4{!llU~wvrLiq(LW`OGC%*%jY_us1+-5izVRmr^F z@||+rUzxnr^MZ4suH1=nW3r_3q4;bEgHwz5ue*abR;+eMwz2X`q=fX7P%YdXn5En= zA;4#SpTq3URZ7yGBm>#+N#T;<+T6p@Fd*D7VVU{HQYOi2MPLYr1GJG1L$zYvG+cwgB(Rd3pNl zMer_4l7qpQHP+>_;9sZ8smN-!S&f<{J%-J@QY!MFwJha!0h@*zd`SPM<*CNl6S6cF zX9y|e(u~`yMG%7y2?Vo+cDBY;eJ$m5QxzlWz4xgXIdJvq43dOu7)1kWG|a9JuexT` zi}`T>`;b7|fZvM<*tYq!^czL-g0@}J(bj%lt`uIktsc0~mOziL8XArYmTERW9!xAX z^MuWTCY1GR@{{7~kqsz`WXDAsnnTdMC}|u0o(++bUN&RAl)Je{zemE-n(W>cA9tgv zRNyFm22z@;?2$-z$Pj00D+bqUt5CN{_Q4k#-^-k={tlA%$@6=*41NJzRU?p~R&9df zoVXM;<6uQ;sFIat*!qsT3CRmSl_oH3(fGTncapY2BinSkkaFmA+5v=65xq}us!yQ~ zod{5$be%mlYFAh{`IC!oqV_bmJogyoA<~#-Uz08+_ehVld>1%(B~N;Eov35U8a}_f z;w&A}aIvd27CZaY_)az<5qa7?n&$Qr=5TJ$}ty`8n=m`$Cyvt#djnKH%8Wr9Te zqaww~ptX_intZvQLe;ay3-d!8bmn;Sh5HeRNhvF%oeo<_Z6Xx!48|(&!TF! zX=>3)W$@^p@zADvkaGuQL<|_*H$j(V+(S4LYX=bp!>H9xHErIchf_X-w!P{jTwQKS z-_6$Q4mIibrDzll{|)CKnJFxUTA8Yr1>7sSd4pRemc0DqSB6_|Eqtn|AbK zZ(?WA%B_0d0UE`op5MYKrtfk$QR`VegTJDq;mIVmk-@%?-=$5$u-)xb(AmdcA){ez z4V4KIcM=gYW}$;IBW`T}%*@SYo^?Rthnk$@_hL$U>?QGPRExO9ky@~-Hqbe`GQ7@J z`|C9}Pad;y2Wv`C7`z3B30F(R*f*=cf6o=pXdCrqjmRW<=ODcQ)^lLeF~9bs)n$KO z^2)|=@ju>mH(CJJSS2mDyFxHX6D*V6KKyIOYz4UYb7ipduuF+yk>_kW4;B=Ip>q_9 z4G}Z|;Yq^)MOOne(6SF$Ab}9!EoJoIOf48om;R3Bqa2D_t?n2ca7Ay(yNClc6dUxFW44oH%vV0KbK=EX~e;ur^#cpa4Cr)QwVCLwCqpi{mHS zHO&Dl?a-YI>mF2^Cm#Anr=(r>XzZzT+lA4uw}$7mc^qx2a%9Pb!O1hmS782Y>aBqd zs_b0CFT6W9PFB}SEuYd1DH^W!Cl`dNPy(ow+u;0{h!!8GaDXqMfb)CU%g?pIiUmQ0ieK40XufB*wh$v!vf1dFTmN_udfk0~ z^k9V2dc8>N=lhVs)``d69v4b$omxyP9=l-oN&2S-@!HK`YS^WUaD9eqvTr! zm?p4I%v(=MM+6MzqG9@8gb==vD}Pcu9(~~?$Q#QRu~$oYR+`pgi!c=GH#b43X_MK2 z`dLed59_&j=+fH70s#0!=+rVHQBNf#cMetcBpH9xmDR=BX1ed;>h+4blY9c&#ZP1E zYfsj>7MlzRhQcp1-N%Eeser|v8@lGpLzl-qF?;riHQyv4I;~zAq%JxYSqa&in6ah9 zJyJIs3Xxo3LD4|jFFUnu!GA@gtPvte@%K4Wn^+N%zo9YfW%qp)+JR70mIAJxw!EzD?Zu`C`CD7KD0S<(( z_&ENC zo3bm>+gmQf0F#{Nb=E-10(4Mc4JS6=*SoFlO0>6<0>zQ1*}0OEW29E$lSU)O z05n{Az|OrW)la3{JMiZ6Z|O=v5UvJx4p3Pe6zNFjo1f&r5w z7)~++p)q}K!I^HJ`AYsJ=M+WN{k86LQk~fZ1hSfpE3VFzQZ=boy<2*X0`kO6pp2n+=J zQHzvXp%O!u!Xdw|H1XI3JV>H=ls-mU^z^1hxgRa8e1_Xcn_QBnQtRI)s6GTUtOOD`9>W?Dk&>PjN^MYK5b%#FJG>_Fl@#5NqQfinntlBVynkwPgKo$S@YQ>ClVfYwz#Ka z8Fucoyo;vgAsuhrHEr`Iae=@F2s5Q0Z^xT<2&_&Nn<(L=OH|vZ-KRkf1`NDmx5OP+ z@79Kh0>N}>l7(Z5`s>Kt<=R`@(pF-@BbS$hHGzXNtwvZy9wf8cEPKq(@##i?}D{I}h>Ga=E8pfc%iLTL2a^83p4*jl~7 zcKUG2DWkW#n|yYhj^xyW)=L(UbuvUdB;09slcG>1&D_Dgs(P49p6I!e8>@FrBb$r7 zoncoy8FWRT%YWWCLYuFw-LMsTa^5Wb$({DOXat(B6R^BU|ZM;`TVo)@eK!`l)D+U;Gv2(j5^pNi5y%9Vw1&NlQy*0vdFPaDRnTZqdl&< z6G=+ah+==}5T{ZB{RJRto8!z2l$>}F;TRhRB1sh#&hU=ojzmE0eV^j&Tt>_51(tMJ z7PN?Z{OB9f*$p%AxF@(YOHRx>bs3|I4lt0iWWv^0KaBV^M z5xmy>8t7Xg!M>+)Y(rZ zmD?oMdeRE7nypZph4G#(VlOcqv^-YBh-mHKSkdNOhH&F73K-^&;)@p>&nEFAfr5Xr z8K_7E<7RR~`Lr<{b3$m?UO_g$h1epqi)T?cmTB|zm0jfTaR&6WeJ^u&T+B$iqX#{S zP_Jmf`#&%HV3Ulw(nraP3WrVJ22ux$08R~1G3L#b(W6vz92QSi-@F~23?R+$5(E$k9hx4{v$liMwaVs>D2>*W@l8N=X;7%8z&5)02Az!vP0U(IfyYxXS`JP9b(Hd#TGKC8F#I0Bo3J z+%bbS2^9F>P7M>>zJ1OdO;u|_d)uTDlWD02>>@XI2;H!?iQk`ifPmkXaV30(>pB<- zSesf6QOyK)E4g^Kj*1VI&s-^beEUr?G$vPdhxF#E&=zJKs5i{*9G&N+gaLF-3i+L2 z0p!dPG+c+~qCv}w*_?{jVEO`X5M_7IEF*P6-iW;ye}lJ#-& zZT-tq*!x6m#6(NG1vTUvt~pP2frBZx1Alak+DvJ|p${~4oD#V!Icj)G!4?~`g7%h< zOCbv}`zGIQRqM*yCjMgafYGkczsOG5PPrsPUKF5!k^`q69NkPkH`+kk>i8laJum?^ zUe7~(ade2zVHljxI~xXq-)xvKg+qTS8Vb*l%5B6fqQ;aK5%x*nk#9q7_$JsTz6{-2cbn$ z$tXkuR*Rqjr6MY~iyN(EBWD5R>k2{3IMKq~%8_!QQhgSXNo%XuQKoNfGtQ8;KY%qZc0l&QfJsUaBPW_4E-w zF3ud*VdpAYT$%mvJheMT+coY&-poMb9@&db*d_7)y^IpKNZlv+d+k=iiT4|GVUKFL ztMC7coOh+XbB)!x;CEi}pL{Q|A(7dY;H=Kbyg6RpP$^ICBjO+z{f7SNvvzB~yeDX6 zSrV0o|M_1fAIm8MBBe+d6Fo`2Tla1FPDpJ=gM-PCBqh)DBVv__>ViI4vA^mm2}JIH zO8QAsyE3Fhw3xu-sXOjjeo{J75xdqM-|kYUJ^D^Ltp`Ocu0QhfOkqKVtZi#JbEfim zAIG2}!xR5TB92D<$eU7{&MV{pW@3d4Yq-!Dabr@bp?Czj{q7E_Y#%bypw`)KRE}nh z$@u~=1(8pzhTky?Y33Be!EwX5k=jgLD7Wb0^X*~V9GqKsw70ws>= ze})np+VRdGf4~nIxIw}%#YUZh%)AZ?TOCW_6~j!aAgdf^jl4m59N!r|GwNDVmNyJ= zl%tQON=4azU>*A48Zkp$g5L(63RPQ0*zu=#l`ck|=ucdEc#?^qdO~`L|BRFnH~><( zQ>YpKocr8=-{8>U5<_+b%H4{0C%rc#PC^ia{D%S>eJd0!Ez-vt>R^CIu|$?a#+d|) zmCEodicr+5C~zj|kDnr>m!Ff14F7ySbAN;Mj=b&}*}WGI-nn;z(L!{6gH?UEvyQj= zCN(Zn@nxq=PIPR9V18l&^f11rCbi@qcJsQJmn(ao??$XLMM$N1F9yQzJhJO0n@VZ| z--v8P!BZFSJ8KYz5M74vFtVrO7A4Ufe#2YDiK-eA>JlgWCt^D}@1vx-N<_i+wOQtD6Ua`pxK3LQ#In|#Ml4qS4l{_X5|UuEL9$mtP01lH|G?^jfA>A4Rs8w4VR zVKRjTr$X1cgqkUQNf9$QpFn$@Ieq_JZVkgSf_GQGyWHxd2j`N@L~5;4R_jtPSYtIN zNe@52=aLmbIZ?&VU!NA{a+%Jv%B~9 zeg@rTKO_CLWxz|6b%na4FKn-P2CL7Ap$T6&@K3V|r-P}}oTO#`vz}AT@tdf}DYwVe zm$5K{SWZHOk*x^7=}+*Wut!0iM=xI3dk*X5weV1YoTM)p@s?NmM6(PFdj9bJwv$2$ zO-P!kPU>N7J+~2(F11*)|_UA9gW3 zmxjOhDLQ&`4cFYYVEjWK7P@2F)Ax$?k3`4?x-~Si@88@3|0Y*Jbq4=QuE6^j33bm= z@HF|(Khpa}DhV4LPCZ&j5pDG-Tj{9C;3aru&M{YUB(jVGK2XunF8fOM94?$JP40^_ zog+PpGj&`6fm_29>-}{1%{@JYyoixSj~#4KQq`(^7CR^1*p$CgQW%>pRQJwV$hB$h zLTcs5&NVgzX<|}Uk-?>!hCL1tp!{Yzm`f#f2kg$ydaZZlI#G74kX%GOZJBT_gnQt) zCU0ab5|KyMjJqmb1djX)+ZWve6QS6pvn{C66@C-Pcf_pOzn3x0)KM#c&qcHdh!_#V z&e@g0H%*`{!g6x{%u%s>!Gp<59_*{azR15STT;}TLD8)@%%l(%x`nPOtzL_QWR^r= zHX2#J6z3;&>W#rhC_1aw%(f75zGsa}>?NDN0m{&%$~@)(9FLy*0#+$sd5 zeG~T3myPxfKQ-1yBKP->$d%07p|o8C&L|>F%0^`G&{0(<8qRfpAvU{fSXW8AC?s_k z9<2GpJ`-%FDoR(6T#4%3+moF>jh00AH)hveQa-3z%3#lUjAY zTO7&4k%y_CST;CgDZ=qy$PO-_z(LCgMD=?qIJBO|U4CG8@JsS;-pi;KLH!{?QtDD% zEFcVoE_vn2weS3}WS+{uoOwy_$wAKMFCEoz_B@foKLX6D>|4hTcdaApS z&wH+;;1x03f_;=DaVCE`TsL{#<6I-%>B!WR0;y9^eyP~1ryoV`qVFE%T+mWCu1GFT zes46t8zpEmN&3K^bmI1BM@U%!jfI<%NSA8W?=>=kCVk0_vqCqAl9i@T>UIp5J5Sij zspRWU7wz35UC0+05$r>)X0uW#?+|2^)d=?Sn$5v9+R$-tqe zhaK>@upb|RQU3D1Kg7`5QPgLCCHn+gb46x6sULoZ^qB(7Oe0o9H>spCElO}P;cyMO zN5d>!!{=3m77_a93)LyPA$nko*~UDkE0wTE<&IRm$6)N~ABm6({dGA8e96z!_Si0I zNF#Osd$?oq5123k_X9pzfIPj8Z;*<))!2`6#g zeEyY;OQGs^(9K>Pq?ffa&$X9^WScpO8~?4cUa?SV`cO+iqO-i2l>+y7KO#?uvv`cf zrK?=AVHyG1KGVs$-1HUI_3W?Dj29mUDNV{Q4&lub+DAzl7kNXb7|Ug5C^~zSMyeV; zDNAGO6{kyH*$}!$Jbx^llZUVOrKbR>f+OTRl0lV&JAA+E>o$}%bh`n6J_-s($jO;oKcU+bhJu;7$N=yqa zA`+843L6_EYm{nyLIQ$dY#TtG=}aYdc9kUcdH`^-k5HePF(X54(!nWd836zXa?~A< zbWC>HT6q(U$u$qWp-#WkQ42yH?{cKI3NW9z$cU55pK&S8IItDG!U1-=>+n)VNGwj4 zS#im%K{!!HDjK$)44SNX?4}TNqK3kFONWmy`ORO<&mKFhMnRkUN^!hjNMd$rZ}8?waheuIFEL7IxE;GbyXvCWqw$lQgA@U6`DEv<&1; zxQu;j(I5kPVjP+QH;fN=qe(@# zxF;t=3db~@Sfy)ny-1DiWCV(+8dNTx!OFgHtSeaz_VED`4!D#X8%|t_&q-!$v?)|V zC?84e`gm^D$wp<%@Sk>k0x|cK#tb`nPs#pGJQA)cOHn68G%GN`M2O>%Fe*eh2|VaM zc@3yu*%)-Yk}C{%#?43c7UjN24q@c5Ns2{=UBMtc72okbCCB#mesY(nTk$sF6n=&K zC{;O==rc~QLc397&n)7{RBJ_^qsW7+%GE?>{>e27C7WXdI0jE~zcJUcwfQd!+o*;2rT9`gKk8u>7JT7h^e3 z)X$3YeN2U+h!eVr89~~sjFg`{KBwNRaYLyyh<|Zk8Gg~8)|+4VfAlw>m$%GPtYyiT z57va3BpXwxE-yPx@OXRg_uA9cUt49z=ef?lO=)ToNatJKBT1jc(}EU8y@(P}5&==+ zg1ZNDSoTKIbNv>+?*qXP$RiKa2REheSeZCNuXB@>(s3et5&cJRn&GfQQm$lCt0Y0K zhf-1vW+o2`z~cfZ9xxCeDQKdg931gAet(y<3ZaeNm`~y(Kk@gI^nK;mBrRY+rrDcd zPx@>5(e~pMez8jB(b7Nk4e~ek<@UxAI*-o++<%lVsgpr+gQZwIic3v+j2P%9dD7>G ztvCEG6}BSbi6Z9fPiRgnl1|{t*8X}vksV9}$ZmadlYpW=&}c_MRHWv!+3NOVr|<{W zJM6r(ZFS+GM|2!SK*TE8!H)IV(MCD@O%iP4P18ZCr_w-QE^lT{h{YO8N6&s|(UD(K zI&H-PeOq}M)LELZ&e80VxgH3Sj0{ZH5x>Tcdea6gtq^CwF3Q*9+l6%o0a3GNT`Ab$ z0XQ}ma>gRnK5E~sh@wAcesu%?9wOC?nbZgJ0de$Nvr>loEVrUm3NZ+0U``NmzTxBw zD^x4r&=<-z5fp$NbF|7SbK(&JpUSO>hyfoc7hf+4C%T!pk1{CmoV(*$q=rx@?k$Wf zC?IEwTCPX8B5=Gd@6_{$VM6qu_$s6?CDrqDHrt!5#^!8o~Q>J(SedUgi%dI926wNAty8PCPm8577ugz8cL-zRF zvZG?F3j`X!bTAUfh=nZVqd;ULnE)OJ9Vvk*$bt`ONoP)8tK={;?8eafKF}c>9Kj9W<+Qufv!$La_MjrTkh)Tb;}I% z0h7jPLS_4T|2SK*DOUNlAEEqg!R@Kz=;RNuFL}YYo?;U5b(=@$I}y zzF+dwf8XEIo(#>bCTkuWkm!utK@Co$?=#DNPm$mB_}jeim|rXCN}R=x(u;X@_aFWZ zs^~i@y~sP*_iAl!Y?PK56gpm84L%8~M-2fS;+6W6{?Am$zHnrER}HF0a@^+y?p7}% zYg!7OPld4d*jvAG5(rFc5+hdRJ7hOH6qWHD5qFIj!_0Nfw!z|KfHUMhq`tw{nLD#^k&>zy7DSJHkb|WyKcp!5;HN2G#kTKqc6Pc(r*RgcNu$R#~`tPIa^YqsqTUK(w_^7RHf`a+)eLq%#$DVe_sCU*x5T;foiAGo$Bi%IKRbSS%TlN+)t#(dP%B z#{+MQzcg_(j@+_{25#&A;PFv(TbkHvbgO%!QDq8&e3DZ%Ss)$Ob#vF~N1()H-|+k# zHBE*;YKZDkcy_~u?1>-lxO2^21WXk}_qC=A3B2%0CM~y|>ir~)8?L7>g$CMG zh#G+5w5Z{z_BBTuNue!Q=}ToQs!87tP8?sM^h6`nfwY?*9d+dZQUbcB?zt~w@AC21 zj$&?S*OTE~OvoNSFlr=I51V9Mxa2F3aVw{7D>z34ZRW44sEliGyLQjDfOzT&jvvik zGdC?ae;8x!izMw@SC)&O8PqjTZ*nloy}k>&{#zI!4sG`i-ziFSWQu$UoAR)#pmofn z3L89p?Vii!XAtmKl8Ryt5WWjs^06WzK9IS|wXob0wxA#KV~bo4?yBvP3`GiErxY-{ ztto@t)_F7-(V*@9UW&e_zCLiK`HFAHR=vs*XcQreWN}cpj?LDU@|M7Cy*Ga)^Jr>O z4e)iR=EF8@uhtj|yNN8lko7a2gq1d&IVPJcl!u*$g~C+x1X0rj3fm-ti8}*WPX2}* zxBSdgS?EyFBz*XGma4=_a93D5smzgr8c4f!=t!j_S1lxSAUL3uxnZ_REW20aePbP_Lr-Sc`|~g`hZfGz6;3sOQVSV>!x?& zp;0SRE&8$&(oId@LA~|m+TTS697_o!3&eQ8?e8=D>4XOUzo_;1d=v0R-k7(QTtK|g zoAbrk&I@EBb!zde#zs_v%M(_u7jF!X!u3_q-(=I32sN2*=E)`A zxMD}qtSlzH<#={WfBr&AmXW;BoTTil;2dcsN{SzqmR!dL)0x{dGy5fGkrYJ)01X#` zS*Qe!z|$kEsVY52DDZ%(!7`FPnakb~wcR+b#7qA#B@vq(Y=Cck;;2*{s77IMFDAd& zim}jg)%kMUu+-VRPIsh|)z^AIv5qpORm=aWVtw1FLFcZavx~0S_H(PH+H}83`@yia zbS!UZ-?Q2NDGqDpW6QXdBkgE4$9^HiVGp`0XjJsW>}-LXZ-!3ehLQPvaN=(0B(r%= zQ)a4w2L}*3zq_%g2V^314fJ2cKpKGUOfC_1KQv$Cd;TX@m4 zjx{g+5GXfd@O}zoIP%tw4F~UB-2vzIg{8AzHe~369z{79%HanE0t^ZZnp2c>Xg>y?XOb!C^)xJD&nd7K%y)Il{^U z_>unzPZ=lx-_fAH9xJPu z2b@eR=khyVXow>_#4d@yi!2w?0mSZx*%KDyR8VJE<>SZ3ds0CnA9!v#Ksd0f=fT&y z0c@+;04^@6B&kxz>Ddv|MwY&yVRo+w+AvvLAnk95Lx_6AgTx?`{%1#v@ubQm)?>|> znXQhSto#K1L=PT9Zp`v3_D9}*@n+Su^NPX2sCsgyQHU-feVykRzRGk z)G0l6(nrN+a-lsA$Uq@znrUIJgDO#>VBV_e0mR6X?7}c<&W%bJ?%y=3VcB~MQUzjc zO`VS>vHiVYuH<{4?=D7`)AlvD@<&}>-}egPEvoI67NH=2kzyI>q9{{9&E_5P-|@=* z13OW9P8_1k9f`ET`i!vS)o1%QY(=(_o< z()Euxw6;|yRcx{@du)Y{F)6N8p^oNMdz|cQRJ2puc(Q!*z0^d$NtqyUAK04Acpn`# z@d3M7E~r(IBo#^O%Ik5JbIrDCz8jI!MA*wRm z?aoQT-Uimk-3RCUyG?A|+TIVG%azRC(D6fU+(5(=)4pjD(4;+4EVSYPRf`M8VkXNvi*=v+@^B%LKjC}68*?U zYQ@kM(FAcHaQCIDFAc@aCC@#E36t`G%%|?WHX3f%m9DG0uUS1|29@`gv_T>Or~B9! zTT3;FQh$pv+4dG@gWtZr`m`Do>xA@iAv<~~!oNX@0}1>#BR(f)*rFIt)A5mFCnxSC zqpje2#akR;%`5LaItn^zCnp)bPV{59Qz$jk6^7GLWB?!=ety>Jcdk-?s{klr0X0@| zywx(PZk`0M8IE)8-5>3H_CrH!57*8`gZq~9a6uPwaAmoCqsWAUHaq?x1U8Ae2Fnmm zzt+$)PjWj!WcA=^v|^ovh6 z8&a*wKjn4orQ|c65Mu;LCqnrwoW;bZ{VT{(FXFy1%7fr_5#lVh=WX67w4?WCRW2Z- zg@BWRX@n}JfxWl(2pu~lLW4wm@h75F4fj}xJTBpZAX256tegzrWSaiNFF859czg-$ zRhOLyNyX|Y$YqCx{*@kC6O~(;^7i7+-k%0}DSY@%s=EG7VdiK>PZI{23dnwlC{i=X z%8`swRl}S53+TjUCklj_!l8dZp0Hv;T}J{iX$<(&G8e4R-c6tB(4m-v^8;$Rk;Cb| z0^0EE0{Bfg7O^iv&9$cY$CF=`j$kz<(F*(NoMLlJzDA%F%DQ*x1g)_W1 zj0hVuDvYwW2+N~teBa>y#?Ip&?^|lUFSKGH7;SDDTmJCi_xTD z=;4)ek~jVD`&(?)Fp&;HTiI~r@3#I{x&_<~?GH%Ex^@o`Dc~4*$J;BI9N!Gn^fJ4zvPGQ(z%74x_=m$m05P&gyxG1$@AP;IQHK2*Di zlm+w|@4h9bCa~#eUhFX0@_}1qec&Q;h351RQ$ewb*zW$_6r0fZ9zs>zAtKMD=Uwc^ zSVHkteT>($S!#&K5p&zLX_WI|B%R|5y|+Z#$(JO6tdgz}>d`YL;R-LSC5oG7bCjqJ zewU^SQ`}jyBhfxwR74Gkyp zZ(nP>t>pNtOFu>v19AFCcjXGMU-rt)J2PJ>b**$0Y!{rg<1*8m7VEy&P&$*m8Z zJMEgA<2~D6Idn(ImgBvbe~Px+Pm>yVif4!Cq%Yo=uVU(e0MftCf2;hRkA9)Ia@$Q$ zWm}HK0H|p-7Lw`jW&YP!j^=iZ)to#5nuasoxgXrPX~mwEul^`lcw&`z zaD47=%Y>lqijVfxzS%BUZezj4&gU=)yJ~TJdG$@t{@~-yTff@abKfodz_+${M#a;2 zJoD`Scke&^cR%lX<>RM6{L6c`L7Z`xC0$W5@7T9e94dF0Z<_Tld$RK~KQuS(WbO5x zd)uyi{R2}X`v7V9|L*C1;imHDEmbUeTFJ3f=4zv#+&GdxE~{Sj|$Mlp zyuJ49Yi-m2m3Oo)tvP^GOkO&&`qq}^_1sd?-%6#!78h^+FXR91bnhqVS92aq{#gv# z;g@s8GssMuI(ALT;tNkIBe^P4+oFV<|C1={s9&o3l6dfM3Am7XGYzrNDl=A+>f(>BHVod-m;&<>Jrdc1cQ(wLR(F?1G z@l9$|M}O*{f6_i^WZtFRy=6sn)$x-0%$7f!T3GeNhIudFsz?3M!_P>Lm1Igk@vo=) zjnggJk?Vw@NcK3MyXI+#dNLHh^6r1{9^W`>{L8QY0s?4PzBrD&`Jc~6KmZt0WNgzk zqa%GsxSutBC8b4N|%;i%2Yr0oAtyi z_cu6|tH0cqha+H`UM~g;H|IO_W{q^67Hr^qf+{5X)m0mQn}?7v6LEH75?J zopSZ^U22GHD&s6|(i~b|-&65=g(4b+FB}8KIF%jmeR~l;iA=BE3k*qT1;BLzhrbaK z!UXVfHYJj#^)jA{l}M$XpDif2p<8X&TiKbc+e*yxY?N_Z9n z*AKrdnq_}ZktXVPZu-fL-kPg;5q21ze&V9V z7LVlRTjKK1@*!&9u_$xt+ zzFn{$zP0n@xCyZ+-$yQk1!LLwr_UT$Q?cT5*2Ktbp zitH4PcDp8B7?Jx^zB1#ltL8oT7O+RD9WZ_^Zz+Dvt&|ym;_O$CtYKN=}1HYkrH3liwB7ld^4Rb>si>z0BfOpz|elQ#v2%*tqWx7TmPu8-IRBd+OB$ zmmoklu@$d+(S>;>fAzwFA7}o>9f^V%8AU=1pOISEa?_ceEOQzC6)0t)hLQr9qRuxYRMZd)+k|rM2JGHk%>>5B=jpB}|4=tQy#Iv^PVns- iTRnKz)=sWh)R{?Pdc6!hu7R1lb^GbJR*!t|(EkB}olPMC literal 0 HcmV?d00001 diff --git a/tests/ut/data/dataset/testAlbum/imagefolder/apple_expect_flipped_vertical.jpg b/tests/ut/data/dataset/testAlbum/imagefolder/apple_expect_flipped_vertical.jpg new file mode 100644 index 0000000000000000000000000000000000000000..cc0f26ac922130a835bbe9f364ef9fa4f4d6e845 GIT binary patch literal 851411 zcmbTf30PCt7B+ksoTx2TvDQ{iRTRWpsvtNZ2djvvG)z{jt=u{>6iclGMj(gQTL-v; z0)jv#6i`5`)Olb?ty%=cfZ&LbDv$^Wkpu{tPQJDGK0$l$_x#WQ|MmebWH@K#5M@VBgnqyIrKLY`_q$iiZf)sVqM@I(GTeKY<+jTk!gsr4Tg zW|JuM5oQ)6%#1fF9$e=je0gxO?AENBMm-WYSifSw9~Bb?D^J&R!qwU;E0|Y5fmB4YPW0%==?Mm@?JQewu^x zY!}x#bLTBu?775isrRygz?EODTK(mk4I4LY4h{(o+qQkj&d6PY-FyE1^WJ^?55y%U zryNO5JDQ%6lbd(^M1Dcx$#dr~T)cGo%GGOs-ne<|ukza!mG|oE?>~6>=<$PU!z6$Wky+$+k)@^*ZUd)_hpXuU-=m=H@Y=^XaDbR?5F=-yjMO7TXiZfs^8v5_ZM(;t zj8w2H+cJZh-tW|3lWa%|`eeJ2N^RO4D1Vg`;wEh_fA-nJmMWP?f8RG5RKjnbx&r-~ zQ*A5TZ&CP?jMT@`W#ZFb{sLYnn}WYl0weY9YWC~jM7MsCa6gyezq!!UXd!ws=DZBu$0^cct&PdgtFjD2C zr>xS6+J*cFzxi4Uy>!g+QGA(^8cctXl&hp04E7YhwUE7Ya^U)ZWB#^57jxtj4O1dk zS5vO*HD#M=t1rco?5PZgIr&Of`<*_-j9t&{^1r2%h?Vv9M#k>0N>aDQ$5G)fuksi{ zF5~vbUE+A>Bv#QCUGHn5zom+nIKHZ0zOtQ98xkMH+QzFHMwXL{-#ly?l_)V%8+vmk z2KBM?u$MQkI}uyn z;aBT1QlO(BPL-6;9w#kT>cS0S{K{GK>B|jiJKf5fE~NOq`M_%z5A`)us}S|%L^O%cZEsfinz?ozS6)*+NaW1*adgu!Jvp)F-8@HGL}Rrqrm3txra$_q zkP)U7x3WqjC3O~UGw>_4HD*R?Mv3BHW~q^qAJ(_k8`_MN+djF+@B-PM>i9}&!o25s zIkA)XT2Jp+^#>+fyG)zhCX4LJiKFcIsn_-RId-w~Y7a{zRie}hM8|Dn6uL+-2=SkB zlV7g2 z4!LWT#+{!cyqY~^U?liK9BPpF1+1J87$I z_rJ)A8wS7p>8h=bpXcD~i>0M~$Z+9HcoQ65uJQz|(Xkdyen;F6WD;O$#K~i`v+E_s z!ri}l92{rU7qMP5-eFORMsLqKm?Ai;$T42DrLU>ETbwAWW`_%BC)&o6t9LU}ee@dQ z7N`Gfa!%az3EpPwiI?PF=Fw0+eh_&mchgt@AfAPb8hIyM4slKpyHf6^>ti`*gu)Vh zp~EeDZ~)fEWOV2x6~Ze&)y${I)o(`+LFX?)9F+mL$7@_9-_e(C+UW~;vvRIc_L=HS z;>~T6ru>(DO*`$diSOnC#tcT~X|wI7ML+YT7o3rf^PCq&wz0|BbiAA=1UBu|>(=L? zUw`ojUP==8z^hm#kRS%8G~&jPBYJ_}VJ=+i7pB07mS1}O!)LHn;^T09A?}*m!ANCrA>I$p5ZQl&_;1!=LoL!t!S70^2M|QQ%B08D@ZA>S z@KESb${FeU35Teea3gGk9T?w-Mb3pt~R zwMzZ)+oSm^=E6C;xYbDYwmU=~HBwE^)Z?sa%z=IS!N2*yeFUdD>U)E9&yyvF^33z1 zE?KNKt#;)Y7Df(o_At0y=jU^y^S*A< z6uh7chijB9zgK7on`b{gX>pd9qNLMEDa)(=qSquUj8q{6=VkC-l4IvzReP!FdcEkh zS;a?Sc`jB7_HN=bo!Bag9K#}4Ma;=ttGMl?68vP<<4PgyT_gv;in#8<{o)d-y82zn ztI|mLNQ$+ho9hfk_u{6458F`j(PW*M;nQSRlON7#+8Kw4Bi4G>TT_4J_aWFLp+}J{ ztrq8M58RR{B^@J=(9iTE>+H1_w!CQfmq}l)S6(V;KGVL;#nbQr!q=%^hE3g5omnAG zj1GvttSafuZ~NTjwD`I;j08?l%*r2@4y9ZB^7bv8x@GtBX2mE`ZLm|oIN8B-PkrJoSi)e2Zgh8o~BExzU9@zT-N$p9wDAKP>3zT z?&!X%#tG44;e|4?88z4}Cqq?fkqSa{j(vpBjn5f1pm{SX*XiU5uS zIS&$P!C+wP3XnI&LC%A$^LOQNR`U(~w7TAWKlm9AbU7%TDZKDT-~i6=z!443e5QVv zep&;M-7Ts-h72%!=u5ej{neG7Q|K_~UpiX5uc6-$?^q353{?yy0d=FNkgcpSL#q@B zN2O*ETr3Lr#LH3fyBWzuP2E((%d&{icl#}-OZ&3T_lqOdY#8z3XS~Vv+UM-~7YWjX z-IA<$>Y$D*Mk+GHp5Mq6HXFJ^T*eNM7^A-+dA8KS^=C(nt88^_!z_jBB9p_erpe-`1kgZ}m6BH&E-$hUnJa*{0qG7U$I!eauKDO#g1z zQRUK~-fyzcjlUyFla3E#P~w`KfPP|_*0jM$?W(bq zB2OBYpV}0WE)%|gYR9@g-|b3l%8E>GwLTLnQnqHX;P-JTM5)r?$tC4Vx5je1@kA}M z+9qk^2?e8RVdhaTIKP;qa{eCY1s6C&0hbD~JH>a`-Z_jspIJssGTXe%`g}||%5U8? zOpm0u&`ob>ku>`~c-oC}KXS#`*RTIhV)IZN!?Iykg{$(HBsw6<;Yw@c=aamC!VK1&es!1w$D4+DQ^Ze;m z=L@4NVtA16F}(%m4s^dd-AFxvjJ|^{Z-fF#Z%?(2N&O{Q!CO=G1>Ks;wC3yKly%(S zZPjvk{WqQF}>rQH2#`mC&`N1+2xrKMIZET-+!#WG05*VsGD_v0{2k_`)y6PwOn@by)=cd${S%3oEgF)3}L z=vo^#Sn4A_=;iU=JVRF)zT-q}EF`_owp4wi(wA0WuG_L)F7Gqj!hNG#*NF>HAGE z3>>51@CkwLC;^BabtkAvs4xVJmj%!blT?%WH)iwnXCV=`)#M~ik7yLcXl1dsBn5Jk zq#3CdGL?H-J;~n7c$yBauZ8McBjujUYER_(SBLH&&DT}O?Iej@LDQTQ@s-C`>rSoT z^Gh(7N2lH1r)ySnmg8GJ`Ih66D_OfA&UvpSlXsCRhokz{Wbd6W6?pTqjKBp<2+h#J-(ffy;rRDz-sc*SfMNF?!(c=QT zjg*fqgp3#2R*>Dc1-17Zsi?TKB*V(Na`3wxm2L8b%9+cFM>CVRL|3JPL1pWu^Hp|g zbfnxP(Mx`rn;O*f)IRayj(MRYSF_h!n++xKf(M)9?8_vde#29*Jnz5!gSFLipQ+Mt z-*HLrh3Xp$KiRqA&c4l0d(;KD*ojFI(+$B}8XahwCz#n^JA6X8`{(+o=X<_UKC#UT zR{Wu&AGi7J?acZ8^4Y(H67|{zj8!nsxD*aN4QzLLi21UFvHFJzK38TMsdsjR*AHCx z-JR~b*`squME0dEdCPDIocF}09rdDa6%D4A5Ai1$&gXF)^8-_c*zPH6lI1zcrcHj# zo-f-S_ydMF3=p&zc(pD(Yx7ArQ#FbCV5^bx2KZ$v^MSAjg6<;KiRC)URb`DCR5@wY zZIWeEy`RIv%+-hRVM7b$hA#V=Er*hqBwd$vUS~UJh3b#%(j&gA7b&IHorWhU6Dx3; zlIoJGny`6@LQt+YQh#QqFLn=G)7$_R{hHKJ{b|=bkJEapz217de9RH~1}RkUv#-Z> zbke~R-CS^we*BEZY3!HH-Pwa~G%S~W^rN0_61uRz)PK*mJ5DbeA38DNeSPVx$%5}S z$L8t$Yb~kS#NKqn(y`PlnY+- zWlHt#sCz!_!7*uDlu>oh4{k`S61WO=cka^5z9hLh^+{Wa$f@!xh~hDJB7KxQUm;6= z`MsuT^IBeG^(Z&mhEE+htpz@oPcnR^6lp$GtiG7M8eF&QOg4n8VI5`=Q54)xWXw2k7% zAYD5tcOMzTmXX97Y~S$AV?@khz+SGxU57Sp3{q8y71#cOnwcJ*i%W-Wi`yzU@i}q2 zENTN*${y zQTgvNq8id=@CEAR^WD{6$a=U4O=@5Q5ndecZOUX3>g6B!{dItZCeT(?(y9A#TzO#u zr2=}9QJ+svtMqJCWF+M6H5*s(E1p#Ha=ZX<{SDjST! z;?%*3tnbhl2&mwW8#w*4ddlw&0M!g|zK$lyITj6$7K;o)m)GETje->`VP#?SsNkat zHfI4OcMgJoDcR;O{WQ0m{N52S_1l))ut8EE=|RfNlioO4QF((kTfw<5v}VNU0b_!! zJ0MbgOfK+h%ZS8TGlnee$@(J941Ga9jZvh1aD{$2cLs0*LrU2exNe488wzIWnB~w}dX_=i$E&vq992=KT-gM6k%A0iBj}C5pL&vt8z=oJ@h4y3fnUh-Qe_Vb z-^U7s7oct?@CZC_l0hof6qssA`<$RggCrx4;_4_ckBEL+?8$q{3{jh2&7LCJOHOKV zs^jwjBW0Ec7|SGsT&fAlw%90DRrTf^Bqs^hAfhn}QfDv(%4@OoT6}d!`7!_|a02XM zNncB=?vN~-C}hhtjgjSyGZ}UDFsHD{Lmc(Iz?d&P~71wu$|{K^~^t_Ia>NYMl>qQAKi1SACq$Cz?=I43CA%sbg*X8Wl?T%P!6 zBEn(vQ0>HBuD7R+84n$DjH#1v=zmu$ms;SHMf_L6i_@{m`0y1Fls@;&_Xxjx{e=vg z+y%L#FP?Ur6_XOtb2K1JV4{btIEWGD7Ncv>EnDImnE)wArcgG1U?FO}HCDQbV6s7{ zZ345F07T*_eg*W1X|YIdhBge0s}=)<_)@I<=smg=uIq-4 zOlcf|AVr?oLHZFdxgFk+07LcL1EwC%iLX|wIy0ykxO8mx7%cKKQjS+T+X&dn!zDKM z4|MCkOsVby&H`$=N)1YHjOa{Dx_xhXYhF~0il067S*(|1MrU!uXcK6--Rw;R^IBL# zPCi%W0Wu1gg}(?8;SKZ)x})){Pxerrx~nB=P}Xp;0Qjb1I%zM&kviSr&^&p$2*`+nFm zyi$QX07`Z7fhR5}-vJ&AxxBeI*wuos;R-(2Es*P91?dz<>aol*s&h1J``JE-eE^poIWJOdjs=+3{=bpOQI-ET z^g@+sew;hbKnh{@h+8Yun*I0RIFCSHmiuxjMem8waEc521kWMg32_!Sk2^9i%x{e* zhYVe?$B5csPZW+~tvI)A0(rnmY$A(6_GR!d-ROYH_xnK52A^40AE>%H_qh!QY|GEL zuxyBY|F)aS*y9%Hae0v(Ed^rbq)OP)l&~vhbxv*u@I+>bDskwBhrcH z2S4L}l8CvL^_JLJCqUlNJ5AhuiwTeZ3H$ zv%%3C#!f2+GsQvH^#=?XU_o?+P(vvD=p zNgfDRvd@9ajBCD=S-JfQU7E4iZWpWf6n{WvOw&usGMNv^S~8DE?kbVeRecQ^d!mDy zB7h*XjblFH&8>^P6gJPrr$ytwsyPGD)`eC+WAx`F%}Z{V{MH%a#0t7tI7=s3!vv=5 z?0L0FRY%{8Z8}7N+9{SLSQ_7dSR=NOcS-L;~G?Da4glUAL}HVHc^`& z%H+T(e%CZEC~!e`GVH36N{nb^FWNoM!))Qo@m_r)00~QhBsPy0i0rfy;gKn;PT9P# zTxv=na#d`^Ri#$8XZ_qV#xb}`1`v?{hE}%25Clh+;`l#+{XaYl`EH@UupT3X1AGzT zWnWc$dkn5h6t}PnohEl94H-ZIdw;9iP10zje!z%F?3hk}BXwRA;8xCdpMc$uucq3XOi=jAcXlN_!t8EfJHfaL z9R$zU+fhN`farrS)MyN<3u0hs-GYS!u}b4o5GN0v?N@zjaMKyD7!hykpDptj2P z-$&L2X+myvAJ2)y1$THR%F~9}{*>m*JPoQ$BIy*EfMf95NycdzCs{(}Q-q)fFH(Kp(Go=sU75qr_{85q)Xbn^j7=x_| zU(^R7DoYMvyeX?Cp+EOF5br4ZdP5Fb4^HN|-IF?Ym$j}vHTWl~VMPOD6&G^PQ4=cq zGj_@WyVRiR73F0ikPh4*w<_S4W2V^sJ|fm2w<=?LdBujXx%o=*9{QlShb0hu`F9RD z>MvTnDj+9nXE|I$zB0Q!X>su=c6VZBx{qnE-u@|9x3bK=LdN`20A7}94R6*xF6djU zDT_$%-yHj~U6;7Nn^n)YJfr4yB(yUkv0+(+eyg86dR;h=z6^f4%lblfOYTecU!kvV zA7=KKWy9!^LfeD`&ih+?)C)YGAK?+Qn*Q-#1eG&V*3os>e(8=ac(Or>T?d5xI8a z9k&aRX>B!9Lrx=zICjj-gNYXf(pTe{-ZmDoBol<7a|mF{w)*wzN@^^#7)1$!&FA<( zpB7zY=dQ^$VW2VPnKF%ZuRR>WYcaX~;;V2OIU{6}*6q9|6cTz8?Si5D7^^RcKAlcC zRZ}rDBDX+p_wkov#mOzRCfW$UI{5MFaia3p4ZPFtr53*sd`yPO^&*ck@Fm)S zd$U)UT-m6Kd9=kyeL!sl^UXOR-r;!UVb%qbot62WYTPb_`P{r2h=c?ELzlV(AILwi*lmK z00{tI4U#B1fTE{zA1kE0pCsj~fW8tj`rRZ#*cRvu#qwq&HJ-Sh94^{YW}DDq>=hL3nuJGYKYajTa`xJp6S9KA%ws=n*=#*qWT(4=Dit;R-CepkgAlJ& z1QhwlAuz!$B$qQEp9pybzx8v-{J7)J7df?nz)0~R zx6tx^l{?d$)d*|`;yq0H1rl!*J)v9k)h|p1_0&6C1%;|8QWJQ+v*CP|zFbEbQ;)YI z1v4a1twei6-4cP~2fQAa7959Wp})YsC~zPN5)?{w2{$NP%pwWA66y*p03=HG*=BSUn=evd$=PR6XuFuk=P2JrukSGk z$)w^YLfrFSyzf{p;$L?x(k#oFFS#X*t{2nSgQq0Cfp*k`q zH4^zUNSXnZK7&B%&VYPv+GA}!!x$nZT9oM>V6S~7N(fCsQB+E|A%+qGCJ5YsQUMn< zq@wr%m3UeSmqb z=Mtur0h_<7y3D>`@EpM$VXzr*bqOzy6PM^jGf0_bDiM;a;wzNV540}}+E;&fJWZGT zs-OT-Wl(5xD}QCz4CaB4NWCNkv_@~>KoBEwYK?MJ4(GrZig8Nd&fg?U?IFwQC}T|g zE{1s$ud>B0h7ba6fWuOFVsDF!^f#Yu1xdid=hJ}n$ciZq$xgy(TK*%Es$Lja0Z%i< zT(h@dm$`u{98N%Lu7gUAZ!~KlN&>)kyuNTKMQQaxWvI5c#7hL*|JKL; zxAYFo-+?Jfw`|0J-e3nU_Zajc@pLv3AdmtNf-y1zz)=1QE_-=HNrm8!H@^m;4^U4M z_MM|`iFZS8mn+g!zy?OI`;8F!6fRE5Hxz!nK?Fr4DyK%@9mM?eB;%3~(Ju&mLTUq) zYMU1Kbw!9x)P%o%``xZs4(pr#E)qF;+5jK|2i5@rA(X>hP=D;<9rU&aA;sKmr8trP znZhsu`G(3F5CNP(ZS#EZ-LU@L*)A6N)W=?gFgc8z_eVX}d#HWjkm!G!lB7`r$5 z1@FI^Vw+KtcTL3<5i~hLCE^M-GBB|L(b#hn|2xbYm1x*igOrC9zvMUMdjbK-u(-<_ zuc2n6_j;wiBubW!E*;zi#d)zD6K4p-I4AfcH)O3EQw>P@4d+VxY^xqPQcxtAFndp% z@fJi4G0OyO4v0jH5bSmp33S2jPX?I`kUBOTex8Xl$+X)DJ1!vVI%*!aPsBeBvIgYi zsgWwaH419mI4U^p+jfwxxDRMn#%wHNudYNgi<1omlUn_FjCw7r1ssLwE*6W6HKSzw zu7JK9pe*_}#wCERXX;i6ZK%L3WL{pc@u5W!REiAJ4kB*g#2WuA_ZTv#608i#>|XN@ zg%D$~jOrA2!49Py=u|sk<#0yVAy5o`k=O=WpEr=;0TdV?Gwf9$!82ixj!ZDi1JI?n zAQvmc-T?Oq&^LzTLk=Ehc0e}8T~6XBmH-ZIJr9+B(!oQt_C+K+C)*rbk^>)03J`ih z!3JUUiBnKnZui}lx)dr(5d&2wcrO?Eez>Z!%B3bu1O>mk%j^PAkY6PM1^LCqib+6^ zO!)3DQ=E}lbh{*94M#5R>CT?Q86W9fr4TUo+P~vHD9=cR>e>)7u~3Fdi^TlK=Uk+B znalyt{0!%Kc1`fB69H2X0>zMe6!@wh6wBP8@NPt9sng5fCaKm9kJ_Aj;jmsbJ(6z9 zZ*1fX*n{Cl>ea`AI}gk=yx7`bv)@ zy4rPvLE@}nW%jl^Z2T5}l?2#DeObi1-To^&cu(gShS=J3=a;+zHAQqsci?h&{vGt| zlU5sNUie-knF_RwWnXfqt}XGeeVG-RRmXDPGPEYs_YQ2yOYWLgq_;Y}VRhQenxQ|k zImvh2%7=GZ$FBR}7m2ttt?wWZO-`WeTV^kXaVt_#KoDB4>yJM59ba9jt78S$ zchlKZY`X9QEX8}lf=SByPIZo5D!;5!{OSXfUI1=w-`&Lxof%Lh_Z5o8UML!2L~wcE z+!BT3^hCwxJ?p2Nm?W`lda=l{mv|dOCkT8&wf;c+-El4=BLN}ER57eaya^{^zyCGK z`vN!?o;W;i53yHF04D|&S0TWf0D)?@>*#KV0niemr2$Ie1#I_Ra+N4~%wBa7&Mn0L zu3oqV&Y1*N)sE?odS#HVOQfbBMvG2+^{o~^(Z4{idj`F(y*UTw7xvlddwE9|BB0tI zV*hYoGf4XHf-om9Q|+($TF|i~t@G@n7Q+#~7`*L+ybQ)A$aVAFHfS*rIfr67)QXNV z8NcZK8=ts|x;trjl-Vt6;`1)JmMn(rkZ`NA1vKjNahxIoF0$?4C451fmXA z#jqj2X>3>7{`70AS7O)9u#CVgQlu43^4_3pKMmk&Zqnl9e6Ai}Hx6)XBjtK=^~v6R zHxCE^!P=I`{W4suT z20IH>eVr#(-r&7*_z9;(5DK|a2Nw0309@D8NcQh@gqx1E@mrx959N{`xf@?Ur+rf~K)SfT@ut5ik8^WihkcXy~gz=5o(j018z^Q&UOGrL-I>`Uy}s5;rzYW$ntY zSwjE4{dJ22Z<`?!dIl_$Ifs~ZFj-Xm9`iwBryp}snns$!=+c2uuo zwK*Z1pbh~Z2?#67#qe%Mt1k27L9xtZdnm{C9 z+}AM!jWGq;!=4?U z!hRfxqvW(he}V8~1#4rZ@DnIMZ=18;jwuzhbQ;>Qdt&NXcik*NVzzm_<)d25F1WNq zSDkgCKPO{#N%hGEmjg<;?i6Z%RT{gy$k6S#Ecw=u%$C;DZ5+`zuE_1IVBO9I-6+%D zcbA{`L{v)459!-Lljljz>T)T8LN%0L7J0<_m~2Zp6S>2w)0b2|BIS*0h2aU(X^+yJ z`<^adelDo1$k6L;&Ffa_B=?}FVOBX$-=hFlcX!uJ>${`GD{9QtQqy232O8#sB=*E5(`xa$5VHX#P5G6#Ef&L{Bw4CH4lE}It)UO z7puu07JZZH?FP}NuV7ogHuIq9)wp^QCYAPq`s?et9Uj9EOo4zr%}B)tyeMF6%N$)S zrJcX{TH-Pjz}h7hb%yQe^b%{OJV7J|U_>-da`!^NQfndo^BR=a{0e|YqJAgNf)ibM z2*!hk4w9&Gd`~XUG_!}c^Dla`4_p22afM8cIb=9y7+roHx1e_AC^8eDSEaih-J(GP zYqV0Rf}-K5t-9qse1ezua``jhq^Tk(w+1pPsBB14PF9wj8&Z&Cj+GEQ7-qxYFj=x> zu8M9Gb&#ygBq(1lTMp?;8l)?han_ECGR2>{r$&z>)N$-&czHG=qN|q66=gIAg` zR)9odaWB3}KM?rBg^GiUt0@~(&rT*O_&%JZjP5+$)95ki{l|_+=nBPyIH$587e>@@ z97SZ*JS4(@q@^6&-3R6eOeKX1ak8|CGB+G((YS|bpT~|x zEHykiiat^U0Rq?8NZtQb1732tFJKNP%|d=mmWtiwXaFmDZ~|~mT-YJZ7gXehB$ojC z4bz@b&vAV50L{&B%HkhuwLTG=f^!s-*qh)Vkeoo}RDEA3F&9rMds!ieH=x0W6ksN8!~etI|D%?Gg8`NSXN5#V zqFM2W^TOdG1i?j(7O=W4hXxT_CQ{O+fUE(F2W2L)-+%sX&9UM@NrPuc?O2!O`vH0h zG<)mTnC_Y2NRd$#au8yeBSHr9vp&N4%A0^Ov>9PewY)CA(RP1NpA(xvVoWiDu2?83DWp|hjodr4BdDO z@1Tii$wBq*dQ<00LL4}Yhnyp=b)FE>-TbPc)gXPUDS+_kr~w=Rk{ZLwpgzVSE_QsY zSncDAIaz)m@EKVathNASF{TO{K^*^UuU0{y05I))64j0Vn9#f@_z^F|}}<7Q}b zBMLUuprCHTu!R7F>?6HA2^+z=GI%86s{wP6+I5h%85J!;AmE;Vq0{qE6bCwGpoNzN zCO$%AE)r88K?F%}U=o15jAXlbQ`{qVgUk$iZ{hqmRk!r0xwDyv4tHeRlH1~Ih)Kp* zaS`ho7V-N;S@&o1)Q<@D5NmyLtw91k$4?7hJdRN7MNJIj>}x3w&#Y9w5`PuwC+yxqtbR-$OTCwsJ#i#=5CiRO{;8^tN~mdwYPW~<}uCnsGkCE+}1?n z;shg)d*Z57qt}^Grezo&GBFpXUPGWg5D>y4@+6(RU`Di$fje*rBgdBO!I?#@524`i zvPexxq1XZqnlT0FAFshnz=L$6*<9LRw za_pL<+6)(9e~6#nk`_DytEMsNTEF2y-adBsB3W?V6QpvZ*V~qbkNkJvU!j{41n!-G zN&Bsh)VfR)+gvC{8wxf5^lIJXlB}$r0TJbn!Pl+3a+KG)9}G~{+(vWS1q`Z3wn0oS zH)yBnFv_>*m-r~~`ja(vCt#aQOm(_58}VvG_sN#Sm)(|5JeaFfeg{Z25u7A=zyGrh zys2JEA6xnk$rcbG0FK%JWt#33YX7$k+LWjkmGK(^l3I76%+zks$?KrEj#1E$+5#$P zY*6-n{T&jTdyKZv&2IwkBzZFX<5@r8!t{bqm~H2a7==I3UGf*uh9SAi<^xg>$`AtZ zOG|Z2q-nacUXbP}>`+e!gtNHuqHT(C2F0uFM_KPbVEQ%5^v|=>owwLMO^$fRN6dkl zH7=eM9N=k^hSiwKSzQI>?aABS`Dv{#OlcVumdHCDfJ_pxW+~J=ZW7qjfndau7swq_ zHj%`m0K82&Id>sZ!Y?@R2pE`D`FaD$D45{bW; zIS*}3sO<&I$7;;@gj*1N3-D<0v$*&TnkIgR@CtMNkW7qx;dDqTEg4m*t6b&z2 zDC^|9rad?P zAsV{rcGn{LvD+rs1<-Xk;kf^HRE1&A$I@SAM1; zN^WSfhZz})sfxd1CpYkN8R>S(B%swn_x*wd9FI{xTO+nQ9(n70RA)ix6xVmFWDP5Z zT&i3Hnq`iIgRVQQlg){J8^lon3jp+OLHFFHb3;w8w`3NgFM@4Kih5!-dckJcp4bQ9TqudOFxIN%$vGA*`E zJ(U%1*X0DWO=ir|oWQvohRP{PyX2}bq~Eh$3ihHm z7b{NA$<3M*ExXd#c^NnmkCBrWPf`Uh3EH%!o`iVMYy1#lwI%&(YIxl?fPgwqLoxct zS&w(%I0(M@nkh9YszAKeB3cKnvzTLo^ay#jf#3)3$Wjy&$2v>E8GZg` zeC3aQQO&cb*iaG2K=|SV05W6)pzHlM?Zduu*sYnOd-2}vWA-!*D}ebHS!dx;&}^w! z3xM#6!2r7tH?`m5^adm69wUP{u7ls(U0!3N9yY|B2-W~iPa1=j*@HEhyBUzY5-i*V zN6w?p4s^mpj^#r%*Ay9mQ@_KOs48=_hq>2+;>;F23MgY=f>x)AuWKhs2n{+XtZ)G^ z;Vx*PFj53$&AMKqaju5uG){a0Li|MB;)ZA=B8A9?pB?OG1Y$+TYl6`#`Q=%FH|uwx zUPr{Efzw43c`#iT2w)2--aILy@D6Zrlt$HGKuH|;D^mleNhrJkcQg`FDRm|Oxcf3hQz(m>mGFU5+f)He2@O6Nzu+o!wcU`jJQ+<{p1Ql&?t(J2W9H+5R zD`szl5KliqVhAjaBROP5Q zY8=LaKuR=5$d9uyT>j%O*BSJ^=xV7QT+5qq2=>-*LKUWegWP15$xw(hOp2pOcgved z2f;hed=K!;Yfm+RVGJ}A0K)S2Rgm?24vtO&q~)j2;_T1Fnwn+oq0@$|dq~*!2T4I0+c%w#(h1jY7R-Tt;SPPhdsVRb`SQnM?)pelpdF ztieC2G>*UKu-F&bFFwYW#r0bZ9A`jV>ESidmFqHzDNM0@mJy5rEt5y{4aXaca0GOp z0F)t6Io8VIbrU}3A^_QVomfC$=6F~n1UR3IK%FmYH!S-n!NoW>seUkQ=l7iiHEkp!m#Dtpy@|@ie0lM0SzEY1{nihIo$RSf#DzaNT3IJJ#S}1b;Nbc7TB%mjrhTRA1ot5?Qr8y!=wwvboR0 zy`1rIFLKxlWMJy6fAc8vBnX+W8TM~nup<*hgwW18>)(2(g>KzrJ#8Ym*>S$_U^@5S z*mpI|ry$E&F{=5FhJFwQI-&W%?5BvI#l7=G1a`A&>TW?mbW=|NA>xO4+-9O&kJYK& z*MV^%Sr~G;ma3bkMtgdv+qt2WlubTm{nd=|4PKr`)Tb z2AZ8b)n+1t%{`%cxUOfEWF&seOc|{{=n^-axC48>bfN5e6|?n9PJY%$Hr&KFUq~`d zD%^_)zDlx2*P1xXQPTnq6<(eo$wN57??;<|n!9d6Q&4y>ZvG>5!d%-ip|d4V;{k4% zwzEl4wyswzOM(UgjyU(WNwjSWD8I;*X5c_0n(@TjI_1!o2*r2VkM_W!k{+km3z{M` z+>fGxTxe(+wWi>isFEGd&YrT$RDy!9MfrIws8-6gW%fD%Ecp+>+Q`hO1DxoEoAOtP zNB&?z6V3+#$>4;~9qO13Ec_1$oXlRcw#!je+x)t|L2v?^27z6wCu*px;4%b$oKd~d#bXSr~V#yuikq6QumTZLq1QiRKf z-Oh`ikUxdqj<7U?s1j^L zOWbE3OpNg2U_e8?UP-9Lfcv`#x1;iKjviNNULCQHQSa z`ZfE#`eo^hrDB*TP;CQq0|9%>CCyZEUqitnU?8bqMp~qEVtK6qu@odT&*C;YRY8sD zHx-)YvBr+_Ms1+z9kC3fo{X17r5S+b2uVlY$%AuNrt@DfM1&}XQPP1XRL*T3TdxWb zg>fCL&OKRkN-ph-EDJFw&K4F70>{ed7>Iz*RAQzOI>{_RaJ@|iQLuu(ngPioY7Mh}@`54t?@BWbmOV z==BwBk!cWs#y{`XqG`x%y9ONtXI?(LnmYgAA18U~zq}jvC2mWBGMr|#DG;{~5TWUW zC6Zo8G}p_+T6KDq;j?+5M4DRirF+YK=Idh;+5fx%m7r-=5w7mRfDO5I8$!$`?qRfQR8!Y)-NqKFp;u?Pj((V*4y*nzm3woi z9-^1!k##1%6q0Hp6(gMK&0&M?U%&Prji`fOmXto@ms&yO*Z|1a-`RZ+ACHTd-8_;K z4rvrOFOIuBA*k!VtG5BxeNEEKm=h7R-4_~~E_%ew9ZpEA_J#E0vgZ-yWe#&WUh~Dc zXQpJ8kkR(UWIYDcZFwSfBeay`={#ug2pX@Lcw6dv>@7v#Uk+b$1r%iM2%}D)6yLuZ zfEbg8eet6N-dVdlQdJ$2y5UCVwM&nyqAOMkGhdi-S)~mX%%)dBX-9&-ccM1b&Y<=~ zFhYtMHwcVNG(Co(LfjwHvw;DVWD4N`i3x!pPoZ73PGeNCJp z)2Qp5|0#1dBLUR{T#qbb@v$M|ZxrBuSnzj|U=^lEafuE|zCitG4eRO%qFof-0!*VV zb{BaJ;tFT1n7Kun$jL;)$-_|1-30l78Gdg>HZK0Jcasb zr1}H2evQ=~)m#;J<1S%Jr3w9$pXRU0KA`jZy_je0HGutn(paaBx+K z`8+ah@kugtHYOuIAl^o|8R?>J*9XDSB%|}|; zzJNIPJU1`Fe!WtC$~B#RFcS_{e5*hh)SR)Alk9QrK|qC~nh2`y`liUBFuJw!KvF2v zno55F|BQVb>-tO@NKT|?dBaFC!V3ZsI=Y_S}V=!Kb| z5lB>d0*fl3{SL=m5uV);Fs&HRp@7aXtW8=EC;9$vD(l zpbG}6j1l+GWf`VK8U_?(TXE?Z1IK_H%M8CDN8|Mp;D`qEflaqcoPT!G0K1PdB{QDB)Bqg}S5apu zY;`>mU6INF@HyftbBGpfpsc@gKDq02;5(oo@3>ck(qeFY)7UkJG|5*gKh$R zB$36u{S!fhq1GMP4ifMN=3Z>)Q~@87@p}{ztU^)YDADu@$3oye#o1ECo4M2uyR@w` zjW|^A0y^;*L|z#B>RLr;U*!C?P}ds5f{_-<*XKiTwQWPc@1U!AdE>Fc2ku>bNmniI zIeB*S(UOMr=%Vn=u{>x6L)|T?*oR4HXwUo^Yu&nwpzaGQ7yXF__}FL5-zF$JMqW*_ z#d~C%{wZI;Lp*V76$iwoP$*833>QmosDVj1V82`nf&tPT()IFxw?P;+2lhkkpPiP}e{0sxEZxBCT1M#5O> zH?fw;HWSve?dDwDd1wX~l+X#gGKZjHCnPkSvq`JWnvtcc9=#tX7O6St4dg?v4Vto5 z$NIVcl2EsJzH9X#+Rf=6*2M>z)|6v0g(%f=O^z;SyU`E;0K+C!B~GtaGQ;adn=*ANL)@0OajhEYY{I!7z6g$ph+V_{0GrA+ zykh;0^f-|T*|_%kFKQ|ItxV@|vmhy^;xl&iWw&K5hsq5-Y`DHiJtod%jmvdnCvlqg z{w`*RV)Z^-1TdITJ1*Y7eZZu-XO4=RY^aMc$y0ni@vlUT&j7OiWpq)gWL*knsL_-K zr$~Kz%8<+SdL4t^xj_Z#Tu#E}#IM`4&`IV_us7uE4HM7Aa9L_6V$^BS#> zv}$~Mr5aDqnk%Cu=Oy&HXH`nLTz(;<9t=g?(Nx@x{Y8! zd$sG+iH|%cTOoQ;cP?MIcAF2t(zxo5INHWujT2sI%bs~lmit}bEicmzQNzmh(Rxvr zzW!^ybX&%@8}@0Hm%h$=85yu1UB?hzaoF0cqvZ+qlb)Vq-rE|t zv0_lCLR&P={c3*WU-6b(*Se3T6UT=gy!w1rMC0<-9rDww($t4E6-r%TGGA6Dds)7~ z$&U<@N3TkzmqqM@{#t1{K(xI16-x2A2OpWhM;nyX;34k-d=WB(ROVcs7*38e5nP)Z zA210DtP_$$WTux?MKnSO|A6|q>ywG1%B<1QD-~Tod)Cs4`zn8`*uw~pOL_`ID2{X3 z#C1#hTAZfu35`^Rw$@M6uPfwCGu9TzAt5qG(mr-IL~OlC4aLT3 zKcpsux8m*bH` zJK;A~K^H*buYkNjcB?hKeEc?0gd^M-nkg~b20zDzT|%s#H=LE`-hYEpikrQj@b0RF zcdxJtFz*KiWw%-WF}di(lkYeI;_8AX#(_6=WBIjT>W_SFq-vRZc=FR4m>bw?=xK!c zpZHuByg;IDGg|_q%^ok+x;_Pmxopmrv`XB)p8(VIAs~`wAj;t;lpbhe2IR^fBmPv6 ztdeOS_1&`h62<`osAwv0piRO~p%0#ky*VAmBbI&EWSTZeraz+2?=G^mw{ znuyk&Z0{pXEO+oelO2C2MzagtA55U$_*3&82Z<|n~|xr#}Jmw zg%|ipnd&~EMa#7_>Qez|sB-VP8}e)awIrE!+8y>i%~{C8K0Acm?b-g?ABg^n z9Z3=kK>45?1^wO)`Sw2S4M-9iYqiM6mG@^YU>WC^`y6TFUmX$jSL}G=K~dNzeQCs9 z=a~{}6JeBRn(0BjD*^S$cbEXieb7pZubIiV$q~e@7T%aGG|x+Iz8znk?xNeAh}(Eg zgaYW`@vLU94j`|Y6w4caV2u8}q}dl|wPe$$P#o`p84pD4AHaDZG%D2g@)~o3L9X@C z2XHW4%XHO4+Sy!R&4ELt8;TvbBA2W4H6fx7=xE%Wjv@%0Dbxu;5?;J65Wffk za(1N*C?Zqa4mfs`(#7<_7|`H{v8OixC{6@?cI68+D&jtI;5Q&e9#F&J57?SppY!Q4s z#%k<`vdy_^1Tda}Y%u6+rq`Zx?7D2|#kmb(_9>I0FsM%Pdr3Y=3tm*AKm7|6>6l0FQb#7?1KIV9qo`Sr{<@3*R9xK{Zy zqLyF8CbGq?aKjNSt$~|(2qhmz^=dsVD{TA(#jCsPcFg-GB7!ZO8r07Ns5#SBv0iy? zD|`~oJ^G=(9`Jog82lC!!uLaRi_rfEgH<>jmQbVAER#$eJ}Cx)ARZ%`a)mY_jOg?C z8F&V-Dz1D3xjjiaw);YVYAZ8=u}a9xU|joW_SZ*|n%N8iHuXYI#O@7<2pk5a(+=1i z33IDQ^#32i-aIbKdVL=sP$SL41+$LJ91{hX;wUIC$Y6?yXv1cfrIeN^nPg%*3Ot-V zx!{p85D`g+09jl!MKh79TnG_I1T{sO6afKYWCj_AS^8b?1|BWKJj2Ap1`9M^aKGHc^9N;taj`$C#gY)@<8P-OQ7&p@1 zBsbmpKUe9LpOrt71{+c7SF#Ti>JT}Et^NuM7z(sl@lkpiFD|efqha~7a*7h#E>oIm z2@SLxr~~n=6oMuK^b-1T7cxqE_Ni?d73m}5MePMW$6ouE{Xcw!W>=nl=`o4E;-4)T z()}A|bs)lk8Q*>>`$?TVVgMa#efouXodi$Uz4d~($h*lSlpNml#M7cB6R1a#mu*~{&8+3i`9HW+`?{mCaf;;Q9WAkKX|moG zwN}gG55F^)GlL@ba*-lwTM&uBINw3dmZP3m;Bqbk^$wH7^rjIKc5N^Sa7zTfSz(qp z^kI|`n;zhT$o(OQ4~y{OeR~x5%O;hXI4Ua7Y877CtqHA95U~cO;!y zD#PgI@q)W9M|)9uOgSBcyQ}@7U@s7fE5859;BD!5)MB5Ng8ly)>+(Gc)tB!tAdwEd z@MMwsu6+6xE==VGl5wLW*uMBc+klcciYclHY-YxzeG2n+`Y2ByM(ZCNP%EQ2G35r# zg80!iUqW#@7uAm#d2Z>xyS9>Md2NE_;AvP69{IvqySi4fcwJ3=;+Xtwv+m?U3^|~L zEYh5687cb*i2vvv8`Nu%ph%BJtuaJVvHDSAIYf`FOo&7)Qd1qAzd7@f`pXH@Z`}3} zUH&Z;HPJl}OW z8N-SjQu3|?BZt=!TtNOKh!qy=cm!1~O*jpz+yn0bOHo^(-9EkPIOOrJ|4|$_EpjmC zv4uz9+8MQLWz`uyhkaN4>di$Zq~Vy8fZ_{-YoRpkI6k?&C9r$&*#`^4nvQn_vzu-@ za^r(S#U!j`Lf+VP*{nPD7_93D)lpiIyZIUYhSg|^HZ;s{$J7RmGXyC5*|86!Rc@E? zf^3X&XK^8;^cQyuhju#tcI*}ZS4Yf*vCW9v@V-9y(g`8DCC6kCzOY4MLp>w$ZBvM1 z@v`fktFp88-|Gv{g0vd&ULqOtrnY4x+$Q>L*^{aOLP

A<8}4j=>*Hbc41hF(YJ! zAsz`X<{hRWcCvJ>XL-ABpH{23Sc*v-dq8$VsCe-&3D_Yi0|jl^mBcK=8+h6AQ^jz! zhi?U0f5D3S^PEj~`$$YG$$pq3u37DB`;zsB@q3vm9q++7)s7)aS#)b0LAxd#g%r>8U^ZD@ zhO2g-v;|kSsnq)L)=W6lIew>dD*&8?sUf51G3TO)=%UJJko2(SI2i;jXD>;j$Pi8= zHHoyB%|q#MMe`ltLt0nzo8*120I)!Zzw2JDtc}Z}GZ%Yrk3ZPy&?+2nnqIg5ezUZ7 zL%I8}19ER!9>FftN(#Wx1%mF~Uz~Kus|8ID2nzs+RSH?^&TXmi>w3+0;*L~zs@_LZ zk82DsRVT-^%lcTs{qY4hZc2os_9KB?9SxTpvF5coU9E|?!0;U)iczBR2)y=qISL1;+1)gF7Agn>|L zd(MZ8$NweCHvNPT9q_7b;cV)t^oL`f>&4#-^SsgX9`p4$rbr9jxGGz?D;%k@R)2PN z@;OSH82|dvFjKhlqKdLULhgNwcZHD8>~_sb>36|-@jG6}ZcbaC6;n*yYmO5W26DQi za>h$Wd27N^DhAmA05k=c^M{b4`~c|rA7iX(aiHg?Sh<+^t9T9Yl(D#yE{sA`1Q<`C zMG&V1iSm(ogdSE29cBJAi$5U;bzyr$kd>t$CBosD9^!DLNAY-S2IlzJXZYql9&Uzp~kz5nMW>Y44}((P9__gCDvwbJ=T<%7h@W?PEgXaz7aTEJ7wVzC3|byZ)3vW3aVvfbf^r@Rvm}?@^ePRwXJX5N&3ZRoM933#*9gZJvXDT4l z-&AJviXFgg-H#E3jd)yiW78(hwFzXD$%3v2j!j$KhmPHBvEzKU*fahx(s?NIQKvxL zvilDU8siLmo(7Drm$wpeFD@dC&6ze29V2Wv;H}tFAHcnL@!7j{%3v9hA4gSx3wn!m zt=tXG#atX4F+{N!+!8kClK!SXtac!{4*zd;JDb@-%H^-%o3HxzDT5%sS89ODgbr9xGu#d1Ki4Fw+CI z@vbkyO2O8}{l<|v&EULvrT$U-MRrtVUF;~#w;_!hkE-JswebKwKxkhNbr(BL`L7h1 zL1RR|j4Sl)16_~bRJsms324XWpmj|iafjIXZ6gK`a}9%x z0)GE}-L*|0<86iw4l;z)KOH+|J&wx64*hbc<9^0wBaS-yo%w^Ey?Hf~iE?x+E5A@q z6vkx056Mn^XB)MqjV=nZoXuaD5*Q-&9<@o^+C6(~69IElebv(F1O&`(TvYi24}raF3Pk0}%JF0;~C zME^8Ga|VXZ1aF>=<09pZq!j7p+0sq0y^LxdEpG(V5J;rb+y9wta zO^bRVPaqMBjWGAMYy_y@grAX1cVZ}2vs34u+b~yNeeS_I%^8A6(b!*hUkMH{qEP$fXg%D`COGt0eJ|c>m^SEsFZ#~@U<42{poFQSiIAmM&Et#Tm3KIkQ z5yQENV%!J>kesj!qi`-^TZr9Mm*X$y+BTd<9@zNYoG`Y4ieX}G+Y;1 zqG|A+rN|Xm51y|d!YO-0uq*d*#S;M_0&SDHamphR=R^|0iTW@Np*tf9e!yFWE$J*+ z3P5yZ2c9slMsD9{kHgTBd8Gy#?Ub6!U%}|!W|Mv zvZ)XGc3ob^0n=%R61TXFbk11p95kMn|{`!9CHbh^yPMy zbPxfKM}!3gJD~0giE+aWQuD>6rLhL(6XCFoJ%yGF)_NR0P=AP`K@|-rxlV<~1+=#M z^wlxxNM(AA&L~^B%p?@5jb$S#hso!K-vz85V3-f3_)?gQ4%=~m%N2urbG#(&IV##d zpBE?s1j(>w~`e*adi4odr4oYD)taqMiE9xW)RiaL^&DEKqQD>z^mP% z0CJf8)R-uky;pWE{}7A7@-XDI#g7QyD?OgWQG?VJGjkemR?C3v>$!_n7cI#Os6DSxxM2z1f*AtF9xjFgd?w zu}Bhk&2}?<1SlN`7w&)@MolSb*p)w)SuaUc8x*z$yT(}`UB^fx+xwTq)z%Eeq~_8s zjqx~)v`9c^_s!8f-De1%FtLV`g8%@~Uy@6=ijBv~!{6d{*a92_3w74f<4-tJ_mTPC z8CxhVO^H=a8tzxQ-)M{39CbnR9%k0zWrlY^ZHOc{%uj`L!!sPwlbjliZ%6g8PHYr} zKGy}W?e3`ptkLjOe^bBR_Cfe#j6|eN`qkq;e~51|n7yNHn&@s@sDQ61Q9-{lrDEa- zEVe}XPlZ09UA0?&uL_Od$o1LVFZq3HtYSsFKvsr(ansguAVqknU4x|*<GKKFj#qWKBYM3MyFxw3toM979-Tef3b&1~UjiPAW0#djYB)VgSN*Y=1O zoF0-6yqbQGt*1cb+ORcvxfby}(Xj1%U*WH49%Si3T>;q3Qs{3;u=%n-& z-j*{$FP^qUpZ?GK^!g;|gwTqdQ{8VV8=1f=$vWJ&S2(M`%00Z_PV~|?4%e8Ao#WtV$iz0VF^yyLSv0ggHpdTx4E#OvSOIOfV}gD(8dL;8l~|ea^^&X(^(|t;-sNw+2*`HnBkT|~ z=3SxEm7n0Ytr#u*vPLe}Yr6L4I?3^;e;MWVdr`%R^g6@v=2M$GuAvDafX7_(Cso<< zl+vV*)xTSaUw)@+W&-dbHo`N!D51r_xu_PXPRFZby#4K5EE|M^PJChb>Si^>e}d}C z(WXBf=I#t|-1nfS%7eCmsvAi0xuQ2tl$|}9oBmIi&bNiU)DWAUs_C6foI@mGtiVUQ zT3Dm^%w4$homOkd_E?VwVcoj+|CmzDD(i;WHiPA;>rw(-pQO@87{=o=u@(owup$%h z|7nYB-hEZGHpWX`JYfXZUoW|}NuPX(47)PF-h)Q!!v%NkYx$&sLrV|PuY3ILZ0RRo zeG07L^Edt7y%J3aryMAxH;KPC~C_N?C`-QWA!d&AleUveOKd7(cwTVxv%u5ahs)y!|L0IT)DQTb_ zbkYW+LBf>#yS^|B!Fl^c-$|fDjOx!nAfpe9+pjtF2$!&=s*a`adt1_kUdw1BG`Xfu&* zXxG%vO>;0F-^;d<{aPK8U38Adp9Agawv4NiOiFAV<)HDtEs89Dl5_F>s zLBTU5|BcD7T4$a_ggOvQ6DKVqCE;@nM>|A)Hp)(_0{ML7B&6OB(bX%egF2V(V-rBGzQUXt*A?Nx!)~B{;E+&0RQ{?m zj{U#UC4f-pE0(XRRgKN`1t01W!gwXR}2z?Ekvp z2iK=g?Z_XZAC2{sVpqv}WQJv6OHLXhkEdRMsZ)d_r)~kNK}SGl?2&EqC*|D{4lG*d zi46%-U+E)P?jxxoptEW}gE#2Rf^mgOkzh88%YqH*-}FtDQJ`iXx6WBt=>&(hjA z>a}cD-^CXmO@XDXgHBM5>t!+Zk(O~mk4RP=A5XZ--X9bRrd47KDDTkt$%Wb9rcukP zlrBLFBO`MP##Lw)56$23^uAMZlpk&N2-Av;lC|q0o+uOrr%$fVI*O zQFynbCTH|)^cF;WL=T4uVO+n+kuSDyte@aOUeKBD5KvW(Yu@|v)_dm@ zO>(`tqOe1@POrLaUYk2l0+Wa4Js1<%WOfmS=}mf_I`jTuf8sB;5kx0;3e$AVUB-5~k5 zjAt>4$z-iYx82^6!fBfa=T>Iet9m0J>_qXsxlB}_+B=fk{6l>3OG?>0)ADXXL$X+2 zvv*{`HevIyza*c-%q#LIL$r1(f(HiOGg!2qcyqU%1~?xHo5!?67OmUTnAO2-!z|g( zK_it>-Go20E}lJnabrvrJYc*;06Og2I?jPE0<| zOA6!}+~$bFL4>l0WA6X{1N0r7K!%QRtrbN|@b7GxrX3@Ul&IK?_4Jj=Xo1hNS3Bs& zKWUvOwf^~k9DKwt^nV0gw0P~pE=Bh9sO5+|^xeSUYk?vpz&;fhTdyG*;(l$zMRc!8^|#9)6O59P5ifr!iSbNE(b>vH=mB^u>O>Y2OfthYqq*JN9gq_ z(KxXOpRZj+zaW=qw=mOx2wfwm>cqr@L?SIYHs4m?sY-gn?}7-bx=>7jM=to$8cF{6 z^D)>|`@>-*W*4s{$QU{}dJiDLgP$)K>x4>zHuzU0JdJ zzm7+Q#Y4FijLt)H&RKd%wvt~ z)k+CCe^^&kQH3MMi8><|{&t~yt$FmJR z4?7JK4-$&&G0_$)bhW%zQ#)mqw7N{6iQZAck#T+9{$DZD0F2)o@HZd&atKx^SM-dU{iN$D*a`id%3%4H> zJTDn!|44A&ELXp(y$HpYzX#lQvlkVavj0ezC)FvBj8YL%#)>oa$36*NiWm}0r{Coq z4j|LanGuG<=7Vi_{a)oqros%w28#G}0*t!uw0xF$wCGI5B@Fy&B zKS@|YfK5{yvh1cU#?Sg>K51Lo84DZ^GagAEEpN{+B+S&Ha`HW>sXo|_*g{h$+0Z_B zPxMy=o%^@788Qs=wYxZ85pzfd)+Id==`v^*o05S4ik?IScBDJ8t}i_tBhr?#^{H~> zy2_pRCuMEBsB=>v_Ip#E>bX~WLRdFIwp)D$X)2mkyCj!LPaKUsg1!RgDO*44E0kUl zdP)4VH`iEaxC_6mNRQQYnB;DaYl2NlE}ff0G0xY0f9BOws;0NLz+&$uQ7>Lz2cX3p zQ|1$Ta+QEd;Ri6H;-+mX%n@yAHkJWcAl{0=ZisS?ZZ2+^hnx)dQbK`*;7~lvq&xKc z5w;iv!HrSdQsGnmwkRgS;9SdkE~(}8Gi2x?j6pfW{6|swHJJ_y2Cfeq<#pZ$UtNm( zdbLW-kybCXJAX-jbW5&NXgA9`pL8G9TdgmZ?igF)>$EsbrHV#vl6p{cq&!aXaU1Gr zYVHiF>xbo@+hkmX8~=2t5zWt2@$6$Bdbjz+dK6n(_+!=?RkLh0$t+^y3QHeLEn0x5 z0s@-HgG0`9Q@jWnvx=~5UF_R?@#nvP+#J8G{}sc({;;()hD{9LUb-*Y+I~NgmZE;h zlq0OCch0K~w;j!`csmIqfr6Pw(20zmd1NfAXv0z6<#rmum=1g_WG|VWYGs-<;x{D4 z#Ix-QD`$}MQFj3P3#YZxGX(2dG~L9gltQqa`0PMg=sYrjy*1OjxF65pMVRfKGm|!e zJ1+cVVbZGgTR3nZ?~3z!>vKz$f{}Mxhr@fEqt~yj-@+h8qFN|l^nx=zW*`ZnpZXa1 zEu;^o2V(Gig&Is5=QtuC^zMYndJ2QnK0*m5hZ8#l*^5qHz398c4pUsiX2zpYI>)h{ zG0U2QpmJj~DawQccsMSR2pWQJ%gskU!&jxU=?7$#enEug=azij>j?`TB1DufhDJsD zKYiUZg`?yB(L|23qPXJdhT5rU!8;qSOLXHv*ocVWJl_I*e=n=5F$5e<)~+R_G`DVu zJ@Mu2o{fwQ$w|;Vo!J$iZ@d!QcG@!fd{`ld$~U0^T%+(eIO^WL(`sLR9<@13pXWtq za2^kxLyr7iYtHxN9ncCq3nlu;r)}+{K#kWW&y%y6>}FwLdiqrM7P zm^Q~Y4_zrU;5kr&jdIuo?D%dCwmv)HkmSrc3~b-Rx*o<_!+Fot;3_w#!v>lWN zU{?2H4~eL(q_e-|nQ3k^1(lEa40y;~2>6y5#S5va5onkrw_HgznwL7!dV2yx95Xkj z7ysF?#Zc}k%OGHGFu;sf*{Cs%T3OKDm@|8Bxa5gdno(Tbz1)uOSo6@Wk(mL~u;`Ye>#|$@Z5x&5x{SXhxpK>feRZ;Z4_agkC7<&Ngo0%J?7?GYsn*`* z?KeTKs%R+9TDRN#iMMke2s*qr={+wv@5H(1c5>Wa@kmCSAjZ%VCs>@=bYm-~)cD<@ zj0oIUTE`ef!&*LEFnRlVvrVp(sm}mZ4}63cLG?cTyNFFWN%@2jd+L?j>UnGzZAZkT z#O^EK%ZfDFw?G%T|No$oprr@H8gW8^6>dcz?x;U9dJc5s`O$Y%;e~C_j-Lp24&)Ep zRF-l{o3H^nJv>e;IA*t7yLyT^JZ@E2U`oe3zYsH*bS-gPAMG7dRylz5b~&aT|PLptsJj zL%Og$dSJ3&xH}_}9HZpY7o%Xr3^JA5|2&csHT^6bF4S3DrbJ~Fz26rWp#yP-tS$3r z?aGvuI*5>6Kfp+dwdA#sIgiS^s)m_CC&>%PTH~$8YUX$$bjP^lFQ=@E(B+3&KnpJnCEp;TD zkdQ}I%i}8)Ka>;-YhKIY`#(fRC0i7hu8kORTL9X4Cb5V4BsdrUhB9C;$w0`bdr>IM z(u_Zt?oMMCTI(A8>wFcSX@fi~K7Wg?N8j7~s;ss7%VRs6= z@mH@Mn07NEd1R>&t!J~K33cIFk`MUb%# zg}M{5gO z5{cR2G+{^LUY1s7@WhJIodZJ8R1na+Mt$o$R{dL+t%pxf4qOoSA1AX&BI%4C|wX)it;!(&-x;Kv6&B>&C{w?EH)bNTttBUo8qMyyWg>p=>Hi&3a974(kH8gvA(A$%}x%dwel@LEHPFWp2hcI9W$|~mi5C+l+l5>;Y z^>@U-i~l`BEPJNsdcBQ*&&XtmGeMo{Kcc)j{^2;Vb;vzJtQ`B}(%%SXLiL0H3F)FFk^1JCGrRvtZW}gb0SvFPMe9HnTPS z)Ew8;H}8XgyGG@vWjU|d&~vDM};@ylXRTY|rY>oLI8 zhtjX$hrZ~|SlSvx>9RA@*MB1-z2(!l@}|yL^u@$|!4Zcao*O=jQXO$C@^k{~T0dy0 zSP3^`?gM-;ycNXOW1Zf51+Bo}Bh%Qc<`jd)vX+?G^`#9a#bVjg@Pr)~4zlC1FUkU4 zfzxM`h_2U=MT$>9I)L$RhMyJoQ^*dELq){q7vL9Nd9sB|kd6k|hZ^7=(Z0Qv@kQxo<%o_ye=bPg?f@iG7Jf}>-^RkfrQSs#{!*3Bg*68CV zVYO%MgdujCK6<~jI&0Enu$XT!vB&mPd#f&2p_y~5dL3LrR@D!{Glt~%c;Nl=OgcW) z1kALy9j(=Sgrq1_vk&T``poFndfdDxM`81Otk`3`U{^BCYnyOpE`Qt7wjZ5ZVh-wT z&M}!~l{xr&gZdH3m=8g9cq&PTvTytEVb)pomT}XS&f5gqc^bVmvIqG1gDcTon3qLg z>TV0a?FFxp`BPQuOG#7}6i&i! z_0lVopX7~}%WR8CI1hVqo<_NGF}4%47jx0rH0C=vRrvSYB?%o=WsF9eqcsA-kccUK zcF1~|{*|U0<#}+|4VNTR7;)cN+svZS(o@ld;M|^J{mdV!o_MzdLFe)mcv%CW1-qu#5Uja&o4&@to(qxuOy+k+QY; z!9?DCA`$TEo516_K%m4J7+^L<-zq|G9%25%P#oPnN@nmgZdLT$p}*klrb97~7s##f zJdtp70#_g))ys!=*hGW%I>!Fm0A?W+VmSdZbwR)rZ6NFLy;?!Cq_g^37q-sU^qVD` z536WtX6;vA6YSLdoowM+bh`eIx=-~#ueouMGWLl3T0q?+%{2E#-M7^q1zxGzJZe{$ zA@?DLvz;jQCDD{lL3G&#RDh8y5>VxXIF*1t|K(8BVz3jlra3KAv>{Z+|Eo-uvK1`vYd_4Vp?j=TPnjI%uvneBE< z{`Oe#^8u|oz+RF+o+SQ#%$f1gi4C9KlL^P!hn=Tbl;f`M?NpAGA; zN6(SS$u1He4DVYl->tnspCx~80D)`BgETVCAH*DgR5ZQVXt284J=WBz3LX_p*G(AV z*L!H_;&&~rHwR!FEEFFqima~Bodd0NQKe*_P3f3K954F|*}_p1#)o+S{hMV!UDiGx z9DLQG5^K|q5VBLz!_!EQ7uD;GHuA{#3Qu*j=)=q{V6G5$f0lV{STmR`BY1fOJF;A{ z-C&O5O|1m<3fe_;WQ}2NQ?L$(JWz$CYbArLxjG1aHCke>g6Og;w1@fzJ=ZbT!r^B^ zJ0T|B_+k50{nh1-%d&AWk^D{W{}YJQc}aBX;El2y>oju`$?MPRQx0gF`v#7BgCDbzsGAhlfy~Hjk0t8L9s?L+q}hiEI4%ahl(#u@1!X~ ze&tXbVnb<%YJXvY*3==;>X^o9Z7KZPwrUO6_$K#J|}4K zJ?TW}lrm(UKXs(-VZCW-vyQScpNFv6z-? zK33B59B3!s>zmZ#cd9jqh(###m})Pwm|Ia-zz1GkPk;>QdpEn<&*8jLHb0_zKD)Lh z`fK(rIO6Z>47i>QCrnhwIw&GCdu}|2>e^_x*xis^6 zcs_5r{e|Z{9S2_26FK5XxI`GD>?eI>5R}qyhQghNWP%_SG16sO3^igWj?KL=B{Hj` z3a-K1l)HPUmVCr6Aw7oyrmNi1ATMaX@pVseCyqDz7TV0igWj+;!An}8K6N+;`@+1v z;Z!V7c!7B!`79BIC6NO)59;K)xOSYzm&>lID*6)G4Y7bW3rwJP9-3kCVbYT`HQV7p+tqnqT&g8$%QS%KwtQqw07f ze>2nH=sA_P-_x&YFaJBQywdke$0!Z~x-8guivpNSe z`{Mz`tuFt~aO`sO+v>7N<1B%z7SREqlTLI3<0Mb^jbtnzEKqBms^7WcZ9ec`hTdjw zv5W>|0301V>FvHF4Y}{iKR=z%{4vMgd_~oCJ!37-H#6?kh)+xWM1*Zo>M z!=TbdiArKl(asi6A9ms<9GWlCqKi|U0vkAG2AfoEnkQLSb&btB%cVF~FB~#91P6i* zPTmW^pk1In6W;{-Ltn-bGbb9ZL0P5G=)ii39pQ;*nMX6WjnJ!VAt2-;ea0f{;xnO! ziR;OTDfKw}7|ugd^awwSEC@wEm+4ojV>g(-i`!n)lPGXFH?YGMI1eo~FIt{yYB-Qa zA;pLGb3~&e%9Q0Lu6~=+dFw-he8J3R=6~4EPb7S0rx=qIqx5F>dJ+2Af=Rs%gUa+4 zb+JdMyx?eBokF%5G`51P_KFqO@&Ka3A)gon_!0k?m}EMrY`C9s_}z{QB0dD)?%uac z&8_ff-S*GT%vdTW*62Y-d8!2jvCq$PTC@q{to$I)5{G05LIMur3QUeM`BSnU-aD$2 z?fLH7pRJaum@DURI5tKzwkUF+awX-?kmp*obpyxg**!_Y5Gem{MZ&WM)I7ZJb885| zk-6z3)`^nKdQd?OHr7)0kpEd({Z7k6-aEdbD&2A=>TdJiglD!yL1ZgKp+U{Y?Jv{SDJlQJDCekE!EV5dSd8NxA ziG``~=A0VSk!|MCA5CE!)6e4Lvu=a;=41=DX{dN_a7X*Gv*CH7cHeSpOaQJlD=4F< z;YCEAE2Hp^iY^hT>w8pZ!e{Sg+FC8hevtp@VlhgMWpB?rq9@Ncvc$;IhS8#1wMd z2rflUiU#uZlz3%Xnhl3>Ya;@xu()6>Ny^Q(N;xx?&Ap&eSnUH2sZ%ot^UV?aKziNqKM$mY@pX!A8S#G@&*qSkCd zQWhoT!IzRbsqsr41-U0Z-*d<9KUGc4Zt?>>ly0p{zW$Hs`<47*qD%TPPi)6;teF{h z_C#X+_sWiG5Cb&_{w0Z3_1@Os-(dfv;JDvk<@^mMEtAHuk=0P(7eq#S@ao`1XZIF^ zXHK*xmhX#(I5{|PiuPW&ug>-wd!z;p(?{#ZAdZHCS}WbY9cWyEN88+RrWGkIlJ_`l zRLH7>Ka20qYf(3()R4vzIL}C^mz^}@KY>>KlIro`YYcnyd7)JBB!<=8^1xTTuw;s6 zlMyj+XPO0i7$i$?=h&=_Z6~ra;-qyh#&1|eV4-cFyVA zj0Zm-Ft?OpJ}?D2n-jl5o|gy%y+RK6bq&D*FzD^S4(utWZ29}9+d*a7=#92xiwXuO z9ciE=OpQPYwSKLqNUT?F&z%7);NldRGB`FkU4 z^f*@WXn2NHvR?ER>kP6;+~?!C*(147E-p8JjVyE)0L1&*}l>V%pNWGDKrx z>C_NB3S^wX148vmbgv+Du0sEr?q}G)8B&-=fFt%7SUGAubz5s&FEuyDji?I$+--NF zySd&;Q}B+eF)Y04ZT}u8iaswZib*~mgvN)6eJ~e~jXb3dLjr#mP z)`$m=$O@~TVJrO2t^||S?yzR7u3fkokBL?%M5$$WO*S3!t`Xnx%poeS;t4wrUAD;E z%gSVRRl z314#fKNeMTxLxA8ku8_b^tWwLUfR(>Z8?HImeqses`ZdGx@sLU2fJQ0Mdx`7vB_lh zf?EOAI-%E;SU-DbeOyR8&yf7oBTMOK%!NJfm_9?;3oFj<4}pOo{3vXccjSg6{*suE z=!PO;6%bpxzn<}+Ouz97D{r%bX8AZ+r@=6nZwc_I5*+TS&n_QZdXAQzSh>Ay-^7jK z9x>pxR<73$OqM~M$qhK<46Wtk69aMhQiT_FfiZ-7vd#-8SZw#rDRVpxB-_wm+GvZ^ zElXvKNQwcgllJHMDLfW0etfu2IR4*|I8GJ0%&Kg;~+Fs6CA&4L3(;yP?6izD|fW>aDOKb zPq`#dtBxOb{!7zQ6dig_J21JVxH4Y%USV#1_21e|(r$L+>oD9mZCSJRHsxEG&Z^IJ z<(r~cVQMz=EMk}j>4ZP9Wb0`^?u8Kv^RU%7FFlM#1Z^?*ff7d{{NA$wJneYUD{%$y zPFsyN?Ei2ro`UdBba2EsP9C~KT{$z?T7SWJ-?xh6mh`(Tbm6_VO{G%f2TqqGw_j30 zHDb z3KtBfH7C0ZcOb55w;cENTG@VOIl~8N(GpA7^HIh02a6Lm8+=Ye)*9=|p+2Jg|8YfO zqa9V+{R2f4=wt2oJUUonA{6jfS~=BM|r$*_`6Xgx3=^k6kY0i9BYR?G-Q? z6TFOWH`y=bKjHt$uyl5%nY!^kd^W0X17zz#!{hQ@;k>%x7{mRZB1)s@bW5BW%eCL5 zK|pa#x>+6X|IPUyO^fF!W7-DwnhaMdRg-+%HMzP^*u_06eQQvuEY~#Bq`hN02(2-{ujS)?2R_D6{_kMU&%8#6tr^2E+%#%x-w{h`3_DQ* zyLd#9K1lXnbG>f}Spy81*k+1$_XKRv^m${~hZi0Sf6jl?_s#EujUN*yK7mXn^X-P* z85CPn^EmJOpZ3a&_6z?wj`=lzB*4Cx3vk7FLMMF_E?3=7&BpsUsx6ViqoJPbW!3E_2`JYo6@AG^#B03tMBD z=h11dl0veXC%zaA*1>w0j=i%cZT45!=SI%qYg$exfg=o)XYp#r$`>}i%i@>Y?jIC> zImUlB|K5Sc#_MUe9`WxhlQCt~$R79VmN;Xb+M<{3o5<-u%&Dc~4H{tL>}?skQajnE zn=|a!luJtEVwzn}M8D-6w#H}cFJzfw<)wA8IKhcJcsZWUu+LTCN6@o{Jp2@;Wurd3 zkT{OpBWrBU7~p~SaEBBXJAo%Vqnp6tL)sot03Dn+$LtPk7pBD6z2O~xNqYvO@54@m zxYrLW@uVHcXxx_yI8Wd`Fg0~DwQ%?U1S*DxX;ki2`yqghIq+-Ui?urGV<5+5Scftz zA>Ne{#4>hfD%G}Jm_sloM_e!7v*(5|h)gn`o}|dKLXai0DdP}-jMebdf4EsSL$?6| zjNu5s@8ConnxlQF>i8)lMEX8gh?oG<^&oxD^Z&46knCpY6PL`IaffFp@bBqoJ3NxR zloZoHQR2-rlTbi1yWP&m6~DR>A77sbQ?E1kuQYACRmwU#=f#L{Kbbe+)PP{_}C2 z(bfz_T#s9B#s#*{Y);ofD~%1m$~lgH9QHiTT2E?YZUg7Gd^ww7eSg^NtdVJOoO5%f z{GA{l1V{1CxR0$}>XLi*@8RO?d-tC4JJELYRRW^`#bJ9zLtJ6c0-o*)77It0$5%PY ziFH?W9ZkEjuEvDvOGq7EBviOveJqfh3$Ey%@XO=MXdF}CSD;{J~I#EoEIozcv(hhC#_Gi(%X z3deH~ko@CJEhz>7aZ3qyEkT^~YV8@$685IJW+hJ7^yT3w4EK?l}abCKqUv!41tvRnWR zu*Xx)mQ-S15t~UV;hpZo#MnuS8$y=}hFFX3FG(3+K1woqyb-%3b0n@_CIQ$tQy}oQ zw*4H6)e*k0IXxU%$$scph_$zbJ!VK0B(c$f_K%=Th%MlXnh5B4b4{PsnPS6h;=}JS z;1juepwMg2IRO-)frAqN%+2zj4yhlHy~chJ`avcBY+>X5+BNDZonbJRJ-D1^HN=1P zZF5mmB;@TIo^GbEof}(jZGRVnsrzyLcC>@Y^JJj)$=tm-7vHaM&9ii<9}be%`Z)4) zaE(Wmf(v=*LDI#@#S#VyEO!$l}3$Cg@2L!iL}rPE)UVl_1C1BD=!Nb zvtecYv-^*36NRLz!|r8~o~33lVG**-SXN4ZW4U7Inn!m+Y>`wy?HK7?u2^#<<|ZL| zOiP@MMNp0P*0{OY63g4wbB?|ku&prv+;})4tp%9&Ob-FN(eJC?%r-yTj&-uGjxA81;zXla5IoO=-y1Y{7M+Vmw&(=!G{n82 z{Ty+58*`vLjd)#{5GV3>DuaApN>jJE)%vp_NmvU8iCtZ}uLurQwjn{XsTBRH`1%Ura?CJO+_$Ti#*@Z_b!|z*g@eZ37~ffT5BSGVe2W3ppRK zzl@*>Dk<|Uf`F>rll*OQRio^bYfbJQ>`}xENC8F$bfJ=1p2(7?WIycn?7(`&zrUsr zj^7#Sm^R3=fgvQOtoQEegXdLuckBprEwEIk`nT0tg6u zbs@~9iCdaJF3e<$p7irJ`=xYj&EYv??=TA7d`HVE{TX4yA?lS$ltqYZulvr=ZP)fQ zT8|lPdxsU%7oTM?fT70p3My)97TEm*Dz@DR0vY2$s+#|jI52~Sbq#)I>95vHtZ#+Z z_gT$p5_*5C_+yVC2?)??O?eXmzs(ff~0rztY6(Zt_B7DCFX!y zv{!Kb zdgFh$mRtIjrPDUg6kni)}2ZiY`^E2G;2-%d4T zehvgXE6b|<(^d;5c2o5sP-!7MoFPlIyoP1Xl${!LUhUfAl&WbFQ$P;AwnTsPfw9TQ zj`QPCqoAN5oepy$?E2y}h!D17-ARIi`twwEBK@HOrhyKXgX+*gs`onuE&lZie^p2D z)|Mzl$^=lVzKVC~oLe0cy~%u*EW?P}o}Z%{x>lx;57}k1-bWnf$)5y9DEV8e10e#@ ze^{%_YZp}Mk-AD)&WwfEDOH_<(y)^C#z>C-Iz6+?id zdZet^r{h~Do;D|9QuOJMxe`I2(7I_aHCr3$Yt7b(@cx$7wj*NhIgrJUyDz2oPC(l{ zD7-0ztggXXWgZ?*c0ayR_TI9%bkiAB&k4&O(&OSDE)yrq|;#)Pt~`A z2D`DMhxEhHk%el#T$$)0q6^F*l+Wa*KV`ZDlCKF#fLgj>EwSM-V+Oi6}P^%UcMUT zz&)5iz^80%dc!&RPCD+CwJB+JrpUH3oEt_S=_6S$?rcS!FZ`Y4Cu|Ep29djVd_Pab@RjKqnz0;*xWDwIbiJaXtX*0av1Wh)WMXK!Q#%g^&EK zUV=H*!yViaf(9bh)h4+V#as%Wdu7GxmoAnI3DM|Mk@T1`i?fk5~v9GTVNqGc5fzf1@dVx~a?^QQ4LAb$6$+p@$5h�LyuWM;uOh`D>QHx#KaG zQ(@N-pJDV~ICA#V?Par1ZCn=D3!R1r)-7sOX5($f>ORr6?n33~$Y?2PW^~`0F#Q8XQFv;{5*0t@aNMyii8$ejq8_wpgoIA9B!(gJ#BWxqh&T3!e|DXg z;;9L{Vjr@MXa5ZNkYG%N|ILBycYhfGYZIDCo>#|;CYPS#HnYN4FajOcQNBL>pe?|R7OOiga^MW~e<8%BQU7c|!}0*^`nS>My&J0Y1jSYP(Qnl;oMf#oN- z^ zrm$aC7xx>M8KTQd+p6a0fsac%3V9R5U<#ykk%|QgKeNeiKMY&*gt!NW$A{eqY;2dw zYcZltIq8b`>FlJzB#gBXAs_?G;5Y&AY%WP$vvRBNi%K}9=9cX=vGTy;-4rGQPISpCDC1?nQradFqY#qYG=QL6q;qA3HB!XO%&7>af$$B|w`S zwAUaM18czjq~-=2R)9&k7opjU|3hM^g`bM-27U~p6bmU#73#C4Mtu6i(`PJ$Jc{oG z943!IWAIc0Ll~faY)n24;T}8FoVg^FT?AkYqK^sesUIB|zRW_Zny2w0{_B}nTwLe_ z_rhNw|M2Kk*E}0C*!Xd2Kbw-yMJtOTT=icW#SH#I3VP0pqbouSU|??g$X|V(h%*%b zT)e>0_v?@krM^C;`1xne&M_2THZO^LogJI}xI|5T8g22e`2!S>Zg^IM2*v3wR^&2X z8blY}8Ju|dUtC#G&^i3;Opk4DH$dHIx3Kd(t{=)95^?t%M#`V`&d1YL=G&Y4#5{vN zIg>V%riSwM^4ejXGn7R(?yThnD9Cyo(Xl%G_8ob>*QEBl7$yn&0!t*7^LE;Dav*SK z1$`Y7KsflCHSh1RlX`7$nG2b0@u#Npgl(Sfo+C!glOz z4rHJ5Y8TrIge}YK@J*3msY7iLc-!nn`?LfM_g%Xnk=v3R{Jf}ALiSP*dGVPrvQ&B3 zR&Nbgi)#^Ls)9|muH{D6x<9!tMDu}8sF0BnzJ!yGCC81iC^9g*rCK)Z!|E@ z^$_-_!RvZumHH=P%HE_$*J5>6H*2_g2q<9-DdZj4&FiEgT7$46nK;6_g4Ua(Nhyfd zw5-a`J2j)eS)T?$b9x#{spr=A5@n1!xz42B%=DeA_CfM7mwxZ8z!kWoI5XMS0=ooK z){?oa!2o<1HtK=(bgs^{F|FxjDxI+_RV(0AXc*XVGDBY{f6|CY2KJZpCfr8x96JvG zk!Q@kqu&fCg=MMmAT|403J5|%*?999Tp7fIB1Xq0MyjlS1W9sbH8AKL9SkEB_q*?@ zjxCjGY&Yl3K61l9x_C?ENdPn*^9qkEs@FJv4lCY6Z|tqhk)nFL@{jw)Xc^ ztp&add6D8Ds?#HdUL$ubH7BmJjOp|0AOwSt`ATj)CPIncHyu>9ku~P)LnLE0W5dqp z+Pcw)lxT)FI_|5uO$uslMYvR3CV8uPec>M?+?xLzQp9qzLN&5a7KgrcTst}Mq5W^M zMyp(=TRra&#F5vjaz-7MrP)rq+ZrI1Ve0`dE8gws*5a-EE$d+4Sn6{kRVfSiSXk|f zn5u45ohD38?$*H%gCQ82!L|X#;S)#O&uF9)SdAieikE!ezt7>d>4e=-M`M)%-tA?! z-q2!V6+(H-4?E=p~5Y(z)kNXcXYqE#@}hQ(a$~y8o~-aKsX>>OR2^ zM>=yzxZ44)6tl3PO!ftQ|MU{m1b3mcRq1dcBz_$MwQK9KO+(i*rvE!A`a!UwKF=EV zZG4mX-oM9=J0Wc~U?fd@Rc&aK>Gm!wQCXu<_6CP|ZB?J~dF7iMqoNDzt9NIZhc9hj zyw2hPb4-Ch>*$|(#YuX!?UZ%>-KLYf)5fDwtW^DW%Rw7Jk0cClT3;)p)|`KdUY*fWyiA2F zu4!t|J;+>TEb|NPp2+NSDx;|6o~>{wct>ag3t)6nx}{|1T^;vVg53~HyoYySaoTYe ztSe6ZC7EAmj>2kHYdX_`)jAf)R55TJ?$^!pTWn#cFZhSAHmf`%;jSJ?kqJzpaJ?6# zcQ5$8D_RqXCWT7MJjCkm#e28_Pc61WlQ2b{geNSKWP}?=>Vqbl>-cgL>mYb;U{`LZ z{zNb#1bOdX016lx-i;fH3@45;kl zv|Dl45WnQNXB(g_Rl*&rgJ6SfL?uUI3?nuo!r^BlC#=1nLDek08_ly{{t-+# zPSsaA;ay^@dv2*e6zMwq$UEMxaSn&pb7+GR!mwa4r`Rfr!#_OkgLv@}U0 zeI{u*8n|+leoeS^SeW&tDe6HqOO|XU*?dM3Zk8d;qF)&~0OT3cI*2#{JavdGlAP?Q zmAq$+b7p5R05JI#-F7p$nvW;`zExZ*ux&OM8pSgUXTpSf{+<>}4SD2YZJQcgjaOx)f%| zVQ|1;eQ6zLkSd|`iIfosmr_m;)Mj4!>Uu~7hOl`pA0r@ZADEd|l!OU*l1WHuhCMYe z*}7)%;t)%98#C-cuz}b{hO97G*c6UeQb~(rUrd)62xmLXO!GojYpm-^I?Ly7o~5#R zFV)oeII>5LY9RUW<-!@6ys=7TS1MV`A(MHVH$rlM%g>4m)D*HGRxQdi2*#jIu;@qV z%3e*Dk@ocZba9&^(JtNf0L+iT2SwPCh|k%Z?lS;VMy?za!Bv1V)F6X>Lb4{NNkpDi zrlJM=I_}5!HX(UQWmbBIqs;N;pP$pvPi&tRm0Rxw`m@{~`nQvvv5?^U>J#jdrbtu| zZKvlQoQ@o7&7Wdzi8u@HmNSk+JOi>P(l*#n(vldunx6QtB{J^_!i}RY77xX2Q@+vX z*Q{&MWJXfFePCY}aJP75(YF#VwGVWb)qZ5R$mE4|4NEK{;GV=1F^I@7{MM&1L^9-B zS$UfEyZUFX@w(h=9)~01qmLch8S}mK?V_J@!lM_jlp+!t=SR>vJLFVWMV=f@?7D-> zkXkuvU}RB59FY)=HcWNbFz~kj^t#eIUGcPWAO7i2%`Vuy$aX61S!SU&gNq05!fA4r z4gol)*;L19z6IM59E&}a#v+3*3wAtII_BOX+y-D&zm;D5>}_*}*jbSmi93^a|LeB~ zrpb_Jun8NnMZkX63PAK`r?0oQ;b3L#0k}`hsCOxCMw4Mr?i(RTXC8x$ojmI7lfmII z%^~f$gZZ>*vb(@}n{y85jkB}Zhi_Wg97J?7cqu4BP-McLcJ98*+VH?s9oR|wUa|+= z$x$wzffA`&Z7ny2oSZ?E7s#~940AtBe8sk zIx+6`T7YiT7T7xS4$z%5edXfd!Gc80=cpMlEv~S|?3X8In;Bb3ywfCw zh-pk2c8_xguBN`yFQp?RleCA$Mz$DNUR?8LvvYUTM=3_tnpzIlCdR+=1-~x@!XxE{ zS;~+C$CnKBlY+-V&O6xlk+B{B;Vm+^8Lx&7zVYP75q2w%7s4cAADZ(=@U^k`X3eZ( zknEx8Z6wZ^K1_UoJQpFJUul5AmW0*fH9wab&58jjE|0^ea6}M z4z#h98VgiXYYu(XY)zeV!gch6U#<7k3iMw?WLQPU)3E<^aGv;=!3l^CkPkz zk6s3h1-Q;Q&(V}XGjqAd{Kx*yjuRuN#!frt)Au)qO@#*cTvA{g8QJm5sO5H|{?%1~ z|9wmTt_)lIBhourj%cehUW`6vYsk+5&Rb?!Q#?21?lre3<#x7)vi`GvY^aXhV^@6l zg2bI#W^zlmi8Ju1F95rk=Z&M5b0q*nLjCAzbwUi8lI+40$+{$QaIq&Y0Kuk33UFWP zePHbK6Z8fY3~^#ckhHP|l|v{3@)xjr0)8`;a1kJcn|y* zvIW)vA8zj5m6K{WS6@{|C!kK^Z4JQyLH^kTgqs_3ZwH{j-FqyqcKFap80sZ|OKLL8 zFYS(hdGdY6^9~NN6zCzId3@o8=)~8_lc3_9N`4$%evmp#b76UL&?#;uWVLV@G0|;52wgJ!4WjM@b&n0j;9cb+*lpCg#rbO$}QxKp| zwnu2|$RPl~uio*q62jiB!C)Xzi*{j8QDkbZ?e!d5QuFJ#{lE!T1tSKxibp6066nwp zjuD5mc7Cd}Kro&N9XF7lVtBPaDKDgM%PzOE{`pKVf@)kg&NdzrCw@Ix#^3cXXFIGE zrj{7wH=VRHq4NP~%-|I;uGrv2`_Gklz*clFNu!`cxj1+c4=nN19#cARF>SNn3*au= z2TTK{)zWOE z`78w&6U8~RXX$B?Ij#RHc5?RK3Cb3KbUiQdhMP=1B-ao1Fb|=gYjr+7SOEt}LE{J^ z)C1QzpI8?1)Bkzid=~HPvd#00TWkUn3;hPR2sQP}Ij)jih0|YEZ0tM@LDPrPQ#+{Dtkwz#+nZ*Fm}IT`J#BSq?S24~>TRZsm;RLr zg!|3umeGZ$zV*GOw7l@W`AZl-sYfTayg#=(czm+0{kB9P2hhxlM^KBUgNKzs`_64e z!V~Ek>vFrK*{$>H6^iyU@4+RC*H@nc3EcSQF!Q(dt(nWO&6_o8_q^|A+i%1T8mkIcNXuQ@iZMg|`?;4!jwv z=&BQT+?JoiMh6FcxVlcLkVDON;^)2fx+2Yu>}@Vdx*4Zh4_Li*1DsEVlo-Q42g%s- z^GZ-amMLMkNx<>(Wn_PqnWbHWN+xm&V*i?NLG;=o6a@OqR|w&_7T z1_C;iCAQQw|I0Kx&5=e~k^QBA`wAK2Nw4EOfPTT9p(xAjYB%xp8@22ArR!|U^|7RY4e}O;8NN47`?)fa?H4P*v@sSKxr8Kk7Wq`CnjcuSy66*p&SXc6Kq?K*^NMC38*0>R2Lyj9D(BK6cr212(cdSZ;i5( z{rUgV_2vOlmHYqrfTn2{rI?k=WP;#QT$R9u7`%#zO9O&wmU2@QF_lb>P~cF03dNKJ z5s_rLfC841i90ZLT@VpR1T|ck>jEPnj4;A5%*^rkdX_V2_xt;!jS$Y9^PJ~-zhCdy z@-emqaxs&h>`SfP+JRQ(kFb75bpw9-Mnre%j|TK*qlD#q1P}S7Wgi@YG*r)h7}LM` z7R@>BJJ}R+ZucNllIc*IpKke2}o=uw4x;0a?wE^k}QKUZKAMo)`nyZ}{|` zGyAPFx3nKO2C>&C@wGZl?K*i4tft~Fk=##-S+}tJ62s zL0NJn@`Dc^$}-lD#^#~q3YD;SY$qtMynJ5b zhRdse63X6)5DLGp80jm&{ZV*F|5uV`il;_Dm7Q9H#me%^{?7z`WEtjy8=*37(q(97 z$Z<*I);1Rc92fMKgFg!!-^Yu91`Ad$$y#$2ePA?YQ*m&5C?L*O06aySJ5ObAZI?gU zQuq*PuR@n^79y;wmJEjjtS!xqZ-3=WQc?yIGjrCu5Tu>bGYC3Br@cGIr(A?*M#vu8 zKhScO48ZAeiGQILR95q!b>*${X0)j&Qe)w$V>Rw28SGu7a?+5Izj~RqZFsIl=NuJ! z;l^4~Pg@5501Wkn~%y%(LSP1T>|1g-EY8u*vDz_r_F3!W-wfJ#KQ(-8JDU zwc-RxdCmbhQbZ{mc_0F@(Uxuv*jy5^2Z*zGMK(rf@37aECx>Xyr`nnm^8bNy zLFLR3$YHF^lRlZewO)<-g56fx%6DqqOp8mzp0G^F@rBgCG6H-qBF1A9JlnsLo{-Xn znND(~^wZ_FQ;=)=N&Tr^G!TwFZ2Uenym(NEkFnhOC84=l(smFef%ldS^6KmVySCgA zr8?tovx;&Y2>L(o*7E2JyMl9G&Vo2eItCjDu{!}t$gidNq;0h&kpH9rK^J$Cd=5_f zC*EDQhMHJup$hyIIWf=rR4|N{wa!E($pLS+ktN%PMVRj|p-h{}FW)Nbnf%~igs`rh zK5SHB+&;h%yar-)?;6XBweW1Q8;)MwU~NeKc~KE!Un~-FTuP}?xJ3RfO8JLIR;D* zG~R9f%w+9^O-tP6IGoo2>Rw#yRB`rxc&hd)j++OEFRI_WTvHIR(yvdCY386~71m3{ z1CuH~xnlu?Uv8zt0KtLNd_k~c>_oQxUU&MQF;BA;#fHL@-f75|k7cHMz`>?4lx=Lh zE086?q`w8na3&=&poamLVr00f0~IS=0lsx+t!d^FdPhEy@wo&n^xy`Te_wNV&LifZ zFo)@IPZ&ZBNGvFxj)^rCqUODs+O!AUkeemjO|&Rd?I8oV3U+ zvx<49*vPN2P#bPi+=#-sn`N2O&9e&)I&F&4IYNL)0wd8W5HcDqrwLRq(HteaYg~E$ z22Q$g!##NorxDq=YzuoQxxAfyrG-ov1?$cfrfAzKVvm!=9=x#UUP?P`A6TnA?2Os? zVyw^}4n-9Yh=kYa9MdNT&8Lu0g8$W31*Kar#zZ$Lhdru* zJz+=Vj{)`<69GNG>nQv&f^C<>PkrZhAN4D$QQ44%ryYJTUsDKf#A?Msw4bBB4n39C zuhicy+41dA! z76@xmD*aEUrsQ5t{Uxd0Dqj`Puw&_yT`ohl7vi~WLj9iYIp%i+4GN>EDoil{XH;>( z9#dej+o+4wA+qpu*w`E4&Jeb;gCqObDG*jaC-BEUM75g-oTQ~Ibs8KkP+cc`9cl=v zD35l84&-mr!GMcW7x&7S^uoiUjrE^N?7SY9H$tl#JhKwpTX&hs*KeBgn65a>5e z4T_P^xOm_Y0p!3BZZ_><*ii06=0gu$kA%iS7c;?$P#rjS?1#Yw<2S6B5&y<@r5g0l z!**n-%v6tX!XbA2_ z*$O^XX9V{`T}a}*kc(YuDUip^is|2-yU@YZkBLu$=!9W~cU1nk%eKr1L=1YL#paJ} zA`3Jdx#UOWw)1o!`)h2pNrWN$Lps{z{GSleD1SdE-q2CBbCwxY%aEiF9q#rtR*x+1 z7E|JoRBLKNW0SGlsc5(GN8Z&7E!Okz&ZY)jxMCaO1GKnUw7w`okHATZl%4G80*CXb z^A%PTylFcVng+gO2%Vi?jo>^&gP;dJ7nL8t*bl$@aAyU#S9-`=cJfmwJ*LT*^z4~7 zsJgEN?2V0eP>ZJ_IcE3XQFbQ*@Po3JcpM-O&|Kl%x_{CKomGx@i&YevI`CqSVW?rD z2dD{YQg+RrcF@`7L-wiN_B$JUE8v3?Dm6R&RlPanDIr83O81_bJAdV)Feja{J zLMW&kRmnb+i$h#4-&W5q=s4tcdi1H)-Y0Y6(PNqf784sz+c^c9glJEsfZ%L9@<0?< zOhf}*t=?oVkUcz9qfpzSJ8{|FVb*#Y^B#;Xb5hzTxD|^>2g_zg4E*(Xf2Xvl;y*&& z^%LhEG0&=uQ`|m}MKdbVOH+_pRwo?&F(%nz*TW@-2Y=;T#$SdV*b65lYwBOlgx8r; zPn4uKRg#DsgXy>ZE!Nxj;J*|vpL9uc44e#yyqIdwq}scPuMZBwqK`lOg4Wf1Da%Gu z>dIha7-DzTvLr&Gj-ESx;r*>;OHS7$t&G`*{{~}z#Lyk@h;K6?0OueaIO+6)YK)?A zWN%4m9SlckwBYFA2(@)CGP!JVIRIscd~{hT+5_}Ar5rutvaCE=+G}OCF6DaPSMN=T zx?w8HWd|S2k#gwt)S6m4sEaa1vB~3Oq+n7OxXKO~g@@X;Ixcn0LRcO)gMQZiH^gsS zgL%&g{{k1QQNrwdI&B#aAFPHT;+Msgla76Hswur@VEf=m8Nh4QFm%YUf-UH*^xOWbv&;~=0Rpp!5AF+MK~oZvawYxP8Vzbtb3xk!`41@(v(9lXCxxo+zpg-8A^-)mwL~ZYm+|_?cxPm5THG$ zCQ2YJkyi|3;PAS35xEw%zOPyyo5qQH?ZLI*4mx5|t?>}wC-M&7|L{C&SuaCfl&Znn zS)ohYjYP_HQ~9*#b%Ayn zw3t^^Pz?l?m~U6`EMjh31o)&n>d?<^8EgCe1aQDxELid=DDp_z8@m0S@YRV+)0&UU zlUCuacY>w-yL|9OgQejBz1=abzTyp(N+$HN3t5&^Wsk+dtdRI2~&9__& zHE!P3IoMIOX2n6prI1l&_~3&o2@^-wA3M(4$oJBU4A_v#K##zdV7t7E4SR1p0dISX zo%<{6)gOb7CRLo$&EM-9OK5+kd||k@XYX2Hwc=crt5EyK}#f_Cec3e^c+Hc z9=kxu0`h@jCuI&0cU17!H}{>{Gu0>$zZKF9d=I-1*)mm0s5g_XMi@1D7!6=2>w2^vzLK-0zMX*h!6Y@uzCT zH9;RAMQ;oJ&TDioP522Z?bwyvjAN`T{K*N9Y*)$LKe#wp`^BB3nBXf-pQ}t^c*2<6 zK1mPu4o^K=oR(`RVT{R=d8{)O2cqtBb;q)7=4fq;O?To6vmBqv98@}B4}R@YM_II~ zFtu2)i6O}YkhhwBIKC*U2Dw=*lmhuccIVv!)AH$XmhkB?oFu1^%W```F$4 z7&t(BGz3b+bfZ%~=%f0xzT+xyg97ZQhzVj=?f#j=UW~f6+;Rr{yh>THW2(4vO_Tgi zxh#5SvG$@NN`J<2<=%G;E^(p9)=qH4w02r)rF9)Kt0DvpZe$;8^H6svLh=Tbyif{c z*r(Qcrq!B+?P0(O$Hd$?jvc}7C$u5V?Two1lW8#yiLai@R>byJ{3R-y859Pvr^MPx zxp3Tet#ZHhQH@f$AqGDT5**aAxX60ecBkpKrbgwEeTrDQ!q=MF7hJ^AAO#tbbxCXr zl{M)RNwp4#!r*13Vzxz8IZ0SA(3nVDKdq|*x+gON1XI|_;M-V2H(037-xybGfiTf} zs%mF>Wu$lL&XZzuXY}7SD=$cK+sMK*EA&z70^c@mV3le)k`dd#DNb{h#dpPK!OEA! zArp8+vUHVg=cc((Yy2|sp0aIpDIwI)%UgD8ms_*yKd}Sr++bBJ{1CQl6EOLF3L0j& zjzNg+x8NN~jJsbDX~!GWkrP5&#d`>Vv!B7rXsLc?bK?a1+kb&Pi2+KuL-6lp8@^-< zM_*-H(N!^Rj!)mN%Y$mZdzv_YaF zG(oJ^-*kbfrE-{App2U=GB#3Qf;{KDHQ#LwsxyL*O!YRtJyVR@h`Zzp2tMe~qi7s~ z!4_>TuP>QFnp%Ka&oUk^EX7Wou6PU6ckgX9L6OVc7+-A6FvBusn}-?bJ-atq7t_QT zMaxarS`w)UTtcWFNG~*GMD4{lGmKTdrLhWNy~FIC`tF~k^PrgzW}U%McLG}%jvVz|0*AwT8qv-pV-Tu?Rn0uRvT=91)hC-W&)o~ zfFQ(JMy?ruVJ-C1ZN~RGI!YTiHa=1#K~nd^8)%{u1uftZ!jAb4GdeT>k*HsRUgS|x zHvvqUs{k%;y?!A9LkOjZ4habrDHdHhqa84FBBV9gjFWA#52$1L$cy|mY1bhci=VcP z-a$*Dqr_!yKu4EwYqxUTK6)K{7I zzJt=y2ScZe{C7Z5hJ6EIF%jkxtGam%bMnCeEzK?M$e<_XUGD0n(0Cl$lJR0Y=DfHzS9E(RQQGaYO7l*qb1O;oQ>oy_J1}g{jMPHl$B)-HpR0O+`CzA!Zmvb)Q zCrFwR*9@U@!L4+c@j>2YZf3<^?`>>MkTL2E5W!s|oD=kC4?dN3%gwDGVEjbvB~7q z&POc0U^y!>`yVH55{Dw^HuhV5PU_fYEX{{ci+~ZrKaBC_NOa(#O)2R^B|f>9E@&8h zd2?HSn{!cBPp|1XTY*75i!Ci5hmDtCKl^waK{tUvVvn<|6XTjE1iawMGSB73gy0oT zIt#b!**^%QI>vQefufij>ENwk1;F-L-4^2kC65S|v06iRYADRU#K+=)}v?+}RnnCix9+g$&pdHt0T-!=iV!gQn{?KMg-W?WSEmGYSqdrAoX~~ z3bm)=N*nb)FMZv#R3?LpU_*jbz|frM3$#> zF9F$nFloG()k4KTmp#dZUb@KLD}{*l^rf>?+`J)Q(JK6R0^#L)T~OPkL+O>SUAikcN}l8AN4MY7J9tq@BisP;$8e_Zs!q+()8#Q#NIwSMkc$ z)$Qj;7-H^QLmAXK=Rb)F9j3WSnDfc}7rZ&#FlS+w`9#h#Eux&RcY>?YSfyA_T7}t4 zO081mBJ<>azUAawTy6P_{)>b*mon>$&>isaS+qvfyUUIM;bCAWVHF_hx8yEwNPddqv2nK z^U)hAeeVzN5On$VpZ<5$fm=BCeG{yer;?()l}UBt`0SbUAxsWjcE(!2);dB`l?B)_ z2vMq8|Npp0bZ*XMQwBVauW{C@({7myK6=~6^#kgf$UhMKX^_rJOeK2OCeUr2ly|U& zUs?0vrHzEkP>c`|(&Q=0i`9KA?@<|eJy?c0^50lcnV5H9S%*OwA)NB;CvrPUbT-f= zkU>!`;QXM!?XZUfEsmf2Dg2a|v>42v6-+(WsSncLci1;vCar58UV&v z*mP-WO4$z|pt({|kY{B#rXHH@P7hg{S#S+brK$hj9x}oCws1N=iMse_RljghcXPqu z`{vY=#%WZz9?UTHSMeJ*M5?xroH*)d4krCb$hS{rNs4WJV(!F15K@(J%{EfKoAa%0 zKE1dDTu!HF>S-PEzU5EB4mKHS*su`DMhu`B^a!iDQEFqZj5Qp&wI|GeogKr%J6>kP zterB=7_D_XvL{`GbqBz!L6kD1G2O?j23_^#%G}$NX9hGl-(QlBVdB6TK0rwOK zU)J2lc7Bp@M*{2^8bqQ(C}4$uS)$%2>xyv+{_;=8w@8Xs>lM*r{;h+ObrAc#7-kZ5 z!m~L-RNNtq&NB~kfUGi=C@NP?A9d>u`S;k*;DYk~&=m3GB`st**)qi8_((Q)La#=J zPPz2#Iyr?p@DY@1h54WYj4GGkP9fdbi{t8|SU0>8+Yk=r=~C=W;FWH5JC>SvG+Dwn zsm7sEDF|RtA|aEZNrEo{<+ezk4t7p7xS96ffsSH`cD8mK#W7Xa;kFT-zU;}JT60@& z!{i=b8WkZBZYNeqe(qv$@kf^oH{3B3{k?>uD=O;>_HJ$F`O^Mu;SdZMAw9gc=SkbP zw5AK$km{v=}7@Wv@9G?ub;rqK%UL8UMkUdTIX$Tn*q?6krWPZDSD( z2;p(Ls48zzoSt@}Fv0VSQ1j_r(pTKk>rBOgCC0U2^pJd7KZ-)gIx~mXMVp*LItFu) zo35CtA;p+mq(_hXYeI=$X?mcT_&Hh({1M#kiP6nA|5gqN3}+@V$e%H3t?RH6_klgg zVSF#++>!T>?*Fi9BE2-F?GaW3sS>foQB)GZFERG@q0EW#@6boHRlqNbpZT$aX%|=9zC9^tV*CaJx#OI%JT8MJOMuWl^iI{`Hd!@*i%BKBHH@cA%r5Bd@$O->yRY#s4 zgCVi}YHYtpUuM@VJ@=CkF(HoxG*Rmd*$6Leq5J7xUm&Ge$!2p=7hSbHzL$X~r14?b z4VR%V%?9nlXe~qk2J>uE)-VRt5S~ioYCkb?xtI2S$z4u`VOl~~sxv~49i`c^CS^F& zW&^+ETk<{PJ$JyL=|m~BKsMyl5W_XKK7RYJd`MZfhyW(+2Lm)?*G$MmsB3uoi^L!2 zxEPhqa7aXQmg+`J(mc{&{f{3oo8B;T927&(YK4e$3La8~`M}Oj41Qm7WI9527-6c0 z?h`ioWQrue!_qXVkbzP*=(HEuGztCi?eT{Sjbjk9XoK$VAdO?BB`g8NIP9vY&p9Gb z3XOta%Eh=^q7%oi7iO6DyQ;vnTi1LqmF$`5j_A;MAQmI3K=~SyF=L`IYUJMgEZpoxdsvyleq4x>`Q)oFeV`Fmplj>M)1~LKDt!%J|mBJax#bL{psk*RQJ*#ny0V?~o0k zG*ziT#bN10WcQoBfg*GfAyr*yy3c6qAWZ0^$}5*LT{$f8z66Kb4xZxHeNqw zCoXpB+}s{BVL!GF#6gF<@=@2tGdjtVb9#sVcvwIGzaQ7?2dTw=yg$VEaoTxyr8+W&^ihf z@ZID?z8|RA9`%yHSUF&K*^yflGJ5f%04)wfP~tJCwe#hhf@VymHEN?`&^4LlQVJOz z8%Y!Y$Gmht=Af+$c0h`RG`w}3{hEEU`I)?2&3KVHn@d{;o}PRKe6B9W4*P|viP*;S z8VQ-H-yrG4k@86aReGfOIu5#vC@UH&c*KNA zG}6`{1@b_W>{w1MAv1#zc4+aXqB%ovH~!Qs${ zbg+v@h<}dy`4D+7T@dT040`Z9BlMyLJ~~52ASJLWX$jd#iI&o_y_kukD|;WGc`Lc7 z={aor#rkFbEKPcbhp+WI-XB1eO0R!rRfTGe{lYy2FqMM3#iH3pi%M>2YN`ISf`d#n1Q$==2 zYo+3QnC|Yx3a9u59NncAb;utvMveoSITkai<&HPqlO1*w3Y5M;sO$yX3@^HiAM&?z zhG3jnOc3d+Ku*w6izDwjN?|qY&4sk7P+fnT+B9;q5S62KF;>T=9*Xeg&qVATF}5X2F3r z7jhzF^+_tqiJ)D8JWrG~$ZjE(Pwo9`)#)y8r-A)4tl5`zTbDqh(${P=N7R6nzZ(_W)qjGIrX&l0FfMcgztk*0cK28KB{RM9^ z%@H#JPn#5XJwtK2)uKwdWLe|WH#*B$6S2dka+P=J{0zg|U1F2ZG%t0Hzgg4X8QgEu z%#Ab ziPIJ$dnr`+E!%d;enHQbu4O=vjLE;*7{q-doxJb%yc?S z!_yn(9vkwogAnA@2~L#h>k3TfPhWOL^S^aws34}CDFZBR;dn_XK9iNlc{1#-lAw#( z=G?-X!gXCQ>R?CY0L!&`GHQXp9B=)QRr-+T3NlzXSDKDQ-8UDF%FtmyC#4T-+!H!C z0tQ&_fx3ltU|`orIv0#6S|WPgSA4JZn)T0*E#TM8t}*3}oHX4t+6aPlpr-FhC*QjF zv;`(~;A<7vQ(5hSAJ$~Oh=_~#p2{kUbsDh3KQ7#*Bfffsji0qtZ$2?p>tY<2a6R7h zva0aARf&S|rghYZfcuzvy<;@q1+Aj~|CJ?1sx3!^LUJ1QJEqrJ z|7bI7@1lH4yd{1Nz_z)Hp@kBaV7580u(`O3cNbFr6$fnKbVO~`Sdsznc%}wK3em(+)1r`0@TaCR%H%$JJj$?> zZ32()QUx0eVPwN*Noixw)#a2ov7&s+gzc5@omRhvpNHio+s@*d4?Gr)Wp`Tj){!N= z`6Q2Wo}gz9J39nq{{NK!F676Dbo7bLidNq?s3WbH0ybe(A2>P-JAqG6_91qH?PRF| zBQX4XMO7Ktl=7B(@kYH5nRI4-uHNh9tt1!Eic#Pw)6*_*s%xhTB0?)EyNCfL$_j^O zS{aa=NF{1g;Yn2I6mPVPFhpNo4z2eYouV9q2GY&5zFpZk8-)aMu7&ZB=g%t7qa=0h zWMK+8TlipTl9hj@BS14@KM|`-TLTJOhDx#NIyK@d;2EM-J0{mf8*g<4FQE`vvJ+#K zxnnNHkFP9TlW$$y{e7T;>YvKyq>+*Idil*I%C?b|&EU_&+USPJCM@Nkh1LB0WdH18 z=REh;>TbDrs0FuVhjJWmQ+lRZzOjrPuU=alp*56L>w6e+7Ul9;!3AXAW5#~CAMas*XLK;en_G+D zF$8U&NG1J}=W`DVW=bmVS-DXqVoOG^%;`(bEvqcc;Sdt7*+qPu9(hM7S8%=x)5g;Z zjv$A>hns3UD}3a^fv}jHrGAS!Fy;SC_li1Vk1G)WX}AsrmI8ftc!#e7`0cE~6JfH1 z0J~R~Y`rL?)kxF@a6lJD+>d8&yp)coI7q5dT2Z@7um}VF&;)P`5tQ2mPx7T+>Qs=3 z(b?9^hU1=yw? zCa%1Eb!O!)H`tnG({en^OUdZ<3_cEih4_)ED2kVPRTrvUP<3x_a8{M89^G0qVE>8_ zXxfSdUNXF#6I5731iBdH4!qZsZ69G#7pJM;3OPDc)YJw_x-H7f4DvcD=rC#4I!coV zCSF-CgXa{r8p<0vo;$sv1ZTDhEQ>sO5VMu#1ADww0L#!j5rZXKJ8$1-bB5)O)H7m` z{|A4XF83b|BGahmWJDlLbx!U#FBm1ETz+1j2}o!B#W4fj7`GgQgK zow&BpD@bDgynL`uIxfB%fTc-~irWvr$DX~+xt1M|e^+p2$Ei9@NSlMwh!oU}5%R}N4t}Eb zL~f6IyQ{}Aj6#bSwRJYL?hdOeROxxCMmzi3656a~^l}wltxlrb4A-Z!-7D7^6FQ2P zT@0kbp5jKF|LFw2frPiy!(;rZ`;j_^na>R>MwFf;}*) zXEzIXTAUZI4`gT#f8TlxC8v)KhJ_~thd{u(u?e0Tw;XC!xZr8bg@m0zz%{JELFZ#| zxOtj`0~tR!0G~5!=^`Qd=wWM96*I-f52V5!ETe90+``Kwh}Hg_W)@EBVhrZpncH^Z ziBJ>N`L@9&S>yaKzF$%;4Qe6a49LH*SFu7hxex5W&(03$$L8UdAAM_wEzvrn5}|-v z9=l%zLm>{Nj)C)IB|jSd7&*`#Vkb$6Ky8>VqUX&RW@s@c$+yvFe888{In6Kxdfyq^ z5VA12Gwr$t8GDY|I2=M#+$?P&=6f*3W{a4buTdBwcqd5T96%(b4+6a~UJtC^823TW z%M4i{JE(DPW6PC_Z_t%CHj%eya)Q%li^Mb-y~EG=&N8sTNEZ(*hz{Q~65)W>pXFb% z9u%XBXZ()mK(t?$Y0oV{7o*f zQyM;NEi5m8ax1WAT28Y+*q3iki#S*Jr@_1~^T(B2FK!}9BfL=`>^Q_r-5J&yz-WBz z9G8w9l$f_o08XxSUx(d_>{wFJEaY8p_qVB5iDV8eTK8}_OF_w9CDsRSM#i&fDG<*c z^vn%pOJL`~z9?a6cTLOZg1@hn9h->h zdCS`6Q)hpUuWm-H8|320`gP`%Wmo#n z-+o|Er|&pHr|WJ|P=r8Z%y~AXf54=m{J|3=MV=_ZmDM1<*hUj2L1$myO7*s$ z*LJ*2w|OXOzkOTf9a<{pYY+MneslNDUYgwyFYTZ0ZHmgfQD(gS#j8$?G)ZV0pz9>8 zBul@n<}&hv@-YVW{_v8yF4pP$B`HT7I#7g@%e^KXo&5QL)JgYm{yKAIi%|BBgB60} z#()T|kq}v6PEveeys$>hf5u5ulJt=68hgH1enF(_v?#jLAf{j-GdrbCFsX}p zf;jLl;cF&?X#=keO6+Rb6DiOQl%O|2`(pW|%LbuO;51rUPaZn01NGh7=&vhgnS<4j z#9-(QqZH^+8jx*(GmJfi%4J3IA4mPVq-Im!6>RC{5I#!)qb&CFvf}%Lg!}P{zgRVG z3Kv)gn7#cgm#S`G78b~cs+wLk{bRU3sw&L}j8t^)AzXIo7f;Q(lbnf0S>T2ZZ;(AlVVI||zxZ4e=(g5y>k|dUwc?%C_b~iK7yyN&z3NNjkVG zacWiYmzuh#vL1@J63Nt!n)lflFgfW?iR@3)x6}{V1F-3tusi4ny8{YBGO8~QZNS&A zI#@jZ6UW7@e}X%o$Q<7ZRz<9pa6BiMzMb>DhBRPM;$SAdB1(J)&X6;#FQiNAlLqc6 zHkPFdK0kLLj@Qyk->DfJVtSx9-fobmmjp<3T~#l(2JQ zS@?6s6gO9ghdNxThE(V6`{85f)*WA)>#||Kt@M!pzUsEtSnkiZwBO3ykDa`omC#)I zj$e}g;=IR)Hod0h9BQ0q?RSg23xZV%pbD5@HEF`N=d;PD z^wi&NeeeKdNlNHRW-aDNZ~N70PB*DTty<&pK+X!gsJduY+*Cw7Fr|5jXrS_kSa z!~-A@r~~tnSGZUmL@!et3a#t3b1uHGSZYxJcYa0>Mww5e=SA*4E3>b()f@;FYrWeV zXvAzxiFY?oXKfxiogI!VJRxkEb=#Z|dd-e@mD`d>t=1!Zu%K8z0>`W{W+H010y~#9 z2Me%)`&G#BHXIk<$Za@_0J2gcr3U{0qVi^xsz%u8p1#t1Uh2u>v@jGjNu5_sN5Idl z?QF{E&X#=Ya7z2h%!+Ra^ENkS^csq509b)4$PcJh>lM{Qb)&kaW`%+pa{FLB^y)qT zs2o%DLmpd|0SQATYja)!o zf7Yol38#Htpzh3Mb}X@UiXH;|2=7p+Sq+9ia6vUz_=>+qBy8tg?YjF1ZIDKHRajD! z5PI{pphkuL6 zm>FtrbCXzUe9u|gF=Y8JVQjc_zwqMa95ql-IiCw{l6f+i410TH5~eyyoZ1jhVlXku z3|4&=J~14Tp)yA>rnD{~v{9vWfL(DoMl?9OE3N9zS_AsG&ssNZtS|~UVj9JhQ3ZBw zqsP8qlKPa>6^E^m|E{nYQ3=Q6MsDH?KkJRo zX2qSlUrtHPhW=Qu*M;~dJ=ZYI8<+vYY?pw2;O0f%%=PV%>D?4 zfi(F1e%mlC>L3DasSExl+8y-7l)6Tr#&D~}d)?1wU>R(}&UlVJeW>3?X6!&96-1A6 z>m9LC&<0!Uk=})n_#?B(;v$cqLG`X?L*x~Uc!xZ7dR8;rURKPK-^dP9Uk;pe(d|N45c4bb9lY{fG`j?!sR$mh0wMt8FOIS~ zkzv+~nLs5*iy^lOC9}7$+vDFzoaCCxwP$pbnnBWIRRb5U5vdF#Z~DBz88D2i%uA^E zFWU$Da%m^AZk4Pvh*T}d&H=skq_Y@BZ9eLR+8iUBDr;|dBRdCX9&%ZQQ0A(LVO(06T%L4&WbYb1}~gG-0F z90B+dC&99ZOI0G^Rg!-a`%@L%PqAx28skH^F)TG4eps0tD6%m~660?{QbXYz72QGu zVIF{mGm01zle{*Ra^%Mn5fZU{mgWZ>DIjP8FR*EeEPIt#)9hv4PT+ii5R_SGqt|hS znUB=ux0x3k-T4Er*l|9!#O5I|3mtbnvs<}xkbBNE_?)DWihp^tl>OFlPFrRi&;JJ| zuzu1LuBss~EBf?aS&kI5}0gXEl!9!%U`9+5A^?{{J zG0utpMZA79zt1YUHYwt+k~<0$;D_v9WXF6a4T}g`NZrJ<%;hy_2$B#xiW~ue7mKU# zY5O_h>>DJycP#OzH&pgHUlo9L*;3#Og6RB!gpyD1i;d%LY?N@d;NRTi>`n&E);$e>5Wo5v^U21{-=p!uzc}Hw1_Q?X(%;B$Kp}3~lBybgmxsReEM}lmiEXE+mL>*A{~jO&Sum?dE|1 zG?gBT5W5Xz7#@3Rj>@S&DU&$5qz~mkMYZyKv?!~EGmgb{Wo!D*U_XgM5Ocjr)Z3WB zhjD!5S@O)zVgnxV+yS&%OdVip4L3K(I^~`<8N8KkG1f2=KOg+rtC6ZAd0~6K&rAO> zc$NR&M3foqk-U1oGE!9thnK4=pO>XvsLH{KrX!-^);^cr>B^)N(P2qS)>q5IZ@x#y zW1)>|xLjb@4QO4xCOZ5ljbCjssSO|^ zeqJ8)zu?!P#5d~{$?ehF(_*dLZbA~f>FVLc@H8$xob1)|{MOs&pYfjOthmhL<%GB+ z9lM!&>%itHv|8ZsJ(ZPLgXuo=)Z(aPTV|1N$uBSo5CaCE$wA+U^`w{?3>)48YDyg68c>G}Z#Q$9U-n(J- z|MheCy+&?LEaQVCsdCj^n!iua|%)l+&66?ErWkc)NLnmMRG?8}Pa? zX7%*MGoGD4ED>ApF>}}}{?5qO8PL@s10>LUE$BHV8>L8S6lv9d9T5%Qj>;8x>Wwi}zN8*se zKgb+{p@8dw#V{AX)O-(PW26Gz7QA&xQc6MkCU>_|T$+lG@lVBqP;#xoi;<}!+`pBu zfSt6Nd=kzM3>hgQuAX_nvaC*{NaNhrfrA&SbuE%)WOPZPh>I2$V1Ti)f)S`v-M5V{4oOLX*ydsC zJMkf-z8JEh8LI4!ZfTY2r~>-^=`?m)<*G16!eAVu)O*)w`g<{t$SB5+4HU?X%+GS# zq&P>dj=+>A{#A-*^$jAe#570mtt;~-ICzxG^ODYd0zM?H!`Sh*A*VGXD&P3m;@EzR z1x=tS2=xMnNsL!IF&69fkr+PNdXo;#c!0-D^M$Afrsx9`a$41!6x-y>wPBn4q3eNd zI(w>_A6wKXKvO_x&f#pY1Vx9X&kzp?oEu7y2!ZjJmZtIyb9%_JcF68-Z}ep-!rXP*D0^3_$hiJL9pRL*kLu z#&fTVPdXj+fxUgwmpzP##7Xtrg_Z@;X%|aHak<@{7^6GeSfpuX zYE?V1YQQhl8Ymo4s2Ybxty_laM0j&0d+g4)x2=sFe9`al(zr|2Ki|LQ-pST(8?6&& z{bNZP9>MjE>vBK)p<&V47L$d*dG2k=`>0JkCQy zHz&HyYP?*tccfyho9ThpcxQ;v8FJGDlMGE;hQqI`-zkU_y{^TXHGASyc{v-jlKM2Z zzw(vaM0iiJH;L&Z>Q$|^m>bzh1orW+5&4=ik@;TrBz=LR zvrXGM=ZYP9lO<#l5_j2V z8C1}u2P-2%7jVT6L1**HTNf@glWm3Cxm>g9POfc`{U*7gRROjuY@$RTS1BhYk(l*J z6|^Kr$c0*+9Y4s2%|uU~^T4{C%89YjK?O*`H?oD(kdZvka-asnjJPtPd5>%lZ^7~DK(_G% z3ce6s0PhxEzEc)?X#YbYE(cR|NjM?2kiMMwp=LbnJ7RH42Ye>OJgG;%_lVEAAO0=6 zN@L$7#ULuU8uEG2aOIJ{Jj0$eQ)EG@u5X&v(Ei1Neo#rKGg7e>#-&2Cvyt7^r=~=8&Xz2IqYol7cCl`O?3C;*bPdOC2cw7 z%XbK6DEj_bZtuVlO#jhh=uBR1Q^ph?sW?fyKhD_Fdo#A=Mb5ymFQ{*+}u;vISurUF1lVTb|M&*C7LZJxl z+(`tHX~y48MHA}rxnE{(8A(mA3x+#teS%o@u3~9OO^CVfuwB|c(I9Butq)a=nY}vd z(rmT?q$~kb7?U@927rMX!@mY@^zM6xAkbI|QJ=PQy;*lg5v(juRm6;?J6)(mGDMM- z|L2=3y|Lt6p*SsXYuVU<{0EsZ@q&fH1#|73 zi7v&FAwgUQMHxKAGM~c)3?EHoP%gj`kQ-)5KqbK_8GH-E9phMFMRN8e?`(8ed2BFc2za@!(gE@VRsUXwF&~+#}{<^mnM^ zs-E4m`jx)T*A!8e(k$T>6T2l&O(**r`ra-BOyB;*+NA5g@~sa6Q(y(LnU*v}fK4w) zixDow$cVCh37=t+TSUmjrUe+U?$t$R)yFSorbC`fOf||qF((ei4(2mx*6?2UO09RwnM( zD)b&BPdcqlq|Qv4aLVdqW0_VqxV>}3tT)Z!S=g~dH#Nx{RT0n*pd(c^q5LMe1l$x` zQDUZ5_IE1nu5K-nZR#2R1hKpmpQK_?I`)9ubG#XQ&bd z%LcN(zXoXz-q;Jlk|3U+Ggx7qMNK9kX^8}yK=QD7_X$Lp5Uj~gpxg54^ZpvpaZw0{ z&Mqo|F!3qlKDKfPCt%4bYd_5Ri=U$~Q*W#xuov#Z5RJ-gLN%^A0VeKNDp6wLWEp37 zg$+NRcJbTfdFEST*HgY&YrKKx_&a~g!=+o7^%2gcAMNpKbSaKLSNro1V{|t)sn&9} zb;DK~)V$yHTC_Fik|>51^U;X3@&oX}ppvT2{zkod&nnrZOE98NT7o%BkSS;=kCYpM zLj^N?bod=nf2nKJg||Y)-sY^6QV925>z+U2+^sY1}7N@xm%1UXCpHsLV@oF{;lvp$zduM(#(Q^=$eZ!f#(o+@R7FDbyi}y_MINZya5>b;k{D(Kx+Y&6jGk&0?F^IFt#xSNMV6pegk^y_5rhouH{PB{O z%z-DewqA@b*T*L;)f&p$$l1p_e56s`5e;Ny6YcoM_vE1Az;^)AWU^%kn1eogxS?~B zUivq2PMe;4ZqO;6&5v}~HvUItoY%iUgOLwV{WEl&M7R8GM@9cPF?7YIPR?i&PQfRl zq+A2@?*#>GSw|cmR%)yr)j<%K!OyxbNN*o7U1|j0<7^^DBm(*Lfapjx>edQYZbC~8 zt>Edp%q;G@rS$Tr)garfnc%ob6eW$r$=mB3Y~|fr%12*$JkdSU|8U;o38@tA&Jr%-QQ z4D;RznM;4rTBAM@-txTMu3DD9N@wmkXB$t0)-!OZS)2zua~LA;Gi)$iO)xD{cPMl! z?C-3uU%@@?2+vuS)Ry&mNphV@@xyX*$|4vQr2FF52yr~8L4$V=zFNrshOHW{XVTbk z<7JxZO%XSWgmGffPT6dYWn+wpRmbtSRCGg6WA+q{ILay&Hqm`=B`uSBQC zLsbxwR=37_dybd5ZqcXps`cYJLZNI7D%|2eB>2%wLi9HWm{06Bb`%g0m#F*lH}vYA z{>9X-;wL(>Tfm(1VP%&28Qod5RfF=#qpcUeZQppGq)4GEVR&c8ix0ON*Z2>!)P#gD zaP>Op1k-64`7p?@VzjulX2o!1LszY*V>rUFkp%3|OGj{?sqlyZGZblC^SmpH-8{N$ zn|==HCSFh(ZlfG88;|mH#F6RkIEnu{IGfZ7a63w{^cCd$ya&uoXnU=PWrc&nIEkRw z8pZYOIIG<1SB6_pN+@=i?zbMas9Y{67FOw*G%kZ{AX58mbI~S*-&e9(6sq;blc1)% znq>~RdI|TBIw^`aU3@4M1(tu&C*;pPvLuG_($~^^`Krt&)p>+~{~P|>QY)MiRUDCY zGUoIOH$_y-?Cu!;j@H706z9gcu&XKL?%5dE=f+(M_hhc$hT;$pL4|5_J`TQC;-qUF zvRVWX<$cZ5Sf8}WeXS|)mAlqES{Oxo%(PxZU6kiq-B&pDk+S3U<;gAy&1X&{uI$$l z`vye{MG8!sKFl!SCR8e}cad*U^&8i1wXX9KPOY8wZ^e3$Z;U4~2@ZLZ-#$XZS-M>w z5}w?yE(yif5pQG7Py~nU@)U2JA~sqhjA;UpRu^4-Vd$=f$NW;7c;YVl)pUU1-Ef)b^w>{*IoUNp$CeEZ^wjnZn z{2S1n(2{9O$r~EhK}lTmX9oG8K_KvujDTh+YTM&weDt0x**OI7qyyfJ-o*KZ&_9@R zl+|f=Z*w^zloJfk?sw*`QsaYL9YG_Krd^FL^$;C9W$bm!eJG8aiCaDEu3pvDDr;9> z5Zf^%y@j~mu7Lt@|NK_~pr+Ksuy&DjO*uRd!Rmuu4`C zAbH{h`n+o0fkEU+tcMjV5@^{BgopquGs7ISI_8O;)DVe?cVs*y-BU7Od{Nlwxwles z>?TK;^K>WkZ%vlvWa`eC)_h8g*>aW$T&Wt?a5G&SnlM;nj=KIO7%)nKUTrO-^6d5NdoO)^BozqP`;o?i z_?ZR~UL1d-WRfgd!w~CE8$&WIs=6@W?JMioS3h$8YPy6LaD^knNWJtf7vu}8nt1#1 z(CN^w1c%NoEn;h3Vu)5KIqrRwwu4iDi&V5c>S()%9LQzc@LnPEdBnSKo0VGPc{oD` z4amG)ZaBdI~zyLdz^KlTHMzj^JZ>s7|!&C>qiNH@dQ6iM4?s zeI&1(@hW47wrb7$_ZdFZq?|0&G_T4rZElJDZL0j%_-w;6P3p-gjlSv7mwnR?TU3+E z6|;hds(%NW7iF$2k1|QG*o`{_AD)uCzC5&o;7}kvz>V)lXBgF6xu3+sDnKM?cM{h34VKtsawjdbEGk@_HgnMu-x1Y$#a}+!J`@eG& z8SbMrOu}0J=C#es7*fi0?Kxz1+H=k+LxrTt5G(n(Pd?O0Co0Kk>SUz?&NE3 z4o02$VF&OjKyrSDQ3a_9$Re7-<7iQ2-Jw*|(?2aa2M zGfEqqPY25a$7lznDVo8EU`@%?Ght?T)Xj>G`??#>ZhEx+C({+>kz1?7$Ob{scfdk{ z<&vFDPp>H)G-p7bF#=N(fd=^}ILPS-<6-{ff%`nKQU(B+{p?q=GJ2XeqdL2f+_Qa>zszeDe5Q-tjMI9^zsR_(1jRY< zc=DA)Fzal0v9!sP#}fQx%=Fgi_EAAh6InWJ^*1}n!`gW^{tHNP=gOOpjf|$$h|E2g z-}JuwIqIHe7zuA#%w3izhbe9@HoW=u#!z?d7}6IX!>K$bG9aqB>0D^(+oL6SxPU)v zr|7~p$41f$F=aMpD%bz$#~CIh%?cYh0V2tWK@z>OWyj$e-Y3c^)I(FWR+00UzcYrj zy(HWrkzi(6Sv1G)dMF^#a0XYB-^aNPV#Xscf9~jjZxG5ye?c zgKXSW*=*!z#ymMIcxuhsN7q&*7`-ah=H1z?+OaVPy|u=>3GZjjUzL{(XIxUmnzcbD zO!ga`<~^0g-a8Qe?;&Bs_I`X;b?(Q-yO!5zty9y4-hKSLC%B~Qj_L8?TtbS3;M+2kWdom_1jE}~E9BY!`q(c`v zsHb$DNSu;XvC&1q(`xzYLv=NYrmK33A>OT|CXLAbYuKD}6b_C;{ z$o1$usARxSNVJRQ(M_AzxI{`Ep$4Y!SHI_|Kd89A@Af#yGf(IYggfT2hJ}4*t>=z9 zQz5Y@hBIT=$<1g~gLu+Tk3UJij#K3pXG5R|;X1Iuaz|#llS7=7VY~Cu@IRkvo9!;QdE$)$5Q z&}3pmXHRMaW}L@1C(>~yb>0B2jdA#A@Vsg~<;C-0bF#4lJe$H+bEIk?BKZ6>%zmB5 zI~_!(eZ>0JO3T_nvd>BT-h_LE=1CG!6;a;nO3?`C(ZiVZb>4uNO8Rf^dGnB0{-{fI zL!h`yU{&Hxjr`8+Dm$ki^C~@`IiTs}p$ak zhel}T5gLq?!Ur9%vPf>7HJK>rIYSWIqkpOzLP8s7w&w}e7UJjlFm)eZjFpPsAA?di zD_(4OGp)!G?UdxakEoxla{8WmA!MhzY!&D0TC$j-*p_sVj*lsN_j*wsX%rW@CJ#?) z_C4N$jes^;tkc3Sl83!)yDNKt!3SOM#3BM=U<21re_G`Gr3}DoKlr3N@Y-J95XcUC zHfI|;ORde?U&))hpef%2v+Y z5~X%2Gu~0v&z#$FA_ZsDyr%1XL}x#p%>M~1FbmE}hi0p`FNbCZm;L(4NZdfyy)tFn za(l%|&m@OZ{q=*)ODwP0ZPMl2+Sx}J<8mY=5}O_5jVzK)|7}T=LJ@MhWI^J0EisLC z;YkQL+GrtCuZ9?bM9u-cJ4&}qaJs7to71l?f>HCB)_)r|sG8YWvfrnXM@*-`RGi); zo9T!8sLTUPsvZ-KC$}Hnzu~@MvN_*VSiL*@uXMlUx*vgXI?B{5;>=GXO)2H;HOH3k zx&DTFle_x!6sziTtbBl8eP5@V;VQ2O$5?Y8mNp+09v2!9YxebvX?GiyGkAG5JaUSa zclU6K#_v7RbG?H1%+a@`Q(Y6Ylec^o0~BEEMsy+!;I*ieKQ3A^Qnjkh0XO!88)8S^ z6Z6Jz?zQesEiwix{soO)yZQ`DgGHPA3S-u^=evQUxpgDex>O!Ib8E_j{Wpx-c>mQ` z6(^xV>o1#lZ$cL6TAzQcs9B}H|6#$7S;avm=^N^@TK`>WZqOhOq*`=Ww=}w?8Bf;Z zF18&pg*5sZPt9?@{ZU2aanE0)ccDaC5Yagi<=LpAV*4fh*Q*DFtdFPVSogn~;qc2t zvV6!nK^Qdf6C$m%?^_WXtovD?3RK;Th7kAw?cJf3fl9PM7WmH?qgAQUuR&u&I&LDe z=H^t-O9iXhB1k%On1U=f_8dNAyBJx(Tmk?p}UICrQmnMy2BP)@f`(%E% zu^L)qa4Y;QN5Rf45Nk;2a!FAXc!n|?M^sF2 zIaBN9n_xWR^9nym`FA2GjchA9e+$nbyDz)Lwl839rZ?I7+dLh>FR{94i z@vc1edzzv?TQJ^{`L)j%PR_xdcM9|!@c1w(av7PH^QR&xXP%>2c|@x+@0Ur8-5xAU zatah)l6Ul{#2l0;pr&AL1<+@uC({P}7X=GQsKTq@0CTR$h|h*f287uuGGQKuEU&{a2`&Oj5w54247|5z672z=S9^NLM*zB zU;l~`>h<~{>%sTbwLfS(UJpz6yNN9$94z)Bf~WPXP-|s;qTV`OKEWG_4~t+^2l(ZA zVu4q#pu7K6HaM$E{^UPO;s(*R44GrAiizfOZG5>(3mqgAuhOCjP`XZ34@mBb(}0X5 zS!H(F{?YN1=EMWSALEpnj^DYNmZ;A2BJ`va><}bT)mPJD^F(O~c}}QJUy02U7l|5+ z0ysruAQ{yhZG<0sAtn1rz`)Wq&6|D)^6 z1EMO|{|DDBi&E6gg)$UeiWC(WVlY!gML=*#OKzAVxR9aZ#BwNanRs&)R8o>5Kv_)5 z#4VP&rG$t$D!2=iA|NV^Fu*X(%<1=emos9&`~4$L1!m@)_q^}(e4fvO?uz;1kVbtz zMcs!i#J;dA01TfPX&%QG47zLlXZ#*%rP&`G(J_k#( zXZzPPHIcN>6Gu|zerv+Cx*)walzi=_{Cz!Vc?)PBhE!AeD;6O%GmFgRZadv zzb-*kRCiqIGu=_UJ**m4_&{12!j7{HLBcz}q?G77P@-|hQa7+|j!-I01ut?$d@rF! zYyWfcstu~925bWB7ut5bXB5LNReP4*yPt=;17(TL#{u68xs{fK3I7sn z!$E3M*#a1wE*dl0sl$8p9mgK*dTYAs6r?9{3s7s0Wgb!el`_+f`O5ZwYr0X8V8l6n zzHWTmgn& z5AqB_-oit_hW*eFU1$?ZZXy{wb2SzQa|oEuAlKJh56#9Y`-3uXnYK1;?{E@L>~od> z{QcDzO*=nSKYh`gH7i&qV^}|*1!7j#@lWzU*=7dW2Anq=sJ6CTjKF2Q0}w;tYh?#3 z9k%p2cj_G&FQJyQ$;QnoZFctoMZtABW#G`{But0<^Adzz9wL(Bv@EtdhJbcPALebx zoXsbT@A+|Bc#pGLoaMlOr=Hn zmssODCINhty{3XwzTL!)#Yr_EEw#f>a>uJx9C#_}k74h)MQ>298@emW(Gk_|5V zxXpR`XFTP_Ap46Li_9N%YWqpyjT7Uy4QN^!TlZbD>WTW1zt}N3vlg`@=fND@?*#yI z?`-*B8|$?0lyyHAltF-;ypDgd;D?1K~H7l?4?iB z*oF2DE*W;CZHv)85DBYoqb6tQg)R2qlW2D0dx+;z;%vj&rPmp9P*#e$l}bkB;%q-~ z969BeJ>5U)?M;6k!p*Yl>_x+jwCIZV!!}BvWUMsPpGhR&(JEr1qiqJo$Kvs>qi_xP z;7jKXr?u>~{fF>ay1CqeVK&cTv5r2Ngk4E?oAm4!MxTVcJ!cdEQzp}SpfmI)VH#*1 zf3nrVX9_aj?O||;oQ^XF@pT~aG-e7GhwLnjeeu|WM*^NGC!1B$o9Z&#D=04T-lyPD z9hAT$@*@bKFUe3qU!^OXiz5fq*b{Jydi8E6JgsC4Yo9}1G>PyvMd?I3311wp+Oh^R z`J-fQOiIW=PH6Aau9Qn$?%J4wp<$R?nQQCa(u9WV6J{@CKaGi z>QoIsBzYBSF)&4-)esJuJ~59k-qHhy>AG*Zk_1>}-jBaODgVaF$_8wkM?6%0ibrT# zqmN%diK4K#xMpnW@g6?+G_&ImD}K4-Jvj5?WBaF1{tPV(-|F@ODpx@9$%wXSyVX$+ zFf;)g)>TeKc!ZS1+OoPe#V11@_YP9G_SQZfUgXkXzhZ0IozsMR3U+b$UNWA4DBY6x z*?|ULR4zJl?e{_5ROVoGagTi!`zHx?Dq#s}PjT&{@`*T=k*#E2KqqeC(LC65+h*=)sWC2-mDr4MEvoU+n_9AlsY*@t z&}BZNKRuS)ysv@Vyx#MMq)bd(uI|`3M|?e{Cm){i9TJr0LOkP}%R_zM=0t`au@81x zF+a?d2ss&isn(I|rXsHeB4s6I)oxTa4yhRQ%9*{(#ezNR%1=(v;pcoJQqBP&xs5*y zgxg2aLC>l!Wn1gymGyY2Z+$A(yvhT#*w@j-)a50^*rP?x988Xkl*PzaIxO|B299hu zq!UB=eG&58EA;)y_=KRB8?<`8y`^=At81XX+8)f&$qtO|aAgMi6%foO_ChgF;;9HN zFpC6~1kmd`ld~aablX9<7`8F7l|~RV9>>L}C93AL7j%|Ky&M>uu6)G)M^5=Ua``vI z$6WCqQ@HHfP*bK&RgkeT7=pchn3SWE!5j%@pebDftESavToCezS_6Z^-C(x4o-(Y$ zEq4A>veB2bRx34j4ARBe+{fu#EVtqd3SkGqtuGhWJ@)9$(ebkzdj}Tp=f~^Z55Em; z9UE+^j!WMBmE-N7M2jc)z3}Y=UT2#Cq??HhP|f^m{ABWD32N}@PtbLV;wU+Ubj{j% z9z9g4Tg>=&+AK$nbMRFZF7rY7)l0p_fMp9WHr<%;O0hm)HQus*UYeMUK?Hcm8dKoY zwMaBvgHylau-yDozhz`vMgnQ}_UgFYZD;BXT;plxQMTkkpbFb5uhWPnkG{rL?qLHNRR^FMDBEYGwbb0)o*$7B}W!LX)G5v3)K}iflqGq zS2tJn|66wQw20CUOag=j%=_UZ*N(sNPGp+agIjTg2Ab5@V@H=}z-X&l86T%S|^kjG?Of_uMsY>-X<)`YGQ2oohg1BR5XGJj1nPLYZcd zc>OMva~}2|oy9bfi)5zn8>SDeX-ft1N2w{UoEtM?A}LD~1g!wK7P{pVjtO^5#Z!R} zz@w{Zutdlkn@E4=s{2&`TO_e>%Vf{Y3>nw$&6lh^NLNo>abR#S`9NImn#o!Vv=@@* zn@k0XZINYw#Z!)0Y5L%VG@n0KRFfWApvNIX`W+0lD1D1JXs&k^%N8AIzb|17uO#bB zYdlA(q8PN;=gwP+WhYl|zcZONUn#x#Sd{`LNd@?*BxabtGI@0dOJJ^|wKca|oH)b~ z`ywY#@WTaP7MLrP`sl9UTO{v;-NE9%5(K&Vhj08_=AI#Tr1T^dQ>$&q|FGt9z@WPe z_m2+s-kH(y@MO&7?AGNvO^vr+soRj!m16LzNp+t@mS@7EyQ-+RrTQQ^waNi|TuY0W z$ej2!pXsRC*DM@{?`ZFv+C0RH*85+LJ|;w%RINh8M$5=mvQ#1&U~~mLywhU5Q`X30 zI3j4?GL}2h*~}`bOHN>rITV;TyJ3K^IFg0m6r-ONUK1j)t$>{BP|; zoVenW;hWR-?kk4|o8EU~`23v0K%KKiKW&@cFACNAHqp*I46LkmU}cS(Jz{8BhqF(| zYLwYWV+m==?`&Axckup*C>-TOp)R2~!xVT>=F%`9p{J!SMKk8Ztll<3J+z&#!Y;47 zhShy9cu3))tI4`#wXh;c3U6N)kL@pr{eI6MQL z$32AVnoW)p=$(?-r)U#l0l-3=88YC*PKP7kZ=qi(k*HFoi69G(BA&g=OWxXKB_7gL zYhnljlw`gQcO<5yXbBSVl`tBw-t>GV9!I3Wm5jJ$oj5XlEYoEQ>efJRz9^&ASx>9} ze!ee~&n0GEr|-qwa}*!rQa@#18;TO`W;h05HKKpwP>clSbo!nC%Wj(?oUe$FLxpaH zM4w8`e9-FCe!x1VWf&0Q195Mmq%VUiGV zcTb1!Cd#$NjaXk^{+46zbqOYQ<2pPX0g~UJ!${scaX*O#ZcR2?hQ-Da!V>`mq|M`* z1dIXmJZXbXf;laVspZZ%5bxt>02(iJHgFq$Pa?R)NO)6nTrA$~W4T zq;J}OA^H6uH(pw0b1T;*F4oJ{`##;f z#AjFq=lrFq20^bdED_vs3O`))Zm2cxGmjK`Ga~cq)vT`hajR$p3@PD3qx}+SBij;0 z6PUUkDB6p`*m=v7+xuw6gRbJFG!6SbrrK@e$nS0v2Guh zeh}Diw&W&v`a}f7p~DM`TV0)7Ke%wofhst^;6huqI6%bZ!aJ}K@PHvt_ex6T9bD?vN5H*{#|*5s1%P|j;aoNhD;1ptNys z0fG9epxsbLn$J3M>Hn*Fg++E~fY*1|+_t8qFkKS)j66BN4*Y#B2YH2XRBj&%{jV+e==+6(gNk+J~g zk(gJ*B~!%Q{V3YF*=$#)azHDIhRzOA;ShhBSPc9~zXbm^K25}UdojmED8GgSi(O`8 zmiRUYo}_);svl1?1>n;L;1R}fk?&?WVPUu8(UV}7#G(_@a(dRtunvkAVdHdxn8Ox~ zemfh`Hwg$on|fvt=K!{Bh6=Ft{85OQ05e;1W-*+ZKYkl*f}{Z`2q?^Xe>avtGdia` zMmDHx8zeLg=JLZOvXSKD_nJ>_bjzLgYc|X2*LO0|W~nwWEu)9jfW~`zRc#wadIbBs zBw5DU5m28bJ@VS{`=<_tAM4i`{#+fTxG#rCxsDRkA~~E*FS3ZsQKGzJO$Ts-q9-x6_#8&@bWV_EIl0&srefsLI=00p9vR z@}e}g5R#-cQY9cj-7A}Sm2K>;E{;pUBA^cc$|t}4E&D>rJ5g|c;1Si=68^-pN)Kd( z@;*krztipX{vl0xLgrStq`oU$6CqhjcF><25&2zebN&;gGSuXxs|@&=IdjPt@eIh{ z{DU(0wEuwph8Lsr)nb8rh&e0j$iBLxp#|wXU&udns<==moJB5fNy+B2IsDRgdN*BhF37x_&3_)>im{RZn!jQu>iKc7xxn+3pWi}ic zDwqqNfVo)9XA|syG^r{pVnV#8&y*~@PEORfN&%Y)5tjmS&~9&e}5H` z3>_$Nau_u$m>Mg2!ltTCs%#_!doZj{{M8bcXduBeJeO5&kK!#X1oq=c09@(oD6|Ff7X#O#SzKxT3vaD z44ov>3Wv$b{>I3uuoxJ+5PCj%c~T}|m*{#p`E0G5o0`Z}P_e_ zH;eb-pMP$vhrE!mo2+qLq$HBjP9K<+L^$CLYR|g*^pt-$H{_e zJ?unSs$eL%M&g`%+gPrA&CAiOZ+JZIGG8zBL@H-`1FizYCmJR;xD!YS-+YA-3RAk$ zrNc?rJ0xo#x9v)u?Il!bQ{tQ$f^ZyhlX&s~VU^Wwz;F2eU{8lwtOd-)(bfh2Q7m?6 z*Z@9pm}5r48k^_bl6|}0le`XgpI^RNU2{lyF`72#Y*A^)2}ct#;U~;PVnlJ~!h@aD zO*iwae%t`)YeP)Q4P^nMhWkizC{Vy4_{3ahk;4{US>1FJ-h)X2YEZM$@y7o6%=k%c z$MC|CoH+e0z@^*gY;F!THL9SkeN=c}UH2dDp2grzlP8~ebI)g%=zhjQz+3?1ob87Y zRO>#gqKlt{Q6VH~hSD%X>oYdyxyiO@Y|1o6DLLyEyC=a(Pj2` z6C>`S)IA2mW+Acs&*B$%2h0)KsDf)9Ar^ArJ`LISW2>AezHTrDW7F?m#q7=>MX16o zpq~wwlhu|?1STLg>(@;}> z5P@^qDdY**le2|jvp~Oucn34q;s5xZIxb5q{AVL=&VM8(%&afS=0n>l97RAW3;ehr zfq&qZiWLo(7!~nWQ1g+aNiK+&=k7A(4ii=E{8zmC>3?;eD8NbIXS{FGfW#BZ!be*M z{ncD&yifa^pLfeDoNj5%B^jLb=KqC~KU{QYLTD-z)4u*6D}ViYgoYIxe5Yd{h#X*CB`|6Vub$ZLW2j~3W3srm|s&8#Ahhi9s!NHbU%50E+p)OX zRj+K0mrRotCi#sKupI2^=4k|XVJ{^L-;+}dWoc1OrVTtPXt~QR+4OI zlU=-^aj(kte`)U8bjwvZgA_8D7mNcAra{y+%Gq`0=KB4Ni$SBRyilu{LGa~cU?8~E zreeWk_?Drpqd%S9YKe9;(SdK)D>#G2w`=w@=;|;!Ww;`L9N(q*b}9%!N%e8x%bE83 zkoU$hftB)aW}w;X0GmS@kf7-!O#XMF|Ex6a+P)FJLGL)r9~zKbpHUVX5OBBjaa<~H$=FBG%3 z8rpw|TC*3I4ac zf_M+l?vI)@f2GaC#G9EWQ)Uw2e#h(jEJ1#Uv)^lsMur*V%IpgJ|)@b^zXtOvg4oE zuNHr)vs|(oH)pNWtrmZ-vxp&0tS!IXja#mf7FTj$iDPM699Hl~#5nT7vRfTojP{Sp z?Z0P86U7;@0X~P!lvlJ zgL#e!b{bazF_;3;FjKx> zKA@(HnVk|c{xhHZ$+Y$oYSvVnZc)I)<0Xd6jbbcul%>YPWx^X@Ga}f&no~*S){blj zOCl;)w*0eY0rrmSB~JW-k_D1=L>JRxOSZkPVmCG*9Py}l>Ez4@OPbpzi%AE5{IZs} z)tNR|iG$8HN3t3h+-EC1i+Gix_fy8jrhjs+Mb8V@1qx)hv&1sYHV(_hoe65S3V_gP&x&fl_QO~pUwwyQ|r%b;8 z4)gyB=);WXz(*6Cu;mYK8ctA{y_v?G8b#8G4uxR(GRZX4zIpY$&0F2Pa3fmAJgK`e ze!9beKE{MQjct_+%sku>lcvX2ndg#2`Z(V&8(U{u)Ug{=za68H0I;$->SE;mU3L1E z%`4}cvWfusow;wNxY!xfX4N!b_pWL`zqO6iv-jBnAo`nuukAfUiIT_q8^G&LEIG~zz51*Ad zf$)O6QqvZ8I*H_ovnUO4q)o!_mJXC#%iHB=FmC3W9^TN%To}j%h?Wb>{oUHt1s=Ak z+c+PRfWl4;Z>E~|27e(YJ-gMJcK~EMkQs9x)%0CB4>JL3`q3{w_@7~q3yA-p7jtQ{ zN|xu;y26r8*s=Ow4|ZA@7mP@#{fyVE!V=pybyVyDgNf~q5^l>eyeu2CH`sm9G{Zbs z_-p(kkM8clv(z;^<=?w7x&TqB*-*KclB%!Jgp7>J%Fl5LwYbiw&Er*tuW{M3>m7|p zV!f3B{6aeJANamW;cpf#xMWmu+T(7MeSy4gr@4Mt49F@CHPnKM*aYr=eJegR(a211 zhC3^ub>OhnS|SW)J@R*C@1!3OebpGb=Z@*ldaoD%yG!xQ^xDW^yCuR;LgsxbAQhZh ztgkhm&bw8v1>Lrq&n#CTWSv}+VNr70`{>@;yJz+KS(7pUp|K61W)IdE_cGN-ot|u1 zH&RNCwKtq?0I%sn(=?b9@GwSsX`o}7ezMp&a<}H4$Z^Ftdvfm)X@ zEEBK#F;I_Q@LXAdu;R2nD#y7gOF}EO>!Fy7KxprC$2&dhh^7RcFbG2%9pLW^vT6{& zw46y#gjaMFglRu+Ft~91&Jsp7vGnFZ3HUwblYoGrr0C9x8vu%!DwX+Ou+MGbs0`Mr zb|tVx2`?I{A4ZTn5A(>~{DJZ4qw@sU?Ny=J?rd|BLwHnjaLnmLI>1BzulW>@uip%O z3<%zqlr`s(p)y~1anpJAZ(74OTFMZAR(4(oT$p?xUE&fU9rqG$XvuGmnO{1fX(Woe zt82EKj$muF1|Oqa?CP`?63>V-Pw62s<*2fSn3JII0Q6w@i7mNGU7e!JuVdu_AJ(kN z(0Btg!Hx9AB7q|g9*|>GWKHA41g#d66%Ka3f6IQuQ&?=uzk;)> z!VFGmj~^jN{>i7+b&&k8$ypKsgFBupFK!#&oUtuI8?!p)+z$9$p{T$VIOyr@ZF_q{YI&^$^=DiK?a8*Z8;(2`ZJ!+*6?D1!x&c8Y4#9$X=;P0wG(^2WslP^A7 zN1@jfY(5QL%YFr*Kfw1 zqHx2&1d!$!dPrJozRu}!$5|G~o-?xrhiJ8r@h`b9JD5~sew`zf_K>tQ$}u6o%=d#2 zp0%3XO+S``{@I542m8^wf*r<}8Q{TvLRe$_8k}MmN_1@IRMXk)iFMmdi^1P(~!f6suTyB@z(1Mk+Zn(-whOz>RC-QUV#bQ6i zV3*9{oKn-0vwrEc<5Ape6$86I`PYo`A^o(sqb}wcFy>nF^=i*VacQ;K)&Fw$=r%EI zPhHF;Y>SEl*z3;(WtGT2uV8VRtF)E)GE)RBIV6@WAMn}^CbIc))VcsLQieBE`@yB80{yg8Zqw-qHBH{4Mdk{M zJR}7$I>5OIvOO+jpD}|s>`>*0jV`^@UYj8K!tB7L$A`y95EPaPz9ck)heOeWjspUR zt$7NBHAhnB-z1z(j8jN_85%sms}h^86J81W##fwQ@(fO1_~tX88{+k2-Gn9QatSe@ zj{)v)A67jvRAlj25~+f34_)cCfad{&QZ7^OVCPN#nm`s_>=LzX#t_vX`>cLh>tUc{hJKn|<`YA#xHeVJ$>uP4Je;^2 zIy2q~5N3a-re~SuY`tFYdg%N6DtJWoka<(syj9RW_0H^<|%HK4K*bCVrjx7%9+tlq|NC!Z7!Hzc3H+vCP|p>nIw7d)a%h80tu;`Sg#Rm6VxABZnG!>Kaa zZ4H{$p1!$8|H-LHe{-p_@^4uhMvtgnGU~C=NOWsjG~seai#c~lgxjfO_LdER5_$(r zz7^cIYD38awfaU&y2j#a8JY0@xxnUDRI^`^O!No0>oSJ!OQZAHYhmQ(hK9cUb>G_P z_Je_@V9X$htwcaXcy;6WG!c6A)=X~f-0gXGXn>YH4A|=+svtg3Ai3Zz;#6GslYWT2 zVxs0c9BI@~#DGD4*=9J@sIJQ)v8?3W7&hx|)8$69#Ed|_OCDuL5W#&WPVnPKVdSY@&z_||;{Cu;VaUvWQPQNv z2#sz>egw*c3?A<2G_&4K{`g+8DPz3vpB-0qPcCl|P2nBBT|Rj1sGS-Tntml$^@X?k z8z1%{!qN=uLFfRn=RU-H;pbkWMhM=IYmEwXNHT#l1~=Kjw0uv^y_nOP_craYpL|Id z3Q{Ty^#zCcdE&sn+U??!x~HJFeZ@*HviGvOtCWP*FR%gTOMxS{j%OY)ozMT%CgqL`1ZEJRiO{vv`&nmH(^1orJ``rvM(zT z4&hdbOA#?1e34OwC1#~-NuB8oZw}nH8&qZA7;+5j<5J!}%xUUyg$h@A>1}mv^d=iR zGe%$c6b=!^9hV^5Y(Ch0GC29n;r=`Sgj~n)`H>Fn^Dhifo|LM0QH4f3oa+B;QH@18 z)I_{VPI$a)6!O)EG8gvD@nHJ+SfZtxNheGnmW)?7m)d{NUtMNr#+uY%&n?W_P0T}b zWgqH`4Tmb*X`PGJGMI{ zNJ$>t&(q_=q*IBL^bRP-qvcC{#trye=6msP8PvW-CS}#fO^$eOJ1U@_8)129S~OjC zhPT@qV_0pQ*dUFk2FH(J1@PoENhK#OS^)0s^``xTy^b+BiQhdpSod8M9aj5?Uh%mL zDxJu6BjM^AZ~tzO1!20iWy2$TCKJdI(S4~1|htbgtc^M02#YcL0CLM&BGGU6y~jh!P>DHoz7eV z7;;Nl(gS(rV+cR`8Uh~y#fsbyv>fl$pR)7lJh+y1DhT719@)cnQH5*38a*}pSg%Uk zB-=)f;md0Z^+xACHJ>J>!apEeRuWMXW9IseaO8t3!MX65nP>_w|{13fGnm!X8W}2z)L|bwD6O;$Vfl z40}g(tT;+e2b5%uJ!4t(_8Iel<&=LVC8ckm7>8AOgvXtOFzh^Gm<>U17~u?*1%{b* zijQ()dmTx4>*SxyxMrNDMt(p#rGdpZ2?C8deZU)#K2y5gKrkp-*&e#td zPA8VTmq)*c9N*&Z%>;p3H2E?hb;`IW+EAX=CZ znVfcEDK8j6|Aw)rltT)tYG%p{BA(EBLOaftB`i233-(tubp1;f zI}n3eC*@3R5MbkL0X$sy73z`f&^Sjgck?t1{DbeZ@kt&x%&JdAUHl0i;j?z%zR<^c z;-=9JdE;&RKQ!mKJkfs3wy_L-zhUQ^p1{Ijqh-`V&b3}&`vm2fn2pjdZa;+%zaPz>cI5pBMsQMfi)6 zIdyRT7;V{W-xv^ik3NyJn^9|5qt>2aSJFGMxm5mrkGI&t;fc!z$T{~DvpSs95K?*} z4BTY;7Gr~QH~RXA2@3v zX9S1Bb69|ESI;a!xuvxTVY!jSWR7o)Nu$@sDu-Wek(@RRH|FR2^X* z0UK>89yI6`g+b^cR85v{{0?SKDvM{DDNey!!-mO}*Mb(12b{AH_$-<^m+cq>2uVjn zTef+d6Q0l*&n@&(J47y>cZw$dLmE!*!fYdL#}c5aiZm0LIy`^#xmC>ICW&d(=?6{Q zwps(P`-vo_i&TF_=+6mwbJiZeXOig1Zs19MQr_Mqie_^PVvWUMjtnOOBhR}u8KjK$m#o@CfOLr#)diLG;0{#04GN*E zHwyb38Hw$+Y5AG=RKXoBfMR{SZ$Hy!IagdQ88Iapeo++I8jyNPx%9G+pTGFA4m?{) z2u9V?rYyGmz%9oZG7!*0x!Xy&RJ%MalKH$-yvudZxV83GWn-?+0XrF+ao5a)?osXm z+Oc9QAIIB}gH?WHQCb6$G3Do+-N>I3c3zc>TIY<%bE#AcNs3>6u2DKaz%);HmE)uX z^~euw_)q9X}5T58-Kidf3&J#<8iU8 z>ahoukTfOR(=tV!DyFqwC_P(%8&O7rjLu~-zG8vh9I_bFslnpMFFOgW zk0dp;A@T^K>BY&z-Yfb#4R`7Ap+x{^26mav8aawq8>4O&hJ|8MglP>8VVzTp8M9lW zXiCRnLVemi`baGErn);~o=O*#maBTrtM4JVV>jLp(N8ILPW&;l0wBjoaf$tz5nyA{ zn&+K&#Il-3m9Wn>(Xd__a{>qqoEH$i?9*)i2R{%QR{NiJl2Q2X?z3;J&kpE zw(-cT9|_0R=fwjdL2)5L`iYrsm0lagv^9Z`g}PH%4@ocN0v8T*Z?PGF+!&a5XP^9x z>akyuNv+oV%CDxHJS!9IBae`-1zk>02uiA}DA-Ql=Lq{%i5Iri8UV!iS2=6%Zy&D` zH;cL5lIt3$!w%@x{3h=Q##@G>AQ*wH(0>^Ydr#WE6UZ$(8>9C5CixO2HIatLeX+rz zvVD<9Z(_gdqqo=j&G|L3F~`z6j}hY_S6aCR7Jk|tUk3e@*T1@P$uICzQCUKh=uwoe zZ&(+j2hV!)~nV%wCvpveKf~&&Ja>YtE)u|TaNe+KBIj| z>`i-_w5?dc1x0q8;L*>6bH}Ab8fLk{>H4t)kuv>%O(ZHdNFFl%X!U||wpEVDsP)}g zAxntJj>xJc-hh`-IFi0=?zgUh9fO~emCn=+L*3LJAsq{gJ&t+d_+MCRZ|*ss@vZ!v zb?$Ygj9-##;~-bVeY(t!$G!1G0xo3`XI?RUF68aU={CYru`F!a6hs3RY`;(sE0gD_ zCF!^n_8KuuKJVQ-BvGIYzk5i43XU6X`x06vaRE0)T&DxkRimh zS1DcAcscJ^IiH}so@i!sg4cobt~eF5bF$^YE`a(ltZe==@(RT z69)q^eUi*-cA5!}=W{(y*4J>=_<4BaUB3_WOKX>jSR%Ii0YEm@XUAV4c}>ssML;Et zpQsB!>me2j&k>xk=+UCn8NC##Qv!^32HCMmAw$rgXV94ojrYQxh!DRwxC;QTc!N*& zpx75_yB5O73FKnY$Ihc0;3vEYs)D{jlhXIRqH{iV#Nq1n0VQYtqq=*CS}+oS;4fwg zdlLhj%i)u|!ui(F8{$&TLBPI7zyifMq??5xMhj=R?3-so4WtQ&eZuv-O*U#~vIXu9 zw0cZ5$WT2FdgASh&TY};F{k|!U5G~zA3cL4H`r2X8hcdyrquyUIwSz^NAb$33Lq2l zs7$+sQQ%5AS4{`JCFEgQH-}Ad_p{vIX-{uzR^b}p8zf*N2{xDcJYJE_4Lf&*NbVjSu4L;*KOCOizN*<&fAXca0 zO(dchPOguOUliwa`l9;j8!!ZY`6ieD6VP(ZvC^I$MW_V3z&LG$(X1SOsm^p0Uei#1 zAP?{nH&IHtiM;5f3&L*U%kL8~8Qaso60MKzZq-N>j#8z5KJcDc@;**4;JCt-f$Np_TZxsM?s8-!su`Th_mP<6NZIrw~ zp5%H-o`*4Sv%IEUfce;PVyANn9%4#zRv;0KwZ`mMj6U0DHB%hQL<9bIj1dyFfx^UPQ)A>GGB zPS}@kHx(_E$u3D8p2!oCrN+=xbcMbGGxCt%8X23Lm z>{`<;2)4llPp6*9(iQgQ*;u5B1owexxu`|aGZ2gPP+f?9-;92j5WL0lYhPbB%;|Ru zt`GDGW>dCsB0+&)+=Th$$RdmxJZd50;j%XJf#B@G!J=~bx-ZkYv}9YQ%L^X;Dan?p zIRokd(0+cN_j&67$(E5`!b%RJXrU7UxEMe(INpW_(~-?XcU8s6v?_8Qtp3-FIrPNj zOKC9HDIft3Jp0w;71el;z-|RO`e2A{in{a>@&CCO8a+WRM@h0&@hTI!`B<)|@)|p4za$ zL$|+%3L&0uLH{3i1N&XZ$DqK>2tIJ|4e}wM)`dRD*;KYvo=>mo%TQZ94@m@3K#wys zp1B+C#Djp^6d*!FC}>8sEu$M+TI#XdY%f||a@c7jFPF;#dax0&=ZwYpMmD^$Zcxv^ zDu9p}vI`6)zYxzOW&@X()t^2mrT5S*kzV+d0dEyzYF+WGACvtJ}W$FOLkx2 z_4J*3bi1rj7c2ju(3thI7^~)0x2C-0zQnP^I5B46p@5-Ku(fGB{LS=Q$>ImZIp4>U zuwm92QhaR)l{H?*B#2Bg__l!ioMezM5q?mLKb^w1mN!g%ENBDx=x`bX{JPUCc-Ux? zR~&+LklxZ0Lu4Ou@-r`+qpmsx7VQVd;Y=T=PPmwFPx3bsHlrP2NJ3)OotdiUKfJZN zmvd2f3_4B=jTBGpWpPIJcK#g%Qc{aLfhv6PXtc5vE{sGmFkcp0}T4S1h zWzjS{q#B1G(yqZ5KBGecBb69$_|{kwIV-E^-(*ZO zYu4>nu!}VTiAYDxS|lQc`p`osaD_Ul77^79F^CP2&tfha|B+dJI!XP8tk0v3qTEy$2iD-$OShD3 zD#&~Y84bN$SXvrez~9;-lqk#o8_Y_{#D$=^+P{jcXM&1>H$5b?K93S2WBC+(3On** z=>?3a_lX-HZ$II|ZEa3=}9Nn?v@ zGs+q;wj3h~gy(EOzqYpd=_Pg5g0gW;%LCwJ>iJV*les>(lHKHa*|Plm9}QdPnp_A~ z2?hxSpUJa4$UCA|z(b+Fqr6aqhuICRUJy(cO?p|2s3_89es(1nz7#GsWwjd~H_YdT zZC453i4VvKAj2+bW{bj{3NA;Gw_vxic+rH0? zI(m~l?>etPX2stJC2ea&VF=g7ggWO2f|aH{JAV55+M>ZpV{12QX3Jo9Jfmh~HBcYJ zZE2R$Rf0GVY-#X~Qax#9<4ySy8BiXfbd6{yZ}hpHQEk~%8~@R#ae>a^_R+a5{eLN{ z`F%srymwdGwv48zO>-@+;1l;btQREGtT~SzgbmlNCmjm<9z<#veB!To9<*4 z5(gXJxsauvQ@=PiApZ96j+Ndg2FnZdQ`cBRi)1hb#$5h_$+bEUc#Pi_GGc?(HOMLx zhBF#9Chlz1=*76pns^TvO}^0j-eEMmLjZ94BN?&uz=-BgPv6p=8b8pWi1{K+|9(^4 zPKUt?RZDqG=DjUtqFanzG!&axdjBm8M*Lxt@Wvfj+_5cPg8oQq_|%p|7vCMWwWK+K zv{#SO?!9eViq^3!;Q`=A9NGn>eq;X@+`H-etd;JI%*R3A`0uD#-iJ$YKq{kp(1*j< z#_$&e{e*+!VGPFDQxm~b__O^Od3DF%M| zj{IuO-!kyQU3Ev(P48(EJFdW%CBVkD@nG|B`s>`iRSGYu6Uu$eTlsN}V|g(pA3!$j zZwxoiOWKOCqMrB^&XE`&PGZ7;q}>tkUXt-=rhY9uxt|efQ`ihLi&4CLoE=qY5?n9n z)q@wr<<#6Mxc;xLD7~Zy?>rf4XMzxm@H~*q$YK9#QMW}xh-s#$VPeIe?qwG@{i68w zqZ;*ZN>#nTVyC*}TilP>`?39bMx4hwUa3KnM4^_W6eT8GWOEHHMj# zLV;)}>y~_{@hp7Q?IdQLM|X9YBewF$TIRL`8KP^#jjHrGTW4LN?hn{~;Qh1mf`B{( zYvA;ALieF zj@*wH*Vgz2@niumKxtmGb;WqFGMY2iAXF?S*Z(QX@)o&nzYM4Vw%T`kS_bLN?dDNl z|I)ixH<(9F#$21|C%(0PcjU0(N_F8B|KYB`mYyp0b2PUUw=gwyt|DS+SW=scrneza z^4tc$8A08wJ&1}yF;UE?2wkC782^5gfCY*gEs5 zLq!i1$nF(44FJ@D*h(xUH)Y+t3>LAs0vC?*v!wOKrg|w|5et5X#DZhS&gi6RbfI+R zP#fDfIcbUqRx*430dapu2#m|-^r1uD6SUupKSJO(LOoxLaD@PDHRCGeW|AUP9& zci^N(sV|>$M^uzRjp}&&GZ7}}+6hB?{LXxOdMcg8{V_}+>FeY6fpg{AI(bj6bCGd{ti1IcGfyfttD|Kwf8P&6H79oZ}+sa{iI>Hsef6#?kc?a-m81kUtw z;(=_)^F%ON385V0MesSp9xKR+11OXJJ&?0ngV?}2@(LA8_ux?eKOrMD$kV7kjIeGJQV^>KG~=Hh znYKhClal3r_HUVwie|^NHQ&YRzR<#HGYbI!%!VlfK^R$!cCITsU>GGOjb@cgx{AR> zZT$5t_Jzz68YVZCw&X*4!!!=ttO1Ja%gvN{e?0`UM3c6l`O3^R{YHxx)6k+Ev&Gg_ zo2DO7q7T;?LVK+^vAz9u(JuN&hpIJ(K|KwdYI(vR7tn@I4%WNhUuzCkY$-O3ScoG{ zXnknIJUu4xtlh+h`G3of8v_=A)PQk8E8Wsrn8ioUvp^)1ASW1RF}GkK>ZrK-QB5M& z8@->N|MWnz38bt~$@U0*L*loyj96XK6W9~=FTytbjjt(uk7}bTrqXUBsqTK@$nv3_ zc$@}B!kakjp{St0hE8GF;LqNj)lF&H)G_?Iuk)hHBoYP3rTbT>Z^{u*6NU%z$ywTG z4rleNYKI9qYgI=U>HC&6N7x=&U|Oo~Xu>fC9lE0_pvnh_duE5FF_gPk^LF`Ckk?6{ z8CbE8#z=nhq3Q%o(sf}+AAPb{Bng9(u8IXmy}u2enr97Ig@NS0Qw$)0eJcAC>YvOH zp$my3t}*!ldNRXVrzQ|sX3$IHy962)2yZ&O&)s}ScyVh(*h67TTyTi{y*r>Wy$f27 zzC%rQFuu`5Z$)WG-f-2JYDJE&fVG)m->j=8{)`o@=l73P6s{Cr6i6on4~ea71mh!N z6m4ugY10p~Hg%Vosv+*rFf8gTeyO|w%i#++Bnc?SpFce$C$wtSU1jwWzg<*YvS-_| zAV#3scU^uu0#eDhBz*;IM2QC{X`vW9rDEZZQ}U=MmXK8w9q8qu0Ly||!92H+@`9#a zTEdh<40cKmhhIu-N5dgfOkZaB1`PWV43$kkX>LinLfB4RHOc2-!~blNtlPdmH8X0T zXok3W^k;8|{h9;LX&E@D;UM4;nK2hvofy)$r?PP^f>eR7voG-I%eXdM8hVO~39c76 zBwjcW7ulLAYVpeNgF>J3(gNE%LHeklW5z!W?lY%mN`s+n#M&sW>k~C448{)6HKL5R zGo3FNzXXcC^kBU6*a2ZDLsp(sH{Y@UMkc=q!QzNr!^APyoX0{5v0h^cDGv%Xu2udK zJI4D5biWkV)EaMtfY+T-_ftwK{KZ^Un=A&Li{^wr$UFf78-6w zNy8kgzuPV5PnC9&i1CldOog9kU_-c~a9K%AVIXWlzB}6J2=j$+48rk%?psyCPhr=- zp*BtGHjgGI+!nHOn}w=q?PdSM%Jrx@w$_ktmt85h6L7r(BWEG8Q%N=XE0!(O7DMRq zk`M->iF&ly^|uWDYa{d-Ym+NOx6%qUaIDr68D;?#*#D#>;*Znn;||wO5i;@I%Be>1 zQS7lcWk(1xHz8u1Axesh_88%&$$m$>v2*syN%bMHAn|e4g^UXNHMR!R1XF7|;K)XRs305I_ zwM8`NmpyK&^mwfd-8=me2UB{Uefj70#&xcd<&w`aj{Ij+8s_)!AUUwOk}Y8!19(}o zAeA*ua6mDWX;$LHf}ZQg@`6a59Ge+_F!xhLxI`!oiQ{RlpJ~1+)B!aZiZ7u>%WFP0 zmX^wPI^7T0(9>Z%Pi0*&;I^%~In<3=HEhu4E)z5xv2V&dTKY^GNWIqQbMQ-IhB>4n zoXWL*Z2Oho;%D<`mz)Y-1MEs-8&qttSxy63dClH=!&!yU?A<%hV;s|eBt2xN1*1UL z_z+B6PVT_`1<|Z}J~r(wzv+tJ#}p~=wH1wVv7}i(SZ59i@=D!)ULR0msoX7lFwR>{ z#Z!{Mo|&qJD8s^?J4~pkfBQkixZa;WU5v;_Mo3^EVCYAf>_X^vOU?y~lf=Go)a}fJ zJqcfwm7&@Uw<9r+$d!XY}8KcF4qPMcR*WZ2HWEiDw!^W56Em3!fxlT80u9fG9aycou zQe$6!D(~0_d&g9!(fpn#(MvW{EA0Hu{Mn^GHrj)))=tXLFhcNfTzzYz4vCU!M?ddZ zvwxD?;r@%2hJ6vjH;$bqdTih2&>XmE&Zpp(0OGW&z14HQ{EkrF>WYiI2~V0VS7qpd|nvDp&AaGFK?EBgeF>lyO?si4>~A8E*9;OowJ$&B}MO>?rhSjxa>Bad5bFW$-d z7{h8diLGpl0PBni5jj-U*U=T|A9`VdrvA%5P-!g})h^n?_A$=Aj&-$AmBFELnj@aKEvO+)0 zuOZIuyL{EuuzRzfDgqy?>>K}sN|)0*{G!u-77#kOR3Sg&IiX^HQCrk|O4DhH-O5bZ z5Lw2$`onfGf<>yfQC|_fVZj>{B4z{ztVb#DrS$o5tJ_3frmK}w0@IGhk&vRO|v;L(?#DWFpJ%f@YNy1KdC{+k)&+nTYykK39; z$kXxDuKVJcD~z)>zbgZS;z=nPLk@Ae0d24uZpYO>vMF>Y#e25JR^VJfMu42Eyt#j> zQ?<$0qKz&HNn(?Gm-E|Axg99UIbbDw_ix$Nm^y-MH5Sx2&5SRqRs7Ia1XvdCM%-rt z!ORAvc|X|M^X^`+H;rzWD%aD}SfaV?_U%Ts1D=c|wvfjSg9v!GOlP_6gjPum3u^@D zW&Z?aqgUS3Lqh4^$d2;2zE3kXuQQMQTSo44sB}`%59mokUideblg;Ol;ul0`)^@Bt z$O8(}ey}75PP9$D4GUH1uxOvh2K;Go^5~blZM8gUup+SWOxe_i1jml=v)Wgt5+KR9 zTjA1yQAG*vH#4*6cwuZ=4(v=xmzE56QQ)yeh{v@Ddj-3c)o({~9m5*~y0k-t2f%XS^yxyH0gr=z#%}lE#>y zxPapbB&Jif5Eu&1B8vMq5O~~(La}7$7hp4Y|4e10XI@iA`Mu=Z^1=w;u9TGHM`%g_ z&B!eVqk*+b+}aooKjY3|`Pjz8eo8_Mz#SoGeK>wG1fESqZiE?%ZDUSiSwb_NGD`w9 zKQT?*zx*i(QpqHDE&)p61IM41d9n-1;1d!dPu~Lr2iddBMkS#ARGzJoUla-mc##_W zlKL*x5f($eINtAuE}SD8pN$)@ig{oCy*DFy*~I71kN$H7y2mRL0LRufT+Enw!j2^? z%9tO_9+t{U#<;s2iBikk%5x$EU>ekWWsDej7x9xs4^QWbAkEV^J^B8cen1_7*9U{D z?J0T9!WfuM8mK{Wmm%Ce><}y3n&suFq3fBuZkIorOU;OU9lyunJv!5&gB$?YuU(N~ zjz?f<%$Qm#H11y;WP_K`M-p?pv4#5gqN+^=SORYccO-&5gS2##_WG&@TSX7~8#{2k zS}7F%2PJRsFuBX4Mo<>ggfPeV9z|J9UIX3?&v!WGkn}UL@ISoX-IBF%1c$ z?kbg*h-`BA>UlS*Wa|Tiw5mCKaM~{t21u+@B*Grq1Y|r$&Tpo?9w8Hz*0o7JU&x%9 z08McW*Q30?Pr&OV>^d??qA(4CYygz1pG0fh)PE3e9`%J|e@&y5pAF=t!94%{8LNZ9 zAne-as#icUaEXJGy};F@haGKikWoWVLOdrq3+a2Rh#|Lykdf$)qq@$K5GU%p!@d_i zJ+F(sxc-n~7~ZB^QB^;-q@1(zvpB&*rWho@;SSM1sq^+YqTnSTTGCI+SiTAznQTF~ znT4~!1VJr`5JBdbxc~Z#a{AXaF=7=0ChTZ6P`pV~Yj|i1) zv!#&_-~Tqh44_6^v8c{dYF_}^H6HdM#oqH7!CJ`orofAY)7g^3PY#wWD9azUo=djD$rZWTp>59=zA*qv1_|0vpqvm?7Rk zNHI|5pRDl+RK13(Zp{+9BpG#x@*qwyi5*aoW%SKze2k64sI7Rfp>k?bno)bhv_%h6 z0CU};wz=5}KD2ITPBR!$7v7j`I(Y`NCqU}il%aU|_9s5J9jEkyPZMV~E&htQi;$XO zXw3PjeF?#Z;eLv34~$c{URT00J-nyI$wpib2HMpR8tk=Gvz0xlX+6Es!?P>a41{SEHfp_g#WJMmQUHQrz_k|{SLu}%Mo0=40K?k|5EFtj>_|X;~T`3uJJQ8xnYTA2V70T{UkyW zJB$X$kMYJ(d_vzan#KYOsG2-86!gJQ-LSB{bU@-6&%#IU<-k7Nd@QYVx7|l-@o~)P zn`YjUHI^JqiiwL!RMoPV|z_ekS!ONl7}YMEmQOk24Z?xGLR_G9fGq@%z$?Hrq4jAPs8g%Nwyv>+V?y0 ziJTEV3L7FH`uNZb=o?K_oTK5l#sFdFNM}b*H79~l(R;wa<7@C|qV7(^aj2zA1QUD> z9jHriHJg8oA7rJ1JCTtMBgRI<>mTG&_F8nJ=6BC6$GNtnQIdAxCHWLNfPMHGSa_e5 zE8K&3pAj27C|;M^{F6Sv?1-|l@nFi$1dXw&*k2y0$Oi z;6xo-o>Q%@5(gB-fu|@`MS@R#h=_oKW0gLv0v4r?hz!?P>--c26#*p_l*xfAC=N*4 zDpEy^0uGF+3PBKv5Fvylx8FMF+|c&-ecvBi%aGjMd(J+4uf6tK?>wnlczopeM>;oI zNw~A8;jBaE_zmG`@wGL~M|}dna2k7`3Q^CW(B!DriqZT!C2;$mqPWHTbvKGcf(+0$ zPvn9)ulM`|%9J{JHL>Dbm}Z}&nvOxmPI!ACgj<*VnM5}CWz@J)vvVG)>LKVirqy!w zK|`hT9GMuQxAvrEWb@tIEs2DKIinjp{UbX-G5PjGlB!mB2HYi<6^l+dq(0IjNo{N+ zt!?W5mhn2w&0?nHc`zL9F=$XDD^%gbelLbluXrMPV&-zGP79?#H zv@isYg!`!d=)NiGR+gH%PAL^DpXVTd>p;dxf$A1=tFc z198rWI}WB6eUZ~yPl%dsK_NDJq6O0(&(cf58G_}V9`ng80Wmu*uPGNbr%4MSKtaUsFosNmTRJGIah$2l+^>NX}BHnAUrH-Mrdg#)g+q z0B`?Zl-9jfSDc&W#@{&sZ1M)u@`18Wn@*`dG!8C-#3^)MC3q>V*Mr~19VH0Sd?N%D zjfi$A?CHgxlr{$`&8KmweI<{kS{}6oxfg-8qX%Q}gqFJg4 zd6q_6S!5(q`+cp$k61{d06ie&Cj?0uKWnk~t}9Oc^DX)t-i0gP+rZrqvlp~4EBP%( zmz~FT22~n149DpvspC7aKuh-9I&Vc0SHs&Z5c!>guG=F+ou*}x8V7pi$69Kd}VJbHv-lA~Y=K=O@r=UH{gW(^w5m`L8V zF&<=g3gJU^uKJ+O$;oA^=ch70D=R36-MsG0e9@|5TImOQ8@hHr?VR;7i{wYzWg2PI zwls$vzCO{k?0)sKlgfxMQ-LJid7Jo=o}O4ngnOchi5Yvs4A%*GNg_gCs?i^oS#vTN z3a|nwSk}9VHqjSk9r;`$u+PCY{;2t{Hb?7D!%(tY@Bd78nWQu*Hr@#c2zwBgKB`id ztI9b)W|_tA73<=@%*|dG6rK&v$&Ii&?VYqsy9qV~RZ4+th4LQ=da_Co8Wq{!rD&8O zi~#c$!W$xWvMAw;xyV=dfX9#X>xeU<2r& zJN}G)0geSz-HoT+*qElJ{IYmu#9)0ut-MqYYez!wbmv&o~p)mwK z4&VgFKLV#jw0#iq03;)ok`{Hrc|ms8h`xzgq&4XRbGj~$qHCm8CbdJw$QvHMZKxihL7d5ab2KacS9i$y%Ul4+zQ87kX?VDf-C}x%#&j1j-2A)QCgD@y{F3FvU`XrMoFNiiMyj^gO7}jOuO#)ycBG%N3 z^UnRugIeA|`isnZ{#B9M$YK@=Y7(i;1u;;tu>V7JEmi+sta&D~*El{K?AqR4B5!v>K6iCPeQe)F@K~m!3cU0x?fA|7DQ=(vQE9;h$Ef?QqRJVbK4=5 zAd32oTH1Sdk)=LDPC&LMbybU=-Ojbyz?S;02#_N2XQL>h(d0a_+X}t>257gyRD@cb zv4uPWfqV2&-$P*O(`Rf*EtbJcy)Z2bIK+%5<&?Ybv2)*-TwN-+fH6F$)*YZhF)8rD zkP}jyJNBX4>fjR*v8k9IGiy$1-KBeq-OCb!kKVEcDJ(>K0bMbq3sEDg^=pY&4TE^u zb}`dm1(iX?UE|4!%NI(WWx0bQ9-%B7hGRt<-e;jlk8O0>5hVHl?+;nrz^K_?F-CO< zL88f%)ix%|c_>-pc^%7x1a#tUw*uDL`|U4kQGz@bgfdAE&M-4E{V{=vn{GY*>sJ0v zv)+0E1KsiF=s}MZW^P%hAf8n(=?V&p1081C&}H_~HCmse{)xxBU^KF6@arSkT@=70 zajOmq`3MF*CLK`&Z7u%(UkC%l!GQzc8+N9Mj0OD6#evKdYbpbDQJDZ#jQpi1=n}F#ly*Ljir%nZzrbY=&;aYO!LK;`}IoUcXBAdFJfL zM!2{OMPAJ)X+Ft;)Zesnw}`d)gIY-*XOg)VBZ z-by#jeaOklewxSD|b0t(Cqgjh|FWZWWU&KJ&_nyQW^Z;45tKwC&o z?`j;8iyzO-`F^5^H>j9oIyGr`@2H(Mt;G|owdm>0Q?|W1j|qndN{8Ebu6P!WxhH!o zTVEg!4Tg3G=wp=k^L>z_tT{URotq#U6OiRzJEx{mp6727T^P@EvKBdi$2jUsbx!86 zZQIR|;bZw&8*y4-?}3U1Vl8t&c>I5OhReESraYnTP7$pw^IwvnFTtcc83T)K3aATt zDPH(&1tC%mPIYkN$|dzbaGD~?F7zO|KIv2WW$*o3Qi7lxAb$hiV8YJms>&~QN0a&P zUiLXBYwLhN5xNbbnWS4q(`A;{K#v((hY@2N0+vM*0rV8EURKYXk9;4!2pAH`Ph0bC zcFa%WBi?b&F&$St2vX<$V8ldrR){NzCJKT>1KX`>4aBOFeh&5n!x6F>c3^}x1jH59 z=e&9C*&oDb2dCYZ_Sl&+)8ER;<_W!`J z5pAuxu=&QZz=tR!tkbOM*))P)Yf!VE|npxrt|H8MY}LV_eggzr{({J z&XH4-6x=u+2(4u*%U)xNUM2uh0mR7*vM?HncSLDP43~7o(+>VAAYXZb+A>RlG0{sE|jq`DH4|inY z^D&ABdw6NNScHNX8ESU|Jqk-FQw|F$#S!7*z_zqo025H6N8&PEKw)`&j-ffCi3YKI zJ%uL5VX~$O?=7*fEf`F~=8{}@gK@G9MJfUxW=IgHfHMO>rRaG^c!o{8%$hW%QlWis28pLcJoz#!H<*l4DkFJHn-Vk61;`LZ@c9qwW1a#WZ_?1^`iCq8W5~!%)Tz50LraypIk>m|WG!L)q>VmrvLyoMDw= z*H0BwB&~zDG?wDG{KvARtQHk52Tx6M$4>Pt`y|3H3VDqpmHqMp_n%zfK}C)(vt(aV zR(QWQ)G4%>lx2j&!IfxOc`a0|&K6zen}eo125mUz-Y&fnZ>x$sYEz|Mm4^Z$)F5pk zbR~4%2{zI-R}DWAA%p|DWe@)3vQS#_mR6IWgr(m>Ge;jHIEkr=YCx)Y2dT1GA(&xy z1`>sfwX1C9Up*UEvlTcEIU_07>msTFp#0^9`A7 z8lV4IYN@dmrsnS@?2K7|WZmIM>d8%q+-==kv+9bJoLrKHX~bs%tCJ5G&beKEZf{fV zVR7JQOfMl7_3$ug#JL6yYPFNJiMT}2*YYx7EvwYIt0Xsb5ON5%&2U?*v`5-j-n}hC zVDbA~EKgXja(viw!AzyXm@}u+Upm8uPOZ2xabnCztuwjM&r%FV|6!!}w1J;m?rNr2 z2LVA@b7kb9M4a zOujLXtAKSOzFLQC;|xV@gHsDN9r&?H?VCrOZdPbQH>S5Ua4jRR!>lSsve%_Ojgz4a ziBe%kY)I`qi^UdP_#R<5!RFaT5IsAvruQGBXNA&VqlOMZa^<*uqq{!TWXQYO2g-eZ zs?7azHwL}l{V(E#%nyfkLSL+Yv=-8zu3i)|Ur`u&b-Tg8n}h~jni!to3Y$ZBJ*8F5 zlkTj!?$#a-f4)2-3X&{DanaF>pM=Rf{W_oRI_4f}TSN5^1!Z9}C!N_UZGMVk)u9?1 z?+`6;Q>IcC?Js_JZz~BjXktnvM?kd#zfF?8+=2IckQ-9HUjOEOV(*d5En*(-yslIJ zx8Rht^|9plwlw9feJ)|8{qkO>3)SVhG<_OZ_XZQ{s8Sm9mMz*v`sp4fW-4FxuFVjL z->#a=-8}T=E$yC6jFeGJcah^~#Xj1A$X{cGQRCSk!MHF>+|kpKDCSUwJ!g92?0r5z zEM{hSL!Y-Sn~#+Eb=Kf9CwJB5LMfZZ>!-{ym1i|4#HS?v|fJ`O_{$kLqU^Op6)bgcWmS`;e23imarQx@IjO=ZvNF^xs_l8BUWw_$QR;gz`p+ zYUE)YyY)cxxLkxU?EXZGEct~`+#vxkg{7Lc1Gi?VJ^#t@(@;X}+G#~Q&& z_=p^+ZH-oHQhrqp(O3~J<#~xEGU!gLt_>}-i8p=F^qjE7y5L$f)q3@@RLRV{)!gFsI15K=DD5=|=f;pf!Ht*7e3&h(jztmpe? zqqAL_dA)%d%0j2!MPpIX;nD43q`2}#&Z|WNU0lGK6hTVun&3Bwtjh3@Ja?Fk1Ac<3 zVO0Jg*{^r`p00n0CVAmVTXa??^J1s^oI85e%!iVNC9>N6@d{nG;jQ#tQtuAs?x6$` zl0Lu0##Pf9K12@dmj(=4gPc{X_S?w6%y(u(3M9xA<_O_x{RFKQqUR7fI;?u3T&DQ! ze4GmyjHkd0#NMLX9gnvO4Pft&NdiavP59Ll@X-@83L`^H#L^g65mP|YOG>$-q6~Ey zL1)|r0TzF9Y*4dla0m%%N1O(wu#+2XNpB}p1c%KTcD1cWs>r9{O_#W5>YP!o`sIlq zQFappnxZj^Mzd;>c?2TC1M`gEj20Hw;2O^4Y z%;L5rr#nKXwP6O)ejwxU^wrp{z39<%7aGT#cJBS+HF0p5N*Bm^CJdAKqY*|w4?;_! z_JoQw^(1top@04_rYmH0cVCibM~28TzE9Ac#BS($h3jR{S$a=8A!7`c)my@_jkP?Ya~yiTR^*k$^CfO~8JDAko;j!qrl=i|sDF4&fY0{KJ+A zQF>@MHXZ&y{wqWeBL#~lJQ+O$;h6EAht%xK(`Zvz?ypN@bkkU_wMda_YIfLI}ti@;(6 zh|h+DnMlGD4{Ry2S3uB&=ppeVWH*xRar7QuN^0BV?+C5mi4Ll4t@JkEZdEN+{}!=L z^f2t{IY<{u_D+eN8bmwmPW?%wz2bS-^rSvSZynCAois5?JH%T+>LMF;wXKQad`G@W z(i8xg*c8l`IL_K<{Q8i0ah%Z-^7bF1RZ885#BOKXac(5k0mdl(K1?gM1cN-=M6BZq z;Ft_Sf1zt8G=^}Wx>&V;GbOOUS3=@Ras;Ke860OX(q`IWzDp)Tnb_Pu+A=idoYe>510ofNt~l1zWQd0YczH{5rTADEvBDVJVk@d?!kZZZ zY`jtnq@$k~By(!Cj{cpmvuciQnZIrjmIqA7AQ`jp(!udh=fSgmH{SbYw?nMg(^ZtR zQjShi{bjnMizCXUuUX|n5^p?!VNs*w-V`2g>T^38SkNrZXF8v==x^r#gKxIU$|({< z0y4Gr$JQNwifLa(p+ADFR(DDlK~yfZ$(d1VSs}3)Dqyt+MWr@_JvCZK0k`kzth@_^ z;15wp8x4idXdzCw7)+=ggo`HNJs4Y=8oqfI%(pP$>&a#fFOm7ibQIO+?Y`YJM4w%R zmJa0|QX9d);RkhA%w$mjU3{9*C5WI?x*O#K&0YF}f-EpIe>P@qFyP z==sOFPdD_=)MskfNii2{HwJ6%rQSN1so?LPK%|^b5`>a~PoTfz|GI1;x6}HG!W^Kd z_Z9F1>9_8)wWaiv`m$NpWFwi*2x2h#!xJSJ0P$=%{BlmV9h(`$*06zOrG z$>6$HfEwt<&`LF!hU*GWDh=`Pt1w}c5_p4adrmxg2CvdBcq92nuJ0pl*K&0Y?z-kH zCOG5(u|zjP1uH*`?-`d$4@kzjdg7CH-2OBSBd1M3^_+j`$9DAJaMVUF8F_o>ZNHYp zS!W*T>F)=TpPE`u#q`bmrudMp`Ik9pf5O(YGV>c1G?@TeQ zA(YDS#HrJdjbf&fXYDO0^EkBFl>K&v;BiYRd@2sS4hG>VWUd%CtUtF?b|7y0>)DL| znQR<|el~=lh2mJVcx6b@hcYCq$q6G?&=f-eHDrHTX29n9nga;)yaPEH$&tlmgf*H? zQBQx}Q$}OW-yX^ZElo-AYaUeygPFY~1=DQh?U}^)E_~S1Fxy59)F$}7Idoh#Hpa01 zGFKJGEx6Y=l(zPvJI&SI&|jEI=e@aRS)KG9l!*F%r*xOuDSkcMewb6`7}VbD?@708 zl)79fhR7}$zb4|N#p`lbq(@mX+mEwVZ>K| zMd|!VIkf!0Ti8r0!AHnYtP4fH@jTER6R#9S3pRnNciTfk`vV=%h#+ulsYg{VcE^(* z`0y#%d&~E}is9F^ASPtkj?PCQa?y`_ul3{#9r&pxp>j^!EU{kSH(%*K=?_t$Zw48m zb-9z9i5o0h_17oyaY=+)gQjac^wKCO2+WVyJwSQ(BVWvE+a#J7KoDadxzBuD>3Guo zgL~W49{R0L0|R4p7jU};LA^|4Lo<~q0XPk~OJ|1Z;3U7-k}BpTnWmo>;*8oqg&%jVX|DGa+vcG*@&sQfQ9XB-#UO1tt2IEJvQX<6ehiCC%MSP0NmJBbaVeuF5P9w+_6D+*7Bq> zM6u6$IDKy`7|tF66e#!SX84djw?UlEm3#zQ9D%FY9iF>JC#Zk`X<&Sj`k%6e9&paw z$61SKt`BFPOh#?HC03sp6(Z9^rb-&RVO)6Hr)<>NC_!!&O$q5su9LugLiXy=VMgAU zPE~%CTk%c7b$^D-2ROtdSNN(sWaT_-xROQ=^nn=_O&j^lLQa#nGFT7}KlplIP+LQR z;d#EzUmMMAO{WMCe(@Y#g)D2ESdrM<@#$3?yKQAnXNqebFoJ&=b`zb?2Hh<@{P4E+ z&fEM{O@-RXpZA$~r9IP@FMFMoKB*?5oSQaL13qUy`akTTfD)LnVG=DcUP2_QNdUj0 z-m%qC*)=Ol`h~2mn-n!8@l`p_f*pk)*6w$DvOCPG{PM-UdHtWKC?)GiwgmMmA zSlTQ$1U-OtV5f)?aGkF z1_v2p@6?)k!>(G!XIq>=yVyC6Iu$D*$ved*4OJ7 zi><#MiUc-v==r*bY6r+TSs>+;H5nA`+5)>S5X~NuyO%>mi=a2k{Mr{1#iXSHL=oL=~)-o;@JK!Ml zakR02sGHsS`ifPIy;%K>BWKBY#3NUWpE#4ZmFwAH-0-pSkJPUp{~>BV<9nSLcIg$>8YI7aL3hP)*bshSV#eRG z|Fm6J9n@hyZp@Kp$qS5xk`W+Ue+jEBvUyHTQEO;hQV>YA2s6FzN$H?DJwfN#77kjo zsn%gzh3CDazUocuTJKzm6LFL@>i)r}y*`1{-@#e4TbT;cyIV+E2ZbQunbX6Q`vja*_89~MK_IB9uheIn+f%it_#KL@ z2sOWJ6Ms65j6u_yF9oBY9zCRM@m2}nBKVdvGhw;4yS9US9M!;SYou%Zrx!{BPZ!nP zSfV;~7*ChFbQ^^VTt4~ca1j?%z9$jPnivIjAi>C*P(|Uk6ORKAlK^>h1@a2I4(PZU zoo-g5@A`tDNkHgz|F}1!Te$Uj*@vK;YW`%8Vuu_qplUDm%}ae}cBWG2f%YTr&0>gW zm425yd0uVE%{P|?+$Z^m3Lq8}S~{NeJb2mM{bnFel8uNNJ#4=7ZN0p{)zA{>a3jIT zx~HF}>QLBk(HJy1A^vsu&n!l&)MTy(IPvy1urt zDMyE_u(MQj{b0`=#bFGB+0CiZRxf)M$2uEAVdw7fZ$VL+p;Kg5@3a5?FmF)xvhpld z4$g{2Y9;?l->`h+##*Ep%Z{QD!|5Gf!t);8N1iD(yu!hpB=DR@?s>>N@PkSHo7g)? zcDy#kSyEjz1Ctxd4{y#(c7;^CpK!dH@~@cwE4g&JdA6uy?TU412#3Co^_T~@^4K#v zuMfcF9YQ#07$)uM8aV{-AFj(4N2J89UcANMY2`7CP z8R}vn>ou~~uvsObJ|e|6nf|>*^UPKGE27H;xmbhUw;*yCCm*RT$t#=SGxk%|-8XZl ziqyudrX}8zS4T+Bep)~7pjvzOIv4ss;!r7hF(nl?6M3tkZBlAu^rm?xyM|gaeYXUx zVUvE(H(FN42kA}m4E}9w!!dUR=c_i&IfgbQYbKIVK2@r@k<>*4n*Q2k;l8DZR5PdK z#ikaau;fnP&@GEWc2wl?@xB#neC$5;YOja`)Q5Rcw5f4zD7hDfI`8wD6!M$KyJ)7s zKQi0g9uihKxT>*Wn$xnl+EM(c@EhSynw=dsb5)J5Ah!7gVJ(k)Z7YY1E^47tk*l zum~E;m<1o;#rYutY$Z;h3{gY-dBA7ID{@t~<}c3+ZR%J*L~%9Bz~M<2rzCk9iucyj zU-uALei(dklvB{l#mhvfz&5zON6O+;oOE&&d_oX+cYE}Z(OSRv#d7zXc8U%&ondrTXD|&2cJ2w+C!c zl1$9|$Lo2z@%mKV2XB!lW0ncTcSyfx1{o6oU(;8HtB zsf}M%e{1)rkrcDuz*wI;#w)|Cv3>zs>&%Y8^Xi+2>M-fusUlC7&Qfxu&m=re58x{A zCwO$DG+-cLfBP>%`UvaN19Whb1xE@9H)G+8d_H8hk8^A^4^)igRV$6)y{<%?MTf4* zLN_M{@|skAtN84cLn5Q`jPWbGKc;LxXly8VlyyFZNv(RO3Ul-44~~h1WtP5A)DeLc zN-3(02#_Q_?4#`z!^~N*5$S~kkQjaoIDKOGFopWX2c!A}!`cKsyP$2zFuH}uP`XS& zSpzU}t!9m1{E(7Iq#bHPbUKZUcyZO7y6x=*;-ehCiy;;!@U9Bx4FvVagye2%)pRTBXa;?LsZP2^6`cEawGg4=}VNqY{= zQUMY3>{Ol=q#_@F{C!ZhHvD?THA_chc{0v33*8XHyfv;3Y_z`++}V3cAi^Lts}etIVEYnU9cw6U{Vx zEIj(sA}Mo=V<#wU70;l~EL2!SSX4w3NqFRNVX{j6NJ%x1(F`FZbb$|>f|0cEjYbMd zd%jLOz!ghM!%8jXJjU7ybU*jKizWpfr|u3px&%KEUvC}?2kvXk4-0U1x=9vHY8}-Y z9SL`4sEAb{TNDZ9uv>{=VJmb!l^D(AI!_Sab(AFJO#Sx`SLH9Qq1y#Ij=xET_;G7k zNn8FL@P#r8=tUkCgd9Zd7I!~*Za=>(_ybPLqTI*&t@VhIl$`N*k~Yx1o={6V<=*xx zn(q$SsXOU^QY{hITS!hAqnke1sP_RzN9gpytlbb_E7Q2m{Ef6PGe%I|SN802$>>lq z3zZXWh|PMzQASaD@nwGla~EfW0u+hrYm0p<^ANJ6x$Z?MfhlV9iOcX*Wi}+ z*hUl_j5-i$dJ0Z_vSm7*)MSm$Tj0`?eFf{^J_50gCI+B4(dvxij;AtMlKRQ_K@yGm zrpNHwauAwRZU^hMdHw~O%?ow!wIyMEm2vRINSuYPv{7J});rLcKIqUgq8hU-()%fD z>^SKU`g+=@n%M-}yZkeW>e;ZC$s-}k0t{PVATij$IOWe)bK*yD@!@xOafN!*SOS5r z{kjieJUOUle%I-><(G{?{Edw;4vA9X*#fL3!N6r&el}ds7lmuQjEe5VT6(9J@jp7q z3kD;%ZBGhH5YI5H(rp*+T1CRxk#$7J@EJ!U5Di1g7l{N4c|B(uVM-$iH>uYz^yi+k z_RcxhnjiM~YQECX{22!H$2udiiSF=nBpts2L56Xf;BPJUp=P!%`5w@79t&ZTw7%vM z^0sjZ^xpAJ1KPXuJr@^^nx|ipQI><)nU_JBSR=s|hF|OF6dzw4K-K+)Y59wbq<$le z1J;x_Y|E&5?rlw3ZHfPlWx(CC(?i#_>MKJ_)?>PI)Q0Ru#jeWf6_^hRD7n^rk!m$Z2;0yyDflYCL_(^H?b`UT+`A zckl;)n5uF}>Q3=kgyS-{$ru`wpsNjSwlu_yIP`vnn(H(7dE`TWu^}!z7 zk`w|=97UA6DgW<61cPAu4xCY-`J~}IA@a^^Pr0spgIX_`Ahb@)DOH$$oG{81K8haA zVqZy_39|x%VGcl0D1g!BkeoY zQq9ivi0Ax%bvnTL7qnkZ;ZUFJ6_#}3HH|*SC+3fR4KhBKdxLa^$^h)G{6n-}cMz>< zDzebM>`^gA9yi#4%snoFEO~G8A$GTKouu-y2SI%Cf6$+J%PFSC-=T)cuI)J=Bxg+; zm8!w2JL_Q*x!0;~wv1 zR%ND=Y*XYS=%1h2v>2S+!nXW*X!5|nLA*c|oFG#_1#OW7__qb%pD+=>LttuerAM*X zD&g>QChKgbKy(tV%06y0O1(DdS3wE&w{!3{*sWb-;C_vQsc?(giGzC#jY6`Q#zghz z)Pk_Dl^(U4TytDPE|YF(qqEb;2XUQ%u%j-N()&ZW^^~F~49KUkZyOaKzc7vM;nOft zuj=?z@pi+4OtWUmvlgLdH(85qy1|_aTnJ&1-WCvxWc5x1!q0@P@hO)K^YkDs0l`Ka z*M#%|bR$ZVK=w(uuD^Oc`hQvc%ruBRblHE~-dpT~@<_L<)Hf^y?PhtziJ|I2$AR^% zP|O0IZ{VEKh5ETm^n%s#_I{JDxYpJ=eh_SPh>B$`kHlOtZBU8!O313o1z)kASLwFT zA*V}Y%(;E7fEAufw{q96$L5(Li?$>2N`Bo;9U- z`}KXk--kvWy7{uGRkY4X0aeEytX+Bd1dScsuX- z*rVfT|KzM+@i-ul1cU(K;$-Y_!o;bIwGoS@;cflcC_~$@iT!exx1Qc==!EyT`refq z*E%icP@85OF}kPz>Pmy|j`aPo((ONOy$5j<`F3pHaRR0h2(=0A0)CK+8N&7)$U=_y zkItuj#utrFxN)wv0zQ-SyMsCeI0+zO-N6x3$F+^xH@rhzTG-)8mD%HX3w~Mb75k6` zgLTj-^hir5a|MI(et|ZFHis-~tk(GpcmW(wz~_yrzkXb@ollL_+)S@KIQv#G7<}7D z1OL6Uu(fPds}e#hwu{t53e{?zYzQJS}XtNM-EMrWi%BDzh6*?WIBF{dn3T@(R-c(GZS`-lZHny1iOqlbb2^aqj`IOEXsW2UA|k^}Mw?iP5QyQP8zgveNz8opAJK`c&{zeWHJ;GJ1EFrAND}{ z>$xA*5g!cEsp4G+Ic`XdrQ?SLUx(^#A46M)QS6cB9dC?(&>EImFqO@Z{HUB#Adcs= z@@9DEiM8)KX?JTby3g)0P0vr!k|tx$7byKF+ID!gM>p&0AqQP|V6{A_d938~z~Wq= zU&aB0TH@a6uF%?czzpSg^?yz+l~gIf>4_Srm)maHSf&Zll?qd2 zs)Etn&*O|!i6B+vtwrON9E`$9&J5CvBXZcqK+75?0u5+Nk({4Lairpqo|00h2LQ5d zag9DyyVF-t>F}=IIjXfew=rA#%h8io$D1P>LnXtW3~L$OzZRiwoJ7^nd*|0T4tM;Q zPzmIcc>bm>U!N~tYrY3?^RO1P?7*0!|JsH6O?~T$Uw})~;J$oajr5@8M((pnB6#ho zQxt!{PsO=#DrCVc&*pI*AJjK_ew=y+u-422Hg6W*&E*5X@Tu4V#xE^A=)Oo2W5Zlf zMz)DRKp)if%b>%L)j9Sh8$s>0iGK7*TA6Q)9!`o>`$=rAOre&981x=JeyYk{tj(M4 z@S(~xWHBed%WteZ z+T%sTl6ALR_^Be5z`YuM=bmTC61T|hLo?b&%iDceC;v&0HG!jRD&>}%QLCR>R*VE> zZeKF!<+@WVzH7XKGicVR)w1(0*NLm{vUgYaBnsV>!NZY5sUJAz3RH^j`j=~bCiU{u zR2&LA{bGvtu(>=?c}~4wpa_J$fizn_(=YCEv=bfgq3=1 z-K`C~b0!3yABYQW5=lKXkwJM~n(%LYMGmW)fwPeYnM?Xas{m!=7+4_se#lN)ZMGk8 z>K>w3Wpf+GZhaAu?=0=~6B}Bb6>0};OAv>gx<$OJ+y4Wv+q!h5HVMv&U)DcIH{^<5 zTuE9~SMzHFs|vni)ivmAbIXp}Ov6wI>0QP{SVVbl|K$8yNs-E_mpjqu1bsoSZI)>Y zfY!!iVU!|hofpkmx9Xt7fxc4{;&S5+3RRi4GVNvZQQ zzlllO?m1Ko#I1dW?55|O9mr`5MUvmqxkm`|MN1Sr{b^gi*X=rfww{Wnp4&TSk6~pR zR_{_+?y?-hIEFKuVLTYFJ*(87;erP{xf6kzA_&ZEbv8OxM{2__fA_@|#;++A{E~Ve zI{JG4CGPpEgI$^YzCdG=E@!9@h=8X5Ey%+2~Vh} zU?i3&8LPvvf#hUXcU@VULbD{KWOHNgbBt%9)2GEPE=c#yJ*83YjcmA?c7EsR!%cnd z-a@_PkhJm8DD|g-p@<=w7y|R71<*3{Z>pE1JT*Y~4sjW(83Q8PhmLYTQJ`OQ%?YZ( zKK7Zjtao~@i+OZ^Rz{zwq3aLxs}bqD5#(POCm>GLp4y+Zj=^O>H=kKJ+^wnPRfho!QA==t!hYU1C8+LID%V|=J-5_BTS}6E? zgb|6yN85~RTI%*v#O{_RRXae-Z>pBuUPz#h%GI3wDRKhB60W~uk+&fXc1iOBd;E0m z+Pyrl{tK&lcdWXk*xy%#-55(;xlVm?b7QUg>*Fsfw$Gd5^kV1tVjJN5g7AbuQWNN; zWRd(SggJnwSJ#G92L19|Vpo9XJDU z2+aEbXa;-)x;WT+-A8>t&aty86+4BTFX^pGE}AiYJD8#(0M9Rv`b^`4HdrBe-@qSE zU4rAdi)JnW`}D;b7oQ?aM6H9NugUV}T$!~^W(-Lb5EW1;9Nlf9^O7ji_!`HB3QO5a zK_bYIznhS}XdAs=0u2F1qb4v*yyh-8(+r zaBaCTg^$4@GSKT`Lfv>ia{LW=R9(cEs3}X^X!5ies>8HZF`GzKc%5iOawK1hRR?I_ z#LNf>AVLxbE)=+p{xL&{B<|wC!@5i8ivveyRAxrk71{a0mGQXvh@H@|<;qL=oo2d* zq}EBYy8r6mnmqYMSy&rXFl#gOg+sU7>{?r0Geef3R>MRuk6!fNbqu_q@EVnVvoUtz zaWJMT*Qz5_YHT|?B9K8L;RA`SyslYqB>-uT($GUhg4hKHK<_Xc)f;S0##03!C#TaR zkaBOydXPRRLpN+26XLnZr9D8BE5noX%-=Xn8Xn92?mhAiit!Zbc_vD2C0HQ30nZ`c z-EX%JTeC!8{ej2@5N5NLXum;ha4sb5UaAn~36pYFa~SehvN`_jGhvp2ct(+tQGAQ8 z4G$9>4_7)%w;3tIOSA!oryky**)rK2Ivy}av<$3nNM5iNdPrnVu}t9kWDzpeX27Q+ zAnJHn5ElVS7Z-y+g8VNQg@~=i?Y+P-zTsF1Y*@VUht1ieCZpZF}O=q zn~l20am^@UFCUO<7L8K~MoFffVtr`DR7vMt$L)3UM%l#v4wDVq;0PQ&6F0Z}*z!FO ziR>)(ZQ_$tyuNq8{fXj!yl~<@!42QdMqm7y3N5xE(N#He46wyX3~h+Fe$Ew!pa{NY z?2lx!=iaUE%67%hnUH zxWX6(LcTyuLe#lC$TTbV6C8&L&gyg`HFdsP%02u-B8$61C6e9(X*%!Pu6yx^$Ormh z-3L+VnI6!h^xuv$uxY~#s~H|bE6OO7<@AZV2@7icXmNGQ)2n8qAGU_@?EyHtV44F= z`@YlH1u-}wFQV!8tcy93=|xr*M5w7Q4`Vkac7uKrfN5oTK}CnxF$^gks>7%W=1?y# zj7BLkVQa^;(1&EOrdJMRCH7nj9Lyq;_bvd0qv%qOq`ffu7lEU{|ER%*&HLiYiMWO+ zqs{jMp&xt#w{c=7$ClT%F?7CV>CcF$6d@4KH`HlXNm4;~50i=OoNI9*vsUpP=s2RB z&5Bg5KlN?8FJG^>vK#Rd`?{3(dg-VBPwS4g3L_&#a%)df19zJY*BZyFu$SL~ea4lf zf*noY@tP&Q6`9T!N4Ky|nRH07o*GQGi=9b&LjQ6aAaozSiR1=U3h4*JN+l3Ehaffub5&k*W)53MXvW z#U?O`BJ|P=%vVMr1Pk_leVE78wZ;C`bheOq`0)hVSjOsI`ZpMrt0M1=6ug^pY1WmW75fB|_-QKjju@n=5TIJ~6!iu>~Y-KX2wS zMX~ZdV!u(3iY;XKdU~DWY@5d_?k5wN{v5%Qfp^bjeCERh(mzM(9B)I)p}YRRI=8aLP);FiK_t(cGz=~- z0(gDOp#W}t=&w*lauRM5z10+9+;jd@A`Xy2gQ=~lI9@aQQ6lx!78R0@px z6-U2?+Cnx|qwT`pepbHr3RLFJPvzDn*6>W`lE zOMd0?4_n*$=^170uA7r(K$MH;k;}IJ(sLAUUm`!aM_FXE+wJ#h6lG6Nu!zZGQXJnr zE)KtYJ%f68EjEx21+wr!N@9$mJ&ir2JL4zKGHii^8NOFSJ^|zUUm$%JZ#}ucPIr_Q zvd&s`I1gS1C8lJsJGn}xZTo4i-cTcdL+|s2yJXCa8jU4>ET#~?q)T>%VXuF4!=kbc z#&>u6)RtW|AwL`SPqdCZ-h5zY-q{q?UE4;xA4|8{f7%+ZSqFfmBo7I>n^m_Lr^yh2 zA_q{Ajeb`&`C#b&QZXD;`>f+zMTlkRYK|5Z~^y8H$kbaae zDoF~YTiXa9S2#QNls(E}!}z32#iXlCv*;vA$ZPY*-&~a|!|zpeFi_yxpY24_Z5*CH`=~^bY_ITx=`dA ztfl0ei|24P+=au?(3wv9o$o8l#BJYY`&^n5JFTWT{AZ-zCPj{EFLM|xSnRM%V$hIb z7qVqs*%w1z_sEK9u*qtR)SY1cAf@1%4hpyKF~1 zuM^oPr86r;kb!i3h!(?>i0!UxQSrt_oK8l_fkd;Xe4y_$pee?=lnSxDHMb;E3G&#NPz^{-25pxwOS*q|D%!uN3fAvCNV zqWUEse&ljG6dBmN(i7q5^_e!0d@h%}y=#|T+ccDNWU=Th4q|CZIvjF!O1fz-6fzqi z6c%&>Yx?T|`GA71$K^S1+Qlm&2@$idxT1e2HfWq37HTb6KAX2TwR2xtXE8gthqYK> zgoB>32UGhO){bh{6tUxT!~D&|^1jtqKRcM?Fy}XOKWTep^{%Ejt*Pg0Y`L_4T=iE= z93OMR{Ba_jeIoPActcnNR~0#Ekeh$KQX7TAU!lD}a20Co0pWQ_N#2z(qKfI4Ds4b} zwL*=WB-2&UyF})z2mY6O_j6bLb28W_);@MV+V*`(l(}RSW9f}jXAx;;#)Bi0mQizz z005LFjy#B2vDh@fYFz^3@nQ2FF?404^B%E0Uh`gCytT#J5z`%q;v zsjv1FxgUeW&Lx;rfQc6yc0pPNlJ}kg90J|ANC}RVH!XJDtzfU2bU6Ox&?ukrh3j#c zt;e8lba)2QPCd-HZSHYX!5jc-i($tXHtjrpRE@S^u=vu975d-8YTWY52zCNq*kVER z{fb@PftSf!`OEG$?~4e-o+5%Qsg^+e-~`g2Jp)44&j049yRD;7;P#Zj{SVZ+nRlm| zp4Rx}=P=1mzy-NgewmW#f4;a3rZd*()B)2#~DDJHu1z6E>>NbY|aIeJYwgh-O;+nn-}ai zgGpJC9nbYiZg+0YRb=r))4N}jakd}Oiy5tyHVq81O;LrFN_@9{@w;|3|A#0xZkyyz zu6$7d&W!!$YgQHVO8-xG9-hkk>35EWgW^i}+o}}q^GL?F6l5`H9~>9r9HX{TNy_SR z9V9P4SpAV+rtm9lr}D~_sengkD&U*Zlnse%Yt;!&+5wJ32f*g-yXp^D&ng;59g+M# zG%eycB>0u2F*;${r;Bm3?wWO&0#V!jDQ%dXYM#DQJZ-T1^#<3uXY=&a^&7tQ9)0+s z|Ij;{Cq1HlA?TqEPX}8ObemK|n`=Mwz`UjUtuF?fRjkt`EX3oyS&JNYfImYd#?|{@ zusONiPQHD0shq)_(}w5&I-8FUY?L)#ZARPEWvXsu>(}nKiZ2Cz^1;_}uQX-W=OpT< z@us|%?0r4yjohBjH~UYvNDf=$9(^%>g%TIx4Yk#(+{QG|q z)1R}Qo?x>xp;`u5#h^d=KppD2e77gXXhCm2o@|>eRQo(oI~Gk+H8-Xr3L|e0(SQ37Gb%-aY3K1dnl#AQ=QLLPNfZSi5_>IK z-@i~SZI2qww^wu4!8-02L+8l(--rWu?s)L-HcV;z_E%<>B@aVA(P6N1uL^trKEXZq z&P~15jiS=Lz%!vXgSabtbN866&L<`VW)Ji6^AC15R2HlJ4j2BQ($u^BL+9LiaBWb> z@PQmqpmeRj$>v}~o4)=RC<5IP=eE}`WVj$$g)<+h-CqpG2ij2x@<%H-Opq~uHJ}i2 zNW2``F4D&OlLt&Gi6-jXG?0g&QSv>NGYv17J(C(!p`zfFRBNaVOx*RQeTo9J`fLn#ZM?oFGI-O8L$r zkUKrJ6MGeYQD7DqbxKl|Ka*eH$eU^Fk^hluUnzsYy|YgIivc*we$CaobDSCnPNwUi z#f5rn%w`eP1=oN&P}$i;W{QecF*+m+Va0^;O=!EoUyek}I!{Bo6z>@QTN!_|Jz{8@ zCf|UEljjfhYN+sFG7!?q%!po4j_B4;E0uK0HeDM&-?AXKmAj~^cQU2P6Bx~|U^IJ{ zCJ5v{Ci*cB%qnLj^ zvk_M#u8~*;P*7*=Yo_|jj>W|82<1USJ(13sp1Up16&2{)d~|Qf4Vfo$uAPIGD(+}< z`S)*OlAX~UV#bL90iE<00s5~)IYDj_ zun1?To&u5$Aw44aV(ukTw}4>JO3bHSL(L*v62d%i&MinFEXT^%CWbxbp>l4fTX$*Ynul0!BVBP(o+f4pf{ntDG(8#3RcEO?RF z4Sjv2s^_^YF18qxyH@Z@5W$9sNw0`!l6Z}?X4liXHiLZb74Y5X0lgmiXmg0WngsFT zR#q_jFv#QJM~uhfsvC>x-f)a(lD;8lrdxAum@QhjGbgL!L_(>)LFidfTD1rhE?Yxf z)MhGA3`y*N$N~7+wY!VSEgbW+?>h@dkIst)VS2j3R}1Tx<(SuoDQ;;Jy9u)sOlAqE z2+I`E`cXHBU3+X|E60p?!7Fbi@}=b@LnT3t;J%+%5{~<|pJ0jM{Z`;%;tN53Ba-N; z)`Nt@{zwAB`K8S5Lzi73JA_|)33D4^onqSdDQWalRhYoR!@fjK<@a6)6Rd6&xn6OI z9A7k)Zyy(PqAC(cv3srCZE0}jLVlVRg6p`H2ys{ViNz{jXP2^?OJ z7MfGtBjh^VC5GR=V1eKBso=5q{d~5|h22k2OZc!^l5mJ0f(i@a%u%dgASfeOA=0Bj zvwfYV5BEQao!p!{ZE#gmh~qM8#pQDIx!2HJQD|OFTl#HH%x$w@&-M7rZutwuf?!(^ zHJ9}&7_$V+%DWl(&H zC>*D$04upS`-Ka^n>TOO3}1qQh@c8V4~0FHpOETEEfHxj8UB`=v3(i&l#hL~j%Ce9 z-2K$rhQM`yRSFriX%#`-E>O;UnuEGFB>oRkpUi1LV-C+3?U3aHvD}*?ttmyP`I-t@ zALfV$l<{P~5MNv_7f&kFZ^&gD(+({&JRWRXYNPeXZgD0+@a##c6J$RpzfU6q4uHAb zn9KY@uJ*d5PbPtMyf}Y6Pq6sB8zENL;hQj-Ux=a8>mfP!C~H%EYK3ikkrv^Y!wLIODY z8LrxB8x)!%8}d8Ed-YSE=-DJUx#AXb{hs@guMb=}`f%aDt5nVoE)yvly-@d+M6K@h zo}M*Ce_j5{Bcz3so9TB!PlPhG-6;Z}HS)$39PA6dVi^eT+e=CyF;nu2o_q<{EY>xg zCAZ9jf#&FJ11pOqca+|1;2%>*o&g6(Y;)+JynF0kxh8liC(R36q&({b7FHBW)xGf>x&AG9#V$hS@vzJo-ICtvD}QK|3fsD1%C~&ZWTw z$lpc*cBju}8+^t;s_0ms9Q#~yBY0_$fZnIx{c;cQ=rhQ~o}&I#aoQ^vNN_sU{7LNl z&pjaP#7j3su%U-Ja`R;|GALb}2gSo$` z(RE)NY?o-GWSwtM{vrB(XNIFMlFV0asfHwlZa;h$5=IbvK$a1ajqk-{O)P}SNO1WW zAO)k}R}nmPTK+xh=4agRQwf;q!B&FgxQ&Lug5knbcAZ-6I^<+++MCOrKPneU-_>~` zWA^JTUA6W@kggez@5k%z^T=&;CmSq_hgMTO-${{sh)qwR)OqqztTPj&>Pp^=ycKzG z0!La*dTLPLA`P6E20#*b@Q66{Ji0CSqn-g`s!ec#F+c+tHx=Klnn}7>7f=;bi6Af1 z>LI$;aZSE45ATPo7iPQ&fD})7dR~GL@PrtE?6k*#n?f=6*g7_0Bjb}ck^`A2o41#B zJ|96%#kyv1sCyhjT3?D@Poc{5^9J#L3OCsi328ap6Tos7nP`IpayBegA^3XQTR~GL zQQ5xbrmt{FBn^diz~%Vxtl7l}#~r)>6iNZHn*5opIfExPyQr~d`Tr}vF7^PVPd=!i z-)eO3OEdGEOxqeIKb~>6iL1X|VH8Do3Y{KJdo<@Wj@^fu>q368gscR7J<3P}2%%WM zY1V1_k853GB$&QnAIg|1nl#q9%=>q=R@SgYq8jEIEGf%-4R-hJ9w+Is@;4Ez3Nn`V zsRJ6i4NEYj!UUTXH~G{W1Hv3tTv%#B5l@2n8FOgJs!$;k`$yX#*^)0=>4vBs!l~li zW&CYOT;fP(n(ypVhYyuh20h$8_9Y)2t(0CKSF(PlgC!iP335h9FjC9sr`SxaaQq;; zE9uOPZVqpb&~$j6{FHSa(n*gnu_vo7*fIK4c(kJBbf~;Bym^-=T_~lVu6^7#f06Z1 zY4i0;VBDXt6j^~X?5^J>=;R6VZ$D&knv2WCa#v+Uow=WGwUyn3(k;K#8VXC7@|JSF#~q3@)&WlXW!og!e;ch540rO>6x0rtnTYAb9l%&#;yf z*U->}@!=tEQqY)^;Jgj4xgmLhJ|~xv0xGzE@f=6F?XFF&hR){}KP+Lsv6c!nOX|c` z^<3DBbBMb~>J&74p-A-)<=exuD@db-r% z0e*!fWSG~sr4KjTS*n}DDzyZoPT0w3|CZXycW%;e=>7E}$&K4UJ}I-z8+|2^H-z!k zkH0s1kEZ7qfN+ZsqKXS56|wuNrqOE2cV;mVwTAt-ld7NW-Fdk#d2sOOk@{929p3rU z3SF`~OqLZfGedP^vZYG?K*NIq_bbIut2MNyMxcPw?8q)7p=ByC=C76TD2`iK#K> z+3;*9ro$g6tip$yo^JiH<`|vfyfD~@*2U~+p3EOQ;?~1(X`r;SO8R1u+&aj3>mzv4NY zEo7}Vrg}5nu7x?$|A(#bj%(`r-w%#jSE-89y3#7jaHJ}uDiEzABE`s11eCNc1VySS z8X4E>$A!fRhzKY_K!&KOfQXFLDk96sR#ucLGmrowB)R>)&$*X=`}uvpf9R_<>yC5J zd7tNb-p|ubAdOPaN!uyjP^q!Lg_ZH$_-Fez-<6#lE86mK=(48BbQr*uw%7}nAzc0X z)Z7J|`(C!MJKA0UZk)0SjKN9=>g={SL+qiMWnPin$X<9LgSdV9tKdI?_1iWkXIoKJ z+m2O)$l|c35~ib6L@XhW30?I7ge7$W9EXtw$^y#U3!lLcj_;sRzf>|#+5 zdt7Q`8*n`#&=ll8^dFlCYS=6+7 z{}V1x*FY)5xK_^-bg#spQHDY+S4AM5g)%FrMsZPo5WqePS*}20H8j;S5N1h@nev~g z))e5fz@#{4@=Vx{a$g8uLuWti-9o-ql?~b4Y?mB}SCZUdnL=>E;KyI2aP!)uZzjW5 z)>5kCheH=QY&E8Y)kpu&PcbN=Yf_7<+W=@N?Zqoc?U0`r^!jK$9yaxw2KW{i_Qap+!>DL?}O4D2>Z}-{|b$ zQqUMHQO`cr%DrW!0^DNq)MhLBr#jfOz+krfNk;8?>`Sg3g>z5|3k-`G;eLf=-XhNM ze0PX{CZ_YDGvmL)$-wRc)zBVW%wK>& zhUl-=WBg+_Kk5KMfBMX~AQhxbmN}GFI)#y-g&$DTbOzPKWVroQE6=+3i6_A3P6d(Yl!41AZ_&k+k~33D->$V z0}2AP!KEk%3a>u0S7QSIfyu(w(aygA16X!PL3U;az$k#oLz|fGf z3`Q&HWks$qn|7Ty_>rO`4Yp8>hEjlhziE#fC7d0IV1d#=r7{@v0o0 zSa`hm4#v`{g%mu?n7;`KBNn1#5X@fycN1m9&iw&fn4eh)@nK-iO<+J8ngRWKqbu>K zOg06#-?dP5Y&$*y>y+{1zC93yZ=6M1BYy;|+rlb?IG%HRI1AyhfE1iL&x%%E#Knx= z$3=w~l5lTQ5=%;D!YQ_Bt%s+A_TPz_00>_WoZ&K*B?t2fqIX6;c>pEH;%lnOl^O=6 zbotNN0$4pbN5sMH{}-rDE7Z?S8hsT!i1GW*Xre+p9)zi9mY|eGAi+kha%r3zp})2i z1bp0<`Y{IezMc05H1?!a(W2{ruz!%Q`qI?IT)l$vb8i^FgvsJ z-k^@XLf~%_ zCp%krshZS_38A2FbTQ&%p+cs%EtRxQv_IT@Xw?E$Ewvm{xuWdcizuY@ix=)PfASz? zdiM8-aB?j@$QWS@_`OR^RFf!Am>-2F<)jC!i{g(3&Fdz%*C||GB?VeVbhiaP!;QND z@v~0t{jjQ{!h-UR4|pZZkr?`~ST_a0RE^M0<$jkJ*N|=cQW-1CQ<-`MkvZvr%gB=e zmm6T5xKb40|Cx?$U@ZTeQNCZf#xI2MelTa5Y>1-^{=n?k)LM3HqWm+pby)8)eJW^N z5dWta)A5re93&~M8qY;2s}X|xXjWz2M9Rh=$)QJ$$G1QaARZJ+G)qlv!snm@amzq=^69i{t5|8_2%9nn%a{B#Rhbp+~4rwO~3$@ zanSKN8u0?vRFdkwz4}C+Xn5Aa@5T+oA^`oy25a;lffhdH}FS#@tV%=XlN`V12R zlG~*X>`JS&zS8Be=YdeU1myQN5h9FZ#?o>shn!$QmpI@m&(u8tq=^k^SJ!O=&*>?R z4q(vY3P$sRpeby+PSkP849k;*QfyKrsW?GBVxSn^i?`y`C38gR6ZZawI>4cYgYAqR zBYPHgPiWqTow~UQ{kgyO`c7b_NuoJJ3+Er6bv!FpAM;ds#MgMS4hI;{ZG7vf+gv17 zNnT^SVw=d1sPtK7i?zmp4rSVS%8P_5&6&=WO4&YgU-BMWwefX!9JH`^a4~~2PmZ1H|uXkT~Zrm ztfEo4prvLEsL=ycu%U({)_X_bv*eg=2(h?@O-8WX`<}G^n58%5hs&iIyFoA+OQw!Mze)U z=CDTD7=+jULUs}2A~%E?Gj_Ghi9CR1ddg#!yNtyFP%?gHplKR_`i31xDrdM_1Lb84 z8c#@{uuxEn=eZSllaP#1aQn7sC!+NNw1c8hvsu5s;!44M#u_pPi|d8EtD=V9VI-V!3*4d?WhDbIL33=>1G6hMf`R)9*rQ!m)$7ZiryJf>*&$n9VAQ~yx zDJ3z#j*o28N;yWg7Io|(1gTF~$MBeLV(Eyzd3`h|?1o7G*dY&}A7LhJhk{IO=ndcE zM1ktIQ!=Of57t2megAL9N==h)CBajdREB%9H$&mpFD7%*cg_x=@k2bVRTJ?3TD1r7 zVfpI#h%Y3pr>DVMu_gFW33Q9HKyMkk7E0_BP+}Xzg8&;Y5$rq8_547b5K3C{*R7^7 zdW)eqI6BB^oXHnx*tkWpR4SIbAIL>;%_&?kV3*Gl4PBT9n|rr`1pm9HkRafdPx7FEwm=x~f(ut^h>L+Ni$Vx-~u! zI@woc4#d^2TdhNxzE~kJ*#QlCs@Z_=jz-~V&u~QhZl0^7V8s5{;6;MchTqk-Y+rEn zfc@_ycvlaN4kza|AepnxfnosMW}Ixc$E*dcr)eg|8))_yw7pop%qAr_*i8^9S^IE*n0R8m2uew*n9mybI<2LTE@k^r43$4fR9Hd^(5_r9pN-kG&$EqPb3Dc%{2j-*>Jt_*)eCmZ*!M=`h|_e=vls5!%gt=oKc?E6 z8y&<0)2{+T+d)3J6I}4{qE&a=H~D1NIr93OMA>55ZDZwuGt2(kA1cYu>`%axr=8Qo zeag@tmHz(VX5}BwXK2B>BHn6&H25!N6)xfbz%#~~#YjY(a4!JK6Rbw>!?y9cm*GZQ zwpzn84ro5x6w+n?X8gs^gc)e}EVN*aX&0#Gm=nDR{^0}pFj`b)BIIm-ew^>-8z@Eu z-3q)wIfxIbt5x^BV1di3SceDIzycn@)zFE-3T+s$iRjZrQIx0`*0xm z@!LT&y~4gTcT`Ob=I04H7ue6f6@L8jF1|nq8MGji8eehPk|=>{l4L~A;V&f3f4Kyn z8Hk8+nBfPTH=>+wYD^jf13b@16s62c{?#X%wS1M}^aJ^FvdrX-u)b@g>N2dce6I#n zasImiOV?O5mcNRO47hGj`6fY?^et76El@+!B_Oy6H(&qAPLXy*Y1~?X4TS2vCpWK=!YvChhZrabTi@@`xhz+%Y3 zf{&_XN*9J{9ZNil%GS-I1Tge*(;6rc$sD6<<~X=^m}%Tt0kLJlCw(DJ0K%)IJ`{bV z^!kS*P~=B{DRoaBylr%dW1iyk$75#w)4`}8O~*~b)V<6L`+IEz)zqrY2+02&)xyAQ z%r+K;Y!DaRA7z*9RXk3E^m;4nDfSlH5gDKqPMdEv>nUzIU5`(#VnJ~Gm6D*#L-bX# z?O8B4WRf}EL$0T!Hr$Vgj^Uyd?QjLvcaB);{BRiR3)1aVAnhg^(Pj+`MmVXAr<>iL zJ2Zi|n1h{g(JewY;vh8gctCgqB5{>RN{ioK5)yEa8a5xeKAnCnpDDy&5*O z49Z~}nZW_Xm}BQm#22?_7D}p9)P*@W12_R0Zu?@nMp~Ml1B`2AmssBD21{g6=nANEd^SL524Ls-}5NefbC44s)0f8iN%F)!4vcDPH4~%tBktz$}Y{u_R z7o7YGXXXXIPw|z&%8;b$WG);wQGqle->OBR_$Zb)Fen=W2bX4Ia0md;zNK~w%uIF! zwH#Ae9e*Hc#V1{fO$u!7EDjkG{wCXwPlKQ05W_uRH(B{5D*R!ih*-HZy2`1kx7(bU z;mT)m-9v=zg-{|SGd4tB2T>ZbXYZrmj$}U=Cd2It)V5j9u2cygFtmvdkkP8a#o2Yh z!JE^Dg|d*hf$KH*Mp`jA+8U3xh3hT&^a3}J+L~PiSu2eiLCf&ep6-L!5)3JgOP8T-; zOq}Ep=q{`@5XhsnLP4_|yyrqYX&!b_`2eOl+Pn8_QJWy6yMTGph>58Wm(2;&cw6uN z*!BF(*Wft|aDjk%er?e;{{~Cu5KQc;HlV|;(v8v#0|zu2$?JsxWu2Ere={sP*+PJ% z|A7;s8}q&2gPZKeQcQv~b_v$LeJ}pmcI$;+sFVjS)oCzxD-LQ zIiqulM+9uHBPZBgSgmuVls8GaCo=pu8!~1S(fBjr9I`Lk+B7DJBlm-qk+G(+N!q2} z)yYOLd2v`hraXVM&w1)@*Td1bwFAvn=EueC%q@&Xr}imBplqOAx&?+~*jZyT)#>?b z@uvFSSZv!_rYRzYa^eBLMf%_0&jVPc%`KTSheE854L6e zMn&{tA!ijL+oYSvQ+s?gH^Gj6)0-S<`#!W+SqFt0n)6)%o0tiTYntr6%5Q?6f$F3$ zsO2d9zt+by^W<$n$f!r?LV@~uRl*law5F16P zqbxk?lmM3*Wj)b9#-5qzI`KQ1oo6CEm@pUMqz2>l;Oq)43i|Z)^~}bLC!6U83Q9Fg zw_B|lm1u{=PxpWCLmK*jHhmx9t!}(N+CSJUYI`Zw>>d1fLY(A^5=v@68X&bCH8Thd zIcS=!YEUp6@)`4q5wCi+hZ%MS2LKcMJTm|S`;gOfbEwqve}s=(G%k+Jodrdo*||u; z4PYH_H((1E5SF*FHrg?~#iYjHiuQ)iL!jYd@M8i)PH%tX%9-F!Ce6uV)lhB#46E(( z5+K(0gMIg1*&--vV3qN&98ozV+(+YNq#*FFg38#u$Tdp1xPjFasS8H z^bJ;SjQlK!4zthZl8;oKg0j>i`u|hp=B< zJH?EN7;=J8fQ~_Cgtkfwf*~s77s3SQA0dhCz!2K;pF!8W6q<<(k9E^VkysSlJL)yf z!?4vNN9b_U!37~Eqb=#8RSOx4jHN8@Z4gvz=fzR9u-7;xN`>bj3d(Xc6&7)bqmIRB zTxK{}^0QGr7|=h1?0JdJ}#ICY_Mk|U~=VD*Mg#}*J+Q`O`D0r*mv_g@zr zFe!ZX|8?E*>0wkika2aVsqmRnI4NW_*_OsEsVg4>csGVJaiw+!1lt%Aze;JrY10*2 zx=yH5_FseL(u}n%iY6qHETG|Q-t$|i}$?r~PnFk^$s2ha$u)J-2 z#1kBa2+~Db5ZsZZ`oq5+fN9LC#@Q;!_F~v4)5Ge&PD}^V7X_h%;E#%FJIWUVX%O|m zVA}xxLKfQVA6YA*uTbHigrINGN&q*jAH5e`S;GtDX`HxHL7_A+UnI|jA^;LWsjrOj zZX+t#&v5c*-^Mmo0r#%<6|PJp=6(zy+h39PFZhP?1PJgh>R{#DZ_8?Wl(R zuo%l`XsM4k7zPyYBDEL@PKU%~oAdLz3J=RqTPb^X)G{$DDaMZdi_VoO`I&t&Uy>^G zpV@5T7Sx9Yj!*Cam%)~0{Xf?4lWv`*Aa+`+AJegsHkYxKkJ^?|bpxXO=&&a&-31~L z71+=ajqdONvAdfzVZNK%P_2hJ0WGL9V8@4unDyUdC3%?zRdsKJ?ctPB0G#6SqQE;1 z8Z@BPssm)hf+h^+dC{x0SDGg;01x8g4-PFZ>wR^ zs)a&W+sCktnYwS4>`%|*=o^S;hFNz&IgR{H=*`BvanzWDMHB7gAa;}LlM|<}@BEt~ z&!4q;)gMFiz*Q+qbxLK>61jW8*8fgVQGt4B;m8ad-^oeMvexy`v-|9qi_+w_%YV2# zM88QvL)6|6>{pT!Y6h0L`=#7=9VeHKciXMnF?<5*XJ&v-CinH#gJIoEd|3X$P=CTU zEYJPG*@Mt$^3c*=eaBq$GazJtGnz_>EDd(`;}h&Uua8H9Sz%i8ldfG~GAqkUXAsGp z$Dhr2kzNt$YKX79Otja-wtagm(bvHJlKpu`G9kW08dzK@K3+;*A0Jyn&Ms^}+9gS(;JhG)opDt?iw z2<)i6DuDN1o2E*091zvb`FP}k)cH_*3E8~Kav(19`$1<&emrd~2XkCKHJH90fdcQk zuwz$HV$Z_9E!}?PimUHOcE5tlJ-fqDG&zUx^l;-z3kk%Sr4xsj8M+L{U1E$J;s^9 zw4C%*q$~|gub!ck#|_ts9_kQ|1UiNIgcoPm=d~LD%?O%Xru#wk*fsFDZA82D4#+~T z^ZOHu=ha)P*Db6eU>74Wv`QPL$RDnjv7rrTtI`FSm$C3jZFdUmZd;3|WuWHh3DW!3 z*)?>yKoSQ3F*s$RTsq4{!wwi4=3kE@U?iFyV!`qbGD+J=SN|otWD{e(OdV zy%$MAh^GmZo|JnA3_S`>oD1@EMKFxM8!-VhC!piN7-EFsW$k`&AvhyKzZ{AY`BVyR zw!`@T#2d>M1FL<4T?JI03uk$Jb%V5{1F1@0?*U$WLxrAN++CWez^lc@(y4pb# zwl-TL<_TY_>#NwAFk>i}a&3TV2dsQ~yILMD-U!|G9z;h9WNE?J8 z&svNe-K&142n9%*f1YThxROWA24s5|vXg;+xHuvt=d;ut0BnQEV zUs-7%5Uw~WYPKQ+1)VN#T~499CK9a)`cZXc0DH@q&kgV9f{$j8sCk>sZ|FAOykEG_ zooH=lp4Y9T@cJh1(jbnh;)$hD%_W@qFa~-8G6YbFG!`x=s(nOT_(;Q#92bJT(nabj z*iwdRqU&}*Q7vc;l_iu0?2!mw<2PvHLoHL^HBrg$H&l$Is^!>jA38NQr4HT7^NPw1 zxEZe`l-vjV5glTSl=h&B2L8f-^nabGn{*xh!;bHvg@BT@CodLGmlrfU5$F!$7q`s~ zF{C&e4*uC{@a68f%fzow&AhA6_ z;gp@b1p#SNS0!)@4=6xSZrzp-aP3qm^@+jHKy>^dzLeeGpF~2zW@Eqy0al}7H-#9t z1jkLS%LkH$jV@B!s4CvAE?ilvKnI9FqlX6HePF48XFKfZ^_?QmUR_^fk_I8j!l@OO z^4F5rw}+njiGj!%GE3T`T!kn9FYJ3V`8PO;(POYr zEHryt$jZgPyXP8MM;9gg7dzK7_7I~q{KD*az?S!;)9&sPA0 zGF3AZgey|vyeh0}L6erk_XF}gVS1b^3gM<(DD&z@&{qo3Rv4-Io!&L%l7ZIaTg=US2$!K>M+-> zhdbu0^x}D@!khe61Bo`5&0Fe-NoMiz3<}|4)A!)t1bO%~wMQkb%<$bjLFckLJg}$} z!mC!K>bn9t;#BCoX@)chb==J_AIHa2;{D6h+k2mis*~(qg7ueETWM% zz2bg;ZN%>JO3KWB`P@eg4g(z{6&tsKTaD-bp@v9!dsHzzfHL-!?fi^-o;mc8jCcXv z3WY%DJOMd2kh{WEp!*vPlLh012SL=;VmVm1<9k6VL~&kSb07`&(vr3C0AA<}DgT=> zjmMb)HeRH1B91G zz5O6KT!lt2gjlbfL=i7ZZR*NAN7vI86h&Z}e#zjNHvVc#50->zs09Ce6OaWWY)H-G zw_bOdQG0oBsEC=}b2a^ixv3~r?IzrktV~t3k5mr2fk~Ee0TxRn@|Xj#LSC&5K$pna zeZF2!!>j&s;EWa_KJ^C3jJNOR1E0oewckK>Kf{4-0Td7+V3z1PwFQBM#m)VZ?7Sjz z-&{LN*)hGIH;tFJSbX^1dR=_IsO=%JrY(xK7pr41!W1)Eo9N6DAGz*u@))O;$93ze zIl(U32;h4i+nK?|ogW4qRIf&jeo0u=Q^O0dh26uShAb;531OMEag_LZ6lUqSJZmy?{ zQbEUKz|Pp99%1Z^=RvAXj&GZQbHO4I@fAFZh~+DMm6q@vLj97uLxYnZG)a9|yi8qUuOGar%(c0sXk-t}G!tPS2HMIL z|II&4{(OvQTBusxf$+5y4g6%6JG;Q5x6F&(FCudnSuMS0CHI6P0HR_mQ63kRsPie& zke1^&v;La_%9yCJNH2El*@3}L5d{$$JJ^zx zJ!ht~!0kw)9EfA)%1l25M7u%3c7-#r+OJqpMC-j^>-;U0DaQ!0R*5CZ09Y=>$JL2{ znTi$e-+}U^;?vn$eZSl+dFdCo`q7iOQE3iLpMngVTl>dKWq=_jCLDX|NeJ_egF!ftKpQLxD;MyjSP**`bW{!OXm*0w!?HN#HAk2_<652$BCde}n#3bhx&RO4T zV=hzoJd2rNqvS_#Whw{RAL5~lBZFMa9vK{GN)(7B>@B;gsO~Y(7uvAG7^3W>J02|D zz>1W*)oGayJ%Nd8E6lX}HyM1h7i!N0ViJV@qA4!4$DT*-AlbRwA>Za=;Rn87*-~=X ze7nWX>h5>Ie_7G4*c9+sNnwAQ5a6iib^C*ft9-}8D@1ZH%3LgbiNk4GC-LMW9& zG*K)!_xkzWalAj;NVNpS#px=86x+rFiDhz2V3P4QIi$Jd4!ELekzBEjIOuB;&cOl> zsE3*jx5h^zCrNAgRpLgL(lG)-P>#@`}%a&ABV<<89TI;_}}srG2`M_Y&fb~hB_;rVB2EK zLjrKECj}~i>7oT{4{W@ggxw>=ca3#;=Bm5Iq;Tj2+Qoz$h!w5HF>!O3Vclu#^KE({-A}qk5tO zl-}I_Wb>NUxr$jU1%vP;q}nRhE|C42_Mu0}F2Ay2=+IK(Td;CvSQ!;WE`0g{sz6?v z9GPmo%WLnl?T#u9;VhA;%LLiBrN@FdwDjziJUz4QrPc~n?|t}f!bd+u)EXc;vX5*d z9xU(3kt{6YE~)UhFtA}sOz&H%VP4#V(0R1@*lq9zVlMvj3Q`1BQ_a1h6U*+7T*`SG z2CX!VlTn&ct-T=3+6gA!kUFb&7kK1Auh^o1>q^Tr7H#5E{^4yCKKxK?%9V;C^VaVj z59m#qXcvtHh6W7D=B-YRF@5;f%bFXnH^k@>eZpzSW?m_Ntdp32iyT$QSsfs*8Yo3>~S#yb9u0#?Lx#3y)D3 z7|+|{Z~BRLW2WK6!x~5qGDtwUU*mofq*7K;j0UX<7T~r%Tg!S%zK?fh>##5%Ul!DW zK+7m_DM_bmZX27|J|YPnjD=PL)GZ5hf;l)gO3c=jaV{7IvR0Ldo}!CE#hz->==QpC zlq4VjwCz&TtsYvNIsbSep(D$~u6z>>a(B|$F*r#q(z31Tl>y=(q$?vT6NttiV`7KMm6zhZ{`c$Q0KO1{aLcL#3*xkTXkwrUfF- zatH3Y`2V%wlm0gy2Fia*qD{>Ol%G0o8K5VJ>l&`$jXz47(014Ko5u+dh3Wy_8;w4q zmqs+@G48~kfBVdUwN9NHnm}U4P`iqMTu~lcO}8C=xUq=}8ZI#v(F?l63#C4(H79gc zaP!he%2rxv_d&7nJDMmy>mgRRGk_=$q{)$jkoEAKi;H^HoqEQIdBsLvT94u%6O+JZ zaGdPb7DB+1C;L$Z*4fMXv6uJpVd8cY&tp5E4X5mz3G^WTYw&?x2=1PX`LYRk4@qVp zieos=hHj-0CmH$JR9Ij%SD1aVkQ3Dia{~pk3>S$u}Xx9`3 zGyVvqB92LziDFDKU6Z^R^G1gA^k}B-KGldp8NXoW4?hLdIBos|b&#qIkQ~_d6PSRZ z@A^Lj1|;ULM@srDjOH=u;PFLs79gpx1LDcCJOexo$2&r!42)aMkJ(m+QX@?AOV&8uh3T+en7FFqB|@sEhTY#G728S*Bt?la^rQ9&D{`UeoAz~c!V zp0?n?hg?E9tzl%_W^49ssQL;8t_PK?VPSuO4ChhoL_7p4UXqdh`OpGsB#pl@(S&#V zrf2@jx$9X`-US&ent7ZLZlUZRy?nNWX44rb1rws$eo-4&xbNrfRsQh+iju_AL#Esp zk#*6!u0sSb+Z@`F*f(Opsg*6B|4IYA?-?OEqyuTTNVEzc0g0qq6eU3>Y=RWW+ zW(&B>nab{Ss+1Og-`9;lJQtFc{j-EaVWg~{m!dejYnHG%dyaRIBlmV3Txsp;DnUro zv)mlM&&8>)*QR|I08HNp=dSH!G338|-rB8v<#UUuLRDA3%1XJ{?@<5DSI3+>$nOM1 z}V&NZ8li}{wmgW!^6QGEG$llR^H6^KdVL?-Fzp&wHP|-$V-`vR7krxBk zSDKodSo)vrCz{?SlA+$stKlHr+KimY?>!`%QOSPms&v`)yNDp(j?C2I+f5;QIhMzw zx&F<9@+|k`tV<^+^sF%nj0+;Nbh6^QCJ;jb+6s;rCvf&|p~}B&O@4t={vFYskg`+d zG3=etaCk-CeL+3%gK)^Xe;((BcxLw&VtaZJy4reO*$Q z7dtjEkO~MZyf7vka;$Z5n162f!rsgImjvA`sE(c-lm*tAoM@rA6 zk(VpU!npS%#sxzqk8gMx41M0j&RHyy*}JS^_E>zn?aDQ%aGh3vTHzrq&)Y$YDXDLlZ8|KV-#j z4gJkhH93-h|J&-qg=WD+S8w+pY8wB?>h%8lD=bxin2VyC6e#V6m0Q!Le(RS*1sh#m zVn6VLJRxE`K@{_7$u9xfws@xM*09d#OlazF_!7 z;%;5lP}rXgs8H`6xPG>er8ASb6+e!etYXoKFP~u zKRD3IoHo0W(=D0)IYyvpUb;UvuGV1%`6ltb+Ne*G&pPkAB+$xqb^jdetd#`g))&n4 z$9F8*n?w{my>nRHTQoI*(-Ey()+^DeHms98-YSr_w78LDgLf|uQd+94`qCSB^8+6; zNZA$}_rb(^*ey7=1VkIfT53la{t^B5CiMFBWGSybPCf58?Qh2T$-}?9F+a5RGqSv_ zGfGd6xT&-@&{$TE0S6AM9OwF00N_8nF}fM?X_+_yZann=l>jF5ILEzX#UNZuj~TBn6tOv>ETsJNnw-u zP4Q3-{L6MYoYbW4r2T_`6zCUAj>>NazDXax@^PLIvBbK6`E4I|?#z$e4rgtX-x^VR z*OEbK5q@VCSNk$6yh`w^ABLDPow(*(UEWdCF|gFN-`PX_?BKhPGdye~*m`LG!3e4K zBO3yabtxcB_cH2XD_f8Wr16;iMfKHf`ukmyu8N+U_;;orodWUBI>`eE^5>SZ-aB-g zz!nkQ=}81GQpV;kJ!SqS%vH+k-Wy%t?vJ3%v*M|aqYGZ?h4Q$%;VQ?-kHxL)$35sR zetPvFAf!4pp_hON!^#=EzRBJ~e4J?~?NPIxSI4qbVh1Sqt*xHho4ONXBH^qGiWS@k z2WWS#v1!%42rWD&S&vmJpLt#w9yk_*_FfgE`4CcW1!rSwy0_XEA2>>_szH`(}re z0NF-l_LZ(5BG;(I4W(}CbO1Qv`vncaKq`v_Js;k{Q_UkYE_Ig*Gg4K9_XQd3Pza3! zPW+x(>nm5TAe|>@@M|8pm&WDU^Kvca-s_wqXK;B^MJD@(hp-Owfzfe*crrbAD1oIH zIgcqRlNCg@d8rImr{;)+Qqo93QL3pY1C|GTLPp$&fan~`NK~mHQ1<62```0;4~8t& zYK3}*gL%2Io80aq(v)%*!$Oc%e)fK;RqCtU{^3fZwoyDhcmTU!*&@yj4uxIorG5zn zx_zFQtm$&XDHs(2Wr%d+H94jw>{jkBP)XTx7_82*m{OBuW~#sosvoxNckw@1rO}ii zI}?+pxV0@6v5fHj!rEiV1IQrNXuwlD%Jc&JNQMhkIvCUU@pNS^KQ4v_-0=}6w}G+% zZ5fqwkusFd8Egh4I1ZxJK2KgLM>N?g>3eiwL`MsMPvW##S$~D#CkRJsh6)y)GwwofrT>)1JZI1xo2hNd=1GQ<&5pl}L zJ3^XR#B07gqb2yjoCLlkNE~F1+vpr*)7Uk@EZsrvEBN&cii9Z!FefWG ze1#hP=IEB{gXJ)$!?Vs*q$?Il6QdxE7AI_^EvTHiU*;CbGt0S8IAgZ|^B59+mlZx; zXuhGZ-%Z5Cg1W0F@~CdTfBLnCe5N=!iX00=2Z+mZbzbvZX#n>ju;jBJ)kA0AqXOtm zMl{6ZUvslizv6v0?T!W&`h%~~)A~WBf9*{_G0P(p-DNgbHjE9~bg{OgQ2LvBR=m5q9k3-Ua;D-fr|N1umK30Q1U=t$C- z_F8!04oOqEUO6zNQea3`I}TmXVqa zGXJA6ijHmmU-={`TNY@IHekoi5t9QW633xzSJ6!@qp%7h$-U5z?xqZ!Q(K4%5xLe* zeGerSbPNUSL=^++B4f{Wxb`Ll**jhxQo7ccO}U!K4QYhy@^WU$cp(H<%0i`aqE~tL z31%`Z$cYD>7?9KD?aIwcyN9A7??}dIc*G@&!n1_PboqIP4S3ykE}scT^<=sT-&iw# zi=xqeWY6uN>cOkbMBZtI<uC%;hkNNh7O8g_xx ztt^f!H{aHa!gHAyP260?%@-8aQj$3a^H=$cKXo^N>9)92ea-)elj zv5)LzElHhskVdCW@eMJHpL)H?1{X3R+6PHRgRMV|@5<}Y2)DG{l<5ChCj`6z#x)QD zaJoBgsz$rEvTl6b`JZn9#i>`)*vNbro+~$ExJdxjF^*$BelK{8RU$U@vq7q*?LgiQ zve60qZvGw8?c;iUSBM%)H(*-VWE5oG^{35aC`6#$o4dG-U3o@u)k6~zfwYEV9^UC! zVKN}O>XMKrC!Jw{j*CWx#tcpyd%4>y+ zZ*o*5KR9%4%mC_TI~;ulR<-gCD3`p7`&h66fN4E=QA`t=CO*Or{SnA^p@Y3vhp_au zz4GAm!1G?tfmtE>F7iV27&V^OMb{35627a?H&ph>nMTZLAh5!3wyHLh*D=ZFa4T^n zs!+Rs!UzaEU}qj8R)q5OXNmnQ4n>`E#S#NDvf~d|A5p>BhIBCXw@FeBoe0fgkZ+w+D{m7CBQBPt2j% zC{>?bM`}u_H`HNPHR0yBd*BB9ZpX5Y3slM+Ipv1-BvoVn}vBj6cmSUPgWSPn+Q_ljAPyDyaxf6Q8D# zw#vmN{qi)c=ghU;jrKgELHsi;l+cy(vuqCA1Iqu8t6>yAw3?v+rG@wh-RU*SScwU6DOavhSX=dNQ>nBfgRrMHKrkyD(#O z&a*E>er@-IuS-^l)z;c-{p==EMu-h5qYOX}(U4Auzi~KE5=y-Z9Vz=thq<2!SKnh~ zwm9=#cTA`;EF?}S^PUjiK!k0r?A7`TKPVpqMRcExqD!qS*-k1WHdba+EEud6f z9aBqn5gNX@dDM<5fK5t6d#W?TYG2^$ocIG?{^>#rHr}~Easjot_9!mgo+Jta{UzuR!*S{ITz_ia5<|ry!Ig+wT7OcKU zP~L4De~t{2lYRlXA&OS7*!}dx;sZzZ)tOm(30GMws$A+$K+Bt;sFR&*)w%(eO|Yc9 z8!0ZmSk!P%-Mwv`z00nA07gHt@%Eci=(@{~{w3`C&O)U>+6xA~22`{$mCGFKcy`Ip zybrJ;OI+!^U)!sB-E`z^|GgDdg*Tj3&|)Z6SSiYulrrnXPkw?icXFb3n9*C2e-@cu z8UlILRR!{s_OUrIhE73YbEb0sJ}yz@DG-a-O~=tIVLCOeqvW0K!jDE=SUqop5z740 z=m*ZwkA4EyGJ)hk1eLFX*GJ2B)2TFLy&KigkhR@d&|8L=g-QXP?V8fQ+qbg3a9ls{ zm`l6(B<$Zp+%^kaud#zxo-u!yf;$yPrP4L;=JQ4yW%d9?HuJOcnqrk-wEEybONKE<$P6tDWxj&kF&LZPerdcGLL= zR46aBLSZ8|OlN@E)2I#A|4P_~kL5zAzet&bIZ#Wi$XoSoF3{}QBV+r z6=f3Dc7&TyyM^jZg|KC$$w04B2N5%dYU8-!bRg)rKkkbgKTzRV7Kx>D1-NC=oy88n z0kIzs3mA5kuzd+*kD+%5U_&ySFXzD{hWWna>mO{Jl_57&`Zy6ATbo_w3pGPEMUrx= zy=csu(Bdm^WjWO&hm$(F*pX-5G(42iqDRCN$%>?yFQCB{9+4Hp_B4ePGX~j%M(^Xw z3WbvA-=8ZYO-Zc{RFemQ1Ui>Q=3MnEwa0pMKhUUgg^={!$6qBEc9}hg5uig^P9d3F zc&?ytE;UTcTBx?Pr;HnV`;_5xK(syYHs`tMh>x+Pw<%)Ee8>LwzR^Z2T#v*Rle*4zTi}{j#FiUxlh7r5+J22bAaK&)FGhESQd}l_^1sN``IjLbRn^X;4u5_1 z)5X-Cx=PDywv9iHPLld|bQwou(ZjHlAfCTGtbUQ1H}|Lv*g}&~6Es)QH=X+KHE>(K z8uMF^H=}f$33w)^8nLNIf*Qq)ulz(KZX;#Xzx`ei0dNz03yNZ<3Svz3GC+6ZpA^7u zu4@5`rVR!e<(z=;E#;-AHF_r!CAjO8v5%Xp7F)5iprVAHMZWOS$X~|)Wf+VKd)!jD z{_QGCwV7a@o(&^of?E1MUO82$8f4$zpxJvz_FTDbOtDrOGsDv!^3{A^L8TTajaQoG zR=V2m$y27;(k>zHf^MKPfk6{%&9W$xqmM@KLj*`+&u`G(mmBr!L3Xce*E)4x!_@l4%9)kuB!m2fSly z=FQnj9H^^be20JmyG#dm@_CPlFu#ueLqqq6FZO@IHZVK3b~0Nbt60=XNFGg)-dEpW zdwby6^FeXe9Ki1IqDk$-bV0sp1A7bH zq)CnXYlfR@IU|FY_+1X3q_#`?v)vvaV0RcDdHtedc0ujWL;A0Sm=|=Xr3|OK^iL7E zRjM|8b(<;w{&(Om*5A+0ox_>?BE`lVZupMF#DmlO1e$}{Unut{ULA1xM(w$G8?m@_ zU#GL-jvdiHcl+)1@4{Ely3tA4{S{vmtt$|oIS#JLi~F@6`x?$XpK@wa8)uc0z5QwC z2RU)fil#LnQh z)AHX3=1*2_F6$OHxqtM%%q4GHSf<_d-($&2IW@~A*{^Pa zW1Y{T3ASe?Hdl&PrB*r~uX!4x*^vE`95!-p+Ud1O+MMxJ&=Bf2{4V*;=aE-@_OtXV z)_$3J3G;&v=v;24CNYN<(A2iu>}}Ly0hu=u)gLE`^S^Y~?pG4`Z!uraEDz!}?YL21 zYzL|V!GpbrerR-GduwyrCZh5=Gk1`bbUFVPvFA!jp?PI~b#J<0_0;&41xLxOd27^W z$Z4jnv3lNHpLu&+f=T{we~248Y_gv}%eqo*wZ(TZ<@8(rr!02U{@7K(R{Xq~6skQ6 zBYQ&CC?)da&WG0>4Xuxf6)Sk|hl#Ejg3RE!H+}ZJC_44l->2zGg_Cuzxi#A(Puj}tuENJB+}Kfz zDzp7hR;Ab+b2;~71%LO4n83Z%7 z+RW?EHR;|RiRu#%>W-`+&7dh;k?`6j_p7&!UF)j8`bGJs7N8ACi>drZtnED^p_9NfJ0?W5fTi(V^`m>`iK+Eu4!zV73kH0hql zi@g-xLHr74b!>Tn!7%@&I9vOVg=Cvr{BX$}*Bh{Nn9U#Zm)^Gkl=1mgK=Jt=uk#`o zT`YO}?7g`D=<_E-75cv(j86NTFDQU69sSmKkVj)x z=qx``MPQd9QP*ry^_ALfLmg9o?jz}PNG4sjcUy%jH#NHuZtPp*>er!7A8bT*i>r4Q zcGXGax(;QD$Q0XNf3;@M`M#p2xqpV2vQ$#3rL1&kruv~dQ^ae@yx1EIP;S~5_6_y} zzlGZQOTJvJ+&IRF*v0Hm5;R?30~gC$k0ZbFlio;q@2})Nd-IOUzZr*#^Sh@G!kx|f zs`KtqrIX4C;1R2=$_x#%C_ixcbwSg-Ke(N(+~FB+BilVUG*3y@3O0Iou~=fzY!uKC zMkc*dx_Z@K{B%nwt#M=y^|FWWrSOyv!()qNMB1QcVhHWaY7pNILpyQ&-7E5Ct$S11R)mAnZf$($_t0C!f- zJXp3@v~ZMeg03CHPv2+cDSD24JP7;aXN1A9_SWGlkTMUn_dmR1HC&fc8_;WZz>B4f z3OerVCZtyS``)=1&%S(NpGYcBb z7B?)tD<*l<)Q32SzG@G?{H8PY$?43Y{8X`W3r`?!ur2j(0}r9S%$@_!QaC}U$JeO- z+D?4@Hsyz5nZ*t6s<#X4jCPfoX8m$^YSZyzqa0W6wQf_P#MpaVL&Lg&*4M`I4ED3SCnBH5_6=u)0aJUl`u+XDkVx9SOl_@C@cr=d5F(xKMvf<`~U z_~7P=vMXbso-4J^@K!f~m}H0NuZ|^B%ZN52$$&V&O7Gn{yDzIe z_;u5N^s9^b*-e@2zPQTs2xr8w?Af3UTK&M{RF>By6Sc;#&35TQ5*ZEdB`ID^GdLAnv{>ja%gmQ|2-?IZT=^yLEe zaVR|dmTu!zIqnhHHW7nTeCeza{p)b&Su{uC0qo$eO_ILEoK+jhTzC+VANbvWyU%j$ zK@0ib(C+C9H1Ly+++il1Z~~<=_=>&XJU?{>(4FoAFMyxjr17#QaySXm;xSiNifl6eKSHhVTXbBE+bxRA_4W-bvIO90xf~<^sN{~b9re$08iQg}NvL4DJ+Rsv+ zi1XiWFvO1UE#eVdxMP^H->!*&m85Mu95j3}gNp5VouV-@02d4{9XM-(mEKY?9^^`8 zg1_Q%1l7f0ZMlclzEkA?BkSD*V#@#j@nKoZvbp3=vxwx@J0Wx#mLy@i(M`<~Qqe3) zji!!e-HK94DmAHem82WW%#x&1qx-#~RJu&l%+yTN%$)r_&pFe+-=FXAkN3WJlbLgQ zU7oM!1;^l6_#Yy;N6%pLg1)g8{a(<;XS*tXAU8zK8a%I*>03;N`C)=Kc7tm`nYe)b@kdOzvW7} z-*|NXZ_BW@Q1+NLy8*o^2A=H#4#x1V#lW&uxXFLTZqs!LtS4${MU3+DHMFn96%&?E z1j3Qbl!^MXtze{tjLK=lB~~g691>@ixJ+7}_0`JYDMhn%E^?mnG%7Ype)Aj^6msw6 zCs8@GZqn~X#T81yIM*}6=+SLu|1+n9mSwC_fIk+U^yLI?nS@}X@R*IQrP4|C59#dV za4y(JX_4NuRMtAA>7yGbsEDWsf3lm?z_i2G&Be;Pa0+_ZKyGwfuXk zv~1@wsu>qdy8VX0{E*S%6K(u3e!80~Ara4+5`1_VW?0R*r)ykU0$v61=40_!32^ZQ z`~|5|7*vLjgSw7`Y2C89yW(idZB@vDYq*-#a(ywmEpHzTX_)C-Cm9;0plr6r%=YWF2!5DO>a?liz9cYGWE4b_!WC>g@9<^F`0Tq#V z7}MO>C`-qy)=oYb(j)VY8R&+J5(MXRo*t7!)rXqN6u;De?-HO@c^Pn(2v5*lhJ1}U z`A}SQCCTx2!w#3bReYsV8*ic5h{=_~PP~5k<+{;{cn4L8u=~znZ3TxN!&wrq8d4`g zJeiI&P3me@HX4xcxjqcRqr%h8T=DQ>cMeM)vfh=_QE6P=2<7knQDtSB^o*9-5 z^BN$bNthW!UnFW|#ZV=uI}*C$y95FWJ$}q$+**&I++=JCSty)mN~qD#Om4*P7N^c6 zHnGHY0Y?^wo>zXAeV)>RO>YhCAEqYYLK}vES>DyKQuyM^LRbqSLCBgz_y=+~(`4U(??H@-c?=+M9TxK` zEwIz1V|7%JK7dtQlEhH&|6H6l%j1uVrG)FZ{ja zPCB6)?ZL~+7DfXk2K_)tnax$DmSd@ZKjDE2Cv|g?fJ40NhJMWzwpa9ZwPRBqfn19_I${152|?g*If-Zq4-dVmFTc1T?-lBY1NB&vQ&SSM>ayp=G3hA{R)*hHnN^4&1wn}Sc_2!`pQ}(-Brrw{ISx*I2{+GkI||?8e1#0GiK^zY z(+T1p^sAx|!FK^^N_a10PhT`C61@yHniPS%Ub1extN4^C-zDx9*8X=Hjh{xCBB)&X z>+X{<@L`^m*6$0)@@TA8sZJsgF?^Jp&GgXXh10(tgdIqOjtqUC1h-=d9N2a&SPV~P z!Nw9tlz$%Hu}V6^!nbl&s5B8cIB7Ms-0=jXAb{#D6dfa&KN)&L=1hwm>xqBRRiSK2 z?xVb74Om&Yap#O7vT?Gsa&rtZ5Ys-R659F{qce_CqW$-%4->;($E%wAEZzwgDqd&_ zQ+`<)Op*2S8dz@(1zP3gV~y0@+nO9NrBB&-oiSF=)D|h>y<@32!LNd^7Pm5LWY!&f z{p+(2%55;VCix*sk9k2!##G=x|FK~(k}JOS%#fwyDQHL_@&|GmDt1=C+DaXRvevQ2 z1CI`0XxnVQ!Pv8Vn!FISN-%+AXNJr(_3G|EiOw8RxUn&1h}~RO19^V?H)DFtYJ^1I4Uy`vK(F6(#xiRH!TM%t}p0FZH^ zBw&As6bf1VT;K^*QD&&kG1F3r`2!Y=0Vw#FqOvsZ#?@Jh5me{E%-@_>iqC;c3^US% zY#0;yt)iD>;I`<n5bK_!jJppTAD}MN1Df^2yT8KB zwPIK!+)~uQ5(brm%u54HipcWj;LAN%R>7bE>p9T&-46I%0d?=2=#?r>695`Rjol$~ z78a()w*ZTbsC7`x3cu_7Ypa4#;%fLC;M5G~ln{%_gD=I*EG=Q>yx+~c&E&j*``n#U zvRm6Lpe2Oxs~ydIUh?=g5WV)Gs08$VKlY3%_5kY>PZt-&h4gAWjWX^bwg$CPVz zr@M-cJ0T>({}$4nN*({Ut^R<|L#&*hq>p#cG3t^wf(4RK*fQ+GN-6kHnlW|Vj%{Fe z#E;9$LfA&wnCdz&pfF7rwkCKA>!#Ud)S&nAL6&7TbjdsAB&ieVwx*n?e^em(ZgB|S^U&MkzGte#AvG}lC2=Of$On@KPv-)qjTooreK=3 zG5s>>q?O^YO!oto?T%nrR9L?Wbxe$f)JNiBTqMwZ<0-FZQdd#3Y?}0S{duqzI4j#0 zup{kwXv($nIHOW|bNn4h$NEx50J!zuDr#oa+)1mI(g(^_;bThe zu{>bBJcff{?KBMsq+&aS!Zjp_N)F z{SyG$ODb7938cn7;&9N?sthA1s{om`|1Zwt@#rTh96lhKY;8h57Z_?V8CR-K7;X!@RrdQ=Ht(+`Ol}i*LP|BR4p2hSOpQ;`4V* zcC2n`Vlh=+g_e>G@F}{8w=jYNXwZbqEP?FYl|mq7Z@_jO9)lqZxife+Drk{SaDlWs z=|JRMlb$34{~S{VyDrYDG@{!-MF_Q4V=6|zi3JD>d>3P?`uHYhuG3(NBAFf(wNg%r zDPQbTs)=v8qK1p?v{weM)euR8Qv677d8FQOG;PRffbsF92&yZ1l9r;DlFmqOnrMu7|1$tS9@JA+n1+ zOkY7V#f>J{;K2F(Wx%P5|6yoP?;gmJQxOni-YB3jq8zJzg6FSJF0sWv6TmUBy1q!L zVj*8D>v9lYp%Tv8Ku@^)s?(GzI_IBuUyzfcK*{nhXeona{k>0UzS=g|CZzBVWJ|T^ zQ+qxYmP)+>;WOco7QbL=*{S+*=+?Qhp#AU#JlQV(hp|Thm4M%TGL%?{K_x&n+y!p? zvJPs+XbDuT1k(UI5jfp#FAG|FRjtoEs>(+e%Oo>VSN$O%x)a*e3B@wN$k6CG5T?TK z0c%%b>BP?`{#ZpvJU~)$k3T}oVgN8yPprjst(JYAP{@@3`!I<Ay~vCx9lyPpvjj4NO?nJK25>J0WAcM&gcq&gw;hwDo0Lysbn6&BYw^Wo-9tg0nV$yfP6pakO$ukem7* zK?ZEpd0>MtkZ(O?k_w(0kYOiC27EXos-R)fTi2vD!nqs-F z#RG_p4easx>6&os+p#Rk?cons!%V-JZ>#f61SV0l+7MMaGS!#%o*aa(5e`PqaSvD( zC+wBKzPSJQ0&20=R^x`{T-!+zjqjP3MmIZc`R9jQ-`XNg9Y5qmzF)`@H)yq!(W@S% zMGRqar|ElW0?ra3 zSD}?--e1~H-$&BK4L@J77X7~dCZH$+i*oHf7cVdAUvhAx71SA8cIwEc zjM-Ygf=!_L78+3l0N#qE5})N&`SerMrQzl`4LHk$j?ssoM;|aVj&#|2$4z?pG4mC@ z_?!!W<)!9F7kpQB^80w(Sjw1%(lAR_#pI{0il2mQvNtggbvDB6?tPR4?y(My#_NVh z+k0dD)B9gv=RPSqWi5-&@aZ(VK-Q73dV8J?{)6LfO&Pl1SjQug9P(8sGsS*S8Fu zE01(|1LndT`AisA{--z*3w|+^>}(2HFy>Guwexj3yczl91#~{xobh_9_3%o^lAb;X zL$Gdy`8PVi)nRWK6D{=}9J%|$-rUuWIDK6FD4$vMO>ZsK>-uP#XJbO>!c=hk! zST6MUJE^z7oDpov)F|(S1@FdEW;QSv8sd9~bT&&q7xOp13K_SV!?bx>>%D(ysE8`& zc-RG4j-4*xNrC_m(aSin1`Zo=Oq#J_S@__EMh$WV>UC=QQx*@yWlukitY%ss*N;ga zD<6%N=de8zvxQJa(m%yrXj0RW|9uC|TV0j$#ovx8y#-hQRt(;FhoX8=&Dx?}#lPp) z4xL0hO>-0;*Xna)Vhpmz9-i@;gEVgAj$|PP!$Y@yrnPL`>gS@Lv2Pk$X?IzArKj*F zzhQF`bR7(vdPg(#BMV=Bv7Os9FnCldDiWZ{)2kYh;=Gei)FdbUd!iyQQZv2NI5R-TR(i?*OKBh#SDG?4qcpSbXageIC;PL?*+bK5bU`4Y0OI= zS5#C*c@y}{_3dqyZ?4Htti9fPHq?0JmTuv2x;1r!g_3pp)JUbf`IHy_C9U@2#j9m# zkE-5yIQA7ccOVy?V~S6wuCJoIJ-c{dK2=*L?Vdg3RA<^DZ%U?k?G-TfI$MJl1UFSVxs0AYUAUjQ+iinoh59!}Y<5l2QEi;+_RmpX==XDYy!7G>uv3pYCrny6g80TRGGS4;xoy*cE{60zg2&+f97&x5j5@o zmh-4X4W8kh$+L>eJ8TuBKa3y!)+lj^NMfDR%w-h^-(U7fxSZZKDXs4xQuFwe+?sSG z&4(IA(HP~r>n%za8#f*Jhx7*=`kfzm1hp~(s)qWsq&_yb7j}EA#Y+6ytvTR_RaqEr zakr>UYfcHAH!iu3RO(Y3JXZ%*yto&_tZ#~|8Yd6(e>=Y9a@W>tULK6E`QiLId>i@G zqM`VKK(D4+`Naa7sFgl&=ms*#<*XpzJ-z1=ltlv{cpq4)+ybQwgY#q>EhnnUs39WV z*Y9KAt7abc)YxwS{5jXn36jyD*+X{|ih%MCZCG^CaG?R}ls1|P6V9riGfM}rdG zod)E2EiLTBedy`yUyzq|x!Ir}<<6X(fl`CHf5E{vAQBv80F#DD*md~xnT z<80*Vsrg51?GgR*-htOSx$=jbC|yHcK6N&if;1!1Bl&$slFk&!$vfDWYu=poL=@*m zezha5q^opGXl{?-6J)AV9Ex+}`fCQhU84-^pm}ai1MUU1?>rsFg(&R`ZE)OvZW(a6W^ z!qt7ZV^4;UUHgMFBQxcgJJ%b1I0dncE{#hwt_rPoaCXboiO^_}mYnIUeHzZ7UQDNO za$svY#(q5I(tspp2T;x}>Hc1qjZ2feyQKym^mDp?J58M<$Gx+nUVkl(RF+hE&<_hw zXUyk?zcGg`?kqw@-(1*c+jZ!5SaYIf4>Hdyu>QvQYT5|BgVLz(=k{p+^DR9irl;s# z9zlzlX=@(JGkV8u_nZ?Nz)N!S_=D0OvbA@Kld>cej0573JEse_c*x7ZI^shg=ZiV$ zXKD-;hiN}Er}-x`EsDZX!;A7iQDN!GvzWtUHyIn8>CGP{VvrVm_}*ev->9z#BgQAN z?<-oG9aM49Oi)uGsWoH`zFr<2d01%D4qX5e?=JK|r=NON)3%*b)s`6GEebvyeO^s7D6Cd-)7Yx}yM z)74UL&*@m|p>u9CJ8^$YebH}0(#&tv;;n}izrSGdlDSe_Z|fkFH-+RkMGy9TKfG>a zr*}T+$0O#swktc7yws9YTnof9NvmaMI3x6JkJ`{siHXF#kTo`Pf|a?kee4C$LR@}L zFSsn9i_0)ML1nw}*GX}rPwBu}cxS;(b z=elhn=Y^=~ss4}wU(Wk#;nc715>dB;d|jhPi<13kVhSbj8B#~v4r`=#!FY6>{C$8? zZGw118`&cy5?feGcD9F%zd0~JBdtrYD2BSJrlCPZlXd^`Q0e}ObH1%G*|)Yt-cqvUsWdu2ssK2-)ypvGsv)fm75+tMGpWZT>U=I|68}*>V#w(5L@9;G-UKI}pXuzFZ9H_TUcj$FWZ#jUGmX;= zm#f=uT&T#k2D({t503{StYmll)gpfKT)*D=I$A>qsb6=O4E8p0wtmx9{#^;bR-)6A zERR8vA)DE}Tc{$CH#`$-oR{YGqNvB6^6vz`SliI7@7ZiB{dO9|MD?in zPZCsMTvjjIRS6U*FEhzU&|%LS#Zb?G52YOC0lMP+gmr#%xbhGlFs%D6^beY6pzajM zE~?C}aIs>Y?u1+*OLCW`IeUL7e1s?Ba1ZUR6u%Tx3ac=ubvR|=C}geS2XRrGP}vfM zUrGupMHy)BTp}N`dM0W+>06S^SJRtMy2}__RA;*Ns`Z>7I@>8r!dxVJ}uAM?) zPosA4zU_+1lf5l4#P%5-k$ppU&iHn3oFK`kevc1R{h~D?SOX5Qzi4?S0g2;LT= zQ_Xr{U>IY@#il^xhDTElk0!?)id9Vcx-JI$4$>jwE3L5h;y^3XX9*7NjCC4zl`^u_l(IWH<_BuObXVPKxaKD;O58J^k zh;Zbc23-%ehY?dtj}$aqS=|AtEvyfiA`6VToPok%^?7nqh}!tI5b3M5DQBsEk?>uN!4nn0I(z&@QoklhbnUkz{MXXJ0U&k0bWZQQMZ5aI=w+8AuxfoP_cCgOkX z2zloK-xtQ}GW_Y88t#hWPh`gu=lM*$J(2zaoN3qdX=4JYl2WIi5knJGpl}C+cb2A| z%mv*rn%?1}RF=Aeld-fFN`4kifwiZp;Pm}5z}uSHL5=M}7eVU$9yu3)lkRYN1m)pv z^vTtLJPWFX8)u^Im5L8H0We#K+5i?6!=)7FHbbf)jp8?)uviF7sWzlvr*^)-XFBh0 zGI?~PvIVBY0p8m{EjtnzsT2g(>BmIC&8iNxbn8<}==wdcNS8&AI zwx^e2Y}*yBrhpyH1!5$@bQc8m2CiSDN#TT_WjHqb=utptq{)ZkRnL3U<;9AuFI>oe z3&3!C+*AA~N$5Nms8OtCHih#*Ayqw6vKyZOghNDW^T%ic#@cgxwTRWNg0ZH|#V1=X zdlw+Js4O)d15gh`YAgwUjtfo``PN84B9UsSYJE+8$#IX7kL;nuC>AN)ciLImajsJp zGp?ny>?KG)I1>t2d9B?6T<|rtO&3dFKO#_sIyZR~^W%1jr@QGjD2k4aVRWx6j^X}s z;*X-D>qNp*8V8fvxl>?B17zWEsnP{sdt!5IP)ngC%X?TaXhT`bQ8OZ6Xs(GC?_h^&m*vSdqXkNB}n`Py>#2drVnVyr~Cl!hqK!)8OnIT-vDl&h4~ zO6b!5-!3l6Op?ao;5?1^4`s|Wl-eeX8W^)aZAupMwLyGfaabB?Z8IZ&Ylo>|7R?FRQ|!g20_UGk1z|`fM%fMj;tKuf>~wL zT4>UO2HaFpASy&I<0(@inpZ*3QuqyXyo6VmB!@i;QgN^SKH)5@=s3uOU@R;i?t(Hl zz>QJ}IGjLRY^gTXE#JeeK`!4qzl`mn=&0LRb%EQmLX;SEM#*E2rWWD*?M-509bx6b ztmS@zmkYku4f&Je0)TMAJtQ}lZg023VyLo?Dsq7vN>}I}{0Y+rK?i51rNoZYqUb*m z@E$giL=n)=X3H+X$KbmS3~-!M1zF1QOX)kOZ#WyDMJ4CW)I-hm?+;c8@{``j zQ6o=zixGql%X!R}a5J6goZr~Z@jo66QD@smQ;p6a##-0Z0h7|XWg8$_wYWOqY~6`} z%@@G?U5#>*y>}p+QuwI$tM@Hs%8uTlI=?yHV%PhWd|h+Nq;3Y{p|}J+o9~iP$eOwB zDqbNDg=B0MnlLY*pSC{B$h|mhk0O`BOg+gH+_7G#`xnUI&#{{-p>)PXaUiu1-PSDY zI4uFRh8k_Y80r2~z5O2CtxiKElhXk~Hcx*f7!(0z$ZL+Yehneh*H&k{mKVFqoz9TB zO1*G&EJfx48DFklfu|{bNi1pmX1kUh%=h6w`D-U56TM_yDP|-_xHM3F5hSoPlMGeW zliiIikT>o!oNnyMU}O2$NjH2G$OC2cgBFtyd3mU_Pk%6O?WY1S_b0#wq8|)3aOKy; zsgJLHGH92SDhIt}7E=nJF?qQGf=}q^>IAA6Bl0FTf^BMAb|7^*WNFO& zStX=RN_{Ev25Faj6XG*>>Si)kaUtXLR8H=G=KG_cLO~v$)n#c=#2HA?0-CjV%DDbR zz9wChRmxeU(3aPhiHjbORFam?LobJv$EF+tT*@1GpL@G zf*xcvb<%ItbJF8rObOf&92c;dq(Zi-TN<)`rVXcuVT7eT&6?dhioz~*>MrtG z#(9E&yv~f{rxsh=q8d73WM1*NQtlhSHld;Djh%{IXfsJ&!R06;+xGBfRq~!HvN3NtrpR8K5NtL{^E0@dwTk(H z!93-9C5e$ot8%3`Lv#md{f%WQ6@>A7oT;D) z3e+1Q&BVx=>r{2nR&v|Ts+->=10cu5>nD)iP7X94(Kt)lfeH?Uf5gBgrNh89_iL2E5wj1kPjB|SvTB`i1h~wMb%%ZkaJ#% zEmc0|Bx-j`>^DBy;pArqtrSSwH>Ej@cBAwiDYz>L$rP6O+_5mAyCkZOP1VDh1^)&@ zr1jpKeHJk&6H^=K>xbqjy#^I}rlC;5!LYz-;}~Ts9>mE1=A+}D!l3S-Qz8dvygJUQ zNF}Q(%+8SeQjW|p{K)AIM|j@P=h`^pX$4QxN2{j9IdLL-lqtionKY*Me*mIrxd>TS z7qAZ&YaMX#AdHCd;Gc`Cj3J!KQDAuBTc{@RO;zwDW_W#AZq%z7rpe)0kZBh)i`X=d z^q1na9u?jQX`RHx4kT?eQ8r+uj0)vfkp?Fpz)kt!gl)?jwuk^j zXK}C-)ZzGQ_Ut<2>w4U=Rc(sV+3tqLmsJPEg!x0I3Dn4vi=6~J*#B@EuNAwdWI zIb@bdFi;$Tf5P{B_j9mRn1p{U0UkqLpM{fG?qm0@s$eE9#{z=?`va9hHX+g_Y7ShM zZR367$_#D&teEaz;S;dJ#y6D%Auw-%?Kd4$QU4dYCeV4f5JD@idK`Goe+EZpC#(Kq z8iC1PQNzoT)NT4P-Ce@8@1TTNVe2GFk73t=kGd-1Ct}LSYUSJ4SF|wmdR(iG%}uyT z;xsUUro-vJ$qC4!5B8#4;K&J#l8O9S;!W;;j?ocS$=%6VMFQdC_>fh%m@J5t1OFY0 zo&P#SEE7K}kSHiGj3Hx`>|riA?6x?+i`RDG)+etj7WBsn)kpZs|8 z43j%c^1Jsx4z_1jY1eYI2)S<S;t1QZm||_(III`C^z{_{F2~M z^79w1u4~c5^xGVbJl<@oxXg=nx+S9ba!PQXlTWutMV}^pw0yG<;0%QUN6amTOK&M) ztvKGmJosP%NBlPUwBp73@~m54P2+8xH8<2cZC43R-}}w+(cGCgT9s(X5)Ej~+=EU| zvRQH}fCwbxC zMvZ+rosMaoz8kDxOuu`u24+gNzouG_9gw0^hi?QK`eD6 zPIIs4dfYyzbh+Zl`t-w5Zd^9w-q^k$_&SwPNZiPnxwziD@Z1hcZ}{&4dD~7`Cm!~j zZTi^vss|_6xuf|^g_6I8Awqa3L%NTiGo*ZdL$(ZeKtJh6zCh+Ht1ps=m%J?$hWvGr zHR|w^vR|n5xYxk+`D$*?&F!CetW8I{K<Wn$b{!Mkn17K|y~A zonPTWo3F4;tL%9y=DaW~7(eO^(%_OSkuEKUdxV9~e;RXBTAPc>(sQe(px%G`rofyI z(tNNzS$Otn2hHcZN>yG#a{OYj#+>50_LM??T?-&dB?zP!iv$#iY^6aLY zhg!O6G)H2fr{6O7MApDhK#}x69&Gnh@rBINpfyF&`*T^d4Ev$*$hr`N#GZE7SScLT zk8@6toZ*Sf7px!ed_BvUXIDr*os)lo$AdF$vi9rt;MnB;?QaU>gq+@jhx10>rb}wI z`faLCoH5+<)O=aNix|32u6aX-`x56T%dAZ4TL&2a??#m#iHA;9*)w1K-6dGiHQ(#7 zU(|^(r=12d|B&=@El1Fc%G_)HHlGIkooi~e)ovB($;&9+k2V|z%SE3PZy7m^uh1+1 zHS_Yqn~7tQhK`Si-`t^%bo5P9s*TmhP&Ec_FO2a?{_e(?{O*?ap*Lpq&?Q5$*-b0@ zNaT&7E;MTI`+&TfzBo;xe&i~L#XY{z=L-AnFtTJI*N;!Jj^5!ZI2&(Ixk_3%zwJ!N zoMCIX7f#&HrOh4G*Da@c-A;j)+s~`JZ@L*PyPOsHYp^+g(MQ^=V<)l|ro&(FkUjEh zEH@#&=>=pdY$ncckoP-(2@5Da_jSLgEz{U#@!@uMd%x?jk+MB&=%O@IS)Q>-ys8_S z?>lc$C|eR6W%m#1_3#&zk!4t)8@tlrkR-aiEkrQ+2xo(B%jMo;`s`NI-hvxPx<>1k z?^a}lH5kwKUFy)l1ab3#g9)rSwU5%Lbq8*6E8A3B?7Y5a!8?1M%#+i5kcwZEx`ZVs zoCp3>GA-g)Q%c95LCAmB4>=01kaNWPEuKZr(mnEf$2ktE$f=O#dDh5E!8>#fCtO}& zJ0np%IcRA!>*v*c5mMUsAB27PAJY4JvpkROlm_<$6dhsK=UinN=j8geMYY4H$So2P z9o=DjT=@ubabz$vUB4cRmh-o2)U5V%Na^YBNj%;oTlYd=xy||^NAQ@{Z3z)otJGeU zPNMK#$u*o^TRBZ)$>REMUds2&PyS+w`3W7M1C;$*@5pJICz<8OtbOWXAvejoO_4l& z>i?)RAeZ}Rr%*cr`TXk&&TAL?a0(}Il|D_VTLcYUANoI}`LxsStmis|F6UD{tJzz| zZnQ(QCdhpqa?F~Kfby;B?P{^N+Fgy+FD2;b0-7?xeU^U`_0GnvW}49kzCD|_#O8O+ zcjq2`gXj%$A_w;&2jp~gQ4l&^aB9j4^6Y_5cxl}QlFz-v&6e%xM`ZZ0qi<4Z(Ydal z5xoLQc|Jw>cDHedYZ>~p@b?!%as#^)Lf{~-_P*?TYiMMEKeTE#;_pbQdDGb?aR))Q zcTfbiqe}y`vF4PkQ>PJ!x7V(`Og%#>n@x6aKP+8zO-epDAu7vXHJPSF{$V` zs#C9GeVpIJu*ECH_Za@Ie}HGHbN8kY`LfKc=3IZ%_YxNWz`OX|F-Q42cS)F&2Wlj= znWZ(n?*|uQ=M~94!<7rKcAn{$IU*fPZ?%penMHTHCf~OG!EZw>-@%Qq-wbt0Jd`n> z8vD1X-EmNaQQD8z7T7}qW}(d4ktu`V)bexq6E>p8&i`mtoeh5$p_ZAx#_Yv^XwewzPFlzjFW zBOw?{OLq^QaKX~M(-YxSLo27zBzHIU-fD3_$ogK}`d62L&K*f%%%hCo)<+`-`{dId z+A3nTBzLNZzCq8$XNkw|aapE?6Z(!d@7yWTAgVRC`wes!ssE7X!6~cH{rp> z@$>!l7@SC6u&_!08i-HzXhSYd+Xtz6B|6+&W-O8Kxq^P+pg!j}z?S^y9x5+2vtw zro3smJH>3}*tO0NfsJ#KioCqS1Je{g|AE^0I~2I+@bBo0ANb2N#9(IScd$54kaXmN#SypZsbMc`*8yD5768e@2=pU)xhU#`=7Dg704u zNgnxVX45yc$COZ@gL1bz@t%3KorBMyHh<_D;9HIjlq5E%l%i#hOML#OH7Bh2 z_SQ9ntD}%S_!3h4V@Sr)LGr+Ay^kBto}`aFqf)&0d1O@o7zG2XF@mYKZ&;doE45nk1E&GU97XOd#?CPvR%&Dv`V&M2>kJb)*udb zFFvo)WkgOJwo)5?7FTAxw6&zi-gn>rA*00w$N}@a^kQe*h~%-w=afy6fN8Wge-b0P zE85bM#@vDG;#|(kq?fGDk8+b2f=lIm>%$F9%Cdk{l`#bckg`f;Fn45?blY`Q#SM=5 z?TAAf6Wf9;g^R<{-Mx@A`?gNAzN3b^d00EV;F#?p{p%y}6_!soAQ)nw;lMQ7e5j{w z;L-)PA=y}B^QY$1TjWvOj@JHEl;J(s??unEJxL|5A}6X=l=5Tic*FIQyF+(3X+F0J zK`(0-atV`@-%wMI7|aMMw@v`_mTvA^MtJi|K45U8x;Rd7&toL`9lggA1v|zbT`b=f3yIR zpMn6CgjcbrTHTWNX~Lox_7b&wc7vv7r$0ViHrA7)Q#3E0R?$;t)FR*TGEL&OYQQ(3 zXF0DWUQEf|urdVJfw&8T((mhvD&2VlPfq|jZu7WNu911(r8fpeSadXk+&KJpaetC(!is)|JOQw;P+sPSybf z2mns%orlWD%XK!N)6wXFt=QqURm6YKbCGy=dpX3V?UdRHP zD(!MF!u#1&y|Oa>g6DRub<4bI@L#R2H@HKRZ9OP zPW^}MfZS<@=9HEt4IqG=*{ytiwBwp^DK!!`&Ri3u6xxydHXseqr3+5^p`SJ2xt!9r zJsxNi14s3DZc1AilhU45gTM}T*ZAa_6XN~cJl~=g?k&Sj=&{0Qe5ChV0R=Aw3w5S3 z55K5SkQ>(OnJJsu2SB8`Js@jdHh~@+; z#wVaWfP}vpI_&0>GO+R1k-tpuz;Hl=PgH2$8Rz}*Ez|{v4O@~$O#VNl>!47Ue@Ica z2v)O3koA>v8{UCX=aV2FTN}aYVAfM}k1?EWA99>>1)Iw#bqlEtS2mbHs6#Ei%ZlNJ zaHZ%Dc!23&zr2!%gIj>r*_X(u7Bsg2oY}AzF1W&qG9%p2g%8X0l{yO|r;>1!n-a#r z(I7%>n@;e_cg;PL#fNdqf|Ac48BNCx>#5=szC}&;tX9vp+j?)i!j+Fe!$Jd7l0eTT zWixf5BAW}BgMD1?q6fr}!!qE@F=ZrS|952r@olTIqyd0pOR?_Lu3s<)LAVR7tJbn3 zNF?f4pcEecK7k>J3?a{;1G;t+=B1wzca|d_CO8*qNJUblQ5UM_fZOVynf!iJwNm^# z`TEL8)VT?1rJ!$Yh_5;0xoIW36(Ssexo`$2wgkt!WJ>OLwnzg;(;YgXL$eqpjsLQwJ z&V8k+VXZYTzrxBSxFhxS5v^;LarJnXd7AreO}HZ5P7W8`WDyO20+cv2S*7D^SxbV1 zG1U(zwc?_DR(B=PKm;-?tDW{Cue>OV8OM$qhE29R8At>h9=@$X)M5$myjcPpIu`ZGUuRRvK&1uGF+F?S6`sKI#F5*lZbHUP06A2_F;I z4yP2Op`zVlEh!X)7zDVN0*|KO*kMQK)Vo6Ph3FaQC(3uv)!FPDWt1K5Br0Ot%*+xL zz%G=A5%IxPUC(a%xlt`u+YUo6O#sp-ED$h% z$uI(-oah5y`3c;-_m-*C-j$+LO)y8+l(=8#<8(;NY8%2x9&hZP!}A66swNSmAJ`C| zC*@P1tHTCr0YILKi6?pDiqUvgo_Ul7#;u_~*asjn8RT?V$p5Qd3bt2pUxGnGn2NRF zzJpT@!%~q8)|95>SY)Ewh|c;-@zTENwaxTyhBFl!VWfYPGnaK#*@779W(-fBYvCKW z!_Q1%3TJ`j%ZuMA#<)+kVxU=^`lsz;=-T%wGvpo2yWLDF8HgoQmzxA zchs+us9l$ku*@Zy_n5=HibUl}(TJYX%0-weQT4QQ{~_&@1J>UMM#Kb@VGkzNKZtI1 zCbBa#2qirs=f2;YUl6Uc`I5_EW}pX%-q!^zln+#HKGU+ch@)pb7VmL>M7VNM;!AAp z))MwU(ZkiKj-eJ z7iPJrxZsHa%phn#x3=sJ9|B8Tio=9JDMq)XC#8~4{R3*mtSD1?TJ$f+AthF4eaiAP z@^Hp~6qw^N3R|DBg{1$sSLWa=6 z)!miSwU_b|!l_EIVTgfE&^=diKoUd8!s7d>+0MmXz={FWo5CyhwfWB*=47vew!7J%Yc^%#ipAfCZEvli( z+(JMSgd6f&6)y$y_(r`kiyywL5Qs1w&%3f(yc!A&7+L-`#M+=i#GI=#9ZBYrAY@f} zM>8sEIa@2^ybbw~P+5uv*TBgv%bb8EB3A2tibNSYN~y~6K)z#=77hjmFHK#I8d~D) zow`4g<{%Zz(5)mpd@5jQ$YW1{E0gyaT9x$37Ih`}$KXmuo7r!>O}gpa<>#F|eh1(O zLL|~o--S)I&J02_GmS09e;|buV5Wc4p0NL+2PI?VLbx4Fd>Z05@3>FJH(6_BUw#;Y z;T(A=+E3H1Rqi>fgu~PsI5W7H^ODwe_O-Tlb2pvr~BTr6sDVeyc%Oo znZ=8qhzfe+UOwBrPZUgKvMKP?IO0p^m2Gv-Cw0)rQQ(2NJ;NgOGj*oW zOxjBSdPt1c`$)Hmf%jIht;WB?h%-z<^LDEZJrQ3P*TBZ`OJMakxi|u=HrOW&jStS^ z@Sd1#Y%P=ix*kf#)aPkv!f9agdT5vdIjsQTNo5x%zri?e`+!k^NZU;}mPX0*uz(LQ zN<+T&ZOs@c5#u;dX0z6RQnuYhDL9N%Oq1^-GeD^#4;kJ6kpw*{;Y8T;d(3jR$Ogynf4)_e(R4|kRx&Lr5u!q*qlpHvp_r}$2ts$)%u23+ zQq8G{j!WZ6%DEi!`KEv^Img;fd(N8B^_1-Uc)Ca~>7u*{k||<)8Hs&?m`zUV!C*tq zLsSfA3Wft5{fZXWvAWNHYuK;k<$UTt|L6!k$KH^AQcq8QF~fSx3nxYBx)7%k5`@$T zh);Q@w8t>CNf`)-S@N~j^J;5(N9-QR81rt~zX#%AKP!F2MyjXRM zA9_Mz06*|mxZU3)Zz}3|bTI)&n3|q)KJN^QQl6Nq366^{`_+ZDG49FsW5Ex=Crm4f zV~>!m4L<<20X!^>LvqRME98`}XwnK^3=aS@;0YRL945rp!2V$|4OhqZqETfqoowQ;A!rItcVKLI?27@P4y&V@ za`Z5^t!YzyF(aHkFiEnliUF(rPS}#;t44^5hpCGCj>l77Y>74Ka5TRg{%s2KcUp(( z5Hl9!`WSD7_5ZVt*}D3K%8y)1sZ+2rRR!$)vXw=uS^X5pf9nD!Xc|PE)U2y_$by1i zNXI7p3J9PZqFbC)N7DmNgD&g>zkcwz z59bI=fD8$>wKzXuaoXC+YA+xKfx){$86Q!3=%tTLB9Ll`&sJSNhEM+s?!(&wS9lXL zKEmLwO3|Yzi2h(ZJ>Knuj>65uvjrdtu##92Bf%#KrPUiO3Qgwn?M3 zR+O5GUo620fQuvi_6c-eMF|4L9*c}1OyxaPZ2mXbnq;PWJ;Z0> zQz`zZeB83o+(H=&;L0m_jPC9v?LzSKBv-`}{PK&i2&*0uCg=ab@j7<57=iFw=2?7@ zs*0Zo{H^brYZHLIOcg_Q((3uD{q^6#12|g2ovBD0KYho&J+Q+lJY`FG3#Bm6|9wp` z+IB*cJ2sqf`iK=+%tMj~c(5cA#1o8xf#n9lV&y4?y%ZF~=JH19J5EPR=WqP<;F5!f zdt?~HQpaae`^YFJ0@wb{s>yu+q4(@#&9@NdovhwtjUlMcnK<($?;Otu6E!#4nu-sgF{?DenQTR2=VGlu}smz}y@v^^i1AM^8b#;8+}p-w!M{kNXHyZYlaX z{{dW+>-Wp$AHjxb&R1loIApfMuBh%w=(9~rx@%GU1AnCff`E)>T+?&vYk1+Xl(u8l zkmF2b*^SJ|3-QIuD&L!Rn!+ei5;D8sHrFrrg z?|mcE#9laDC)QFZ%6>ILIaR^FasHR*#C>2t?ra(Yv2T0g_@?#qN0hejso(6oyHU|q z(_Y7M{dcn#zRjo6$09npI}w>A*MM^jd_i(Dm@e0T3z08;$6kE3gCgsv_#*c*-#j&V z=xWlG~b>p1=hwZCYq4HX@cDBxJKWXF2t~o5#xxHlz0HXQJtM=$XjZl^*8$*J_-XiYm|3S_*7K8J8-@8sZ+Y zH@^`tX4hnEwNtzuhZ6^8W~EviEDl&{|NWja_M)S2H@f!g&FxR+oLf5|JSK2b0CS%qKyM%VYkL((J{^a~c6J{~deMO}cF`|Of#?$ugDssE=Ti&Tj<(rHgP)o z%#)h8b~lbYKW52Y;kl=9=LV#GbCEgl+k3;8*RiPzNgEOgAk?$s$15nO>+*K=ka;>G z!i<(%3q<~LRlCx@PqnD?Y<@ay`T#B6;`N?&eD2y@bNgrc4o;ZYh?lpUd&k_yi>a#~ z_|@?B$M5GxE=|wfw(xmHk3Gph$06%$*|Sc$&#k-!%gjO|D)Px{WjJ%A9z_A{X8L ztckiSSnjFn1y-8a%8|g^8Y8G`l{#{wpQ3`gh(y=vRHw3{Bl_tYhSXX!$Oo;=z$RFGSY* zy*r^}4{epZ=||<0#3$N^zSsyz_jAhOAbN}CMrIkhAe5WSZ#pbIN}bJ}NuhPDU2+?x z&eMl0Oxnv)C%qcoDT?B!Rpm)U!H2t!L|Afhoe?3Sm3u!6B1ZTMG8_Vz_9=16Op`~D!L zvXXQEQz;qMcJ~cwvhc09UhC{#717`t2zMY<7DzBRp7b0t-0h}v$&}ur{I?VH*&-zm zZ!$RH8nw5JphdCqpuPJ#z(Sv)mAx_*mXDEa?*~eE^bc6PKo5DF>v^YkZTynQ4Kq^s zSL$kpcK(ER&LvXY%;tS{-_X%g@=OZ3clgk*;`oPQU5|#mvx@?_|4P-~sim(iwoW;Z zB%U*HM!7raF{wu0r*h_xw8*V_cT6;O<6kMkUiIEEUquZUPihkR7qsu4Q#`Wvoc?Vz zSQ+`pdQD>XJz8e6u=1U5PnQLYxNq5k9?tzA8B>eW+a34`?>rkFh?hwP@~Je2lgDjl zxq%*|EiG;A3V}K0B2rr7yLD*YYR^3LjNj0ajEKYWKk@&fzv3RE{}Nwn-L9^OZGP?Z zNeR2l&RcsTPR*APRY9V&a`|rU?X|QC?kZw$M3Mc(K7)c@{{!aNkqSr@y+<-n{mNOF z**MZa`S#k)F-Amv_N08HcafpvY4e#NYc$7G%}TuxET7Z!yqr_Q-}#uUs4{_&T58bZc1i zcc8}8?MdGbWNh#&CvIjlZf;i)xh69!HU^e!+l0pWGrOD>m#H?6%^ZOA)%!%c#iQ@F zH$3n2OzDMBsKZv*P=%IJo5>6pVquBX?3Id^)paZfms(x!DQnT4pHJ&=R_WeS5`bD3 z2<>twmcI~92cYu`642)Hv=&kWR73x`oH_Qb$T5Q_%jFTI|LajvFgz3>PP2 zmdw4|7}Pq#YwEaf^xTJ;^;8k8KZ}w ziZvyVx~=qy)eFCZa_st^-_mOd{oK$i@{Xi0^>x<)tR8j-WpTj$pk6$e|gWWKx#oEPQ0WZ{LHj zY5JM(9lMt_3lFN@XPhqdh==n`?l7J3@N8zQNlpLB9+LZYQp z56?f{oG^8*VHABLrwDTLs&)s?q)*Q~AImik%JIQt`nhi+zyBRX-1njSHm3vjv$x6F z7u^m2m9n;hxOy))SFqfD)X1;7<;|MaCelsQ+r;UrQjQMdu#rVd%d%jO8TeJ|wxQ z4Xgpc0UU#R?GZQHaJoP8kIRv@5>PDhIvMIyC4k{@Km};|dzfHw*fBDjPOY$ zm5bSv2PFuuR;>$@GU0Yaqz#C5<1`nMt8t69YhtVIsN_=r!8-g1sfd0sty9^8Liu-_CeVu$zZOvB2|pLQu<@}G_% z-s{LE;{o-N~YeV4@dc|AxcFP``>!XJ28HN zfeu1LGo{ihMppnS5C<<>KFYs@Lo*&lER`vaFP8)1y{QvbY*lb%ZBh~nc-{U5%+bG0 zUP>Gxw;fdw{<(FE-gOB1D2Z8)(g0OHdtwl`f0d$T>a9yP*T00auaW?S%jujVbE3^W*-_ zJAC-ZJaYE+u)2Huy=>J1cmY$#3}c@Z!lBR1StIE+O{*$c?8+SdQ~p(#1F9!Y1nk7t zBMTI&;p5*T%r4ZuzN%4ld<9?)-c7IC&_eyy=;8=>8b#E|Gn!uvloHP(06QE`Lg}W0 zc0lt3vYyMWi{Zj|ORbcLn^;+UKxp@d?GB~TN~BVjjhlai|AKh%xnkNCG0y@2fSY9j z#t3G>A{>&1f$iAywMNf*UIcL67);8A8-#Z9&;{}1zT;gyIE>vdF7?rbR|PZ+b%e1|p+_TxEI)0kcr z%nOuS?T7l3Cfe~~9DJ380oeFu%Yv;o{QI~=JyU;xXxeB#?yJ78DvyC8?%&#$6@P2x+jc-%keRE+bsc)wNi;CBA((aQ7m!4Ok{fi zHB#G_fwF6kZrb+z=%g+|m_1DUaMz@7rB(yVt}CZ?R?ftNmnej5(mh+qY*?(5@H=y~ z;ZMO~MhJ^SFl@eMzgEw?k4njPpoPeU^p8@~hhSk=@us!hU8Ni!x?#tp)TF=};82oJ zA6s}+Cw{{ty*%VC`x&Ni689!!dS$dabow>f+|Rp=HT*=!UjaMyT9BO$sLEA3A1A`Y}wjfaAL%X#!9kIB8?l0)*cOBGbFu z$xI~nJ&b+EiQehtt*ph@P!m%;1v)1huIpTl(7`!+>v|^Oyph@b_XeR-R z^JAWYC*N~dk*GH}{`05eHk9m~1M|oWEuwKflDE-52UHha5U}pbh%-XP(b5w17ayro zLBZg$TX+td-njj_i4>_U_~CLV8xLq30ET* zGS|!L#YKXX-l?K$+!J+gchJ`iPsXd*IQK1$gq8qjdxvvBJe?(j3CPyy&r@#BGO+y&V^KTsi z&pXMNaz{J_OhR+V!e~V{PZ2LS2bi%CUi>Lfdb|*d50$$@`|b<$s{-AKB6>jxhVar}$j@b$t!W(HrmqkVj2G)`LU&ME7hQ7Us^lX7Oo@~5>v{!WlO%0+a65)&jt)xiLT*AEQ79sc;H8F}+;3%u>NRNbe5P8$ z-M4r6d?fNA^>Z&E#C9(`-gdjLRux>Y(7>_J*wCKuucu;elbs4;mT6pc<~$%i;e$dRxl9HRM0M@4pgIM~(yv zOT*4dTVBZzzrAh!=oL}Blt6O6z~hlmOfsCHw9Q4W1!+b6rNKCBhCiRNG{X#^eqs$H zz6`*ep|*(sh=VFD$31ZKwfhwCUtrQdA!=aoGQ0&jNLYIpJ}eEi$u;|jH%pm_q0c!T{pSU+4q*A?{vFXss72%Pli)WlWK_y#lKVx|U9(>n+(3Vbf-?wr{GGW|#cV~9>RCLeTXCLvAAd>l z18*B4WsD*!Q%E*sVc z@cpV4-aLh4RtqPYsI?i*5tOdvL^YoxgfD<&*SHpP@|hUsyeluEAYpSIvlfZ>mm#uC z*&hHhlCUTzNiGB6`vFeYI2dd~nwo&}1CoBH*utD&8_L!Riw4kzP?^nVVC1f3Hv*_MA+AMH0B zpfGYV181?Rw&YjhVix21!=d7nNTyh4ec_vVDgddtZ9!0Kkj^bV+>dFF(-C~yBM2+w zIkETe!ztmXIYd*;yaS|8>K9mhF>aWctBA1|WujWeQBy#*C5xBs)AnLa+_Y+oJWx!5 zea4#ZO*n^FKgJ7OfV{T|**M3!h`@u-v$QgiRWO^RB*^EHg@D;riR>k}2(-Vk;>X}G z!HDq+0ozvLBqBzH!G;X{8y@`eUkp|P^M1l4#7X#h@c&L_aj^H@?XmwOOqS$?<99_; z0mMq6(f*@8dYFZz6SO>2?P4L{UaI7L@H%RhXlB7F8 z$OzmcS)2-EByoiRlpO;jb%l439KIM6KC<9Ipl=i8nxXxeBRY57noHAxxs&U%(wHS@>fwLFpO#_9OU z*HhkgY13UtR=wJ*=0d>&Ha071K4;K~eZalHT>Db4}O*8|{ zC-h^c5GvuJL>>*f=y)8s2~^YnKXVZi_c?z+S_LX7`P28S2`1`t@c6V6$azL+^YHn7F%=#bu? zG7b1J?rT1E+|T%@XJIui+(ix*r6DdiPzxz?p8CCB@)LCJtQ+lGL_dW>A801n!K5uf zP5PQNG!jqWrsbzQJ4SCA|5wUS&4BugRp&A(<-Te`cn6b0;)755p?M&K|Q( z)!m#!6?(M)BM9+LyH_;%mgrBH&4Ddws1)(Pt@Er4Y3|tC9x%%lrq(%q%b!?L+q%fW z;Nmu!nGC7-l-F!4b#L?iwSo(&k6z{^zrM^akBH7wd6M8euF!SMgYwZYH>aN7NZjMD zFY1@=I5+Y~V@vDF;2C^rM1J1~FS5|7J!esVhC;_Ff&YP@ z&RQ4g)R~(IyqUK4c@=~Hi8kLI5xBMw21!;`zus_nsqtO2@MGS0_OkOx`G+4D=%U|v zsgcdvurE7`;*6WFUM3Cr|F!h++e6VUo20cXb$Vz|)|xf)ax>Qux}t=PAIdewE)zX0 zSQ4|;wB$)$kaESZ(0NXSLX1x)7cTvJZMKB{nM!C+j!m%57qA8u4>%gw4-PqkK~;c?MBndT?dOL7x^ zw6E{}vrg3PBAl=QDDr1-+dD59w+3xCfkw6L59)`)L&Vi>vu2G|4JuA85JEgLF#Yzh zZn;Hh^36?#H(FN#?pBjcD1&XeC+J3JxKQ;wQ^sMHee0c^9PmKRPZco-&e}KXJCY)8 zuD{6OdjHihOCC{dFi7}n;gdp{sZ;ZM;HzG+yLyneqgGKK?f-S+AVllsHAXmE=k5%U zt>6>c&+1IV7s^rm}N9}JX4Ovy3 zM;x0vaKh&QRkS%eEy-e7gJ2i3nl#dK=R*7WJt|vuDeS|SDsF2-u}avt+Hk5rKdd_U zGSHO=o_u{b>!!BueD19lcR7cgN`6!?ftGNTl61J0|2Uq)J>i&;Iqv=-`{Mph_jdGU zb)V;PWo|^~(#D>7@EgZ&uTcKbi|*LJ3tBM;(U&gel|!W`OZKNGvZL+IOcC>VTAYuig-}y!otHuc81Op4K~N_f^IW zhK2w2?)7?SS2_7FX;)~~<6VE%Tfo7*vWV~hKH@X1M3vQQ)(@mJ2Kq` za?&T=`dS6-m6r97l*XPRm*x$0jE=mX+M=>ake?bRC*L|lQrh2a^Gr?f$&YQez0=Bo z7{2QqeK7JhsP3{>G*qk$&(m&8s9R0@dL+&rxedG`J4d@Vk=t0ZlUkcI(%Z9}c5cet z_0{yKs>tad!b}^&pRqNYSa3{qJV`si{Ms$SPXTMtsPFk#{eKDDN$v@8@&6bqU_{nM z>@E`&B?qs66IOFIY&Z)j%=#;es<>_D=)yCM)r6G2TD&V4EHy%X?li5wSSHhjd@_02 z_SVS;-J%R5-l}?mh6_t(7fnIXyfCfl17+`;e`YGQowN;TLG687w9Yh3!ji$%*Hydg znm$c%%)2lMby6?!H~)j)m4#iSxv7Ku1Ccb*qbnsO`cvPK1ogmdn)j$sso31!$%PYX zw_iih|tFky}swedd0fXF_e2=LloWqos{O<{&R3@CsNrw;)wFz z+BOe7KrMlHDD6XUNY3F8t@O-WZ8&XK@#};t$K-QxMW}hZN~u|*dUiA?OwM+C%~S4L z^GROb*08n{N-rCV_wiB?=`(f6O@&S+`uN5e_#Nu$yzyixm-U)#A47|zuI`v}ds|-H zA0<ph$GzjHsbhm^x3%c2k5r({*#xhH}}aAw&2J#$I1X{sS>*lO3iYz~be zG+nIlj&|8?Li2Nw4x~;ggbD4}y(E7Rw(y-l>+*J%ai%=txj%t7s+KQWaUnC{JEN?l zYWMD~1uaBt)XI5L6Ty%4IVx@TJ5e{x-Z^ww^!2o&rca3KNoy$&#}9>mUo+AN=+?YJ zm9L_MZv#(k+TcvT{WBcpqC|-)*7_`K2c5{K4B!vEwV$#yAV%&UJM0hF`9sF$RDoxr zeS=JvrDqU<&3h%*V*h4mMN%*8l#z?qE{<~D8^hnI zwSTTw7-tU}`D7VABkcM5Q-|PdozpRwIDHlO+6r$v?@CcduPMF$tjKruOiZ-C?4{P7 z!EClg7y}lj=m%iX>D#)Q@zW4wYSgQtCZ%SzQ>|*dYd9uV-d{1YUGgQ5%jka6xTb%X z${2O*GpW1hb%YLe>;m*H5=@@>HP>gnF}%Gn?`wx?ke+D62}INT>|i~ERkW7S`Sw71 zJCWxY5o+qs6uDN@1LYc|XIx#)qPNzkJN@><2p)gh_KJAJbsh0VZm=1g+{Xk-Yk<#QEw$8ZTUdu0o;JJ_A1RCmA?6VQbEVsae{>?Mpce75P zU^H(c9_@ygL1zC2+{EJT0KZUE&)Yny;wxcx9eg9L-J|SGVd}U4 znXT#rg|`0DlLV!P<1eAo_}S5Uv%}a}qy0758KUbeX^o_ow=B4VWrkJ;2fV*_L7eGy z^2BZ%WSi4*JzA%}3%}=YU#3D7i|38F!wj^@R=G`_KUApuRo!XTfkbFZzxtWkNN9Nw z&_>Qo065HAXY&XBLJa|fR%D?Uw$VdyI?SV=p5J(_V8wI$7y5G1HlbbU;N(Q!Ozipe zO(LaDgR9Z-E2MNsXK=}nfp>5YZ4`L{h1+^oZwuor^+GGH(}P;BdB-P(eKLBe1HIEB ztVDSg7I_n&pQx>}e^*{#M{swhC%_hb$S&vcWjGx1QuRIK9d7EcscWjDkA*~zp&{im zzu2?UgggMyGS41MDTnP2njrVCJ@W_~>AUV&)4O8c!9Cxoj68DD?)*+mbUxht6itN+ zN6*WGo$((@{44vMM2^4Ld|ahmxwUY9jiCC0H2vr1odq!-E+1*dO=_Qdg9Q`6nl8?I zVeI4rrO06zxc9Dw2`=k~)%-=_cKLrQ`l6n;7TFc6MDLbujdZH@@cBsoe)cbusqzQr z0-bKczD3>4uFpyerK&!IKf$3E0OX}rp?pjG;_}C_08<=H-QBZ-1PxniDD_N$4-!Y{iC}*uU`wY z3{%MvDnp2{Zlm$A=)x-)OE=H2Rtbe_AeSgy22CEzuysb(u*FaDNMp@Dmv^6Fa-VL^ zrwRstCMCx#7kCw&N2lH)#UWa)1}ddionEX6ZYUS@TJR~U+!IRKx!*uMnq7qjt!o;72R&L$vM9PtEpIB)R zS#5F;xujNaTG|>*a#OJBgSz9Z2xDWZ>WgWL`Of0Al`%RBZ)XvPSv=rqQld{9X#gXmVIMuWrYgpbV_f98asv%%_enn-*&MMLD zeO;S!Ow>LH((aaX6Exb6!zuGK-qHc>ko14kXvO}`>N|1Q(qDt-k@vX6``o0X^ZkPY z-`xQjm(F!*8PHF%BND!fZLv*dfkX@r+7j4zA4aRiblm_)pTj8Fq%_?84GWt73~96v zE}w)pjW<2(W zm$Fqm)+b>GI8l6)Am|#5RP}rM^6FfI8NW)=*oc< z?v;_cNrTlHZ-k~;h0)LEdDC0&-=lwlQ*;>&Q?H1v9<=EqojKXScaaUy|K#+!6M+2CV6bpDjOXtKqk_69! z24_^?n00!4>hiy#jtv7NhAc^tUy2k;xD+s3H<4+DjGfK!<=mZSwNEk|Xf!)*W|)=P}7jo`*$D4w{P zrlgs9v~HwN90CdL87;9e0%==~dQkybM_ ze>umXb;_cTtB2Z@1T{(sCI1-n3weOP4Si3ir7`x zN3Ua>i5qoT+~k;6nh>U>EN)v$14s2i8c{e-5scj>=wLm0$T(c+zstt2sy&W*d#j#b z*km4?)IO;HpX|NU&vsp#L`~(a!RmQl9_LSFs*bLdd16qDs-a998;QLu!}E!~-ev)( znB8u^8&Fr^7#y08vw6SIQVo=R0f!Z(LKLWDWknMb)a02$G^u1i`-7wnKacbsCTPPG zk{4v-F_3_bxmVx=Qwt`nC#0W#SwNc@L|X?CsByzZDK)Y=}`;-?@#AWtUY=$szk zOHcaRhE>F#MX{KPMI64nnG79{KZ1{M5L;JPBKjqbM4X9Z{dgbkX8{ix1ktJE@ zKoPw|m-FXTX=7$Fo$Z8cwvy#UJ#rW76{!M$Ro$%T%OpVnCvXKe3D-hP1uyfU_XiVZ z39uPN;;S0gB8OIf(Eub|_dPdx-gP?q+CUfVwFKs*kV>l+?wuPTi{5j$f?NOL!umi? z^abdhgSSHB@Ns&PUllpB8zu*b+A{-k<zPm%LNK3X~Znf?wBcQsmBQNqy{ z%8mlaXs|--g(T=04cFVF!S10T>-u{K9=Q%)D8P1*Y~F8-=G6ugIyWVw7Bm^<&)Mv>0P1YK+}NS~d?j6`Wm@;tW)2(yM54 zaK4ZIi;Fr0W+(HZ&YF^1`p3n>rBM?1=vAH+*qe>Pt~L~pYceJhjxk;vV%J0yiEH0Q0j**>MkaUFW!fZ{#tl}# zf}Z0+5yb%}#-tCU2yLA;!M9;`h_7DhJ4V@oQtsGFvNLG-_7vyevP&E77uwC!lx&_2 zxLWrG!hjhKN#vL`6X1S4l0{!!LMBOjQz0K2GxHY*KI;SMPJ_4{Jhcb(wm1zXS(4uV zKZ&h?+%1O3cXrY@2WKFeCu=t^@`V;oeq7KIvH*6Iv#zV5{>WevL;88hGkdJiebe&K zF95WO0aFbL2g>9>b%)e_Vz=P{sb*9MCv}OQ04%H956I#o37B!FYRmRm0L$_qFIZx1 zE`=jYXLPXARZNfOLTV9fYFQn_Gw}q@h(B^eLm<7CRWHIcIBGQ z*ffNk<3yY!BozQ}!`RMiroB@T7LYN-)JtrA9rchrvw9!$kLr;nB6xO0f&;K#N6`Y@ z-{H-*-fHn;L1KuVG8Tk494KyjtdR?H;#WPBG!_>#QZ~XxpkiUJ+zII1V0PJZ*c6eH zLtx_rCXpsyHYE*t_AGpYcyhHZp5gQRCWMnB zfxC&cIO^8LcZjHwv014?SB+p7Q2mg5vsG4l^SMZJkTuUPZX#tI58B} zFHOR8e#Jv=j_8EvLdDEmFSe{VMk?t*h;7jqeFohwun~O*I*gA8>k2*wxYC%6X<#A1 zY2&!|DS}4Hi8&%GDD9|dTz}GWXD!A{5YDGTzV>;spR?&=?3i`>D~g1${zAOgEN2!5 z)D2gj9M~(S;$f0cjM#%&gp2W1DP4O~yDuNuqhc2!>L)0nWL*ed>vjdq8DchUMOclPDJ5pdNj#bXR^BAe zOE!^shhzK!Ns=&By$eAKdt@a~%=6swKjP3Yn1CBXVsPmuR8HU^V+J$;VV`n9I2sjb1k9n=Sn- zy;wloEeoP%eJcMFzX?9jXOR+;lPDY~aGQ^U&bLwI3K8n{`>zI6g9k(46rDc^uK-f{ z-)w8(mNW})Ck;8VVTYm)7=4j86IM)lW6F3FEZGWyRorLoVl&Dic^H5BIkBzfJOvOUwGdgRuCN_2|#y&yZx!MkKY1iSg zV8UJT7yd}FX>evh%w2scWqK#ml!fs#B{UIyU6IlSU|4?u-q1J_;~wHu=e7biGQoJ< z5W0a=pHSB!-)Y#bj!nhDY9=dIAOi;o?yKuUC@?XjFFSam;*A#AwMW)sFNwWvqgEd2 z#iOX$|C=yOq)qf-8}GEfzR6BPp&LEF>&-whbVSTi^vTKfgOnlNbQ@QG0^F2ardo^SWR~N-yq; z0{aXXUhAJjd{ITU=-rU<5PdiyoJXEWh+cD_F>3KPAPnv@eHQT3;>L2{ZO!P`Vfy&f zS+9Wr)Ou&&IK8@9llSL97U^Wwt)=G-o9kLFNpdPLhE~0^Pv%}E@1AW=J3V~T|Gmn{ zwEnig0dO&2wUb53IMuYT%=S&0T*&41M`)hEi`e7IG~?XyYJHEq)nv7(s2hvF*LkAt zuV+?%m_im_sB4<`xpF?=F=L}c)?kDoou7L1H%9M2b+<~lPMmlVGYGYc@8%`TedFL_ zOH=avy`9{Xm{rR@)e?#)DQ9N?Fy9lF+n%1u>NuwbPHYvH3cNF;mpZvgHZxmtj_?01 zx3FZIp4)1wck|SoO0pLb^Tw(d+N|B)1oJKI?>&5pES`>rn4f zo1C{s0;f8%*6Bjoffe=IX)$7sn2RzRa%?2Gn-czFx2VO$IXaW8g{2tH@srSSh{j31M6iV_s{Iew8#BDRx-HnyiUmpN)Nn&YGnYY-g;% z#bu%V&#(KfEZ@nnl}&356vuppUL=dQ>}J<{ZzkTfaopw8R zQ#c(^YulM!9JMju)hjlh8=XN`cF0qO$~%ERzxFjH?rU9tT6vHRn^JN0aC2#bio3L& z{_5HAsye$tl+f&bM7zS?zL3I&Pw4w`p--FE^Y-oB|~W!7+b%E*%{>NQSiH$dM= zA>G%{?zpd)CSUT=nAAdb{Q3Kgt8Tcp=oAzbznU5pb*$8A>&tIPBVA50lM?w)GrnBQ zX!mfdNYh}hXhaysqy%#t9?WCHG+^6b+R#T(S8Y0&QPIf2apZfuuVK zV=J%etX{q76W3R*B<)!6SKUnYgdjs=q)SrOx?C4@joj;>!`AWB^r`g-e>yk9RMeH% zvGY0f%}V?Os&n>w2HMk{m?DQ#kH0RwTgPBd9)Vh$b(7!lfl3tSY#&d0102b)>jaf> z=<7gnk$B9<5u_oD3J>-WQ+S*A)4pBY*-gw&PhDCinw%k9PovCWCR9d~Hm7fNZL$dDwv&1pETIFf1CHARp4N|s{6s`U z=w{S-Zp8-sYky=PVMimS)Zt0iZp?5>7YmFG=kT7E8XqT@4i(rtdfh~GmLt#Xl)sj{ zc3;TUCx%mrxth(J46iz@th#BldAOYBMpg-IPT1Z5*BSkFU*)xJk&}5cPacIORwSPD z@HKkErM}Xp2;FiAp48MB6r66b47jwp*39@db#})FM(+`Sn}fgW_L@5H$xz=ZIGxh) zk=}Wpl&&N*s;}Gqiu&Q1N3g5?;FtGK#PFNj5;HF}<~T+kK54yQ(U@@ER;awl$zg-S z z#8rKyKZ)@07`9a!eY=Os|3cn((PT&ctnpfN699)&%)X&Zyk_z3RnTaElM;^-jNko12+e_@FLx^+9x` zuO(jd;h%`NvhSK5oJ1#Ji6v~e9!qHk6-hZ-T`*bv;PzIsFFlIPSr4D0x349P6AFyd zznnp{Qe293+{!6FKEc%EvE9ZEbvNXQXnv?O@JOIP$ zcxDUg+uw6{uf5y%yrbwmbY1I_Yzs0cx0l~o+~E$|z9N{BCacNL^g-oY4Mj164f<|H8sm8Y24W+M{o(UN54Vyy zxtnyl3F&2BCT*-KdgjHLpQSsY-`a4a7$I^XgbVYHg zuZnYX7c=YNq;K{nctt?nuVJCzKlt>2)-We$KcUJ$9#y}t)338ZuCHl)=$8vfW&gXp zUSr26iF%|S^IxoAzYUKm2p(IJQ=2E>=w-M89W4kzOZwVGO3S>erc>#MPPN$D(FX#S zqXln=iz17V8KYv_Bc-&@nac2qfM;7IB-T=+En=v-RWMtzh-7{uH@r9MbUXRC&QJ7_#QBrSlu|~z z!HzQB4KKhUd(-;2%SukRle^DZ$;1D9>r#h}j%Qhd8n=@^uK6>P8W$UJC?Dx<-$D|1 z`>C@hZ89TAxci@_4S0HV@u%Mw8HCF)-ALm~S*ec0?)IeyIzaGOtz!&6HL|VR9_M%d zK9Ofb6L*KwEIc6m&6+rRsAS5&x3r0Y#$E*fp>qQ492+@n);aOBzFdfX?+*pSwWe6r zJnj_uZO7%0|G9Bw?{M_e3Hg;Z1YyCc66D)?G~}OSZ@VgPquGQH35`M>E%)^yN3=7e za-LL;^t;4Fcy(!lx|?LtaGoG&+}|u5w^v{0c*3vtZ=0t8Y`Q_AG9Fb@$={#0s{1Hb zv=G#F-ws+>#R=HUtmPXJMv;Er%mB~GHTK-?4ci{nO+KfM^<-4IMLbL#>ya+eFq4T) zibt>Wo!^U?s-%V|*`mvY>8SmvDyTlPM0Su#8W>r0!g}WMx|tM)Y;OLfDe}!QBGJA_ zYLaJuD@zbkVc@P_ge09nX5|#Ogb9BCJjotuu*_vb$v%@zZ?||9BFG=5yNl`s^n_z8 z%8utx$M|QI@w))fw6Nt92>xq^-Ap}mh)38 zk(nmn5)7xEw}#WJ8LQUMY&zKAl^6wgasMI2qP;QwTQFNl54@GJ@b5^c8c#CZ@0ZBQ z9L^|xveikNv6?>42%xAHF}myiM%>kZ6k26bQrdmqv@2W@Re&ALhaF88ySzIdF>)Qv z%Y0Rf$BVx<=4|k$v9keZ1?c0FwakVQFL(U3m2F1xD>jYyh?d#cP5~c9R|LMlu^~w- zT&D~dyT;?^S5v%`1YM&iN_dwBi|{nLcljPzlWK!CDIS${9>A=FRmG63{05G0P{1v! z^tc+dVc9F#8`js6pe@EsFj~7++XSVY5w*KPlL>++VeiK}ZEr3~b{=VGOeG`3KKExtY;!0`3Xa~;thhJTnhR?V&@SO^Z^CQaH6H{>#;?GxM$99M|)tbRK+Zj z>#QS&9scOkbcNsaoFOKiCzlk%w0KB{PAV~!sX(`g_PF!B5Us zq$W}#W_(Qk zT^Vc42r0@1roGpw0jo?jAcXwZ)~aSOW;Fowz3O`*A`ZCw;sx$?+=gj*i*0H*dpz)J1uT*Zm-SG6NjnrHO3qff_R< zy0|Hw2_P1xsGBo=BV)$BFOdm05yMjOpNWwn=<1M>*-@;4bayH8ie4;Y@)^D_s}<)} z$l}gjz)JvTyu`CDdEm+RpR?ald)s-NGbvfKLma zkvvN&+?QOUh{c`OupZXkb+{NCWVF)HKh}$Q8osi8MVOcky28ExO07$i!b|kVy;-7D zFA{|^YG-JCTLLKa7eb7+7~*t;feNm4F^k00GJBNT$LqG|UyhV0#lX~08w*#K-hq+# z!Dx-0vv~cS10O)a`r`X;}79v4p-azY2 z?A7R9#cT!bHgGQvw(ij7ZreS389=?)lXnD-TlM1vlc~r*d&yW*-DbzjFj>7Bv`vQ# z8_I0ejCkS3NKu409tgTiE6KMl&+$s*?Zp~dFNhLwjVeM!s<>afSgo4WFB<6N+3g>A zY=DJB*6VEAqs3IN61p4#NL1XX+5sJP3~xIJoK< zhnFzr(l0QaY#6iFsGDpBcZn-9!V(^BrcFZ(_iuGxDZf%Qe0~!;UuQiaenAM+)yP$# z_a_du;tG;^vwp*hBs5SMyPsZ zxma`*@2?nZU06|~VN5}IeTb&JutXF$ixfcR7&DL9O=i)nNky2U?Aa;wC4R?n7e^^s zN8-V%-jc<)08nHw&z#HmF!9rLCZyrh^iNXF;8a4f70nNh$G=NmjRgfDZqB~l6 zSf#5cxTWdCp`FBU{x6WWP)3mC9iGp%vU^Rstr09EH(Dp9T0;SbA7dtJq9IXWLSiOC zCtNr3Gp(N^(Q(bkBhP_B;DQOB&xoMx08c|>1MuHm?cNJM-GZ;)(b3al?N`!wTD9Nj zHrn-i@#6PoK?v-Pfb?15YcTLNJUaleF9Z#TXupN@-xbl4n9$D!14^|RFK3>AD~@O6cS5xSr=?ak!x^dgx>xz zRNO~`LROulH>FUyk!o|45z~A4-YE*|6%0(KG|fh!*1YSBdoTQ9K$e6WC*dD*H>ueZ z0^eY~j(Pg(R)H03EZ!N#WyE%t_vG;*KcA+Gh6B(k-jnqO8LGB*EZ>lZl;f2_$8c%A zst-M~-;?GA3XG%dh+r`cYwB@(Ne{a0*zDPUUFz6L%kkIFmsR#1Q;~CK zEfy5nO>LE7#r;O=>^S{MKNQ)&)|}AQ;lC}^en8;5;~c^`Om8VP)w^3qh0=`NPsTyX zM8l0b03xW)p51?(HXIU+E(8xFshx7Bl_AZC?eM(sCw`GKA;2HHL*rm&x=l-#sz72RgjtxT(;E3?&E2n;j^N^2YQ~4>=|Sk zVRp@K$y$Ty=guu)mI`v9WV)>pr6E^SXP)ihR4Rnox!m+mQIoc-mY0T_)bGzN--DjM z1&);3%`#&XUY-Iv-==jY!B4b}xStaF?B%;2QvOByHxp?NPoG&mxwR{c?Go5j1g?Eh zHLIP>%hX|=*GueBgBWA6xGN(@YXUd9of{icP*i;t*-h?8(tT);=Kgjd(=~!M7_bd0 zmjOM+xe|7=#L491eEL8t!jxT4YW9Xn&b|p&hj(S>!0S#NOc0}o$A9yQ6S<5D73XON zNdg97u>fR&n$g9&tAb$yKj!J-UvocZr~)( z$Q#8%hp~i&&%lw^COc;XD2UOnbT!7_Oce_74MtazFo2;R0MlTQ363f>OG(Us&*1DG zU9q!}Mmw#cMXZO7Do@R4N!4$M-sjEXG4gRo|}@<}$uZbd7e& zT7k4-n(g-pDa>9Ox0`->Ogf0D-a&zY^TWVbflPBe(%x>@Pw-@Kqgl)Z{XPn25G`KS zT)XgTaaMe_1j#lbt?0R%XbKj!IQjvy7>ECP{yg*bXR z!W*UVn<|@{ZejV`*-|E0*bycr-BoJ&uas*O6O&*%fUmHxFJop`)>5_b!M*$c3@?teaq z$qlF(-->xdfKZLVnn>1tD>J$x%sB?nR9nB)|U1FpB+4dDMB$Ci6(|4&X<6SU>6I{ z;!8jrtBYq^V_+%{kpZ8sjxB1(YDttr8LP*y?Cw(h)m~$dw}{I1jDuQDjKyK#r<9DG zHJn2f=k{noM8mj<3oDkkN*?hwVm_4GhhdxIFI=wrAOe4F?pFNv*d_e`zi^Hkw#kSiX2jG`;D7?}3*%VU6UV+!o(1e3 zZi6_uojMCxp5rvG=ac~d*nNT<0shEGFn;zMBuz@zKu}fulIQayR>Eu$yBYlVx8QP- zeoC+4EBF?zluSf3#r@mCFPvS<{168k_gcTt$2(vN5wAF6tv4x6f6h={0<8YW&P1V` zHucH(_5veMnhytEvjaa0NQ=#ekN+l)0S7AuF$X4Q9yWmf>mdwB6o1HWwwn0@Sg-j3 zz@{xRX}lOSRhH8S@hW^aG5=Cb>l5pk?b4pToI^{ozr~2D$x9IQnqy}0bD75yDgu6F z0o&;k@%0;nA$${LRBy+5DH2BE4}fUJCvxmE#IQ5&%p@LOIpM5`(RZd%@eqN3E zvl{1)F~Z0=J3DdVMDp6;Z^KK7w`(#B$vyL(NoOt0DKWEz6T!xXC81qc(RnV^kHr8H zeoW|1ZQDcXdKiyP)*NK?R(P0BcneggJL7gdBtDdx*q_`(_irV%p)&c7z`Yd}@Q?{6 zTH@&8Rffuq?+M~tMQwixx4pIFw;lXno1?aUG?e-M8Eb0Ea^}z<+f|xX+5;cG3`k}k z$S-_=YK@+BdPGc_Dw@vF;um~*N_u2g(4>&{V6-(eU9n^3hj{FOX0c!Za%Y5sQl54 zGGQj?qaj~sZw;4~E~E-h)|oj1b9QXv8^VNbPP0sKjbM~W%i97AmChP6l<772n<>h%JOD&oTs<#0PR_ufUt47-=kTOfF^xBOyLtBSavxXI| zFfpHL(7(;oS1=_~9C$KnGkZ|i^;~n#0e?|AafgBcHjISJUEMYE(KfXoPoV6R)_Lb> z(_sT0>NK~o70D+q8-FD8YzJrS8LM03{GMv8e{y@?$B7T0-yv>VFJ?{i|Crn{8Aboq zZ02LwcbVlsB7FMO6An43ibgwAkib8NG>i<1;n#VbbYG81FU`vrKSm$F@?&ho+ne&v{2Nx@6cD|_P`pB$J<(KA_ z+qw){qe`TrznXq~zqz*5w3^LLR_R5?;NMwaT4nS)b}~CZ9FD)fPrfTNe(5}sw{E=c z1MQApBn<7l6T%_AvnH^c?=mCIEJYfy0x51MOn}~xxjrgBipFk?!1XMb}a+3WuAw|Z@Wh3zQorS{XLJIER82~c3GIr z?DD5M{yac#?lCWFPNX}99_sTG6zw%Ov>Pwu_>mNn*K`}+{_b^gyJ*?zXBzwgzWMf4 zy)adq?QH;Pe?}}Sqjz2iI#?-U^T&{`p15NB|BtRWkB4gizo<) z^ZEXMkMAGRqM0*idB0z;*K^};4a(u~M(Ph$04p;J1`v6kh_DT9f2|RzMd}$S7y@o2V`|erT-3;u8&3f zqyr7_L|Ow4xo>Op8phhzEIr_Eq!>Hs-09&Vxij0V4h^Tx0I%B1yt`&uLH3)VDa6YY zZqwkppa*8ZeCQ>V>KQzJ(f;6r@3zv|ms(>3BA@=hkl`onkqf+MK50||j}E9vNN;A% zwymp*X{%B24NJMbmdMbx-v6MD@YTevqQBeUp7Ln66+FWm(Cqt+ZuJ7ust3lyR)Yx; z&s5RQuR4bi~PXWAB;Z>t5b5;nu4&0ga_NJYF zUWR3iC%t4%N)-RdBvZV-=e73c{-gEiKaH1&VFf>I<*wi1%vo^7c{*z?s`haeYi!5J z2IT4;&O}mz8cbV`T`G>^_>Vxp3e}SpFKmm5-vphI~S<3(sF7E!5&N+0j-qLx>3B(aJkI&(rQ=Mb8H*Kq+Ha)b~6Z9>e32K{S zPn|c;6-5bwC}TZ#pr`dKdHWRiEx#Qk&^IaFL7ceQy>Bc{2cPa5D~<5`qo*z?tnIh+ z=gkH-_YfD(dRN#O_9ivCALI)idRfer49uey$2iZqRec)V^e zt1TtFk($iypriV#;o0No6n8jYlGl{3r{Zq5wQ2wb*MG?6dgb|qT?DYetILlte_({F z-T9K_8boITqJHgog54z#WZ*wk%%$NJSoH%)YP*&wSnDc$RRR6s6?t42cgDwK{*jV` zh^YnM5v1OYy*EwrIiuc9%$BYZV+wX0KV=b0yc`-hTf`-26zRSV}o{ z#>*kA=$^e^FiXX35b^PU0TDvj9LW7x%gbjmV@r15CzcP#EqB*3CQTYg@3?BWHD2;| z*s{y*hEbROl`r&(m7#B1`}3meONz|Oh`%cp(94k;Mjy&UQkGnkJQi(o{JMTYE8>{Y zzLYuo#~FiX7u^+loOA}j1HGQR+Q2zh#NrG_iA<>ME9hRBv?Q3UU+t!?IGaSTj&%Bu zar?_iJrbYiINj)UBwz(Bk=nFb>*BO2&%$E@h{=Y@*B9QyC zu!!Ot`2#K9SmMyDLwuVpt{IhswMv5$cA!sM&~vtE3=q~2V_UsM8oYg;B3j5eT4*m~<#X_-q=vZ$|Lv>Kz8Ino_m8C?9Wlp4V&X)vs!rmpPxpPB7B%(3U zL9DPeprxOy;)M}7CVBMBu*J$%gcdTcBe65a&hFjn)|>^rUYq^+Wq!S3PNxDZcs_c zR`20O`Fq9JOTNx;Z(uk12-YGy-maw_F?a7-(OhoG>|D{fq6D4Vr%5kW2sjkVFdYTdbmjw$m-``umNo21nb^c+?Whzc!Gcv#MGN^>sMF*oWvcrJyz z^DMz5`dPYhn%fz=Yx5+1eE#8us{ zPkJ8gI-`~25np&HbMI5p+aZ-;?D@FqvAf3NwP=Jna$&s4U6pyR8yTa+M61-}h*`78 z5cf*|BO6howQunq5k$SBQyy@(om^)lFPIM4uj=Xl_~qoadb;*X7(qaxYD?i+=g;a?NoAkR2zQxaVA^L zZjwH~yJ~?nr%^Z`0e_TP>fI^u=b74ebuQ=*VFi3*5WcMSiF%w7eQ@zD$d0zrt$a? znWBb)WNR7tydce?4Gq2vI$a^u;?hho#!)i2nk4ECQJb4$FbKktS?lm9gryB~K^P_M zu+&na%tY3}LJ}p>kPgQZT-TCHl-Q6Y%U#tbC<=Zo8v|JnhQreZhI8Ay<#sCoVJ3rc zdjpPd&|xt1C9x**DC0cT9lGdGDRCcCpwx9H+&)0-uYye*j+64{1*sza*0MW#n2k?hJ(^l4kf@~nVb}P=lPL^6|ZND$I%zShj&Z4JwQ)t2}tsZk;G! z;!7N#1eGex@zlEMb+mp$&xqk70kfRVZzN0)cZ#OV29TK@IpCC0rPg%|et_6BVkc*! z^J+2fZjymOvNT4fY4Kz-|P4K)!aa)_GF_0eNelDCq!PF@B+v}F@awL>rTkJA)KH6 zyv&4n{k=z&h{k+*1U_D-X<1)PnJk#YcDTn1G^v}d2 zs4x{9!!7?AGeFkvfEyXiK0DNi)$# zZItPS?gu;Htt1)0URw-ACkm8mdBCf{)5T6324Ouw0*Ny%QI8#pLC-r*JP%z|L{M-A zW5!tO6-!*jYq=%-IqsSs8c3|TTRY4-+*-IShEOJ90qh;E)*jEI&)2Ml)0RhVrSC1z zHQGqO!iwp1Cbwp8a5`J~X9H}?X6+!UIw2c`87=I<))L(1j+3T5qMmv-i$PqXGY;lWluAJR()} zSZJp#dbCLFav&)A*yUzXxX|1gygIcf;SEEY@H6Nj)lFpZbH?FRpPRLu6E7|Dw{&di zV37HIOEJVMzuNar?ux_nsb@7Y1?9|f&n)6}b-jd)ox73K3y(XwZke(_#1LjkkHaMS zoI!s8vYw2M+(YCG%p|i;yBi7YOBBI!4fGgmh6NUe_#j!%h@H!3 zYPjQ_i=|&M7u~8!yOE@=Hmg-QXGesjTK@V&ZWm#(Y3zDR3fZSUoK#|rQ z5iM0yREpBev&(&+hkTERk;^nrHM68Hp^jiqjR9N*JZaBFt9m#vWA$79#MJOTECa%M1j|B8!YQg)g8UENR# zy#eG&*oJ5cUFakIpv#U0eoY{bn12rB!4pPJ}Sn-SObK{;TeO%gd;(^ZML6&(AXNP30|r7 z(EP$mfQ^*RC7FxMM0;2j2(x)Q+fW4uYEpku4%NnObV)IMl+s?(&PO^@;<}89k6&qO z^Tl;?C+70$q-v&+A0wN-E)s!b+&zo$6S;wZK&y*EeV7s%-^+Bodu(C{w3~zQ(`~q5 z&Ib9zQYD+op^0P}Oq<~XEFgwmj2MDFB;ekRU(Ae{-;6x?ipa3#k@dCw{!YQ^Z z$ZI*RNukJA?d@wip&r5C1|JlIW#R2O-5Au?aUx9u6*Q)XSs6T~#hSvFm{~M`u?n=* z0UzLU;$mKG%q|t`tnI%m4yYVJUL5L7*#J8!7)(mBN?Z$A&{`-L#t$YV+A|pNkby0@ z+etfBfN-Fx1~j2R>G(3=ywzkNta0kIKLFpZ7YchOeq6uzUt` zAL)w=9D(NyeBEI~9o*|VW6&pppt_uMbz?iK0l8d2 zUNV_%+Pk|$ykX#0!7LQyp!rgVdl}3CYQ6;mwqi69@YnxE@ZE6!MZv38|4PSdaM^DT z##+LDVYL)S{Q!58Z*rAuUMP0SN0l`&7%`<4nT|iBw+STme zzE9(BnX=m_8l>V8h&VMRYt8$GHzM5wG>|aDZQ|&CC=zQV+>=}Yujz>RMu;-D`#Qo+ zgQHbBSk|*lgl4!5FK1oiyj1KhcCrI;z}CuJPAQN%>pc5W>MKB)Gb{$RJhryTXmbFH zekH>nBf!Qa7~zxy!svo6`Re$WP(+853Wcw77?cx^dl=`_W@p-Gn+I++&}xl4Iid6h zV_I#(A+SUu>7VF-NFfRPi3e`36SoJxd^(gZBShri;xjx^$u5XNcql#4Ij+e^Ds$>E zw-!@dUYTdYE0_w*QKNG_Q~hx-cUg39;&-yZfXF17|8KIi;tNrJg6t%yuJ&aBjE+Hr z+tp#%Gj&J2-6Y;obk|2thcI0^13GyY7D`USC9z-A@Jk7m_1U{%Pb6WFK=tDm*2pf3 zc-lzX4bS0->f(v?E%dsuy7q(vgAt&_#pA)H$R6CR?0;(djPb%{!Cd0>{WKhKUVRwo zNt_zS_7ar^k7ePR#wsD|?YiPAiAx$*ReynOXVq)RZ;k!tujGkBGPo7ubPF}$U6s^D z-ue~W`u>YBZP*5j2l=IfoU9l;l`OJFSn~h`w2;d$&7*xtYrRxUM=$?axxLZhU6`wKbF)N624R^Weue1(WhgTwQ%FfH z);H!tg$JEIY9cL|AjCT`9!s=1 zp`Q37?6jD<0B&rHEho@OX%_L*rP}V~IUD4VWzuR1jteuOfi6Y5!Qs&E5T?Sd1e*$} z4NF0hz99g<7?_EDLU1eGTMWuiV_V!T_x}$COW|SZ!Uw@RSizT^C<|6KhbJBL7m-4$ z&`e-kCq}=-OD+84LWVM~{D(IWlATeF1jqY<8OpHzCHTdaK0NwRXaWE8WUxZqAT!y| z7`o5jZ=H{KD5A{h5p~c_nI7lOKLqYdCcXGNnp;Id^VD*hTNyqA+N2Vw-=Oe7KM{Y14ZrV^ZEybop7()^p6+GH<9K=y%;J87=v& zA6<95-H3cv8im>|u3Oy4n##O!=cRM1E-z)muog{DxG(U{-n%Gc(&j?s*Ph{3b%dFi zonrY)>X7}yK(axD#748$@lLyf*sOO`K(8vR-FgY$INwEQ;h8=Y|6slEUH!K+?kX#5 z#Y?tbowf9-aI8cL@B2RF|B^L`{3QB2cdW_%OHzVg@~&F(DX&nje{aww{R2r8ajU*Y ze$~zGaa-L7d%uNzH~H_}y`WC^v2eq=@dpmOMxa!VC=EtMJ-xtC5$Fk zr=HJ|7w;>o`XjU%<~qJ$IE%z@oILpK#2>u1?uT^#H2CS^NaM!Y+cn~Vd$fB*La#*g zW=o*4|1R|XSynqTluH`6(b`eKdeD*9F;w(~{YxRV489w@gjdh9R|LAIeCmKvbNX3F zbVY?bJwYwyR+2@4vZDEQ0_8qaWtn@v;vnZ}lH@Zuhu&!Uxo*QFLBQv>2jx~DDwF&?$*_`U$9)mn zlRw|0zsq5}tnfQ-y!@GWR2_Y){g{rx3E)_Wf$gh)ba)ekp1H?`*tSf_B|gzyVs)lC zrqLB#NWj-t2kb?cEY0mQ+f~)rgu~zRZ5vQZ=*rTn#pLkD=Y@^%FICyN+ z+N!O7;RgxN15M33$u$Cwh?n5Bw)576HzrO`x_+M$IqGHN_8!}|Nro2l=_&ivQ;VV2 z{eB(7ESo)T0Fjkz9~Fs%AO^$cS-2KdgV`Nh!ncIgUlcT%Z9S&j9bYT?y0p5|N8i%_ zAeVDSH6ZP7gXlmK;tMQ{kuw2ZiL`g&XGK4En!TBgAF5S8dPoPJ!~X=FdIG(u;KyLS zY$oY-ZXQ)+Y-M0&K+`!ptFkt?N+q}0BR=;*D{?RJT;L?)O!AI2{<0s?CjH2DlZw6b zM~ay32lTTxD)qkh*z)n^rRo_M*nAkQVK~0Dc9%y&qUXoaBHhF*)){G$>Ve_yWQ}I zxkgOg)r1=@Q7XFv6WrGkOY&d{XX*+r$b+MtCeQ?KdvPFz5r0uNJlPetf29M#Ym(Aj z;}qf)qto@yvpLRt!(Epiqi7`70fTyPfI}4HWDMlU4c%9&i{0D#Dy-hbkY8opn48bF z?c9wZzSC)D8=(2n+9d$eSjm(x3LjkM+VQHEE1a2i<+a=HY4ZacnlDXUSSwk?PJj7L z?$x>P%iHcRF6_w4vfJB2cMk2}LGP||9^3X?Yl_8r31A%F5 zpK~pnMJAFF3OMOoJW)>;)nVPvOTFG7AB&cTn)HiWMR<_d+sS49{9ONI#|%DaIhx;3 zyxVww9Kqq{vv$G!y#O|@|Kx+YmrjY!1h;^%EbUG3M_Wp_8Q5^8tNH?FbHUdJ0Bho< zc{^v;aV~M4fgf-Im;;xIWorBbXn{&AYK5K|rE(meO-=T$e@Y+Ls7;O!pDZgZdVRMs-{H_L8@t%XQRPoz zy8x~+Zl%k=MS{3F{2SK$y;OS6n-w4xp|mmAFT?|&#T*@eQt*SARs2C)k`MQ&wV#Ym z=Znxw+*e|wjHAO>(d6+=y^84FGf9oqf}mI~y##Kpqz3o5fuH)sI<}>xo1^0nA}_)3 zYLZ>Mhi1}!>W}aFRM`GbpD7$Jngw~6-i`*EnTb{^*G`u@K3-)ec~V-+N zAM;*Z)R-rI4uz~nPaB%{_jV+`bd%3jOHSjE-5laPSRu6Y~I zBQq<eC@T+!i*YtIlXV-k$VG)1A+DJb!yUFGs71Z~a9UthsCo4c}uSc!_nKq<yZ9zo%1gKcbz{1Ki8o(eCX( z6buibs%xToMbpH1<3S=nd?UcY>#vRM%SGzzA@GeP$JpUEn8VuU zsy{g>q=EAL9XDmFoVi2dbqX`qitvk7G-w?fzlp4m821}P2L37f9O{vL9|}o=R$T7h z@a!b?#eaL}&L82%=W@~cVuX|=psox3-s4^NG<=) zH18_(PGwj3u7Xr%Gu(2NocSRtjwBRAZ#{jcy9UaZ}`PkroN8e256uZkJljv{SAKt+*5D)$8)J(QGy%=X5j`721S z#P-xFJ6+MxnoS*teSLXvw^jzZjO^o-sEN6>0f}4XC&UG*dCVY*>J8x;U;Pk~uI^8Cyqjz@vE#`B(pk}5fhSZz{LP0a{O7{A(&u-5j? zWgNPwO^_Y);1QA|xWTshlGM^S)uEe*;1kFD;tRrZ_7{R;ZI4 zTU{BDJZa%Sx!YLf)H)~5ur-e;o*_mb9x(0dDzd;k`U3lPFfYp?0cc?YGn6GVotlZo zLJYeyk~w>&^x&2>K1KScuK780ou>+aMFRG!b$*k(I^YkRV}Os%@BXKSb|ij!Qo`m7 zckt!ANs>(we*GrL|7f%XLVpT?_ATSNo?a`>>@oxPck(0RKmM6C9HL9(!m(tVL2C!1 zLQOOtnfw|xW{Cu6RUR4^j$77IMZ!iCC^N*_?4HDb4;55rnKQZ=rAY3p43hhTXLMl- zM5*P`aU?)Hx;Y_#9{yurQ{>}!*n1x5W#t1*9B7^htOK=JfdlvjGO-lITlKJ=nQvoZ z9-lLW>1M)h(9&0#3{=7Mcpep`fAw<)VDULsu*@sY>&+Yt_*pV)PdRnK+-{MI2*RAW zf~HCUXfg(hXoLPBG4BSCprb@0bUxiEU|>2z=3mt6WH7`U>3IQ`&(LUkClF>P`XtS9 ztVqMv!Cxr{a|&QMg~`Cuv6Qvrn?<@^K>}XY@l{BDS_d&qM3>yy2C>&eeEV(}%&b+t z?3W zXtqh{rQc+cTwpXS?>9kMo_2m3o_otIjPK0=cQ zGsOAmm&ApbmKB0XQ$Z;A!EiB<@9jXNh0BtjQ>GDsMzmmb_zDc``N3;8nJA!SHn^qX zUHk&>8~Q$mZxqiQEvtVAnL5yr>t|yGEi8X*xF{BElq^{*hVB6pg^zkMRVBoHIjn#) z79WcYRm+QdX~%@natvF|!zOPgzfPfj<6^L^(-1Qma2=(hf}`xxn7R>`Es&C>T3mCf z&LH{~oX7vhcACM(!pAdPCE$4pwK*24bX4j~B{hFCl^HC`1|8&XV3f2*;tX5!L4(P7b@W6*bACKg62O9uUUA~B1XOKnO@>{AIv zv^lAqYVd%;C1%S!&!%#49h{GWjZsY99W9vTEw=r1NjA8~P6O>R%pWab>I8=RddpL$ zQm4urv!=UENOA9edOkH(!`|~<+7|j$4ZtT@NS0c(d{n)&fOV0%|78(=t{W8 z__*(^{W;{mv3OC)%2ijnU6r{YY)t$&Np?XYu3eZ zQ&!dch)*ylHopJPNW^4<7qxOs9hi?o;ujyaJ?rq~hrrIq=3}KHt3uVK(3jx?ZHfZJ zCY~N>DGgcFTh((e@Y6+B-bTwo?@(>sy2TN(V5+7#4y?XL7^(V44ZV0OO_i1m^Tp5d zM8c%{qY$BZbt{BPpM+P18w9>u$qDUOR~5WAnr%6l3P8U`aM|#T;-}Btz=opcJ@d4> zB0J@yqdP@qS78hcdhtkg{Z3YX-6sO1;6h1;%V8IWMvN~t8cgbocse%)UrhpS%@x3Z ze(3^950+SKDNU@qxgpCB>f#2`hkw1|zy#&qVi{UsP`}7v$e<*h2OB1^QF1T*x#1s^ zzR-1r3q*|%##8QW`2L@%xX-an=BX*ILJ5sR1h>TWzfSnnbSWXFt)N|agc z!?0AL1-J;~@`ix7F{5l}3xqj{oIL%Vxi1<{jHB-dy1eX`$?1W;gSS60M_UWBQ->;-Yha3qh=7hFF?KCe2F4yjxjfBJ2$#pg0Q-2JiM%qaD7xOjH}YuA^j+&OC9( zVgqN1j>MJknr7EgTYOZXHS63w16-Y7>ZOWT3v0gwbyTE-{OMMMpW0vtFpq~y75=Kr9~q$~XOYtO=gwf#;z9qL z5yGu$+zPe%j!8iazKNOu5zo8JeT?{l0o?(JzCnw#;1OvsXu4C- z;sXT${VEW>H_5G0Da+1HMjMEJhc#&j#Ux;#szD%ZU<@4+008A1{O=$_6AscN7 z3k0Al^T{r35tGxK1h*~h%UJDNDl%Yo!-~GwP{+~V{xb2$6ZA26@YyHz(b0m#hRdhe zqEwotM3%3R`D21#h;{gc(11h}^g_n^6(SdwRH`-j2GHc=^OV|UeyT!BzwjWbH6G{c zyq_n;zTmOlyT!cb=nvpV`SoNr9rgekWliP(7@$q;{+u`ILR z0;wnGBlRTC`vpXI7;4zfLj-Fz@+EN5_3sXu;!p|hA1U>m#3B?hs@Z*MEu=1!xIS45 zPz(Zon9{ojs%#t~C+Qn*leF~slHG9o3M(%Xk2<3Pj^diDA$?^K@EMBz0c%Y6W!|`$ zrWnoK2==A&90%@=t4*a#PjG{wb`5$P+`PC}jbGXlY}^4Sl=2yquL`}}VLI0+hv%W< zrU%l~EB%@X$=9%@oDQaFzY~gCl>lXl=$KvjLzxagWQz!j2o^NJflWtT-Wv^GXry`o zR-TZ=RrM>+=(XXqVaxm%ZU#7eIqcrt7Ssy7a+poY&(jwjao<7iCWaf(blZDJN#MYI z;UD8YN|edIi45WZ4@YG{3NOW7I}jE`cmVh?r+HlO zHZ3Q)4}E~~z{%86MEDii?#(O{(f7(J;H(JH3hFk(KO;Et2IuLVz~yOB9peiZzDM~E z%W6r(AC3~$W+BJ+IcQvg9^wL6qdF|h7=zRX~J|ur)uOR&;Q$GG6jpR%QCJ@YeF`=jotD5_{|HpT%0i5gDpdcaP{<&6ibr^=G zo+x`tp#;G2Epyo!l9PQtI5NNj)#Xx+l1qLexCY^9AQ#jE0Dv9{f0*pAL;1)2nZmI1x9fPNhNf z*1}z4_hVDvxi&TOR{7GJaLD&FJMQSB(vmZvv5`2k-sn$4xIdu^xmRSrzeVLkzGmbT z4Sr#vFS5Na>x@G1wi|kv7x@+CMo*G@*VIt6J9=7=DA6vWDM!Xv1>Lw@-g&0=c!G(& z^%6o*g*)lZfyVggG~TJ0wJRjUzBA%o_C7go+Ud2DTNgC~%IGt$1TCS>ySD1;0G*kJ zyXt9&T|!ZnC;Y0}S(jNSwr0?mPkqiUyq6ROKdQ-u*$ex%d4)yy(!_|u{$+yqPu3Iu zrpViM5olJhiBMxoJgOK8(UpG4-#4gl#(weFZg@1@3S;{f#ULL_JK08OtsYaiOj}lu zDol*Eg%`+=&3F57_AhW(Uh3;9-d|-u=^SctUNB8rniC6W;@(LQ(B6H7zZ~5oTKNxo zfuXWLvF~%V;O=A%F~!N5bf1_zJshb4b=s8ZGOnw1pwv~5#PQ-kAJiVz`fgSu4La!h zhwT#gJ1yQXEvsiA9Po+N8k7v12pwl?*w<^j%CnG*=vSQB5(G4 z*Dql%_uJ{Uf{q$wAFm`fi=Q64pIsa$NH=%B*{6&e{G>L^o{+dV zQVNs@Yj>uPXvel2pLm&d;XBm)FGTpo$ms%W$1al{%Z+2&OB%0n--z^=pBoKUb+8Ml zzEPOQ3s@VS6gr=Kj5@{q+)?6jY}WlhgpPTRY7-G3Z@2;pr2FYms{3=n`OiVCt8Wao zk5jBq68?64$7u_FGxNp#cisI#iS??T-AER#zK5K{*mdd-B_GKygPj)ErIFUV5socR zlhw}>hvRH3-4_nkYuw_X>w1f{gO~ZmRCesJZ7o=^OIb2-&euciqS;|Mpf6Fc9Ncw{ z-Cbo9i|NKzmUSeHJm?*>S0(J7IbYYB55KN;Tta}?U;B=Ed*RbuKet|v5{M{>d#ivcO7PEN$ZMD%#E#wx9DU!i+-pi1R>TQE76?RoV= zYgDSyQif;pdWr{ZbL->mF7*+%KQV}f3vz~e+dCjErQLs>EAtNK?0NW)%)d9hK9?Hm z$IMGEUqmv4DNzc>yb7S5eOBC)S$&Zo+vd4FxfxdzKx*7O1GGwyQK^ty+g3j|a;AWD z&vr2H;x@YKi+$Jc^KH9BE{m_8?(R4S9-qo*014^M9c<3saXnIA*SY6zLp7(G>MFNk zP58ScZl#ZWtVE2QsCXgJeiN(^Gdf$gwYFbKQy7{^I#u$+YSUEPBWf`XAuIOJC>69Y z9y#W?PPyBZHrMX?O&D-ahaNacb)6s8adcozYa8&#`pyv9-6pn**H@-J)k@kWM8<~p z#_Z|U3rx1_5hisQwmmVTDkHNoY=Hyy^*by?0U5S95~WBZj8~Ml*!VWJ+Xkhs`wc0< zV6cv(k*P^~QIa)eipV>@-8NpxM_nq@x*MRH>%M8JQ%!pN9m4^--B;tz6dURjcmW1G~n{e z`0B=IgH*<@xbI7M6}X4x6ugZ*vV*j-haA0ojKy$^!P=CQcigiwoQj`Jo$HUJ=ma+Y z5n%cJcnWO#4~YM)d4H*P>xPm(!pQOt7TSD0!PI3`ReNLSuA+2w#@lxJ!?<@jN^!sT zL?ZRh2d@f{nGxd}rSsaFDi3x zK)N#NZeEIyD-DFHCpb$DJZgtT`D}b(ph!8dEvo9GI>l&UTkp@pUjEo&Usr=@0Cb?-tg;DP3zzS5C#dfu zQVdd1My9{6g+MF1yG6li01?#teLH zve{QYuvVcN{VqcA;5{vQ4du0^D-&Hz9kxc{rOm+l?i9wvgUHcSl+2w~w6HIKASpkb z1!;^UEG_hjhzs`^U5^|k&_>zTHLKig;*M>P^g#Pe>BF|G1ciS2IWw`uZXiwQq3LCd zY8x1q4@85JO?yhp>uZ7jcDXXM8^Scp2;syr%lm;XVM?7@fSXI~x8?tAv{)R-ApY3I zoz_+!r76PgoghTvQ66IETOl=~d4i9mv4dpY2JVJo7>Zt;A#JT%~_Mv zsXL#*&$9|FB1K+m2ecc0n0V(w2)$gt)hmE*j|rkXxxC+`ug%iBe4p)SGR*W42IJuN9$w! z3&I+?MFEa$6x!SgT`@jZEtK+4<%|Sj=4m0g&J_hiHJ;D-yZEl&t8 z$s|I@lu8!NV8R-#KLo%q$ZlDj|D~|H9^(qLrPX?YMIJpf_OY_zJFL4wc54H{CNc?w zWm7OzQ^^ray<)98^uRRt_#+8#neF@3aDzu8(QX!0cHrUl4zt*{v?ZUAZ~Pl@EdafH z`yVCwE)0oju%6$M;lE1QbDjP7^XS%d;+%rdNJSCADAx4Tv5@N21gFfuICHtldO`^= zvrSRMLq2wEE@fQ3apmPc$;nRUkGYg7N zOx*a<<5+d8N#GkQrNHXW=gqwXpJ4h?qH?mLvA+3u*I4L_3|QW+5*(?;SIv8S`~$OoeJ>hk2`ps;rscsViWeCXWTBhUIZT-R+CzIY zndwT!&0}Af>4i=m^bnpSQk%iuYGb6T=sdN!SnS68PnIcEUR(C;f|k8AOKof|3>gM> zn>}n_12Yur)gUiBcSm~UvC$&ST2X#b?l-w>VWu3U^*MKiq%y8CK;*8;e0`{Q99{cs zUD{-o4n5NNk}KlWH`*LIBm(4dUv#f~<8I2Hhqhn%2H!t0>4 zF{rS;6c(#Ql!}Zdj3Yc5AJ_c-sS9E4C67EhtIOEfHvPjZkA-g_8J$2#+Z%H=_}DYv zouY#k=93alOXJB4_YA7}OD!dZjf;sYVT#u^`iD$opT(g5CQ8VK)}X3>{%?}dfFnSn z2`0tmM8GW{On>Pya-fYzoprf=GU0yJdhas_epfo{Z8P)EWx2^lE8W>tTAI)|1Xk&a zM7UbVCb+<>O>BbEy;Ev&y^Ve0%T* zy(4OT-bb>Qb>S(|*6C`uJMq2WZw%(?nOE!^EZWktW9JpLQ8mO1wkn-jhY%glzVmvF z4-~p)+nz7wN8BDc*E>^jUF}!*`h7(wg;^@(u3_`xT}KA|EpE?_B`)#3T9F#fqq*v# zgX+!tzqB$Z1x=ku+kBIuL84Z(a`WEr9Ec^}%n5a_)Jj(rkiJt3{jl+-35?m(Sm+wh zJ`4D5-`GB+_+tax^+3-9r?zRMB?`z?7414gKnu=);5fBi;vEcHJAy_M8%0 zu0?{1v(QTw|1|8n*;A5zUF(-#Q`R4uRF22X4JQ4B$7U%OQgYCGOH~Aw05*a7u_#oCIhUR;1`2w)6P8^ zd`B$)D>END+SDTp%q273f)BOIs}P(snhDJfKsMXo=z;$gN_Kmw*2s(=^hfS|HGi|p zw1@P0I^Ezc*h&or?}x&#rL&hS?rx-xk7b9bN&(jv)D{#{AWzTyC9+Dq9r?#v_Yp9@ zoPu;uFUb4B)8L3#DRF!uT9N`M;OfAYAyIp})`IKo;32T=Qka#?J+qyT)hB0Ukty?3 zbzqHTSqd#b!?Dj9+2SHO@Gz53&ka}k_IIwGyZH?buaLo@yrEHiF1KCXCcS zzH=}ck~{&q!T5opHL7!y9L>b1q>=-;hlv>d<{h!f2Jf;TB$yJGQ^3b zzXY+MDqE3HLxsdc%wiGq zNAR)|-7lI<{C-TnP}EV^^?3gPh(0>=}(lfl6c4821R({*5EYctjg!{VOn zm{lUE(EDPIib@zsb=wZ^jiOWWA~<4j*=%lAs!*X0ZHyHr;5XSx&#Ul^Jd( z8^46e&q+yCKzukQ@~5X@(`anAnoQJ=8464w_r-nBd$jB!JR;hyq0Ub+Ar!hUtr5md z^E-}idlMK!Ex6Gj+r>hpSn zsvvFH2BMjtqK`YgxAqX3OLWEZU<+uL`2A-ouhZctsU2YD#Q~)L0`!n>RdOo1$!Vj- zH6kk(5w+U~c$Kkis5v)LKfR#nQN8b|`qh zOA$>vfZkI$+&m2qXi~4kHTvn1LAAf23W3mdUMORhy?zeA!KSQo9PB~J$IV!qr63ys zg8}d7$-p^U1tg~%LD+$1abZj&Lhno=-tg}sJvJx--RYK9JySpFr*A&ZW;?T{QxjAp ziNYav>jMI(Xz0jQZRlzbi4b40%Vw~J?Uu0Oz+)TYYA|Ynxpc~yd`CX7YVciWP%fk; zfk>C`jB0b>KFa{j5nI=r%@@I0JN2(`8K5rabD2sUdX!1D6No0nOu=?wl=u-*Bi9 zGC0eTnYP1VoV!0Yh&4l08!?le{VzE5DW`sSW&I$tacs^SSEflZT#ieZz=qBY18RSl zOp2^FfhgPHf1@$PoDY=VnOVxSM4MQ4f#Kk=zcI<-a&-%J;RkbB{E@%YPpP7 zCP^D(4NJ$5A&X`HNnqxu7~LF^e{uq@uU4MGe7Qa#;0?o?1=HkxhrI?kVV7BfgCR=VTT1Sjtk}I9S>84H40LY<-TGQc>^QC( zr!qm-k$QTG)>urvCb`~(lU))Q_CO{Q z{cpMg8fb7uqZmH)UIM<*ZdE$kbA_mmj>*aR8^m`WsOMm}q`Sp`AEV`sWQ^UhWdJGe z3eKN*5hV#ELT2}16xOA`fGCW>Mfs&12)dREI zg1Iqys$?|S%6JKc;pE?cx2odJT7*G!o$j;{h7SsZVXYxtEc}h~lgX4}jk0(Vg3GX3$ z6gTPw0{5sSxXcZiUO>DYGKwvYa>w{e-VCP^`lbD8wnx@9Fx<0QR|cQO@K^drUL8Br z8&2jYT!u@*qz)SrJkx;nbg{3yS^`AO)nosyFqCDc7zZ%#3O331s#so2!Gjawqk;g$=DH7WHr?Cu# z!j^IH*u*MT24>SRr*3T&Wt5LIT4ZpZ;i{lV1c~=&e7(hs+I%{r#M>SRC;lj_Bc*^}$oEo@VLay(v!(s2mpH|wLhysWVPukM2YD>m zN7Wd|xj)Lu-XA}KR+z&E7|h7BCja-Q1MCz`-Ov0n`P~mMmB>O0a3D!WJEepL2OE{& zrRQ3vIb4N@;({tWbQgoXU7pTPkv;71H%4y2-&Y2DS;(qHP&i5p%^$Up;hfL1SXWu} zq5{~ff_M%CO*cIDVs(8}6@<%TzFdr+Lvr~{*am-;-(9OFo*o;$Vu7pnaXamIVB zT!+e<(rq+2)0d(kciMjn+G_g9;l6qpdePx;WF@>5LWlb2f1qh2u#X)Q2BqUlXSefZ zOFp2Ps&LzX-7e=;J*X|b4jgu{K*D3w_e|A`&x$qJmtrLJcOWx0kXqW#wH>cngF$bA zy&**}Z(WqalNPcL>4V`yX=5RWeG+qHa#yqB4xL}fPaxH zC1u~2oPG$Z%J<&*>HIIuD($Dh>SUTaL)fbz4RV#~B0-M{9oT>2=3AbAw-wBBzo63& zYwTZu7FJ!>ibP<-S371oRKVPFLG6JL zt?`$yroaB!WrpV7a^5>77$p}7*3G8LJLRTKZl&G}Jmxvt@1g$(QeQUzNzBC9?8k3% z3;gWaf9-1$n@`Rc`Zg^+XS{U?C>Eb4NVoC>&JzVEW(?)qflpFhAL1S_pk}xo=;Zt{ z){dqe7Ftel?DhD2E*@KNIIy?z1h;yYPrqTRo$8nU+V1@E3Eu5Rw~|N$HS#^b(nkym z-WgHc(*mC~50cCZ&TuWCN(L6CC8#l%>R--RzFU7sV_T8uGd5F7B-P0*-Odj)3V#Y)Vw3>)j4_CFk|V0`rKkFupX zy;^UZ!n`Q!vT(USyD4wq@9J}kX@{+MC|r!2)Q;P`@y#<%*w z?9Mvf`II@3;L3|TG*ByS{&i-=vTm5rbx*g?IR8INP0HSz4o@$$XInh!SbRKqTARQI z3aWKl?;$TcL43Ax`mn8Q4DoC0{?@%=6D(qi!{6Z97(@LT2}p4l?Ckd5bfDL2Donog z%x&w~rv49I zZypcT9{-PPxwk#HMJh5kk)oPgwicPXB}tly6ylb$FGub&8CvlNpRLGv|I^=gf3JpYQMS{ryvqO0%5vKJVqZ_8U|%*j(H0`gKY`ztFC$(60r29;)Qi~y)d|capOw|3RWBctH%V5oiJ&XIThg>Fk zwN8_|gxlhjO4YHKh6W#U;Z~qH6@33Zfka>l&ql$%2k0?qWRkqem zta=4~o+nvApPHy4=bQ+p%sQ_bqOwDfQ;Myuzuos-zeq0+#49u}+H-9=MVI~5&Cq7p z-t^=FR1t>tB4o0_4AY`B0Xrki^#>`JYPcF{OHz})*X&c!C*B>%!c=1XA{=Z8PTj^*4~Osrgd&Ou=PA7rm^$DAKNyI(b?SU0`96oJtY4 z>*=ARfoZD-YI&bw?+7&Cm~qojPn1`^b5-)-^~ktwv8=18?cz4^{*{_F>%mMQFmy0WP%V9kt%m&BS%byeS)_OPYHGLj=;S(n=B32-9y4j3=(74g1 z9qmD|lk+@EFn663`jaT#OW~VpL_8)-x#UaK{b_=-CFp$+x_xP4P<0&#|hJ& zkbd-n$NY*^tkqYtWgjGwTkbtmJz&q(9RTGJydU1HfS;3&LXDtBfnaxo!nv9}dgZf@ z@Z0Xy^Et7shE((E^F0^hEvcp3Nz3EqGJ`w?(aWi)zW$uCuCPMa#r`rnWGUuG+7Fcs zmBupkmXgZLHNkxa)_-+&%u0%q6+m?D7ZW3{adN&eCm#>bw^jKAmbjBXRpU4k6P%qK z@AP*Q&sqmkk$-{t-QJ;>WRflOI&iYnfG47huguWN?S4i`7 zJaqqyz2^eXI2Q(?7VAuOGhJn)8b#(6>vpbU_~dg?sC-*+$3h1&~hEjzo3je=Y-|JEiIKdBtZ= zTC{7y-w+t%9pD2vk@g_Bh@n(myMpj7yd$lA17xUEu2}593WlKlStrxp6tCIFFBPPv zI(CHDKdmp-n!?h+LIQg%%Bu>y93~X+sJps17Uy_*O({o(4_Mv=(r5VdZ4qgH=6%mW zinZx*JYCuo$HyUO)V8}G$=eQ?z`XLPKKdEd8=w~XrRJ%xe1=17_MTJI}?7hx(`MkxF8pB{Q@E&WZQR>1-aVxZqb=PHNnF6BWW|1;BkebOODYkNizb5 zb*C)!Z>PaS+^=~Ukc*&x+$zhn=_NO2iiO>`#H5{T(^>;b#haIHLl8>Q7V*qfYtF8v z{LR<(hO^hIc4hMiyrl8C1A=zjYqiiGMhbCQi03dRWA-~7^?`0D%R;AR+drccvgvaA|R5c9u}Yp)Ih%+%3Bc1S=KjbS6;mjI z#Kb!MH;Y@tc97cNF?ltH67(IMY*!vM_NMi?D%n&#c#jbcN@B0&cf$jgsVWIlinx;5 zMso@h<<^yfSq^6;L9aLI+aC?NOxw`yhj>DIUyYQ39vFn0=$z&SV*98b(9Ubc_Y1?X zyhlKK5Al)?dp8XgZ4(Xe1!?NR*012N14%kLP-7pT42fvLbkKVO_VMa_-mb%fX{42j z9?~PZC=yf{_PoEeLFN62X&{mBP>U`wX;p?OPxL~1TJuRHPl)sa1`Dv4+A!SfgoR=} zHgGG2Ci1NUgOZfwPJl09Nh~CTly?AjB7rs@_`vGBjo-Y8s(=?G!Dj}$0C~rQdeK)A zv)ICB8)^;vkAFOltUYJv9fe+8GuL{JRxbzuV+T)SU)qDTqwE<8h!tjjpIo|!`C3vr zDrNr^b@8AeV~2)QNO*D~I2~EFDp6=|(VbRsW=ojS^~~KM>2Y$$sld57CBf7WLWhM* z(>Un7o6zCDwD#Q8oH5pLWdh{DN&pqd`tNQD0&6Ri2Sdl=InDq}PChvFefQ`4`jL=e zSPKb;_}$A}eFeVU0Z1m4`rdj$7aJ$2 zBjmw-c8G#xlPkf#NeY(wg-lZipu(Kpk&HDafqo_4zKN>dlVX6uoeL|#xREJi4HzSM zZLq-TwE0}^o)qwa6SZ@dpOc%lV)ud9`yk$7{`_Xyi zA)|yP500(BK{>{Y%WsKvz-@%aA)61`@P~!gg*c%P8M&B`9Cwn4OEQxZMxYe1DH>|A z;f6@-+JFhqL7qaJZkSYdWgK4@Fg>vuwe@q@0cB47FID_hV=X0u z6FEv0BQe-+>366o9Z=~zvl9~nLA>p^GDBj-(1Iqb=0&@uyZwb8X^=)Lw#~Tr*06HZ zP9c#2+HpFuhx1~j{`zwJGRK~(FzQK8s|avJ6G?)vEb8ucBHviBmMzjc>J4A>7J49^ z^^8o6b%_R(p!8%Kr;N%rz3~bLXKnsqCwQv9!@sC0=ot$AV>7aEg;M-|r6{yMmw~$* zFp^&aNFQfbK4=`liroxhHOh<8t`WgXTjg}96m8`ux{%q!8Vhz9=tpyM*`)3fW4@ce zFj}}%pooqkOjEomEn$&w6PkJnn04q8gkV8n6f=NPR1|&bKPOBA<2c-N464S*M8R1p z>)wDpzU&!Vb;uL31jeOUpaB2=&>FgCb4bgju>a{PghT?oJ{bH^cx0)Rq{1^kP%GBs^*q7K}8HocHfAhnCA5pU(bHY*JI5q&bBJ*4NUaaksG z5ay!teA?@*<^?a^Ro-*x@-TA5Rerwx-r+2Czr?sN<&6(7(ohJnB|=z+#w1@PP)Zfo z1th^M;oY>D$-8;+W62gpOeQ=+lKh%zgue_{;%>F)ZVP5FAf+ZI6c8yJUUL3zRX7%m z|BxDr21x_BVT8=LF%ORQhi%$?QYVe-h|W6Gy57a#4tFOXde3>#sZmd!b|&SM8nGbY zq`9gdxTa7;V6s%5f@OZm}@ zNZJihDo_M{%#|ak(Mu#(j%0DQX+8Z-HM+>=2<6N5SFCCl`xQC~`KW$aU#`ioz8S*) zBtgM3L058YYGZ$a6mhbmX&Y5t&^r()8K({jKZ>_`-r{`>m;t|&b4OP9quy~tgjB~i z8q}lD@PBQ4^iA6%EEv@wrU~pSg~#VaH+>D-s=*fUSZI z28P_lf|FHdUd*US6gu1Q)uB453{~mw_OkmngW_~c{g(v(5XAm}Tt1DD8gPC|J$^+c z!iBy#(UO$0lFvcOn4SV#K6gmri>LwO&C_)flTV4?!!N?Z#{&TQ#`0>#1aE7iemJ|% z{9D40HQ=gl^&b3ROk^{3fkMzM_9()`43aARS0|jZ#hE&Mhh>b>$=~T!oL0nblmD4Y z+k}vlCS@+>d&OzB3&NFN0-pv&X2dQ6>O&K-VqYqOnkWk}Q0O)w7Z{4tSfZdW({L9$ zQYj$Z@|+-5pe(4hQ?NqyBW6KSTSVQ*WQ$^XG$z0m9SMtof_I-zbQI=(f>Lde#xPaJ z%&?UW#C_sREaCaY$P0d7Ej-^W!wEC7QQAN}u>?ZW%6fK@w06U5zr9BAP{Q8N?6N*2 zD22|Q|HlF%&&lDXRptK^c|N=V3)GkvPau$(wg5d&?@Rr!Kt7d^hvb5XRQ%zNi}9F% z1cCm16m$z>eD(K0)}PQ>8@05<&S!2zpXSGk{%1E+U$Y_FL1iknniL@$k+0+RJsk#yKY&fRvl_PBgU)mnPWTX^|HYS3U@U3to#m@6-bhSYyKKH}ky4=l)jT_{$YVyxo#pTe14_<#TJ-!Ta; z{!Kjl+l)nt3} z95EyI-Q~NW+nhu}VV{I*fF`Fh_I)hPYXB~VI8(&)E~Ne^aLQLagnADtG=E z5YRFY|JQ}bZK@rg_12r+I5}ns|6uta|Bl5EiOG3GDzg2he`$aAc6o0xAyYVY3Aw?-oX@6$K`vId8ge)w=*mGFKk(Nuw7EHX8s}^u#+i{>?zA=XBPCZ5QtN&d4NRy)?$F zQkwdqG3mvrTa-iFs6|L+jE%H|i4MK{U^G{0W}A%jc+r)*eUYEm3u;MslG+P5rVpO~1=a+fZGcUtk40ulnl~yjs*0*2t#yfZlCPIsJ6tO|3Fo z5G(X_@Jg83>rKk@=RD5uVP|RlE((5Lr`s+(ehaE2<5QNexzDnD?rfiYZ1&P?NBVG; z7WxdLS%>Y)4Rq%xgiih0FM6^&qr7g^tMIfsHe`)|V0|2d3KBRCWRbmBZ8+aa^bNX% z-r5G24Yu_qqF*NmE>v_~e&XYj?PuhsW9Y3AoYwm~K;>4{5FyxpuDR=DpF!%!UWO;U zioCD=H&)(kp)?}>;B4=}3}vVc-A&Iv8@Ph^;GNy_!zcN4o%WuR-eat?PY|(~=|pe~ zdRm%u?(1{5i(PEMm9XOu+k%tPEzu8=e@OGwOf4UyL+U+iX?=Yn5A&_}CbVf4MuUR( z&N=sgq_aO7xO@3HuMgmzAA0SUF!E&Zhm7H$Xu3qL@eQ+@yF;S2Ui7ejk8Bdum5?HC zcHgI_>3NAF)&lC@j}jWQKX#8=7}a*~v3&%9TLrc59??}Xgqn0zdh+yV`bba?vr{wA zeQHn7tAoeG9lT{%d4rI}VcuWnL9UHos>`AP(r}`@e?-JnGkR-u+Z|chrk(xi&$Owa zoZGB18+zC!qYM1FO10IKet8s=?fnN?S))7E>siTxRXwWD_Lcw%5wXudLZM~*wCwgT z1pCr``aeBd3!b7M1%>U7x@#Wm$w&S43rDc7qtJiz9QjyD1=`+m>?uUeUU>Z_t+>)* zF7}Oub(CXdv>Clj?dhjn3(t>3UVux!+*QTcwhArD3Od;A7XDRyJ4`NeSO3%w3)%u?n|q^z2(hrf4YQ(msP zbsH&vbk=j3_v}S_`$=WtUIWLWr4?OsvzpEVtreshq%Kd#$Y(v_YqQ~8;Og4~fs7Db zCwU_uZ0pGtq{@pvy%xU1OF9Y^^Ge%tl1tCSky*HZ%v7^PDTvIyVpYu3TXLkxBNq@h zTFB1sg2-D%3Z}-Byd^T{TNxL1-akAh>SYyN6D#-Tg&pcc#7qU70a{`~|Ww zu1n~A?9IW7&0cMhb-^1@Q`1zIb%YUcjr zmqvc+Q2S9B{JVM+FTZwZzS*{Zo5Cg`-?!7IT9ee1?V7*|GnyD_Np}>l^lA6scFT2b zk2g6D1-|Y!Z~wS9>E>d||6^6&vue@XLu3)q#2q=_=jjjYSt&(+a2;n}OjKKs-lEgX z_X3zg7v0HN9lnc60V&Et@Z~PE&pYY=q1YE;GR)d87eNa14$JNW4bj?*S2)*-YUml7 zX{#gB^m)I#&;20fsTH~8)|k&trii@ zHOiN`FwGCN4Df%qTK8)o6o))~rQD!v8yqa^ zFTDG+z#L+@xQFut$1LTAVRGJJ>~h~yas9+=-OnB|jiv3j%&igiTLJG0^cX@9kM{Kv z>;Pu_R%6JcjwxU)IYFc75?1mwz}AB;5edM1pOXw~H@ z7u$r}a=xaLCJfCU8A(SkGw&?+qMnJmi>@V!Kk;YR~() zs<|92XZ2)?t%q(5=}ukt=txY({A5a!zK2|Yevc8F>n_X(<7Rz1ZGtwoZYI^TTsrj3 z$EI)E3^5aD@mJfrzLOl?tKoA~E!go9cB&?qx?j!amiFU+B#SA#wG@I=R2{Q8zJviZ zgu^$loH%z!QJfl?)L+^aOnZPdme!MJs@&~W^k;~j$Bc-mj`Q-r*LT<41Wl17-!B#%oa-g#i%UUI|B`*FRk>)wJ?^AYWH@;*W^%oB`z? zsV^&Yg)6*a8K=pzC#x9J47B|wJbZhOiCb0Bds=a6e$Q9OuFPxV8Q1p07-~3~Gm*%( zXn&CJVJs>iujKjf$Lc3#`m)ZqQr;gT6-QCW3{6BYZGj?qod`n-z~iGz{upawVCGvx zp5c?LH8m^S)=9^GWA<4)Ck(C|&!Z@~GobqikV0aiE!4GD(#y@X>2bpp{gD`{e3l{v zD$ugECiykP-6kVP(BUYN=h=F>TIrauNK#*oa(A(C4gvf`19#-;!fKoeVSF{9=h9^U=hMFlMh6lB>%6 zOHaMPm_li)G^Q#28N1eyl~zA-{z~KpdH`AYV$Nqn1M-sq;O|awII7_mSuYL3@%mIg?diUb<_- zzg3aOb87UomMc+)D~<6}W}&z0heVHDxQADJT(}jspQ~M@L+{8P=|pX!%69kL2re+B z9BMRf7|IugPjvC_>&{#(Y|C10%k8O0SxsZ6*TNQovZMnw7p~CgF(C8IISH%A4B+|6 z^<$=3-Jh!=Ms+eZAFTQZcxSsbfyoN`?vKC&E;&#x&JxYMb9g|{8;x61FY0vZI9!zJ z^q5~>Oqk5oydhT*Z=-cYx(LdeLbdWgXKNXD?Y(}B)c0|3@yv_a`NpKr7q7nTG)W9S+>_aIu!NWaIj2;Wqf9XzZ$U;QQ$wTaQzo{t9iHxpTzt8bK7`AN7W>-b`xtY+_D-r9$o@1F8=kg6g4qelav6*oV zksdrEqv^Wg;uW;BIqf1ca(57(nz8eHXc<;jDN12c$EkT%tk^{B_em9z-m+`ooSV$B zn#?Bu^s?vTzf#X-JV{eOyk$-$EIFoVD7=u;TDWt7vbT=Z_pATJ+16yr?ro)>g|xZD zs<(ldIFuPKy5u=teZz&(29Xz}* z@ajGdrWeV=ZwYn#^OAR$27>B10fRYC!e8)SqZhA7h{BhPrZUe;aKvF^RuHTk3HYOh1RgfYqzMI!Rm^gstJ91LtcZ~pd{D@XIJtGlzcV?i?wT2|PhaOqe3qHCq#Vr> ze@W+`lq8R2NT8&!CPTFn9tdnDvjlWT{Je%wHnFNX(vvd~jLYlJk{}~lM!$c zfr>`SHOf7&_H1Dw-u1J<7I$8(typ`MmE1$-d0>&F0ds5nCl?~~aI1!gUtS6)O^@trhxsTvgf!1dE|32DVPYH<5tJ|| ziU}QMQo8NZyiqdW?}cD1M~A5f^Kurv_@U{9g#oyKZ3mh&#tGiER&*@Zo>SylV300C7Z_sueOd+3)0Fw7sg;OVdK+hbqW0$z8b zI~*44g=r!$Ij0@A#X(;LXRhKXopG_wiC_0zgCvK&6M^9kV2meda}(a^wrj@$IE#gkiFk1^UmAdf{C^C)1P+riA?p602Cpqr7qIrYxg#myWbtq*>8jBv;HrNW(6W{Lo+;X>@1ADt`h}7-9y`_4Un8~V=#Yp9NLv*0wcnaa zgY`vkmvKeEHOw03^UVu1_SV3nt3KDNBcM)F>vH}}Q@s8*OpjIe!)(IRCHra#!fEyh zKDwSP)iRV1*X@hTd)D?u>MRLsFr>Z3M#FKUIHmN+5?FJw-XRq>(#A!3VgUHqE1*Jh zd|^kw_sEKsVc45KfqljZ%^Bf;f+P!X@l^q{uIR&&wWr0AuIcD2V(y3`P=XmHWss=} z`#Rs|MRW{4gKRxe#X;KRdef-uepHby8nI=u&&hz5*K4>FwkkZHX%+SZ`WURqZb=qP zo|-}Qdb3=aTQLFIz+{eg4eOsCQC^^Ky{y0{Zo!120hm$=U`k!4Sy1J4Dt98E&_NS- zGm0Bfm;8c{s{g5bTKY!rN|?DXB`&WediDg@x9QL~7#h}T?eunvhyRs(Z(hO&&=J6O zhDqUTB2^qJyFm9HjoXVbHQ1`o?j^u2eA{Ir3I$TE6xbriKX$l+JU1DJ;g!3)C=R-x zYRn3}i-PJMiXm0qExevQ87Ot%Jd#vW$NUDXH->YEsdOk# zIQm0O6b>(o-Ove;orj@RRP2|egcaa1z_H&58*#s?*v1Tiq+c8kz{KxI3niAC=Ebp} z41`^&A4%np+AQ%1xTuET0i&^GV_%l~vX@e29yQmzU_ENj1P>K7mBA`WTCRMrK|n43 zCV{tcdnwio>;PsuplP{tAH_K?~4Ka85;e^-`w;n=y#SMcP9%>{=_l>WbMrxS?QZ~fZBTdG9 zsbF3Pi!9pf4;n##RGyL-4X4QM|`|0}g<&71CU*z!E_D^A`(}5M`fCg% zo^3Ji@8M32Me}=7q&9u@ZzsJ+n=HQf%j$?`qvN32Cxt7=;0Wop8dCFGa2p_bcn4D( zH}S)0Q!q!^@GkQ(Acah3?R3&RX!?W6{<{iG>O%H4Dtp5HZLA|@X_RycOc;mZ+ekf_PGJ?=jIrvEB;(q?iZjq;E=HKBmWoBoaIU}PH! z+&+XIXf%aT=#gteyRo+E-~E z)&2II)UYklOy*%#vba9{du!ia*$ERci}26E_5uP{A*Mq-itld2dBTHvW?!Y?F2RZK zJnpk{+W#BZj$1s~v;I<3@#J>WX0T|ADLo0zOUC6G8$SB7i3}IS=bFpHdS@LIzNI?b z#PmiVc~_O2(8pKsh$EVOnTBx%G7Qp1tHBhxkV^1p%GE21CcTo z$EIuh;PR|iSp8v(EpHX19QPge$DO)`VVE)cIBM`Pk0ouSfzn1GZn!dAShh&wC3 zmqsQY|L!#u(Z<3dBsUD=+NI;Pn}7|L5Z>SYzhQ)a2GkfPEKCxsuS`SzxojcO;JJ5( z7vfhS885zcT6!ghT=jwu6l_2whe#Yg$MTuZxw|D#2aYuaE7PA|4Iv81@|*9bka`nj zxff>Z6L2ZbLbPv(I6Xxo)d2of4gf!_64RJC`I&o;Me2i;5gBvwHE7Sp-FopB$W7CT z_gni--^)fXf8pcC(8D&n=U@iPB|7_gn~Yzqe{W;F<6%DI*esu?{}FIMo*n z4$#!8Ec&8*!xvN@;;|k0=77>30hK==h0Z5ktC6-@Y5C{fSv11eaY4IG2IW#eSk=O=`J#Qc>Tz_`K>Z?HvB&s`bJM2RYI8c}& zq1}!)gLM-6Fn~i_z8pp>bl5gkYbxW3vSLH-6TIX0`_KP>e?O`-1)L|w++v{|@T2RL zj1g!)l7Y-yrjr!J(1Y^OuYWFmNHpBbcOsJ@AO+q4;<&{+D@hNw_`!#5Kd$BcK4U-f zWFf7jj=Re-DIKJGbN`DSz(`jn1&gkY{S0G(p0ea9&b1p3)x)fAj?Sgihl;~N$^aK8 zPzrhaG@OSZaGY_(OESKfi%Ok#iK?T0;f(>@2iHvJ9bgc*>}PwxEGy22f$Ikcu@mo* z;ixe^p!ngdCd07{7*MgPSKFtSE!3#SR{x3RMOyXkYsd#t}Tq_y; zGn?R%_BQ$=Ut3 zW|NQO9=nGp;90m7DS$LMg8yFYv9jtp@9Nh51>9STieKtTl_AV*FM@*Zj+&&4Rt_;6 ziq{sE{K9>I$`oV9`hVJ8~N>jQiJ{mCr53c@_|Jw+lX}KCH7op%i|8&g_hS7(?d(~BIE2i`x z0!9)LB>JeYjOBZIT3maSH%jP^T51=8w6RI5{#!;b`B(|`QvOl-X7>(bIeUMp5NumJ ziOxc5lEE*ImABf^ttEmb_qUapbzK*6Td2#v7T@VeQ?7ZT-cya18PRU!>uw(lC1?2E z-~Xq#IO zc;~S%;{rGLC$e+&o-6JVx}(wqLTA;BlVLW*L!-RdAKJ5{V(nG=w6Xn-s)tVMy!Uc^kN1dym z5vm@xJY7iT-DoikBDLDeaycpM%L@V0fcPEkI9QFBgg`0%$ zDhiFN_-RI7_O6awo<$v6OLmKBaDKlnu|;S%zp8WgEzK_L=Lhxmq*d%k=5pyZ>2s-q zD@5U9EhnF`qPF3%S_8S=SK(QDbjM$&d*q<~pVPUk*}>ITNR!pzbOLKArFT%b)-y?O z$S=;EDWjaya$#gG#n$;fyCv#;`!ORp&~ysAYOXnH8z{dCO)fV+G~ays`3|=W?YUd% z?fc#5WHxVquqREn=XCVHQdiSv7zSxywfxPWmZvuLZF=i$J26|P6*qKj&x$JJ z4utId^W!UJdLvinm8j zy=}8G{+-@pxkymJcYpJeA6;AI^t3Il9sEB7CkFw7cxgLtCOELgpNkDjm_GPd>Aw+HZZmHva7+ zi{ zK2@ZnjXlwa)h}rUZ-3V9d(V{0dx_MW9jN_m)foOA_q1@n&@taWF1~7R4YTu^8w+(1ytJ6=L7T?KAy)bYZe4E+ibO(#K z^i`E;gnTfyVzHI<+dp>b8r83{-W#-UM+gK%H8c- zDIf*yXPCEbFZzSHr`yG6;IGnVyXtELu4hHX*Bxbm0W0}g89yi zsm`+3yIopS*JL6&0S~|{eSUj3@|$wlaVhtSkxQz{rzuMI^~oYSY7GHiAi<706Gs|~ zoNq43YqKEkTX7cc!3sP47&M`8ib4zIP(Gq59wk2QIN}42u`yFm2aCUYKXe(a`xTUR zk+J#I>cF-(*Dp|2m)0cwF(p8A$DfF6<=v;Cg+?q6Coc-qla}{gIhFg@o@t_5&1`36 zr_GpN!6rCkOrglB)^APK8L2k;mm1aK5}-v+mVOrK?1hF_mk{ENN4fywUc4b;qMlXb zL3N$>IT`P*rs{N1{-A4n?c+a6`}AQYuz|g7ex_5gdDwf!BJJ6+z_)DxnVt7)PTB7U zr*yvGW7ypLi5!b%AwJ0=$%c2%%W$aX%X>nMw}H^|pqXj#0IQxv*qPqyKD*}XL^4MAnmhs>jHiXj>$m%$S>?+@|b!{Ehi$e{aBLY}pGLeJfh zd(&YLP_+U&nhxc-QH-vTJ9gN-A%CUI+;2w;I=`b*oQZGpwb+!`Zz}b}**7dHRrDewZ_8aakvQ zo0Ogn_mD(R8I`2r$HK1<#-|Fx44kq!$A%_HsCSYYvzL*tEX&Hy9^yQwf^#Xw;SwA( zItPm~7)v6A6rmbeJS)PsVLKPlFk))R3s@zbQ>Ae|(ep-fBYc*6Mbc)n7J(03QSFO{ z?cHR01ZF_Y;Jb@j0fDcLJWl=PPb7KUqOU<^$psaW1LS=dHtf^ogg!INUSfLCYq;CK z=%TrRifUR<9xtgVDGem;wijeP8yEVt10}h={13Tk-?r=V9~sArdM5qSc(mRgz`N3~ zlv8x))aF`1tqsEVRslaD?Do}@kS>61cF~?XuG>C)1i?7tj`WHtv`I7r`+7|2pRlyh zai}zkaiVLYZ?kmr$v9hf#RD{@faEh_e1GM6*1@dw9~c(dfVJ3S zE1!hxSeG2ga9kBUEcE?{bdbFEwp&m2=C^VBl9sBMj`vh>|^u5K|zXtt2-0r&fKNK;wwf-X2S5FRCB?;R4-T8xpX7 zK__%4DZVsqZ&F_>I)>iye-cj6LbMaWyJwy;3+0#qlfKxJ#0X(F*z6;JYC07Ts`*4^ zWKLlR@O0}?mKFI8!u>!IsUAqoKFM9$zn}7xHE|i_nf+~O1kZ`al^jM z)24!T=pAZZKCA6CGFn$GTq5Hf&fBi?4P2IbILw*wQJnE7DeQ>9qB1|*HQ#*lrReuW zymF$QTv!rScU`~ANvQR=l^&}hwq)dD%%9rlq>GNa-SXMI+S9sa6yeW_R2j)sK@~WS z3Lm8PRfJKlopk@z*{Sh3I`HL#x_LjBR7zdzAMl#8i(@8SLUr za0ISn0RJdIbQd!D*Fafm;HW#|n{JL&d0dR~>!C+gZ2^1gCDqRqS7q+*#B&pLfO5|N zz-XP%74a;?^<2}t*hPOGr*k_S(5E@)>h7U;v_&)Lsxw}>%SJ7@#$y^!DB8Z?X5D{b zS1Q*f>Nj+3OS^x|2x&Ow%CBbh&V~L84>Rj!&qoo)wt7DDlir^dyK#oF#ObF# z$Ct4*>HcG^i;d586yA7>RGD397cusSw|+&iM2#J+H*=SsdwRh-l1eUlP#>mpsqS!X z%HlZJJ~U|H;kKn~mxZ$pi*K>31K)iU_4IF?$o)~L>dUakr;dDvl7-*nM=E`nw|o0X ztSZy2oi~b^_*d#$Nx}-wNPdmSkY3%-&Z5+yj)*=*38Iw)I+%jlhJZ(5cxeee)r0Zd2n3A$cn>7S~2HFrCY}rMr zSax*S`k0-*A5>|sG|`>a&{yi#g3MrWN=w;NJ8q}w2M)7erncs~A_w*5BiSwvcXC6~ zY!4uyuyWJ*0;}1>Hm?+vjV}3H_7WyTV@6Y25Lfp56Xu%Y^rAiCV zt5%95ecOIlHkhUMQJ}Ws)QwraZ$=mTb}JhE;)NRSP#dDE)@ z&EtMu=obl(^O!8S0A$Znv53@*^QIAmE^x+s{(ghdcTB0zdv-c83Es%_%ec_f1nivH z3mBZmmIV`cyos_0Yq8({{j*@O3y@6sA3+VI^hBIasV8#;Po6V{@Ab{%0}DfnD>h0X~-a7YM?Jt!#2fWtmUo#9^xge!p1 zfq)s%X<*85tYaA|7EMt{r)q%z#>L3VFJ_369M^=<>AB>KyU@C+zp&oCRI%f(s~*6q)+jOFYHHR9E?Ck%MC5hog%MnFw zZ^Ex~R=DGmu=FbcBjQgf{!}O4SB%i?i_~#v#*wu++zsd-pe!&*OrJciUl~qmY^dt- zJhDQ}es2+v&*sIRU}{V*xW}tNr=es0SB_JJ*W8A7D@G z>`qoB^ZiE3@rIOO7>dv@d1M7k?+d>;s|%g%wy@b}9?OW1sO8-npkaXB*~Ql@jFELv zsr?_slX_*a2jyp^1G>oTKhspJNeU(MoK}?RGPIvj={+0Mu8g7nR1x>cYOLW3E6DrL zJ-R^Nra}NDhWbf1!k(s+96IO3bIO!r?Yp)blEt1*Yk1REvol88&d zy}|j-tmSzeeG60qM%X|&tlAf56v|RPn1?c_fXBYO(j}t{H)XISm~&v;mUQ8^ z;R8>-E|$TW&FuuL6u;!DqB1svd)+@mQzeqtw_d6cGpCv`ZhsKH^Tra;fee?#CSomU z+Zfz4NO@o^5c}ca0+&*sfMdz%&+tYIYqq~5Uf~Qs(B-G`AJRs*L40yFMh;5_0S!h3 z3{+OZK1#rV9duE*vKXD*W=m0j6|N}hy6m0M zJ;Nw%BK0Ty$lr$E2BE{5`x;?3yOVp^fp+gIsM^l8w;bt0rC6k^M#`uXju>in z8_`+Edm@!d-)Nx&Ol!M%Gq|P!j3B;^J)lt}4H{R2%{!c~FqiT9{)KC`;Yy(}!JQ2E z9yxnVOiwH%m#*$YPXcY8X48Vr*3jcdrgO;LT2Uh{Ti<(0SJfaJg16V!uZLa1sEj%K zgE6KBWq=m+DdV8cOG9Fm;1&B!frtr@5_=|LA))V*hz4{#f>Bw7ihZ}%BC;v^tLjeC zz3w4Ct_F@ANTZrlnfnT+*_ZrF2gaZS5Ibq8lILck7=qsE5zIyc*|QfYa1^Qh)e|Cr zNF)J$x0pFT2hD<5eIS-PtUFu}W=tAmlvIwF)UieG5o}0Dj`Z1djeEr|B{_mcoir@}matIvkDbEYL)!!&6RFC+RJHLj8rwZl-hc>_3SylgdbyBl|YLuZNFX`E4vwN5N}m+bJFYkjy@P?}aP=xLZ{(OASd7T*`~)>yCv>^+!!kJbQXD{!D#3G*i~ zYsM1x76+d(?Gm2v`r*H}NU2@9yAY2E`X>$Cwd1ruCwmYyO4>ecPvi%!_3zu{Hz_Eg zqt9A1Mih?`yMLy__9?_6Qll6E$}Tu~!C zwHHopt=UWyZpAyh5by>qmpQQZ5dPRE?)U^~#l*(h9yoU;HQ|vCiYFC8F6POO89DM3 zmB_AFLwd+yu#T2VHqmoo2&x2kT0~8lh0btgSS`!#nk!6w(+P{}q6fI!%@Eg*$#17- zFX$q(WhP*kwYKg6)iUNG+ZKAb$nLJPK>nR?$*E*oZ=)mAAs@CduzmDJX1;QsH$@j# zG_fbs6pgg!gRUd}19u6IEA836J8Pt!frF%8?Lv7Bv(E*EfrLZ#wi8vk<7ECQNmWW< zkAIc)QG+xA7)Y@G(CH=d5_!;f(%{rqfjs?fHDtr1O@dtYi8X%OZBMGKnq;FE|3s?l z1vDR)UT{PxF|m+TF%bu1H=j1-T0DCWiolo>ahlPJCRt$-G}AA^*$4Zc?Hu2VJut=k z;lvB5viye|L3|t_Ar58PTRApEJTRDv-Imbd`>#|)J^=4n>^wHXu3w-mjv@xuUqEV0 zg*6cjo-jgy3>4Fz;$L+?0AmbzO_lR+O1S$O*7;sO+_;snJ(J}5N?ovZK!vgN<>GLP z@83iR-gERE3`!kIPW$?oHSXy1Xs9%Emc$x6EF@x-+4qmF2W{85brXf6`4&6j-)|_Gq#GJ{Q=Bi ziFM$^n4?!^2ic=`|^FRe;78fzoPSjcM(YW77ZN_pFAb|Egh~WPCd029Yk`T zqdb64G+En z>^`<5n|BaKNSI#NJWUgi>1*an!KDN1|JRC%^@bi$lAqcF^L~VM z99{Qg#@?4Usz^998;>4wf>^Xg(aAuI&dwx>N#Wlm9Uv)8V%(QWo1AXk5P@HEyem|2 zk#n*&eyiSk(C>PPxF=G0x_-^S*t?_5Gg>A;NY}ilLgO&_0tQ4#4r6l+6xOh({ z;jks@vk$OsmRtm}KwaAwR;P3Pvy1*Ojx;fF_1zRn>VDEyn?85J=)Gq^{jDk;fPLC9 zmX(o=F`WN@_Du2L|3Mj8ub@N6l5vj?17e$sWdFET{2}2pMs=^gn@W*Hg8RXuEuNH{ z7hMh>f~}HisLgl~<9=tuY#$`CVj3PSNM5p8B+nk_RW~ri|GEd~f5zRm#6Fl4VsKUzn9=VU+*no%%mPUm&n&Ab6WyXqq!&{of#YqjT}ywwIVpzqL>_FGwt!oD|$-Z`S2|nY2y`Upnu4KWam5lHtrut@a`0Un^O9S?3|CU_$3mQf=nO zdJ1#YbgHaW)EzMt(U_Ml8#5|hK2)^`-ICG2cWPOu7ioOrm-HgQbWIZMUA1aW!%r0P zZS4qeAn8i?HckJ;6U9PJS7N8pw~n#I#{ei=C10aWO;~%R>)BE=4SY1~Pjh?*Wj+Xx zwq4AwVl1l@Z2pqv%OHxj&$p2TYAUb;%{y#WyyD9R^h12BD7WSB!c8fWn-rP%!;4cK zjBdWHVY${8IV*CIX?nrKKLw3 zJMRGZ(3*dQ&Xxu(KrdG+GOc!3W+9wr;XkOnS(D7DmLf%CVsqEv%aBXaVOx@6ElZ2v zKE1Q`{*c`5KR58An$OgH`B&vk@w;OCO4p$l6lPxv-{6FK&8?bNL~Tv+++ zf;-=Fqx)E3O-V&XdEm){`eQO>-u5uXrpW}1o_)U2&8+;kPVOGHjQxT%?^7QHG9B^d z{hf6-Y_ZaWnW5*GWv%`0r3_UNP-AS^^Yrbzp^vn_GpdJ^jr^Vk z0jBi__bhkoX{uGrsI;TIbcEv`=5!DQ$k+F(}OSmKrd2$ zwARqoac%UHOO5~iA~USm z+B!gqfPjDylnnw!_DmH488OJ-QC8R?1QJ3<-*Y9g&;8!d=lO$H{nTb$zj4m_F5@zS zpEWDX#X_wxxL#j_KPdOeYc!Ed{Tuq0EVme=KNCyHX{EZDh*$h}uh(1Nqq{%kk`}z@ zQ~Rj9gaR*4_|`D}@`~Sm{o3QhXIf0!JZ01u%}qsPH{oG#=(_w%Z)3}mY&E|}c?R3| zh3MR|86z3_wa4{FZAq8+YxsfhLG_MTjXGTrvwlaF`Y*}xv35h+$@gK=`><#&^Vyrt zD)Ie+V>-k|2MBNZ_UHbSA>KDjucB`UG5q_hE_JjC zhw0w$Tqhh*N^F^aRtTGNcjhq+zfGH^@%9zxXIf`UVeW&!9x(OcT&XI!v+&M=l3!l@ zYI0sK%lm=Ci*hp46uWWM;$fVfE5ZNClW**85$wF4{%34r~B?L8ecPY z!bDm^vB zarPsB>Hw-}kxEx*&SY#F&xPwlnRUI=Kb2W&-{m^4*edU*`pb)3#dIj4%biTlwQFl6 zsM>OVay8{QmXoStKF_M_Laq$h>5S(a$UK;s$rEZ>W1jrVFZszFOf0%CA5a@@jsL_D zoK5edio)$ue!zCI-GwXedAl2U3UrAUl=N|~N%e4#Ls><)C8aqpqyF`yto2QXKS1X7 zB!L6nYWo}4ey9?ocIQXABU1$^x}1+`!yDbSAab1^DW;alyW0Eu$0+sG?c#7|{Tq2c z0` zerMph2d`3;yxi}nE#E5N8Ps8$quVdTRC}>i3*%7 zmP54f`$e7JSBs5UY4zFK=5(4>;K+F!(KI#U4D-AO{^M2_)p^`Ex-_aWUw856YMZk_ zN}bsVOs<5Juc!x8LVhSu+o!~=OoaB4I5>Y_pm^a|{yF1@Wp~*ptdp6v=rx4lkJ1szW&2l&PJQ&!v+mF^p?|Rx zw!~zq#~!BCyves3%fhxE2a>n@RogHn;ymjZT4{JpFFQwQmC zj@Ba$WTj0+8eGs8B(nkhQ6EkU2wL{&Mq)5OaWhIsLI+I0B(j!ovFC&WJ$&-0K(Y0h z{8^?ArOe2&yRGjd-ue&spFk|&KWwi#@xi3CDo93!SxNtpevQ|Wb>LIwHQ?o3o)kr7 zDZgqy9Ja^wDl@*o`W(xY@#u9}JMF@sN#xmn`%|^wXzXmn_;Tmb2Zuv@-)mdp{q5A` zo3+6q$qL7Hw=i*dqW$h-$0m*FPK}F4=33e1bQOr6O9wINJM!&R#5%`twy9yHyo8f9 z+Y639nVcs8J*}FENz?>5J~A}xt&VFn6ye@KrQ7_A!Xd3`2KxrjoH#LiQoWUP@55g9 z^_PAw;9c_BH0TRFDKmY+A)UJ8{JNl~vb~4=ixW5cG1TxzAp~OEO>5Dz8!%f>S9LvX zW!8Lr!r1aXpRPmZ^#2Rju@RmQ9y@CM4ZpG zm^M^o<7TrGcTg%~8e^1xpgbcDTCURk5$P?+&{|w@$+me!?Oz6i1cdebCg1*Czf;Y7 zVi^TyE~h_Rwt7_TuAT0U#;!B<4RYAxS`N0};0>ezBu|vh5J@r$%43?X+R$TR<1^5;|DK>sUsdOp_zW3Kit7HW@X9bteu1 z+!*jj12qJHsPYS%AK-{RBKt4ONbLO+P4s4`HbNVh5qbf}S(#7CHTTgF=5?JpVX*$6 zc>FQVR=#s`XFM)^gp_@-?2mTr!n`qhFlJPvtes&O7D`%&A>3eZuZ9w*UGh` zh}VqzT!L{4KCnRcOA`LWdZ@&`#ch2K!YIbOQ$)*xq}g8TVmYTVNj}&s>R;3-dAyA1 zTcoQSo}?RpK4{(9 zcFkoshUZFNSg+%(jkODLNgsQPtlr#3$T*RI#63F|K;Rb~)^OeGEK<86?GUbsDbbod zdHwakqS1&Ow(jxNR|UCt&;>Blv-6+Jn2`{0uP51yhWseow=l5s4;%JY5A%#44D;vH zfx;?U%p?AjYoyBut3gwVdt_-qlKwU?k_&e~X5n@?^^qt52p!|v0wFZF`orLwz$1q! zMc-XXP3BV;XLA;A7t9T}6s#6idH%Q-ZXy+-(#sYKfjiu@VD=}*O$MNtW8yd`beer# zssU$*9vo@bi^g(up~RoCGm7cnYP?du!Dk%*c&3(D_zG^f#y?+R>SE`;U;@dvoS3Jp zMJX=iEDvI*+rq@fkiJd(OMhhkIe!d2&8c=fvAsiN3A1DfSvAmaoA%&Uen~Am*Py;| zSulz2<06XXxNjAVg0qZ|nnTL<~WxAvlRP4s04ZxvKpF=Y~ zKp!(b@GQS%Fu1Yv9dV1+Rlf^iH_FIxuZqpzEhy`rU|6?v4?wlwBl_2hSecdJ4QqcJtjs6@O^sW6Qw1Oa~_&>axx7F*F@unRqb!9!~@I{ z*Ch)<2CC9ZuR!#+dh!hm9Cce>(<^p)QgkXz&mLg#-Z=E`4ff-^H0 z1uX<3yL>wt#fSfIfT?SVC>3xdd;dq!Tdje>!&6%V+sW*Iz*X?9S6KB359QY4d0J*s zZ;=!O$mrIHj*$E%x6%DoLv!3W2vqt#2X5zZhC~*S1PpNr>Zu1H<~7MA2kCUNS#p5u z=`C)h3l%R~hVgVRw=GU5MRAGlBMC5DaQBX0fEz6)QfaL{WjGi6OxHzJ)c9bS;~JCF zDGf>lQ_&pGp~F_IiLC}??!$u;`=-~MmWe8N(Wm1InwFDoif7B|m>C4Fj9a`)bS$2& zq#@pXeV`^{7QmQBGQ|i&aCY+v&97i8>(;!p9JaRI*n`KtYw#+X5T z2Rt1#mx?-VpgMpn(aI6#Q2|B=GebnxhFIWA2aMXJK{{LkL^+_m8$QAOLOv*(KelZ< zjP^ce)77vg@p}I6i$Nw)Ao22J6NjU?X^A2WhUhw6pjc6b?9MylX{`SLKp>aw!ZzSFkbrd) z$aB!W{{s{veH_VEdJdMa@?Q8ePd*m5wR7x*s7 zvV9ku=#GVqR#>a|<=+6wKTXtd3@Ko;aJg7UnjmAFO?hxBc|i`PlX+J5Si^|Rt*O$t z(1gxIiYziM#YRv73Hm?6Os;rl9HX5^WDG*2K*I8ex+G?887tY3cmNNUlBG|-Ah*B7 zgP0_GhKY8e<+-1do5D8RvGGIbzKJUYK7_Fv#&lpce7zWTI7!f8ohfl+lLVkGKatFKuEg681fjW`Q{UTiWFgn}ed&ux51am?)T4I=~m61QeQh%Xb zG!M??POWdNPAMb{?EId0_$Gpl`%iP|>qhRHm?765X}LzYCfJkVQ5d!qGkd z>dW^bEExOdFHz{W>R2$Pd_A6)9N&&BIgbAjC4vRsC3KYeTXzB|2JArY&?w+(iK6Rv zA`2q2KVT$2g{>8i11A(F*^oWs@(hufOU!%`4J z2&g?$HzBNK&x?-_4T&f#J*|xy~JnAN# zY!QfGD@WBU_d2mQb}LYlDMH8GVZgvo-ViNha_*t{8+iVmE=Byi)%ud;@*Ru|+MTwY zVBhtH3#T94KFMc#8R%d6UO0vpmq;)bRK4y;q~w$0tiAD}|)W$j?} z5&iD+k?Q} zq0-gL*T@H{RZ<~zPoF~sd9;7DXnK5;jE2yFA^a8;RUhl!?Uju!DO+q>x?36=9AV*< zR$e!4K3ZRY`A`Z zta;GD(_>;Ux<+FJvv~e=ytGU+l@dPFmVREv?O-RR<-+%@lLeO}cCqa+wj!#2jAnrn zGbQZb04DIe6~6ENutFoT!i=K;c07q=L4N?PzY^lJy=uixTgH+$G+@|>MP(R?cQ0XS z=0|NBMLrnDDF`eeM$x_;-fh$n^l=B-&@FmDjog)dCJDd9sry@?oC1v=e zoeg&p$)t!Pn8n7aA4~~n_PPMP-6V0tkZuww8r4!ng%wP^2pUxxe$IR^v5cxGE5g&k zvH1cS3=>Rh`Mtp3W0uDuQ<`7pQ^cuq`=LP{jX6)6)+=Kri9;_vP`Yl>VYR7o>d1bC=r>P2*D<83Pk<#I7B>Sh1v8d_PNXsQY&S$xIBWjTk1-feK_d zOL>5Owi{uY0fMOTJuF)BhXl3`e@bwu=X;tkupq!oQ*4cJf1iZD+8QAyX8~UQ&J|pL zrCc@fMIfmgQ2O6PpU1&$#I>J_6tTd?F-8tm=651Ui9GP+4QHU50H8?ytXPyStr`-; z&Lf!M?=mluqzkoUN}4v)iJ}!aD0xBl2s;lfQQbj7xs8sA#e$?%YQ=gjc9o!r?ML(i zMI&g^%cK^hFA%Li^1?wnj74VneD;G^5;_D39zka=rj~VRHpxoHCb}@D@TP&a2|~Qm z-3FJ|Jt7@`DJF!RUm!Aa#&o6fQK)jy!MB>QRP3cLc~Nn1YYdWWWEo<85tFSJMD_8&F|NrBWSjp3;;cer@iJyov6Lqjl12!1`p?#u1o!v2)Fe-_$&;tK? zE&>YcA?5Ir4?dePiYt_uMXIB}vTbII2IGrUT%-66aC@}Qh{o|6IVTS;snn%MjC>WU z7(&(C9gfVB()gwyh>5rZG)``ePQwYP&){}XMs3yw0r@5FjtY*%e_3js`Z$%Xvj{0k zs2Fj`!qteKD!g~R8%Jq3ftYpmw{99{=62GXt??M13;6>Cxk8`-jt|OC>=Rf^#21?@Y`n^ArUWt+&4yyyXAH^l( znuGrVwWt88(zHpk6l9ImiotgFW|&s{dtF&$v8FRbALyBKe}9J}zyEuA`f{?@F3K~g z0RVB7go=ZDQ&L)k^S9VDW6e;gf_TtVB=<&E)Khv(7T>oIn|0cCRgdlLB-s~~8xOmO zL`N3r{9_*swb~~SyK&aHmpFb_4cwQyI8QdLb%?$?QJ?KbihlZ=hXBzC0ar(Z=at#?Ay_HUl9$JMvZ`t&+GQryU; zGaO^w(gHL27Wv{fwL1BAUV2>ws@@=4uJ3(1@q=9Ar+&w?mEZ2eh?7`WR-qq|e){Xg zj$o?#FD3l4Sj~*UZG~mUP01w{d$L_^>GIV&zhcX*h}yWU2*uxeqfXEK*?kq;tyS0= zNgP<^MQ0^iwdM2lAiCg8K#LqU$n%=Apklka2i&B;hfKEaUOV&FS2!0FnGrikS~yih z-2U-o(gt%g&y4|Z7Aln`Z#1wi!a(F3p5~B-gbtvVYf1%I_Sk zj6?FO@Zo8@h5+{t1q6kHDUlZcnpl)m?2jHkl$Mv)!=c(wKpztbbCjVAmaa1^X5I;rUw z8)X}Bd4{Oo^$WHvrmgu(f4frfllu-beRreNlvOj^3ivJ+e_sRoTxt5a)>~u4T~7_4 zlYK7{ zoYyWJVr9K1pN(K=rp8S7gj^>MXOI?F+}6u~{KhAX*lhD_f1lE`00VsM0d*F=tZIno zw1Z9Z9WAI(xf*%+9JFD+cSRAgYxY+F+iLlV05<5A+RmS|xEvvuMTt_HCR!F}XTC3e zXEA|x3NnAn)4@j%4+-63{)yi1VlLcvkJ6XjQl*8xKZjpa_tufSRXI~nL*N@P{|j81 zmXwzH43&?g_t4w5&D9GAwke05f3}(V%eKCo>*L2_uUJtYMqQ^b&V$Owo>2T(6Ph_MsQq3K7JipWQ{H)h03L zf)_xd{l#`0|I#twGg|U{ox$RZSVdI>VXti( z6ukl;fs)_ltDvUWcb^(2&QH8#XiV*C8LlW2HF<<$d#fFsv@VCIcK?g|_-=U-w*Q&P z^>ePP_Uq+t@;*nQW$|tIep2x17VsP*ABk3+{*b8j(xPB8Dk_W+=7yeGY`>`Qpj~PD z?k?B5Ft`jfwfhTq2tIe!C~~5$UF0k8zD9pdmEHBJ`LChksD}G0e$yFTFAW?fO4U!e zXjb{eVFl;)PP6bI1fLf^D`GbGjF)an-(Mgs6)_h=*<=K|i#MNbSy&YXpE=MXC(Ux7 zLeF;G5m)90z2P(X26s4tx5N{L{#aG;mk0BGP7AiVKv~{R<7MbqfM?v{SX@hJb$@#Ijmt+wt(*n?v~8~TXP+ql zjRP22AIwL3Pw7s#^YNE*r_obffnw10Dr+ZxNi?NjGj*(E%jUEDOZy5xiDs5&o?{Y6 zZrp>W+!GNw2Hh+F0+Pmy{XG*}4uZ7uzUaXer;CI7p;JRoYtGgRlnm_yk2FUt3gim>q76ywsTe^Kw_f$#{n?_*4(&8|+v{oA6Ga06W(^VtQ* zUsy$NIbPWmT#941+rCzPwkq~y*?D^3goRhwq2LBDTE%I?{ubH@?N~8u%lF8r9PbJ@kwRjh)JJ@nj1GLZw&P`m#)n=nE!_gU8T8ZD|&x z8-!PBpcOjSB<|J6#Dq?9sJ7!BuC6jM&s|-or>i}7dmY$x0H z#<`~ni_RDQKw&rT6ct2Ysq$!=+a_>S)EwCT`RtT6PKg?c`zI*JdvEQ!dp&x?ytuqq zCMnN`f*Y=@^9K^k&i_)oX&o;>==k{1*}yx)vjv!!Tw_96A*&_pM%lya+w4GEQIDEc zv&f0`T)eRS+m}JVX*ykPEc+di(5Q1`3R_A^#GcjMW!xd z)v3<3Dp7?N?j1GtwE1MkKmVeh`=&$r%CbbMOUxbh2--QO_m!N`)T$s&@1Ih7UTLr0S!xe@&r zK*swYG2l(`Deb9$ZNXM*m|n2!l8MtI#CKKe11osZ2A-ACz0m)tyr5@sWww&NMJJ2X zw2MSz35=+c;ukb!82f?YqDl3tXfECLo4%?Ox;NbW+nL^#0SPk2U(Ki~nR0EZ5koFf zB0pE!SkMSDfNAc*6GZzL5^G9LFZ^Dl`UE4$s~t4V4>+gmws$8%NGTJkSul+Ze9AlD zIDmD%s_-IV@9e87F64l(*Vgw0_WljxSIwk{Q+p?6293I9p$dW_fxijmT{fxj1^ENG zT)b+|ZI!>4%U%tB|CQK_Wb@EF505A5SnCNaRjx;d59mFKT5-ZI>)InOJrLhq5;>wB zw!>lMhNTTaV#zliP7O?vYhp(*u_MUzG@=E36LTftQsTqeHe&0AX5v3bFXfj;U|UbY zlX6iJD)wHQ;7&%$)Oyd!H7*YaVOAKmHGmxy_ER&S=Ro?9SAbSWmz>&?LZ*t=1*+|u z3m>JT<6|EH2PhtPVG$$rC>pe5PDibNvTn&b@8K=!JU@t>PA$zg@7tUeqBt=`x^% zf*yQ0p)Y180H%+8s}{bF`$L%3Th{)=gm3Onf8Dp9yjg7DV;!$1pKCE2PhR=_*3|ye z6H&&=e^FO$PbE>k?()j5f#`>%qBd3HFyuZcAnfJHH_yl|-r-@{@MX zAnn@~h(UY;2~%)>s_mEXN}o%@Z|vnEf3Ij3uy@2#^FLdq!_b;9kvAfc12`f%1?daO z%#llG?O2I$&Svv~HHA{FHqVqYoKc^Qk&TuVD0err3b?zur@(^jDS09lCb3JZ_g_m` z+QJRQht2{34oPEV;xGgR@d5;tcFC4P>c=BzB z2=G0X893{P>z{#*RK$=-GhwZ4yt9>Fw}v|1GC#l4xj%5K0Y*-=U6l4h#$%aTEQv@H z3LrCa+*Ki`LNbGEC5v4y?~8FlBOKAx77y_}S0ZA9;drd*g|3q}nV{Yh2xSNxo)}$q2}YWe7$t#F z4FB4V=Q-+qyZngzH^EYdnKa2lTHjAJ%fB_}yJGfeHPV6IQ4+{J3k2lB~k)d=lq49eGWaA%B z7o|D|YC@o6j<32zsk@9}f27I6l~+ ziic5r@FK>wBirUXoQ)bsNaYgivhU1tn3XM)gz8C2_Y!i@LD(!}J2PZl!)yk3Kpj`g z?BZ<+wEZSnvZntB@+=LCN-?mLt>`5FlX0Em$pntL>c|h}J$Jx!25bn2y#ch{aXm4( z{$3U{CIATJfrIss@|b(Y`4PCgfi2l8SQm&>t_8wNIQ0C}m$M^00=AEe!ivq4cSsNs z*}t=f$N_D{olVQF#Avhri0+7xtUN;Wc2S15TWmG?CT^I*HAJcZXE`g8F|J(KARh}H zORw6Cs7KbYolJPfirU0`SLlBkO6-*b5d{RwNE7rc9mjb+*K^rz9d_a$^fcR4@EA0IuGqeWRaAgu zuYpy}08GS}BuQptQVUg+?}idm)r3hUW04#Vw10`kCplM}d4h_*WO9Tu7B%>N+KXaJ zcA6wE3r-%i@Em-ybtZ^VI;s~`A!0Oq$C0BPJcQAXNXBns;ex~!O#=8o7Aq5^{-F}2 z2j~_+=a<%(_g7PpbGH!AYq&Y-LPzOeelCo64Gqw0f@rK8UjiIvYQkX9e<=_U5)VF> zC3e5O^l-<^GH`udojrJ(D_{HV609lkX)}yj`-J#dW@ zLOl%}s*QG04KXL-8pMij1jH_2%W&+iRJJyyN|h#o`VcCKieUHE zN;|GLC_R1zI!O||ec>Ct3ZFkNe087n{W8hbm@cn`?*z=LkwgmEiVij*Co|N&``TaM zU1s}ie=N1HeO{UB5vy)5Ik4dDTA&8^a0jMC_ty;FhnpCA@*bq6fm;b4?w)95j|(DWc6D z*hK8OjMcun`NrW&{B$RQIOUnSI>Yk1Vd?y?eWHQtVi5t3A>Hk?*)!0v-Yo;L^OF#0 zRut0oK%#bcKoBub0nB`e&!l4ZRo!Iqv{y{DNfg~oL{}@gUKF2fnc#nloPXDnBGvjE zvoyGcEtsDl2UOP%0`Vb=l24T3f@si?-!|uZ5x5V++Soj>M;FW4K%ghOE5epZ%|}sT ziL)>{5L1`Z1f4_)p9#j@$W3&`LL2P}g$-SduspD%2^(8Omo3s7*T$s@P~f+NUUoixWxa>J9%>AR~fMi-d?g<&SA&U6O~dP z$8rR^g301Th_(aU(G^k!jwxhQSLnSC?(c!F;sG^w7~2On`V;+c1I7ehm9R1+tWx8D#&H1XieDyLOhNSS5xcHs zERlHWCPWfdkQr{06wX8WYdBy8e=ifQxFlY25b3P6z+Si_jKp(@TBraqaqE!8P9_>Q zvo>6oeZaq9S@UP6Vf;*-PbUsPily@lBp?^DL2P6#Q5cGYkR^YPm{c;B#NVN?>xj%! z&<~LVf?C`hgdD-R{;zDE!6|{#|KH5@zXZ>v;WU^}#ShLVOOCq){&FiyG76_FEtsH; zI1|Vf;;>9NK#0D(i>5Mh3ZG{mwSYiLDeNqmc8VW~(~m>XrCc2lZc5#OkS#7@gC>b3n;EDC7J@S2nuBB-hIw>_sD+V|`l08cy0Z8WwW1XU0E;0$pz1#W z%BTx@&UmGqllD?3ldQ2A^Mf#1aOx4x>&kuy*-$Uw`kAK#-397g;L1VpIlLcb=wx^6 zOTZZH7#v)?Bv$|cBXvTj&A+}0lEyG5tTfi$DE56tS&L%yfim|6k!Zn>B-0TAAjR)q z5=JP-<`8^NO5!*u#di?IOY*ZGw2Q{NEB%o`#y`$HL9YJI(9|Fx&;J8gla&B}$ZNvS z&TsGy3O;QR%`ThES=phszcokDaVDV1 zw$lArHG!P`+ru-%H@e%Ny}3>GOzYD>5Vzue*!2^GKV^pXT^}vQb57y%cTi@fMdba_d|(U>Y$V3yOjxZQefKi7pXM6gNCmDhys>mTV}u!*V;lrMUB-V2#m z2N&%vTU-2fh7+~_^D<%hGzdLuwXaOEe_2=;J>1Wd3(n+`$E#|K&>;@~hKdv3xnpK6 ziraL!nh6>0ku{ge>aPdl>>BRuCURL3f{AR_s>iC>AMD9p>QpwBo3d;NcEi7@xGOW& z`CGPZQhRZ$8T)OtmX$42TlvewLwB}^hwl(}tr5~cq?kO_P=9^qcu>%fZLhpPDs|zn zZZWcS4U;KPTmS`Au0VNtI-27#6#0O8A@Rh&s10q)HQf6h!>kS;qso)3^M1%k^oEJsMU9)kq2pp7pyj!tJK6_l~}w%)NPG7^ppmC$C9Q^R4(>h)&q6NAal zDdVGYY$A}Zd7ge{QMddyS}ku^1RdLV#>E?N8D3*DV`8SO`Ff6tRIVN z*TCABc)5CMrrCxnGiE>fZEf1#+A=R2RDaO*z&bSZq7V39y0~HN&%rJTD`^2)C5zjNv-w*G(-8tG&SYeBQynES9N^BOr%l-`0v^+N8P5zG4=)OKG zq5qkSMo?-P-_+(bNCa3J#4LedV8Mppraf-$a2a*qY)CxtdbsDtpLnLVTC+f9!Fm9< zc_O>CV)91^7U;vXCj(ZVcbpDS?1|R-%vy#?QA)zeb~!9=wyav!H3fI>V81#krnWDE`@IFJ=xFGhyYB?Vw(&tz)gvK;N_$I9=Da$+n&5 zvYx{*d{+_c}+mS`50}?iogo4 z?S9#jG;@;tXkSP@mgnYQoZsH+gI zG_*MIE&%zY)*DX;BzHPGKHw?a^!9Vp^;(Cke^?Jf>%Gxwe(cq#Hq5re_G1XlbEg$1=U#tCX zA8FypW?N#MY3oMkyMfIQ8eOM7vbH$wT6##nPy;jK|Z|B9?FgOsiu(M+Unm6r+Jj^9_}xP<}Wg-Tu2m9_*?Vn&$HIXMf{<`xwPyl zyJN!m5xNJtkLnS>P6yqRJ&h6QCJIXr?qP50#SJq#pEqC?YuXuM*8{(-y^UkuCOQ0g z%yuKB-f&*=VMR@}1(0AEfEzZU8>}O+l}T7{Ydf`zE@o7SCI{u)6iq9=@ww9i_+nGSEQ$MaI{UoS`530VA*3aM84|c^ z=34C1vd6Zu7R(+#41~rFee@z_zWXGlJ%i9;HGj8dm>*V8pKmv$HyP75-?fkV<-KPL zy;k`Z*q<(G&n&Z|W$3$m-)rWPKL|6F3?|r^n6G5#_P6Tc2A(8kJ;9pX<4gN%4hD*3 z_=zscy?MmioS42qj-vy`9TUl5r5$*d8cTg@*A1}XHj_FT0>^=&hiCF$3KbhQ9JUJn zrU}+$1SD=E;FkXi6aX8&??Y?TC4ov#);sT2Fka+RM_jsI@jX+=H#f*c35)N8A-7b-L8vD{y#~oU`V1|8?-!*9yqAO%X_oOIZvrOWLLzAhDHwj}l^j;$n zC+AZDa2lDe?Jlp?_tHbN1~t)0!#cT3d3H-iFb8%DuwQQoOX#(`^+8e9Gw> z(A0u8J61l}=zTycG1FfpYv?>{FjA(-w^Mc0FiGtVXu}z=R_N`e%xHb=BpE-b7io*+ z;tn@aHDX3P!IrEI@shndQWA#wh74P>ACNX*Otm&JW37+RbmAyoIHMJmjX3Ll7K z;AAiG{`3UfnoVne10>G*B5fSD3zusJAla8IhGLXlQA=|GC;#L>EzrS-?($YnDRk#n zPx+>8%Eus#CqreibpJ!WsD;OwEf+#h58tg5&D#=SVlG-w4>@F8M!3b!h5VX(7hmit zah%m9_@{qn21Z1w!rIKIeKWJsA4WUQCZQ)M-!_!&g4RO)UsQAHhNXv6XH(dPc0H=S z7SKaB0a4K*S)qjA1U7t^X7|qePf`Bz*f}&q8#Wl3q@7f`v7P?-Zb5a(=X=4&{Or_z zuK-$bs9>Bx@x*j%Pu2+qM!(Hopz*OB*F|;a3E^97f#b&}Fzq?PW(JYoRywh#u2|B_ zra_mu`e*4}bAbq(xx+u7;`mOqkW#g`*I@XPw_nrJHuJO4(AkC(`<+|+5oY7DsqGz% zTX4iw9@MPuxvB5ktmn#&RlD87ofXE}%npf0RIZ2ObyX{GTIfaz@5x!u-0iXBW92*SoMxdNsF6`Z7Vl(p&kE}+I3 zUrqGZctzGIFs9Y*k@kHKwuiQQVQ03xlCiPFUjw|_>T>+)(=rR|mq%!Kx(`pgs_0!IY3X`+C;Pj;^|MaDQ$|R3l@zUD3s)7~7PYy2I1WviywHyG(`!+A)?tf4f<5 z!*FNzvW$S?vC>jhb-u?k;a+DA?ggp&qus|tHoKEKzyA;-!(0~gSu*V*47i6`Ks3{! z`5do*5u=uLO;|XR-hDpaHFm^%EXeu5$WP_TBo@_Z`<5`xs7Bg1^IOHm!3#^EeQ9@A z;o;wtYf{{!uz^j{XH*DAA%Bq=_J!{9S$=4Pt4t?Sr<6SNj zE?zbNEs$GJr)M8rDhkj3aC+E8m)j zRd+$yXKH?Fd=Ik{_EX*fnJOG<)DsHe94s2agpV-D(?~dA%wM$;8FlWPZ0?9qzJai0 zMP;U%6mJY14jM%l!;s(dH8*(N{%7gN`4tJZXW23G^ zX%O$1yVyqmv1e_C2H)x8d2mGaxaFS5VqVAF;glNs>6UGo4w+Wn8uj(Yl#T;*ib!@z zoVAD~dOC=m!l)g|+V{+QvYj=6Wj+ud;@urA(K-F#5x;?4W+|)t28x<`;#aw^R_og^f~^B8Hg3OBov?i- zH8e&HN2s-U@?yu7I@?YdN(Il`j~`o793*y}2zNH4TE9kCI{5%_>o(YRi4|t)hrc2- zS@0)5_X5TcuULnPV(pl+vK`yM4-|91iMWhWfSl6zl*VHQsW7Mj;jPu2gist-4LIc% zadruK++5p9AkVeTf$JSIU<3qF{yt)>3NsqOD(Qo@-wY}QWQBU{&DQ&ixu)NM9IY>n zXG7`35M%lC?)O_~Y@ymmB9x@#842?g$+ZQ#v_f-w0Qkl4iYT?YfqM4NH zomD~$L^pla{d{vkojN>lAcBD(DulXW2X)b~LI5kLEqpNrGHvdqpwHL+(qQYGO#BK1 z2r$1h9j{(>97X|PMRq9s0@yfmfq@nY^E%bPKjjJ+;CFwo%k+tzEasbwpcU;%+Y2#t z%%Yj(*`E4Y0j(Bj(|fVXEkfT(p`$OkK`*v0&YM4%29uMho<`w8`S8i`q}|Sk{zeQ@ zVB{oDHcRv$!^W!itm3-GV22w{3he>908O}19nxRzVR*%}EN7~C&q+mqs1>bTtS2Y1 zV81zxq;VX|#awmkU@9)Q`~F(3FbR@uTm=zz)0J)F`3E8*6h~@GL~Z|hF1_7@XZt_6d6mf2(k7CVzVB@DPs;}9FV~w zI@Z*HtQO&=71*!W6IQ{k55~UY$?5JJ6H}->IDS0<;f_do&Wyyb_j1?2kduuS*mTQ?b&FwB5)GfK*2kB5O>D-V+Q8C|0K2 z&L#q-mh|C1CsaPISw=}{Fev;M6Be}LS0Y^E*$iPnY@u z`KQ^c5uOhqY&zD%y7F=UmHqh(cP^xWDLDzEoPS~84%0zGB*b$*WQSmV2|C^{bfQzF zUk8%;!z_?6uaO;vlF(MfwqYQchl1~HrtE=H#~#c0X7&M>^14&D)X{H}e53{Q^KatawL21)da;0G)b%Kk4uF$u98 zSC9ktjM@MKDT{~XM$04Q*i|yuUJ{G>>Fv+d8gpw{ii9{WL-zS;4i3}c4;dHRs!^Ek z;o6?REA6rm%_E8!#MLnjpid3x@Zs2bcnDnjaoGCVm2ld{s=~PN%}L5Lg-||8i3lnz zlvv<_w#TyT(vWZaP**vC2g#6)NEG^_Jk2xZtp2m@no=m_ZBk}|tnYY=?!Ja$5py^6 z{eip=k&aF6w-}TTBWlP$9zPhPGY2LO$3;Ds#hY{SkA*m0(@fZSPM+&()~+_3^+{ba$5zJO-m{j;Bp3NXl5rYAVlsOY4;rw z74Q5psmmgEffkK8;gCcsFg>3iMLqetkr3K2;%6;;$!%+_{FR2O;ekZ<&lk#zl{tRX zRQ(@j2WI21-=H^&N9WkAAIqCzXWk4%CYUGY-h95l>1vQ>dD0bd?@T|hoE#?46MPmP zy`sA+O6CO<_OhSc&f_6?HIb_$3fm)+VSWKU=X;2A8-e6bpf#Y7e=1_9PT zN@l8e$9{UmADm74??734$S|LLug6ue7DnB=R#u>GkGb3OWELH(m4*oI)xOl(ONo&3*3Q~}vwl3j&Jw4m&{M;jJ? z>|1DuS4V7O#F&i47}&`gfFh&{!GWnuc?FfkbOEeS;zSNC8Zzb{gWqGXzS*WGumfJ~ zANNxX*tKA^xMNy7N9nV`?bLt+K#1{i!1$5G-y+x9lV4{=gVTb8;(o*la8B_hxL?92 z$hvi_D~oh*6r2gXfE z9W%G0N68cL-}O7`=k@f|6A7Y!~@2WvD#GEno-2LML zpX4ZNZ&iG(Jv;@(AibF>(VVqtP~h&+L-$d4hHdZ(kx19x1U+XwKI7l%-~Wi8?;QMj z)~l$)^SiyJo>XJ@#{)RLfC?eeno1uo>VM-*@Bl_pDUxRyC3fuqx-huzH!V3QmhB78cynQgDX~5Ld~rqbok=K$Dy3i;@=ey%`KOC;Kp={n<>es>lt;xS zt(4I8KR`%lBt z>{SE-&nLcyD-lyXHFio4Nq&b|f-u@Jm9ZD%h?lOT%J=+>YU9*egtUzC1<2ahCiN1^ zc%o@VeTv^Oq_2EIN@|e$BO3%{> zgbwpa-mw_+MivZi;^tRbX(=E@li`n{bAm$x@elo9?xaY$*=l8OrG;G2Fn3=CW#|E(y6gTsWj zspO9ShW+q>RVUTQi&pVX_+vHm7#*l+ev; z6$@RRl`|2w`BTqheikK}uuyJ(U~=)&qa&G5Ke8=G2en-@EFSBYd+)d+Q?3OU{+h2I z{Gu3i(Q$aC9(6_$H_yUKquuPZLRXe{mBI|~Wi4dqcM`gUJRHHR^ZXkK5Efd zrZS_g*2M2vUtffwdXYrozUmmRO3Xk`KK1O!MS<71s_r_0GW&K>!P>dvNA)h%6{y6j z_Q*&xwT+8=4B|M^hb~;`fN&&y07Cefhb# z^pQ=!76p_n6W9M9FKVY#N@DGV>a}xkz z!w!6Qw=naLc^C?Xtz#-!g;}mQbK@^)Xrwruk-zXX-g(7WZhhVTFze4m)7Cg z!TIOr%9%bvC!RSP_@vGb5X*kj>=CKG9uN`=;Cje=UD~LVrrv>0J1x7F?hG?SP0j2>Gm%oR|Eq+$tPyySo$<^T~M=Cl9TXJ*>p_y2fokH{b z=XhP`XNvFhNI~2%q0i$ckEkZh-u0nlrteQH+YaD<>M41MpFh_892022h$Ou8?%jP@ z&~(P%v!L5_oU>tZ_2{LkA4Z!@yeSQ#g>5h6+Hozr;~dqzj6SEN8n{O^W}TFA`scUt zu$`^@_8YbOgsh=Xt*9M8H#=?v9*h2x7nAfNl4se);LvMuEM2^+aV*SjJFW-sAZ>i^ z=9LWdA}hj`NL6gKpS$-q=1qZaAkDJnlL9G6!?=n%{jtU~_@;e%(am~OnNn^^e;H%d zSI04A5As>3FN&1I33G!!Mo_a4#kPAb%8uNz_1(DRv#@0@rh#^qc{ z4oqB@O`MXuraR07pU#>n^wZ6@`{WlYXhPg@T1|E<~XyE2WqzY|DYMb*szoupbCYaLT{%d;=cwTXb%5Nd*7J z*6|{foAd|xfvnTeI?BGl*~Xmifz?xIPWs2-0F&mcKyfieSVng04LoJ zk7nkeuqjm2T1($1(nVWAp!U2XQX!L+;rNj*WU0$C7@I!S*v9D@2236_9?GDE&YW-g zOt2z!aVDcC436%pqN&7T1wUZg3j72b+2e_F$#==ik~VaZUhI_vo8a%N&5^^pv2eo4 zHpL(v1B*@eErniI4yJo(ogQ~RhuMPaGA@+K>~`7PH3L<%Kej%a*V0;;9GY0}(wrV? zN^JilsS9QP3jB6XRuABXY8f}SjhOMz-FJKnlj_!5m*m#|NU0AEj zxYfaZ4J2kG_VgDs#Row|XW}F2(~sv%H}Y^R47oNpx-S8!ZVn^N#O$+D&!mIA>0xOp*WcdZDqcj=S@j+TwVB1rGwO4-GSFtIwJUBbWxeR=ban-R z6pX@Yx~bRYNw0ZNCnp-M6q9K>Z%-9o!3$NEZ#|!*n;<%S*te$`!a2dQ`KS-j(ko}4 z$DE|^uHfUSu|j(B=d&b&r?u$#RA?hA5kwBGDv2cUA$W&wIDIR!re6XyGW#!a~O&K1c{c5A#(^BWZr{aUc zUar~1)H8!_Q;*3~BV5A#!?21mHAcM9qz)Y^QSCpP_U^PYiLvo!*NYc>*fVInGIac3DJ*7T z5|`0!Z>rwGdI%+hnHp#2?TBb_s--Yv74dzc#iWVBuc7z(GHwOMy8gms>|SWDd!5NY zho-hrV{k(+6iPUH1PQi*ry6SM6>jLXxouaDKzTHfxb0;JM2iV@K8>PsQy96 zr8kt3DDw6F^%c7B$ZwyCk3FT(w%J}~!BBa9$+qHOsguG^0h<(%xwS)%f~}iuyIt)< zWz`9h`@iZK58?QXu!PT7Li9G#cb_L^`?Xq;)#^F4Z5NxsF&^1vZmjUB+I>Qqn|*NP zwsz*=QuV!g8Y~(L!iX>R5rq(Hc?H(NG+)p`WUB3$?Kyf|v6)Bme}d=xzbMn;$IqdDDBd;wwe14`gxpd+Y*it)tw4(>zv^OXC3&NX z4n1QI)0?&?nMhV<`W0W_y3wV-#vJs!*r3HPUY8?S6~N@K)JvBkEB0FMtenfMz-<^) zYSY(E`PCmZa=pbj*N+p39eD?AQ&Y|VmFmy&Q@$mvh%eTVq>DZA7GX zXKK_Fd+`s4!?rxWwET98XfF)Y-f_^tilAQzJ%y=TN#W8Ta?kQ}DuMK9$^!lGw(FsD z#g&L>yeBf6m()^3`D5PaYb1-6!x%mP^g=m5)>jZ<>o?$NO8lxh>ze%PjKdW((!ekV zIw;q|7k%nScYS?daY?yMnsk>Gxm8!t&umOiM` zAJjFx{`IK2I5{hIid4i4vGPb_Jpx*Fw;o_lP%3vuV3)0~nxTuA={3`2C9l%%r#6!5 zfqk{CBrHjwY9Ogz!xhCCJm}|O+k$FOUv0~iL2M@Sxdlhh$Eh9150RcsF@)M+j5btS zblgj<_suWy&H&{C!I6zN@ldk)BQhI92XCJHRb@RX5V~|CRc2 zG&x?p-pb3jUTB@w-)~`#Ckro93X^H*_++cs*SIlq@ILAyfrN3<@pxIB^*u2~cUJ-{ zK%^-OHGC&2N5s@49AHIo8)Tv?CP4qIOD*;7_ZE(|rl_vE0G!m(^wHYWd*@@b;)e(+ z<^ADuKm?%$YNirXB$wXkAC(D#a*+5jZX105a*2on57ITsv)H|(CxU5G%%*h_1Swr*7Cc~Cz*1ZA-p z2TK=iNZEa`qS^g##@QJWw<#7LAAw#%NOP0$c4WlG8;9i@U1Pk{2AAZiR2A9~1+%cz zj{R9!r@3G)Vzu^X`1x@MK{ubh@{2JhKp0d92$2n1XxD27goErJl$?}Vg6(WZf~EOm z5BJHtbVDcK@hY*z#WRbIEM5|Lf?R*-bz%IJS^d9KKqihiHA7q)JT}!{#Mc|u4~E2~ zb_E=LNr1mVOC_JQt-I$~e+98erXb!^9P?>B$j`cKFL$L@uyy<>IJ9-&!^=FR!f9_m zw&{1a3$z4&7iTx&_XKm(?m>JEhfLt)K>n$~=aoa8E`IlZqAilI|T5zLBu@1v~U(6rv?HCcY1!Y*`yi zc+nMFaN-v+<0p;6t|B911`4^DgF^%uLa4EUHll;i&6kn9;?YQ4V#`%AP6FIv^bmYC zhPKE;))xsdx%`IMNJ4>QGvKFUHVJF8Dca$Zl&ZKavk7myo8w1mfb>!E z3#Anm`W&i!JuJBPdD$$0iJB9e`|XD8Z?X9vc}66$)mLKB$!Qi@7epYXNM68%TI5Uk zr=3#7CPf;Xfr&ZCcO>-M#@BL7t!Gmmct$pQ$effWOkXK50?J=ROephVq>6H|6QNAY0Ilb3Fi}Q4^&#r;M0yMb4#+Z1D&pb#Oqq*=``7s#(s7+S@jh z`MjWBCXE;Rz(YbfEBuS*vz^sguugTVr3?$RrU6)N&nS*Dpz7Eg;ivi&x5D^|*$d*9 zv}K%4NGJ$4s=Z=Svq-Ueha2?eah}$1193{6-+Yafae*jzGN%P)b9W0_rKERl;h43z z{gDK%5LyTQAf^XMR^+~e;<#Lbf#ad^N6iKwAcImJ8f3!v6F?;{=DMFYkf9V;c{&Tb zh!unpsZ-#N|`T(Cv7FN_}J>+I{%j*^jmNuQ1bHI{{qKsQNYF+{;b2{ z*_@FHo0MmbTRQ9rU3h|-UOfu01Co@z690zyi9zkXM-67S4(rAb^WZoYq$L!`?}lB3 z0)|3=VNVO|fD3Po#;sWbF0pPu{IG94>EYxZt1+B{CSf0sTO+xwY43N%+0lmLs0%wy zyj-uEbUcV-GGXbnrvA29?&0Byd5$LT_DGbAZX=^aMT*Xi9rB>EEKnA+c^YKqvtXvV zi01RHDv|W9L9jjq{_PnMda$skr8hyG(ky`6%hS8MjE5_`gax8BfK7WUEqKmmCR~na z^I3UzN1XA{?gVbmg+JSHn#3hUS{MLVMUL3TzhKQ-Qls#O^S1hV+W-xmt0ia`$5sxO4}|-ERTjem3+D6c z;ekfOAzh9T`Xyva)3NCqSa1RsP~PdAI0jD�#hwT1MicFIuuk?Ae9jLa}d4S>l6F z6dfQW%TV@`=CJWzGOWoyq9fRuS1!fEbni)Im^FT66}qZX!Q36l0&gyelT-=)%dWf` z64v%CSW`vNxyZnkPQ*eG;WN=<9yDrEvoeXx4C+Hu+Lh;?K)WXC07iS0KQiShV!5MK zTq*RBn0NdN?^84a7jZ!Skz909GnMcNQvYAhfY=`o3ljkL@L4MO|4M0_4EmyJwb-@d zj)NAeB;ZlJ42y{!6%o&n_M@1p0GB0}G=v}Ee+5lP(Xh*2-bkKuXFG_BsI`#QW=9HC zr?f-t1^Q4Q*|>i9T`X^e2}wC(D(=3>8z=>xcibd|m+_=r3Sr)Os+i!*Gm*O zZvnQhrRc>o-iWZKh^;`GNQ+cPq6;II!-yVswX}QSMMUMXZnCHs@u;zk4T#rZEV}6I zJ6YJZ6e!&Jsnv!3M=x!85?9^>tbW;jVsW{_ zNAy#eV^_ODfGaBJK-+jBKTQ>V{!K+~D2`G^|1nI3##$tQh_JSMae3=O0~dA+^=%5| zFwH*+-y>uc8|$*%A@jCboKLNft>#pg8Bw6$AY2jyt}og`WN(W;aN1+!0>BuYFFptRlHBC+xt~l5Zytducfx2G`QppnRoU=er59 zmHdqO_Ts~i6;)xwXWhO2@S?R8oS%9@s!bkDmaV*2a;0vhZvDSfhfMBZrUtaxWs&=F zeIAoU?0u_@s`mlywj0Ngy{`+J;nVs1AmyPQYZgdzV+{pQQ{)n~;LP-hMF z8E|0sm}#7qZ#OtOIvsk`_Vw!23rD&BKtoOEN&i&4yqNm+pHL2t8@{Cj(VA3hjrz#0 zWg8>qR)}m~|4!A~cgYu%@4gtAl$L%_5F1n7vZJ~)3#)9^pv_P_`>MBKMkE9O1oSPb z-_P|z4u0O>!Fk{7($--*<(FtA_%Ft|?d+igMRXmFjN01(#wIR|!ksaGXKrtur8BQk zJu+`%FfN0PKU3;Ld$*3$q|`NH*$3g8W$cpFoKX6<$7VajWYU} zJ9ZXnFw`F|lLuLz9lYY}vt@lr%$heOtNX8OLA3uD*qa zrj@}CPDU*5X7($XBBECS0!hsa}^>>@t%Skzctpy+f%?IrebqU`Gn`u8+s*)~mGjW2Sjt zZEsk+D8z#wJJw8=%ErzF)Fg$e6yIezN||D&BgcphJKre}{r)>!U>Dk49>oyUa!7%z-Ia*Tl)>t?(Ld{&k&NJIy7?*q!pu+l3AWYQOI;e1b_>FwyVGpgd z{vL|*R)uP$5O;5^#{S>grI!ZsPp1Ya-#g21o;Q0GI_6jrd?V{4M9=g;`OeY&`moa; z`SQrv?hi-Cf!E1h1Z3V@dcT0I=EdMXV^Hsuz!~gzWsD;4{Yj9Y&tTORCO7)UW=g=;o8ZI|O^Pzf^}10kgAO zxG^DT%sKh^AkZgHLANd~Psm6->$|LuOW-pz{f#urB2V(SeSF63?n-_?v)4Muo?k2X zOOkal|Dv2_5viByXe7T7@jQ6@@ygOKyM9zBwHqr!ui#aR%`5!GJ%2f909BaHQ(s&A z4;^R6956{$Be4`l7R`ff{(4%Zv0tez(C49cZuD!R7vFe!n zPwy7j57f}XQttf7EZsJ-$qpGDV~UXTGxJ+sgz-8SKL;l5*WPbAFMTj1bH9)Yhn4)Z z?|}q}n9aFL*})j*(UhNd8qCZyUI{5=7QEYl5WVyC*5tR-VT(4(&Dsy;D1k7KU{5Er zK9Marmt5KH{4zSuYvojBm!d%Bu*II@jE!Id%g7RPYTUgy8Sm_U{~GARo*JK?LE|B1 z`MDJBnCF-K?v9d1;B@H_u6lN5f`A}UQ(%;N5zBT^ueVoFgSvl3v`=5O%j9TKy~l=P z=@Ww%6~!TbCZ7U(Q_E`fv{Ib8$Wa?IDA`_gG%{wM=?1mi=JFq2AyqjVZJ){GM*ht4 zRkm~yUO7BMTCjODG3+Hc*k-WfvuRfFvmmd5i7$ad&lcYoxT(p>N?5SUDzwS#^W<*L zd8zL&$_S^;tmq+sK3895V!vl(`#rPmGLy!vNp`Eh7(&Ff_Fm02t zr&znoxfc{udyT$6?>W|Pal5+tWvuX1$0c7AEkDfSX(QIWH*V6W>p^wnR1OBBac^0q z7xO-hMNO?W9!?*)``|_K%dMRAp-mOD2#I8QaN%%k>TW#!F@9Oo~Ti)jcg;Y^}H#pTr1|h?EIg<_Wt!nw!YqrI}K6arTuc$_uK9yz2jcM^4wPhEX>&7bI=N?Q%^}CFI<~@ zc8K@n?7@B1%YIpKN*?(}Vrq3pbD5Q4M7Z_H?bB&vd!Xf!&|{d>B|3Ug&|H(WACbQe zn3|KMNG4KcE5eoRGlFTil23P~DjCG>1|2z(s$E^7YKNmpQ-DLiM&*zg@Mu==jibI~ z3}>-ca<7AhnHqOo+z@D#!B-radg>K_a)z|t`}TG%s77?p>2*FnM99!>mKp!niKWVRJtcNtBxD;ld9p6Dno~S;;;BthjP6wuXn_P5$ zH2Fe-v|uwqEgPGgagBbYY=42#$@9yxuOeP*!bV$3mS5&@+ zeEpyr5oa8H0jOJ}<_-8MR<=^ar)WAe_3#D2-$=u}SU-f7fI5GMsdTJc#~)-`@+yoM z$%k4bOd3Mqcr_kX6-s#QU5-x(ct~wM+4(N=%aiC&oI+MfR`Q7`lFp~gCfsZA;g{N{ zGH^-tB^0BDjKXp%f6mt3jpO`NXyqs30Wcr$CT&Z9)Yd!~%}}eq(O|?wy^|UCLvi*B z4eaBE`NxH09&orMU-E>B2`gy!AdR_KVcZ<;&2}q@#Wq$M0L$A^Yk-ox{pG2>!aIp z!ZWT@q!`!Z1|_VW_v?!4n1*Urlb&JyBQt4wihc|6r<)KZulCKC`r94u?!5PE-P-`+ z#7ybji#>r=9`BNidbiK)Nw4+Cb9o#`Jeoav2S z5#RswOphbj{Au1#h-{b-e;=Z&41;}3X`cObv$9m>dv$u%HFRK;&)h! zp!+Ix_=B!hzmUn~Wh=hK!reX4HG_7%bn{S2waqIiw+)~kwS>d1G&%#Mtym@BHB|!M z$t(a>TAjtUwt2od-WXPO^FKOZz-y%yKjueMKRiPL6!-Y$3_-EOxZ{rD%*W-Y(i4N{ z2@sNN^XM#;RQT$KB=ISv#kmA8Jx|xD;cG@NB=!7po6@RB8uhE*#4i}C-CL)E-<7DY zwL<=Dj(1h&JD4L&=3oZ$v-@;utcz4t60V>reC2XLTM%$=IN}qfkxBJEcl);>aB>+?&)$u`hQ{0H)EPA z9M2vN^r80Nx0&lo?k~nj*ZBlt?r+nE!i7V5`O<{EKz>dD5Y8cb@?F!>@9Q>^+??*z zV0)G$BMa?Mgp4_+^0*5&IK(=!-m$dQq!W3Vf4%ud*SVt7vJH4IguQ}KC0_>--R-E0 zbK)qUckl+qp^-Uyb$fr0)FTZ=Vz-gdSsyr7W0*cU3!i^beUB}t)Qv;hGa5Q1_8qRj z3}8;Ht0+atwlG)Z(<`bKw!xC-U5ozzCY=|NF_6`3?^$*OwL7rB2)iaH&?vIl3#_g+>1KaIHLk zW?(vJ*D0KUSOe5aMp5D}!CBzlK3`o`AfW2gvA>=p_f$HcpU;Hky~oTS-VMHkgE{d+ zo*3Mda<9Lc^eK5KBo&$vRsw@(T0KDL5TTRW8Tw^VRX<3&%R1uJ_ZqoeN9}(91%8PDI7zsa7$y>r9-__}b8!{ZeT$@vY zry??~Z;4Yq#aMIH51l8(jM_nCx#im;EtgMZQ%+45H1x}P>N{I^jw)G&d<+5ey{8|arT(hcjsqI#Zz`AV{aH`OjSxe zCZ(f!4iYEm7l1ao4U>hCstJmFK&s#d6Zb%=EIo`;iB2%|5=g9MaLTABv_rx<`e^(Q zk!dWm|8J4e!ota@>1Qwq1W1XnHFdi`ehfo=dhFZhZAyT48>6J#)p-g15DSY+KNjwP4rG;}=qKkB7 zMGgBR1v9RH-ReKyUesC|uGRvdN028Vg^1~7r3E5cjXnZBh)W;yf%Rhq!2QZ07XCmLaEO|e{D?)a-+(S758_%Cg`)&kYR9R}8CMp9pk)BdwKjoQx6{Nt9 z4gZXbc{L&Rs=PJ55?1ytX+QQ6nuWw_4=5jV9eOvU=t$-knxNl3?tfwevzMYVq{d08 zj~~P0n6a7srxa)@ICyKCowI$jc1tJ6Ic1cp)liIGW5Y)4l;l-VHMQIiC@ zE~D1s0wJB*mPBOjbKhF}fqLD*HE6)RbV&shAz&I)(1j5`Tn|9?`gQD^?OJsOJBO`? zF+DGe3&@IpqGeoYBOHqBGc&JjmySFE#@&$_QKBabsG^d-T=RPy-oY07Uofz5)<#0V zOsWH+*X+w&CfpgIDNt@8mB9d(tFlMf-Ft=D+afXveUaEUfqjEMV&&9b#!pW;`^-#= zH3T~t%VJG7o-J*dTH$YVLE@3ZjdbwzE9W`bkyOUGxtJ-nao?@)G`g*b%B+!6q*YKF z!B`y=8%ock@hRCwJox6dm16}V2#M7nzB~8{Zq}#Obt*48C_y)2*w5wZ*a;!M_vZoO zl-IvfZiVbhnqGC$lFgftFtU&XHbl{a&)LYGp9 ziyDj_)jpljaN$h=Ww?>IRFtd7{;ZDbmZJz5B;kW+vULclArow2VRFL9XyFwVNE&s& zN1P$V3lP#JisuqyT^wF%g*CS;8FHPoz!`-uctN8l>I^V@im;W)Fey(;R`DQCMjqe6 z7E$z|pP&w!j__!H1N;gI23;cWf`pm~=PYaZ3>v%TP&vXAJo6HjB}ggp;5`tjdEcAR zZU@6t3MTNFc%D2yp1}<`g7Fl2@+9sRaW5dq7sQ=;#!@2xG6XK?^D%SnLy{29RL?d@ z7SM!jt53RC;&fd}0Osv~;#~F)cR@DGN3qfBlwuP7)HS|)xHN2nK4=pgYZCKL$bf=a zDUsqv;$kF=(rsbY7ul?#4ez^^iTFLml$#}BV^WTynjM?JB@K@<7)m{-nxr{hBeMiU zkrwb_JZx1InY6?Lkpw9Gf-e+Yd&b}pf`!034Ar5UgW3n9KZ`De%uLJ`j1nH@iI#j9 zBtxel>Sq(ZzqF-PiMg`FVt^l)D1fFj(v(5oBNXT|0;NrAIy)QrNab|HVU8kNN5gj- z%NV2A;`0(?lYNLwH>*flPk&t+6H5xDv7U(*cdL1kB2((1?Vs2R9oAd1 z>(TpE+>gktK5d)i`-JS7@ATFXyzj#p5I~w}iQInAqyRpH2wiYjO-hETZc&VVF=Tp_ z^~1h$nlIGck1|;_-XppEz#nZ#U=?U3eAgXruZPQ%WPp_qw4lOE%1;AxJ41?SotwPl9ydD2Z!d7Y;pO&$zc*gU?cKh##*8(yXO7$i;j0 z;JqH!AkWBRaX6)#iG-oToSd1%rriED3(Xp?FeRXWqgtj21nQ;>dc+J~)wjXe&oHR$ zh?5eH-NvVui@y!g_6ji|3OTzZ1rt9Z5PZEfN<21&D}atoT;=Jx^t43hRlHNmPU0<$ z%l!@E>gHpV>m|Q$5>~k$-N41`?Q&^ew!MdEcDS+J;(kZX|1vH|?o7>miG7$dnq+hD&)ukoe$&!It%2X>uyS*=IiaOfPF)Q4JVr1%DyBx5dKpUpl={#)+a zx3GQdEMI8mFm$$4Kpv}fbqc(<=Ge!&9>Qov-+3(aWT(&lFZ&Lp&g<(YT1OfUv2g=VCm-jZS(jRGUV*JZwg2bm zx*9ieH~o&>-wt=rHmsMUk95)b8-$fTTSo1<*JXQ4s(s~t+;6DFBYh|~(CkY&Zu?rn zmb6=%_suqFv?lQSd_`ZLqG{T6NLz?SF0H#@Xw{TAm+IEdK*xlt!)^Mh1OKURCRihwWLwPvfl@{^&E5 z*Uz&z8Z@KO7(@0@X;6&Y8XWz$ZLvLicHAfQbO_cky~ODT*v*(wMTpyXH%TXtU*F~L zhv9NB34c*jw%(iDD=furrnw`Fe`52jOURDZNfV0`k+8SCP7hkJne0vuyg?QyT(hw- z!9G1C__p43%Zx+z&4wEW!^o6BqTRSqC-1U=5J?45W3&9%`&o*2r1CL>SIYSV=bcM) zcb;ZV7CJ(43h*#b)=<10_^?*@HA>>JtF!nW`@%C-Saw+ciU+y(#!0KaXq`yDWYOB0ZCD zJd^do$8r0vCku)t)Qi8f-Qqf4I%B712gaRUy)x^ARwn=cPP+W)sBPwkx|JObzuSK1 zfDy*$nALTKjgxJ|>Kc9th6Txtie{Hac6BrJ1;FHfyygAuf6gL8)s&vRHx&A*x{=Ha zfqc6lja?_>SusUn<5L8xGFQWadtLi;G~F(|ws-63Ik(8py#u9u9cQzS(IY3HD;KJr z@2K}#d+(z)FZ4@ca+Gqa#XFe$t{(6fRzF<#tZt}cb9~{B{_=mNZepfy*XBa);0Ugp z>dN66f;p6Ke#%lnw@<}Z7e+)+}>QTVd_ zQ=Rhk`WCw3+_?z85r<9)9ej%fziiLj#g?C2DaOUWjx?P=NSz5;@aFOJ8Kc&1C^(phw8%>2QH3c@^ z<;K;aB4NWW2DdOOOD*FduXI1zdp*uWEn)wDCgK1a4D+0`z2GBp@N9KQMunpiZ~qIs zI@sM8t*RSxrQ;PpRnPA9-l*P*n|H((`gSu>SDdNS?kGKCC68K@ny~hj!9kv-37Oau zOK^z1xx=FVCb|0XXSd0pe;>a_HmLT!Y4kO#WBeE8MXg`pFz*9RdERHUaG`P!rg>o< zbgP>te_{@fxL*rxMdp_Ay+hw#Otmv$DzD_1fmO`}$|fBOjULM&_)oU{id-d@P)$f} zyHS>7wpT!_rqH(KXN%nrvEaxf(-CgHh4+Sx0z#_0VS&>piywCT*u#d~^0YI~Zt|Z> z;jF$1H^B0&3}7$ygRV2cK?!J{lwlm}+!7zNR5d$dwpBg0x6#MdzNwLtSxX4I{6z2O zls%#6e9POtuyJ>eiy{T_cB7Lyo*L<&`VWy>(sy=w>Oq*T%JJKzp}W{^XHDpd?N70* zn~Rz?xIHTe&dvQ5*`%(jc54sU)YCD_2>GDcw9jOs*O|cYc$P$(8C^J6{~9|qPO9#< z35B7*oLzElJbB!4$-|(~z|nJ00T**jK9PvguZLQjOtI@q^N~j<^D`2t`~dCHcJMV( zzfaYzGK;j|&9T{N_^2ODVfBm+y{^>ifU^}bN4~C*<3C9hx4Wb#lXMp}7FxF?8VtIz zhfZEZrWW(HXDexvNcH=qX2JDS(d`EH+z)y#6cIcrf;%e!43t&&<3(-|xL) zXx163Yg5^5pLhaSaS&4l8^u@(IoX|qUq6J535wJ49_z{lnGYGgLB7N34>#jp>$Gk; zePDass*T@Bxt*Z_$q_J|Kxzo!i#@y?B3cgDr%==)|DKe7eiF~}OE)jucE8$pjCpp} zsl&vNtH4~e4s(Qc;`5aYUe+Zwd52Gs8<_GeO=hP1t!@H!uW)44c2^J}Dw`IXbw$To z=s~ENzf$Jl*{j}1*Kfe}{!HdxA+3ODN2SNtVuc@dp`U1Vdhwdon!s~!$2y!&-Cm5F z`wU3i9~aU@X<2DM&UqR!xENAjR17wBh_{1_vn7<3T^A5y!+s8S(B2 zRHE+0zb&WU48%|Le7Fk3^P6kPLv8J+S=kQt!_j`dCVmS&i~hFQ(oRo&-Z-(|AkZ9XHwaT%1DLGydq0&;u<0B+c-q)(#^ zhXVhsqkNhsS)HG*m$r7iD5`5p4GZ7JB{|az;1oSHntXe5OU>}fF5x4;tatqA^QN`3 zp9A7q*3k`rP0WvjCL6`+pBZm5_2SeOS83Rk)`$e1o=Z9-iUN4q(5LBr<`i?%j-wB^ zi}eR}jRaiDSSXx0M@a#+JSjip>~(siFQuE2`Nf(R5B{;bkr(|!PE%&#H7s#B*BKyw z1rxK^WQ#9@kwg9gK3sPw7RX0*S1b{nDHYWu4zZwfQ){JS z%-2f|{sjEb`0M%m)9LCIMc#au#j&J~{4i}-NiY;dGT#z&5MR*G!D<3NHDClU(D?Pn zwo}BU?V;cFc#6jBH>w|46$DoP#~xTXtGavZ=sFlwpC9=I*)csSwNu2NO1S6XH%9*? zDK*D2Lt08m>}KR4QtTlL=z4s~)B%MXtNpR}U;P7Eums<*pkos}xCi~)wV|C5KI&oF?N%E`$zDW%kxIYkS{h82k}Wl3}bX{lP+_IKLU zj{65#tbs*a^=v}YO5vtq7nV)w%?!Nmr~8Q3k^`xum_^N<%t@=Tx|Lo2h2+z@x|FA5 z&epsRIMS+}+|pzZX^!$=!A@!Hl|R`(RXhDe<`iux8+*fq;0p5%*5_ZTFxuj5TzV~c z&;A8-Uzg^r4sJ8VrA-XNj(q{4c|lDa~)q|~gLT0uWw7%?gD&g~cu@sQbg3GV;0(v}OPUUFq`!i39mfnDydlJr~G4@Ul*66tY=iZ zSE%e{Rn-uUz7}|cH+<@V?PJJe3rMx@Q@M0zq((Rvs;`reJZ*uyT;^*S2~KSlLTEEI zHW`MS*&qqw=cs_=T;lvo3K4S!s83j5Y$);7$%qu~3?h!jVy(z7oi-vT{>fJAD`f!} z!=;;4VMp!>Mh(EbnzpJ>*kS}w0gSmKoCUH=3)H8O5xOPaLyL4dm-Kgmg$SUs>I-(| z&uEhVo-t7jw--0)sMeyOJLG3Fy$itN@a%{LB5$V=<4BeuUhUQ@#mjtd|W(=JjoM2Q-!mOZn4CqS{&@FNf{!e%-KoiFy;W2IH zgoss61hzuRFkm6WofOv5;ddPB)NGoR!SMzjg-e~7g`_L`$kq8wrYh-$8*X8}7fMq} zK{F>1J)-wNfkr$PyXN-{=f)k85jijO!XLn88qi)|6|LgPa!GRP9vLZX5yocny@Q6t{-B4A!u3IXtBAjZUupuZ zULv-y)-}42&-7l>!e4#4JXV)D|=sJ}_;t!{?ROLdXU1imCL$2i) zQ1Xn4{8Zrzs4(Gs6X;IUeTYZ*Sz3&g3D}7^`Yeb>3kPQfn7sb`jN+InCE-Lh-t4;g zEz3)oKvi%o%Te0z-WQ`+;5-2-_D!#mOTmzCmxutlqyA)L1h^BPRAD5UF!5UWWE6=atA4@zj9 z<9EBQ#NQjfTCXJf6A8lY0Lf>1OOYovfRLjBe+BXME!6Ms&LDyYA%{^cg~E8^2hu{k zVTNz5wq*-$wF(&XGf`n5Nh3+)7SPaWtuA~~ynoGQuw3dc%q&=MxvNsiEG9U2 zX$rlWf|ndvXRf95_esko9maDT)qv04#A6Sz0>-1JIiFS^C6Kzy_2I zj{Y??pIc?9>XlZSgg9>@bzzYz9nU36^w`RLU-xng2>F9`LDZSH=MgIMzz9k?A*5^1 zkBftk(HHtD8>z>Sn5<{ag$ljgWBcKE5Cq;hRbOE)WI|Pxms^0Ve!?TV;Q@Xg)FXg? zQ-`@(^xZXbNkbz{txv1u%+Pavu_3nMC|99&;8%D7;2oe;6;qA+C(|2Cf-67%>@h!T zX-)MCjQ(`voZ2HAszQ*BWmj}G ziMQ&Q0^G}7EBA9Xiz=DV~ig8DFd;FyyysFn_gCQw1$j7Z<1fl zKa3PHU)2L$cK}ale$i)&DH#cXlP6KHGKVCzelcV}&g? zC=Vy5NG#sr=R@+D)xDvr=rWIbZBXx385$c6#<`ntW<&eAqt2LKz|T#~|pd89G^NZtW40H4Ws0IyN}f(L+#mUxlzOv(*x zZ0u7%h}bnM#LQ4YVR->^8;oFC?DT7m32Z~5aVJrS0ZA2^%WI>AvNRdyjMa%<& z$vt71AU#3|`^w%EAe*i+2N3{iNtaK_)AjmnmAv#A*atC1TuWOvxF+YV2x*I;t>4 z4booyP1i8w1SO0<6EFZn&}>4f!ST4G5@*+Qp5iIdb@*b;tmhtjl7SZ3{pZG)mIRVY z!W>bFj`+)w@V@90C;DMv8i5oR-IQSSm%1dr|FW#a-^fAOwf~2SggFH4v&K>q?U#&L zS!53T>;L`v-`)BP$_t9`_i(5O0V#osxXes#NeQs~+*F znGyxmhzh&_bB4pnlo4IHw(c1zra!ru2Uo_T8lzmtA)$%Z``m8wsc;$(5$wI@2#XtN zP|`MvMUzPv)2E)17i9Nr-;IounCE~>Prqn{OBsR`3b;rxq1;+-swL8qWsiyUPH5g} zJ)cp=ZaUU>C3MzJpa};DQD*j{ecHcLcAscdFUCN$OpM!j%O8Xjg5Fl~2;90IK0Tzg zt(}ObQwQhAjkhr660i*Tnjtn+Jc>yMGFZA!-$g4}2l;+X7Mh zplG!I&Y_Bq6hLgFMakyNqw>*|4q$h~6;UrIM2F;l)_x5`<4&L7o=9rP^UC^RrZYBQ zz6mZr5SpxF(dCp}R-D$6!@^$gvnQvR?|S85w_Z}*a4T$zL>8zt|HJO*IC1@(W#0;d zXHrs%i;^wRPeoxC7M2u5jpV8{Es#z2y)`&zcCy&Oqn{P+K^P<)Sl7TN72hEp9_p_w zZ+h-l_WHJKC8>hi_k1+J%Xub=vX3#iDEl$nz%?{nfVuTGcNGmUS${jz8^6 zp+yp2T$u{lc@jBPG{_+~{xh3HwC~TQII2(h{d_5ey0Gx>!gImi>B%RCA2LINA8Sn= zFze111e_lURJF?5Ubj8DHzQT9X`)gSuYx8!<<)NYrbQG{oK8;Z4OiW(5DqF&A%^4k zt3c@4w`XSm2O56wy$0;H34a*d$fFLOC1rAd85qug-Gdt80t;iP14l2XWfi2%bk!zy z&Sx9!di?8!R}rjNYhfaLj*Be4ZW0-&x9e`*HjkUXm)UDZ2q#<@+m_kNZE`kM3wOr# zD;#>k_*;H$)8x(0$L5a0QhGpY%jNNPk>-wxFQyLsTJ77uZk8Uv?%ZGVuhfGHmoC|y zem|Q|wfsWI64v}5&*l^7+dd16^L44bkwVN1Po*0w$FCIk{H3cfamCLRTk)tF^Y+yV zEa~Duiz;;A(Zgnx(#hmFjJ|NJYBSll)W5rH>{WLB!X85v<*tU|S-*n>;z#xUJpu;r zLQeBcp88M63>5mtLv@{&szAOq*JJL*Gu%(#I?pjWiYh$e8{mC2v*pwf$Yg;@XkKuX z!LHOsy~>ic-O^$AWlskBV1vi7p9iNKu2gkz)ED+W^2^?Dps)IF59j>fnMM;~?m=PN z^*u}{y5FASg1dW275vIcp^xi{mZd`KWPsvB{zu!bLT|$iv;5y}3v5qTp4yM}JhK<3 zviQ$ni#fmAts9u}IPZDW>w;RO;PCK^;JD7WEv(|o?f*(uT0uP4)!565u`Zquk%=yZ z)p!2Gqqto&>+U86?n}Ow?Bs-v-fFebb}jYxf%IL8ziN@#@9)xL>?!9`cE4_#c(`NH zV;!&U;sLhPsj+KyyZ8w8#>&iN8zYCSm@&$)O7He`-NuINp00e_&plRJedl8wcJbvT zQaIcHD$hVed#V7_wc%G&>4z0TD?7`p%=AO8ekFq^8!>FvH^0(v@uptBATZEBU~m<- zgGnB>bm}pC#fvriB4qRla!*eExbMm9#H9mWvPt~hdecjmza9*2sqK9cV5e<(YtAsl zp`|D!M*Ghnae{(-c&6Z4$selr2-1K2=N+o*=7t&;l@4DjqCXa1Tg6SNc52PZ8U5_} zljB&Tzwc`xiYzBTQQt;{1>bEM@LA=WxOWLh|CiD8NM6ogz6}l43w44~ZJ(5rY`cNG z|BtNqj%(`vzsGS@+d6RI9;pg~pi&gXfkf&+L`8<6%nB%5L{v7{O4SN#1Ox2>zwB~=O2O(#BwkUd;n6tsHS zw7j&isHL=?$Xq;BL45zZGE*sx7S~rl)lD8!wb^iLk$QKzwS0N@8;5b)f`!N7{4l4> z*ZXW`v|HL!D&v@x&|Vf~yJU0s*~ex_1YT2vaZMCz^fqO3Fn^(R&B%pFKX0RgKIl!U zT^Y?U;I$;R)H>lclNa*7-iv7<{vo8Fj+BS^evR(!dR_g${lam4g~T z4BjyGV~4vxc`)K6SNJo8-Ym-qVf+sg_uFVFdr-&v&D`%*@itgjR4 z-FAj)Egk9k>iTV*sx+(b*C>0Ma(-!Xe8c+!H2c+I>#HfmSoPS_&7FDqwTat`J5nMp zJUON)UDjPN^K)-{g+aia>Gry^yH~E{dhIIhF#&%f)iXreV;3dQcqaw{`uU&l0E9#I z@4)6F?25vV8GjgeaAK|#wQj16P#aofm?6B?-m#8P4BModGJMM-#lK$h!O-|fHvfXv znVajprTLRDEhi`9?&ZH$GrK20s`}acVtCq%R-#|p`(z_-VQ;?6j7KI<+Oo)ygOolz zf4-#jLgn8lbwj`8o>?Jf3L5$L&LCElnr*OI^kd1z!0s4+5M)a@czAfRyDh&P*{~$8 z$d_ECQkBNKKic2o7||*u5!@X#H#?HIvlmsAiSu_WiCub|jv3d`9+7?fxF*ic2FRMY zotHO%dXsHgoCAPFJyA8$_xY>mS0)a0-mNV3(>0U_09oQnt9T{aY3meU(S23T6x3-6 z)UjG8SD#m;36~Ic(JAfon_qwVP!$4)jtioyTMy5xRan&)OJ=}n&N6!JRI|`H55NSW zON^I|j;-~_T<&w26H<#fajbx*VCxk5(^RcF00q;;G7GNehroQ5iXiXvpglrU{hH{@ z`olNwm`Cb4T2Qq&ZgApI(fw1Rz14iA%KCj`Bqg7J!!aFA!e-omdsuSo&&Tq|kh7`CTc$#O=B#vnwe6g;a82BK}t4zjDV>kS>(|i_ozTg=P0Jr~bR3AKd)TyW2{*^2KEtt>AX7JZMD{$)8aysza8ecZwb}O?p z*K>udzs-d0Ii;W`FSGPVTcyifPkO)!JE%iCo%FRod~u;)x?fZ6BZKxl-?_;VG>=4K!{=GuBMI)rd)!#f^P$VEj#tdM?H!J zJel*flgU`Cv8@%u2lm9z4I6ck;#_#XUkJ zLZ2DlDVF+dwQl67xlaw7;YVE5lw;|2$b88BSB?v3PRo=O;>0JSs28_(`F5aS!#%R7 zEHFb@Hf&K@B>nc1qTiXuHx}?))@DC}BUA{@>H+&je7zUhKNsQ;$3o20dp!?!3lbTs z3H=y5wP4$&ik1)jz!zxfB<{upK6jic>4?6jfPFn ztUoUPrBx%C=Xn6^=1{SsVu&M)-#D3!_umg}ot1d~F`K88Gs8DuFrAL20*4%VGJ@7G z^cB#w6r)-3+d=8Vv}s#I7I6=OrN7+cY#JN|3c$%{=m0*?zVN_C+_C3EYp@ZRL>+MA z>-KtxT|1uL=l-ho**g|-dc)Jn4I^`}W}I2r*~`BaRC%NJqUbPHut7TWl7F(ftFN(c za}nI<{IbfyZ6Y%(N9iuc4zhh%1Fvwp_4vk}Fg?j2j0=Mh%hKh17=jbt7i1@~cR)3T zhI}Ic$XPlM4cUhrSgSTITpMYrXvbqg(BT~v4lMPeCV)-gOfTeE7Wy4NJPhoJ-Wu+j z0ijiU4tA=1DX^Pp9M9gM#0yNwo@D54agmnN1cMfu^->^+RyZPlk6kNqFTx15H}*xO zdrDHi!|3N9hATF7$II_AJnk*|KwcuWe_bTHP5sv4MwKLcSxy~{Dv42_N)2d6EDpu+ zaxVU9+Q{iPAiGA^sb4RS?r$4cEKG zAeB$%42QnEr~fAAW8z_EXSTq{GeACSe?1U>ifwT}7Vp|7_^u769;*~ID(!eJ#Jd(n z6%KAw9@Md&ZJCaSv4PnRROW3Zo^G3Z6xH*vS6td2rb}E-ocRmq^O2+V6>QJuye@eC z)U5#gORoseAK6w|CHcn)V%e$dm_M2o(_89Voj2K4{0(de=(!0)@LgA>IKs1 zT5y}6q-G}9YDbl-UgEE+3d%f?D%AdEP&@9q{AfETGRz>bCI-afHqiGR)@Pef441}~ z5Z5m~%G6yp^n~HE;#;$|ABiwiq;sX>(CME7Gy0g7#J??TX0E1fsx3-Y5Yfn0whR|9 zkC$sl?QcDD1mBGPIev|Xm0q>{Kk z8Z~t?WK#uaMEF4zRG|0QaR5qLo?mBu zC`~Y+p$Tx2jzHks!1duE20;|z6W%znvG&hj{R;iM$45dV zVx6|hQ#TXW2mN;uz!aLCo3n3QOoLH86J@h8am}wkviUZp0rs&=x<4d3ye;w{rO6pd zS>r(KJ}S(Sx@))-7@wTM7xM}?#)k2l4>??L=J|adf2Jus2s9m7}W*M5j*#HW4 zB<+u_yo`d*f92-U!4Im`;L7`{6;6^XTvT^yjpUPj`cD7pr$3_Wu1~JuAd}e+_P!1$ zsrNNM{LVLG{w%Qb8#abMYuep3=tHM!HXJFoQ$nArKMrsvMZ85ZA`l+SA!(V_F4iHSpTuYjZ zZ@l2khw)1{b@nLQ)hzizbYPimDXPdj{Q{8DFPPe)?GZKxpN)En9-7F~ZPONfn|n}3 z+$JrW9Y}XQD2KmFGH&UXp#-`u%1c<&!0?g5-aGCQr0T7SC2nq{Daf4(>jgTGFA5lCefDj6q)3GA12PW^T?rt|WiX6^e@T}M8$7nqgo>)~L_(#mda?9gU z+rN>&GxzpC33|yAplNdLX`vVG?|q&uCVR8_OLUSV;PzSzB!27;Uo zI9Q#$R_9A4W4bS0^$fft6%qj|0y-)st%=fZtg}Qd4+G>QTbTIiU>C7nXp5#|;ZJWi z!%8J|2*!C!`E78aRJ5L>sz(LF@T&VTu<&1B?1EA?C3HC=`VOEk<%027`eg+~{qm<oLn zRdy#HhUinWH#=nm2<3$mwA`aF`)yPo7CF;|tLKh&K9pqXdQLgZVPBhWxWPa@BErLy z+eHl$mI;0s`4|UijGe~fWM(nd_q;EhIxMAM^GzfGt z^b#yr%;59?A>^>ZEf@`h<&IOP$7kn^WXuuX*+66Rf4>b~EU@fRuKWM@4@%zRBqiQi zg2xgaMX~fD`5S7Tz_7xp6h_yutU>qt_25xmtx4Aqf9VApzukv^YT)2PY_hD#G;lcI z{|2j^CxTj9rYMK7V?=}iN&Z->=rCD4<|*PrF)#$oU=SCFm&|GPEQN8fOpvC^-r%O! zENPHEtX807q4nCeQW6+XT-<`K%9h=IzDT>lK=4{Y14|rcP;}ZJ8Iunj}7jECoj}G4M8;z{= zm3CL3W5#Tg+&JR2s*kgZy^GZ1dq^e9lG5&`M_DU!{8bpUVV<9^(=#OMQV+|<^*0;K zi`t4U?!^{;*QAqIUE8Oo{Ea%3ugLcc{TQV8uWs(w?ba%;G@@*9zNNo@NhMsU7Ag`UW%pLeoUxx zQ?3bCxhK8Qd+~3cbo@lEoA12_zckWG8*>#(6PQV?oa7PZd&RZC`&yj+{N3}W{K%%A zJxTG67A_@ytQSd{?;cL=c8O^UvwMAtY44xa7<@CtjZ+^K;Ca)de;bXXva7VKIQU66Er<%!OEfK zjRR}jiC^D)pZUVMsN}Mja|?n`h_Fsh8O4w)tm|%MecBMMwdBfAz5KvCK~O^`Uk0r- zIGCH`|NLb;(`oD~gH#uq@3JK8U%8{cdzA&=mIq%^ikkaM3J09Vm+sYTyyDIK=8|q} zjU>NGpthC(A0u65XmH0W*VfSEt_j3Cb`bCSuw0u6rz;~y+VX)SBHhqgZawwD)8lfw zUtFe}pVO$>CThNx&bWH_>FZW!+l>dmS&dw;X1~w$Ci{1UesLMM8f>F^Bl2w1meR;`)=*9@*7G^=!~J!uvIonbmb#=4 zu1;bdW%pEU`pEpJFSPfn4spJ>zM5eTap-)VV5sT{NXJUOux~G3{sl4doeObq=Gh!Ek^ege`U~@`kQyfR-<4Sk%7 z0M1Z@{F>@^_7ymX7nQ(P&bLewpKgAxW*XNN?@xG~uQMl|@<$rugsoB4pWTu(*O*(z&C7ai^c!3A1j>0`@#Ym) z*}0sp(0bl-+w-ZtcO>}a2T$%fH~obtRvpgnDR%Kxh!DEK0VD{rvdBhb6YuSI3;zu6 zzf!ED@uJRQ=XE+EmUC;{+PvY~;6+W}?OP@n8N8@8>^3^6H=nAhFM7kB>>Jdp*D(oc zd#-#ha9#Rf--yL0=mM{ihEL>XoZSldV!Z2%2IB#x$y;vO3_PV_^Ub+AbU5ldk4Wh0 z&fA|3nNOepD9rsGnFwg@ly+DaWY`IfJw)4~?k2`l(^WSaeRK)z{_Bsaqj5LWq;b09 zROD}!L+o`&y=E#`XUWI!8SxyQ%sGh!XeYFE3=ucy zgdRAnWFv2vChzZexQ%oWgacA78iSEWo?+MZSz0dXCM~gQblqbo@p33drq7*mK4rBj zI3-7KSAJ)5S)J%N=o;5d51cK%v)`lN=?%i=E&OV={uWuSk!^ddNfaSJnMde#kMoS* zXc>_ChQwV4{tu*m`)z~5$u2%~iiN%Owrp~|h9TRpQ&4Lijl4R}_GEFJ3wQTH5@v)` ztNmkx&9#dU-s^kl;w^}A8a9d>Q4DK1txj<*D)m(Ext%Zm`GrJKwJ}1oKiqAko+FI) zOjmhYtui!cGBe%JUaAD({$qF8RCg6ln?-&9eP=<{T`zhBT(3i)!yG=nvvO85M>-4m!OyWhy!Rl4S=#5Ei7IgqK@ ze}OAbZv@kWJCt=s4ox>`!ERqFvw9yKZu7cO>eB%U6c@ntt}xUiwHqOM7_CrjM-K8I z7k~0=eKkaSZ!Za#7_l!3x-QdlDDK)Cl689}ZnvC3&hdo!=yz9*qw3?~aDvJUDcA7+ zQX3GEmBv@Dq`yul<|;ADgZcfQ0Kyfwr^Pvs^zxnJw5C|5hKF-r)$ji9mrw<5)s5>X zKU9(d1bKP-(N5bd8YAYi5FJSv+s{|*W^a(Y*kW6M@fT)Ap=HPQo#Dq<)Rj6hH3(H^ z6a4_bX-Fo8c=J0^l;p!md#Y|%~+LSQoW(xW$J09i}W8td&t9(nuBI% zPzO19y}Lu%S0@H*Oiq06Yvc~w1gG4B2YNz-8#gdnwwuk@$sFhSZ;q;3_x{iyAA+Kn zX9nGkoG0~1pfc+=Z7+tg)wAs*mdgA~2XxwXGyRzu5IRV90xkb4 zTbTkh<#}W(#dMr|7R$IfzXaeYmza+w)&Ux|x@rNj$V2(*RThemuM<#Vq6_91=K7s^ z+C+F^Av5b(;m2S@1FzxNCE$?Ry?U3fCzXb&JLYRt5(CZnFh3STpSphf@j5YS(uMRs z%ckcdLa#W7mbyEy7Fq-_7g*YXkFy zJ#Nmu((S}C)sQuI{0VRk`*xsS{69l=ch;>pqHbor&#i{jjock)rrp=A85J$Nb6%82 z|DHq_#Q5|;e;*^zZAI!;HhLj7isK26Px({Xh;{>?xE^$YO1?=vsb~}i)>2`5 zVi7!JLxs&wtRU)LFafLjwzogr0fQX%rf!cLC}{sIk`D5~_mV@N$_ykcBYnmVPXLVD z{JC#DDZ;%+$JVSxE#Q0Y&d=Re5Rn_=*=N8}7Ti7P$*czTPu}fqahOu1)e1WdXchwb z_XxHm)V~b|82S@YAW+vC1ZN3v1j(3Jq2bQWdLlDgspR?jIVLWf;s1OQWC^O_<{+o? zDTG6x0MR^8-O6ghd6X)1GG&Dc^ zA)FK^Z1W|6G`SBJb|1(Q!Sf2>eCM`kdC$TY^kqs#k3M2}O3{mO_$?>wiOifv^b6VU z+zzcPD$E^$^u4-YULN?Ebw51QV2zPFHTPm66yU0JzHvW!?naIZK4sb zB56R=L-7Uxkw8U$E~>au&Q}g`^b-kbSB`E~o|#$^9F^kYo4R>GPiA3f#N|fpVD421DVS_4Ua2L+_;OIJS za2dJ9gnhp&AU+;O*%un!+@?M}21ul;IbM{d#CMbol%A6NQdnjzxMXaj9$lUu?%~bx z^PI11wb%)v7GICM5pE#pjFHZ>SsF--QK0bNxgyeXy`qaE#mb8bVnuU~#Q_dZ5$uB+i9+^GdOG7j7vR7Y4qz%~X_Bw+qCQ%n_+J8t5jHJMkg**R zw|}06_D|yAHY!0t#j-Jt$Y|WDQVaz8HYehXANjfsb!NL4qXGYYFZ;tk>BE;`>?BwN z$#g9XXx`)LfJ#ABk!bMB)0E?)BnNq?%l7ao#|?I(Rysy` z!SAPdN}&`Cf1X360~@_NSWvYAf6b4>7Oh?i7MB!jD!&ckhUOW_p&q3eVHve?ho&8` zk@)CB?rqYCI%Z(-Dy22ZAW)MMXOLRx8bxw|0hm%3KuHU8_La)PoHTVIAu-f$O5)C5eB& z6SnWceBw|q%#0(CP+$r+8s`(_AqPhYRL1m-h9#a1=_KMKt+Nc0B`(Rr`R4O$0PIiDp>Z7_~dX}}7+$b8X0Hw(X4A>?JsKaH@Hp2vV zM-~2aKhGzl_{|PsF%&Oj*kDYtOunGxSxV%{Pd_Uf``J3fJk*}qfV|c@_82iV>*Xec z_KqxlLG)ISUAUP>Q#j{!Uk1Gc0Wbp0;xO}$w6@^ zeZlL}F=;`IPATE~>0kx985Pu*H4AFRsik``!QX5~=HuInuztazO{PVKZUoiDfL??M zIXfr~5A<6&LWfp{G9=>>Z*Mc|sb#;dtMJAtZgh3xkXF&0O*p@9EVH)S`<-+Xn(iqm z?B1rWpk_rDAnWBG15FN^Fzj~2&B6fTMYzYRK3Q5f44gQw!@#AN$*3#miQwUZvCN^` zj2Uef`Z@m7$f1uULkJmohN!S=_DYJkNrxpqgi${B{y&|wv6^$Pm4tFjBFDbl#C=@IldX0t*6s_72Wc1zDBY3x8{u1m?ivZZ(p$8+gdZ z&v*c({{KV_q1SL24*x*(hem#X1tvl%xleoNQyi)MdW2i<`3#Kg!7?{Zw9{@R_~xby z3$Z-2@PByJ4qYN+T?)Q*R|@Qgd||_j<4j$Dv%GrYQx>8&9Qqi>nld{7l?#KyAaM-= z=D48_JgS}==@@)V4uvl*I~J@zV+&^uw14uI7Avj$qT5NVR~D2vp=!LlU~$LaPXpSpshPzBwJwZCKFs1XjK2V$S~F{-{ZxKtQtL^+TAa}jO`Huhk!w_3U9L5y@^ zTy#p;vE(WPd?-YA;pqE(0iG{UKqZaZ$FeEG+(yX|GF!dfNW1CZvpnGxiV#~AR?=7e zlw%)5QI*+%CSTB@z7bJ>UMy4ug7wQdTg)uWy0jWR(pMA5h|=OiE08+sQ!3X}`V9Yd zNnbp%;yU;{59W`>JHX!swIp5GadRh5_{-_<9ttOmxCexxtcU*ks%YtA`(ZCgyEuXX_R&AG;@qRP$!%SnyXi~Y{9c_i+O`!RMZ zc_Z*jAQ~%HI5)RiE|U8;ijS+GaOUl#U~J}B&t;x z;K&Mwx|YDU^hs9O?M7&R$+tKBtn*rE>#`AOKsufXW*|SB#`otp!+76bE`4?l5Q;y&-zmM=7cB(W` z1=sTe$02r6;KaWqgzSZuD-Ve!*ok@NNi>B6o1nJxUFaUCj}`HdUV-0 z;~Shx7R)H1HlixA%3;zA1LMi%OAY|niH;McI5K@OT^nw21Q{=X5xIc>40QS?<9`Km z*?}IR+gabJ`d?D8%tf>#f$#n#QCoQlC(izIh&Wl<@g%=w*0t!gOezi&n|IKr;>ZA; zc+-(ww6_R3``@)g{Nrn3)qz3C%S3}8^?sY3HUc7P_I}DT2k~APCo`d-;p{FZnnlNB zV2}yk0Gu)YABE%psvXh)mX`%LLOnz!mGE6S{fIxE{|mxfxYsW@Yj600=1@E zI#1T>Gh}CKBRppO-2DBIf?L5nT*_Vz#yWOG)376lYF7^kDV_vI?Hc)29W(DQNr!rG zFQo>=5$}J38;tPbyqOP+2IkPPnY51miNBsiQ z-9afN&1e*{a>2+c>6A(Z>WfPz4&opCFT4Z7#fY*QD=fw^N(paiJC0yqp^r97SXYMq zVf~N6`Jq~jq=Z;(be+1^T7(HA{pdW=`Ow)k>*7ta#KfmE#i>l9C?_-0H$KwN_E$;H zj?lOKf)mM^-;&|Jf#A~Yf7PgSWg_IZ?6Q@|$0W61k7O-m1VXDcmbo?LBTpI#6TE7V zrx|_gcIJRQZ_}MX?Z~Pp+NP2&HmYiTEG}GJaDn6~-Bu^v;TyYqm#0?zJ1c|{1SN4I zUuD`Wg?_k}UbD$$F+CHh%j**mH&1>RA4wdpp^pDja6O#9@BJ?EJkb}fesudjjls{l zTklIw{ULDc_d1`B9C3VhYX50%byNA8vr?}Z9|9vMGcA!N(eNm#lZ>=wv6 zI%yx!s`73_XDz?b({IYqaA`4$DkGoI0~85&fI63hv=_nB6>log=$^AB9E2rQ4Ag{M{ z-8Nc!y!G1#>oS)$mSgqiZH(x6lD9I*Q>4WJ zZjpv(%-bWvF2jBpvdZ?8Cd{%kV?qK^#K}M8An)Ons@rk~BxnpC3|uB=4KG@J74$+` zUSAvIkwrZQt7<$n`JV4Kj!X%1;~yP7pvisJ^KOhR{6_tH(KF+Vi+Xv(#fL^(zdLSuUNN^H9eKoo8bCXy6tZ3jqcdu9;;I|gv#k*h#}uzI9A0;*_^O=hX(1@ z2l}wh>KEy2i2?2}6=5Ajw^|uAdFFOF80>_5ZSCOnl?mHPd&Wu=x3%0nhl0|Fbt{gt z3l<)#8cut2(ER?_wS6T~FSLCUPb}B=ATR|pX1bd{Pw6R^MtpY^rKbA?7nP?E-zaUl z_u=EluN05Ej;>Vx@Ta|G)t)8u9C`{zG~DKCAE zs=F?ihIxuDJa2l(lPW1*H;JR;wKrDBIWS$U7LgB$Qa_vQmsp)Hs(ej4#r*P^|A{q3 zWzffLRFqfg+fZ%|e?_g>U@GrK{Qpr?$7-?N(=q{)k)pz~Wj@!M-s6aI1Pxx3zW56^(_27ulOPmflh0>*i*5 z$86N^7r#tt+1Pq3+9jh_(Qum$SCsA&Jffd&>pXZpUIIYA{&sekiq^eYxI- z;Y-EGfT}we(hvhlSN`^eqfR_xU*|qJF?w5Qxs7yOx6)tDCQS=&iq&>=hJFJ0u2XbCKAQz0>siym%r3uQS>GR;Urq45scGw0`n4p_MT7FhL3Hf7@>ux8 z&kKUOhT5S>ytYb#I@^y}a;|c>24vrV^MGj}9VY(+vm2K)J0ED~Yq2krq<#=J8roLu zQvy$6RY$9rUi$i{i@Qntq=>2x4Bhy>$vD0+M*casqFW)RE5h#hMw4z2K9QZQt1)95 zcS27oTc4~x{bWFM{W3J%#q_9)3lPoQq;PbY9E?9A`g&hHH@dmEcSm4$>4mB(PDbJ{ zPG@d)%-y?qL#c!tKicYj0;*f%>EKmbJy}i)5mr|9d`VN`XcIdhg67fId=(ko#WGF+ zL;6hjSr)M{`~}?69%gK=q&23C@zb0i}Z8+ih4^%|vCVrw8Z z*Q44w{5ae1pYaY4rz>mWQGEFehCzauIqu1P<+W1`>wzDfPkH`ENfDJfKG5|@khsOQ z27akJD!)Y7Y3bUzF1=1mfWsSIUhJQb z;zEZeUY7io=pO!iAZiEUpNn;5SpyHicyrf$3p?f55~EQEEgfRHZi=W_EBRsU$g&UEV;?TG5}m)^EVA@}MDjoQ67yD)gj^H6^AxmKk(QDav_qMhgf&0jV3 z2sKn&T#oT`iMW?nfBydD-FN@{`H$0|4e<}l7| zY1_(6E$ST2HCZ0mIxgv9p|YWG6z)I|rqx#cmSM9UT;mY$ud{6PB14&B(ikmm+hnHK{;I&^DQ zq}vWg@3XTn3-3Zkb7G}YA)1)uUTSH$ho^40dn_{b*!ci0t1?$#diYIdI-HR7Tu0VQ zy4{J4VH28{C#b%rbi183`kz3;x3q(OHoo3&@(a@Zg5NFmSC{b5(Y7^;FC_I(d9O6~ z-L1sG65~?>N9)Ry!~xRq`ELh)#1V9Z--}E6I_z#vaMVs#x!~oRt0U4k;A3{6j)r;f zVJM&K!2)@@lqx`SZ*1&69j4r~!Nq+r9uxDBf!!gG1V3H+@s11FjBZcDSy(N$S?p$Y zp{vnXy^qINzu1Tb@AG^Y@r$Eqv~jw%(7JxjSQeNawO*IDvLn-o1^Us{6&@ zAvNBf#@7VY+-9v&oz~(Op^4CHFWtIn@e^w#uM`@=xj+Rxuyv6_49PyJrXQx^EWLG| zCLbU56bi-uRFNi`X(t*gIZ%hzqH`0X9(>-1FevOQ#nGu=fbq~)4kO+ z0hTxppzZU38|+D*USbuE=&Gw;W^m*3F`l+bHY{19NWy@atV@s5!W#J++)xbBCcErB zIvp<+g8b+BGr+C&gj*v{2HPZ*a~@EPCJB`im?>^DAQyny37JMv^fv)tQ56ppw|bxSaVyE} zVn0Rewv-Ty&(i8W zw}Bh($RE5wa15{&Q5T!P|02#;A0a;OdOX@z>hODAF4&!hUQ2;v->#ElwX>fJUxe=h zT&tw(b4voqCeOLhS8!3V96Od7Lu?g5BMTI7Dw+z(L7;nv6BsO^I`%-f=0esTjp zxP{OXWLsKc9@6l7ohMg=cJ5xEqpjJm$D|?+Lj_My?a0OPK@bJCy{as()w=X8E!by_+nKkWrMS|4VEVqGiW1c+gprqE zyS_NEI(@YN)CM4?aWtC1goA73)5jbBjO1pk!y{*Di^1dBAa64=J#`)?OK{#=K@KP3 zaL^w=8y`oVdCHmgX-ef^Ttti!Ndrl zktnu@`-}P|Y>jE){V|Z|TBbM;9Y$ewZ2(>q@MIZlFMcf|iVLOejS+MAVHigl4z4jr7x&Qn~ff)iK zE(jefM=t<_!K@$%N~RTsXc#$_Og0da4=To!*NIvGK+qk~2iS!|su?8eg1heVgf#I`9e9biUcSc@{GFKN4SIn5=QND&+&rOa_ z-PoBm7$;|kg5cPqoMjy4bpB_+N9JGZEQ^WKlyGMf5oT{__&CJjiwhMxo3=pPlz+afCP>0GGI%KblHN^x+cANW*c7Ag2Su9wVr6(dzNdpMIC&Z6>nBD&z0{b0~IPXrum1 z#x#b@lNA=Igy?#~GWfus^9Nk+`_zJ181*;qO@`s37N;P3kH@ z1s^G+0ja(Ns32X77)WJ+K_R;y@R-2F4+|7AsL=WTp5q*96Ed<8hL@Nz+QL_|9idl2a?aQkNSR~GI@m(*LT4)p1YS?G`&fpM*tx?&V{>3#U#!d zo?`z{pRQez5`P7BV}e()E5zYhWt+7KiYbOt2HY@QWNrP3HOkjh`ft`wxd*o!-Kxg{ zX{1naDfF=|u|1eSk>}~87VllN&)Uz{GZIV~Wu-ws2BO66T*SBF2@FBPrF>)uv)uSN zahu@v@e(73fHxy8zS5NYk!cmJCK)MW6DNk{f5CK^>OWxY>kL30KIBa;UH2iDGR3rN0vA+?vpBQY?u zG%#a=9o`7DyMZA`n@jXT!Et!_kXGr=r*8B&L){5sdSW+0iQq#4H(=Dv9<4I?#EXp4 zRhe5g%jGBf9J_bGxKtCV9zHE1j#ko(j!&>ohc@-ad(+FP1JSJEBP(0IjYw}2^)n7p z-=!L{j&tHg6>tqIu=Dm*`C4B}Z zL~vacs#20XNLgNt`%p(>&0t7smvK87lL>`R0)<3PBR%$;H+(p?drxy=367# z5Rfn{(T@DkDNUOSUY9w1)i;v94BwK0FWqOS7JOhhH2Hxb8Oi%gxi8KUrd!gvKCsG` zBXtTxa-zAZ{3}M2`$R+_zVU9G#s`?wiZ{~>#y?AB1phu5+yOomq-B7HyJJ?|M|Ri4 zfFG)NJTCeW7$Xe&*0~Z!I?R#j)wmI5lI-LYfw>i+1ginGj^aEPGilCr3@wB`d4~%l7exTY?uky ziLMG9bHy|e;30MI9gHH3MsG@7qW{=M5u?vxF+^^6a1Yi?P?-7Le5|O0xDPT7n~p5_ zKrLqp+TgII*2pJrgs&W630*`hrT(Ob+v{-G3EB9Q$va-7_!RLUEt2AG%*_}+n!e1b zxkOGB>DLkn@v{HV<;flA$P2$Cs?q8UT0fP>HKOH}5h z53>5%W_#lDdhzC`AbH$qZOBHl1;a+X4#H(V@t+HD(3y`>;HVsof^Ao-M{s-R6i2Q! zeG~X*sU%w@5~io(33qSE;T9G!|0xs0C75ESsVL8R3*nx(SyDW6+#*xH_$@~JOMJ&2ip)3<2c8sBSgUJ zUFQ80@?52J1(IhD@q7PwdzQjE2JIV8)KkF~`}hX1ZrH;WYaCe!Gt}UD`V>!c#2C6zpYLZi<$cE=jiPXwTZw@K;`hsiYA;p#YBs^ z1{ME0{KsMfe_$W+|8wLAZK!AelOKG0-K^>o{rBv1M_-KiZc)+yNgj>xx3d9#6F}uE zQ$k|B5^m2Rr-1z?nGb<=1h#Yg44M-Mp*C-NIXUa25DEUH@a=0O{IETL6Y8}JhX`oT zVcOvTZH98r%@4+2To7Zt#5ZO1@q(Mea=c-{okY!^Zdxvw`Y&l23okeB*gt#6o$Mvi zzG+7#lGv2dGyT=PfXS2%9V4cVYUEMRT6o?|p3LZ@b22mk$z@~Vjf!H|k+|;3k5MjJ z6nT~0A1U>l1QMDn=8Jg?4eS?*^FH}) zL_hwt@nWgFEo6FttvDQ&&1}uZ;~fffmI6Wr35Bw;CnTE8P@to|6y8=rk0`U$H^w4W zd&cL==F()FhMKMcJfc; z$8TW1s%pg=@55+|M?WG8|oZ!#loO^?fo|ecV19^|j|2mAp8y%<^gO zFVUk0Iz_tHzjEV@M#-aXORh`ze@B`cH531EOK!`%A=&WmwafU2)fdwkns3 z?PWuF=R++|570CBIXdK0Zab(S@rS!A%A*?i`NX#tt^EAPleQfBL)Q~LssX7zk| zph$IGo~mn4VO5P-Uy1BXYuC!snqO>PC!hr8vqK|$SHH+~+pF*CqVa3S=vsL=2)n`L zl$q{EUvnF^>Y4Q1ob7LwXR4~x7JB?>a*yrdWKq>RxzVoc5LM8oabe|q3yFO>QeN>= z+}>fa!XRkD3pIf2+2s7b^Sk?7G^rJy*N_nkchK6Y@VDI?Vgi31w;DEPRqhL} zS2*-O>GFxsyz9^}3^+~BKh-;7XdcS?F11caMumvPznwDHeW7k+spy%3Se2o2VEk2@ z)z{XR7A>_+{=^c?hJvnqW&(ZQSo+Y4dPQPqm)TGu;p-B?%Z$gt+ngh&EeytXhkboJ zKZ{gs{lnie6McNGGdjeZPQ4oxMFyf{#PJD*(eq+wo$2Pxf8|1RT-{=B9VFb__NH=a z#|j_iwK<2C?iB`KRp0fru3s2Vok$($KA;xmrUR5i<>98z!#Th_4}^QLn#<^FQu_ntYcv_Z4OAztExEI3!}&6P$;owl>jc!9 zwvW@~S_|_flE_r6=!!{s(x8>kweD0C{g*q#t#uBqy4>d?ophT1)43~<;P(2j9vg%n z=pi|fG#Fy)zcDJBd^l}XDfV)VXcS4i>KeSfxyXaiSl+gR((&JB)uf4Q!!g&>L*`R%#Qb>gLJC}87;7wV*r0%SnZH{10RNsoY z*j>ljNoJ1Nn3Hu(YdQZPUGE+jQ~v*f$GWU*iwMyyq7+)Xi8QQDl1%p~HA~V>%_d}Y zIX>))QqfhVCf)BzH(h64D#T3m|bCi5fQk~i@+5k8SZBO z=O7oe`?UYXl&K5m@Lkdx9zGGiIoPS1xG~1{*?!y){n~NQydNDfS}DN&HrQL>+wOS8 z2YsSv=Mp%0Xy5lpiH>~m++~&D1b&JhreucDr*YWxp~)4pvFJp1C~`H<>k95@TwBhH z_x(G17zg>2d(K222fgbxU&o+=!t0hov&jDWrBi>)t%UkoZPKQ?OOYgQgZiDcjW4w0 zeuylDhO66|W$KYPwgepA>bL}V>Yyb-;Ms-y!@9ie7Qz>pF(*Z33%W>)o7TTZ`k))f z-Z!0EAF`rD#p4y(-C41_ZH>xp_Kv+FCTV8JH20b8od~(pIejtq)7hPozSB(=lR=xm z+|jLg%5jN;VkdnShpCm6ZC~~sN>mKT5BAxDpmu)5msS17jR#6s zHKjIgy?X)GB)qJYSI02|3WRiS6|lb&Hm;`g-rC347(pkI#_%%N!$~uFIR?EhhOWHN zP<2?OQyELtoLJ9m1VWMG?z*gbm|iI^3DF+_4`tbkBNwK*$MS~#uYiE%I@`Pa7eSQp zw#cKtPtmw}Db~qgZ+6(;Uf`FrG(Ccs7<3Jdxwqp=tBOzKn!Yy-S7D&~`rj=nrQ*Er zgoz-%tg~m3&mX)kq;hX`wA#N=07ZxpqTlE59^=+_XV9^&7d8}i6C%k$Aw+tMm49qGhLHP@Vpy#yHhu-I2y+%nBEmz zZ4i5AFycxAQ;$}P-Smr99>ix%8X0BxHtP#@nj`Ub#xr@`WZQ$TZItZZ(B93DRuROE zG|m9%AZYY8-&w3WeRSAp4@ca0{^?bv5tq`xo(Y~DLNgkrb8t9A&>pQIG8*3QLFQ4bLA8t*Y_)T#C$=`AxgszX#m^b!1 z;`E1o?)xHi@DTg^p&xm@v936y7ZeF+oA+#)?h1%+tliXVkig))XI*tEAqR9nk)KGY z?@*l{p8lYxndvc^58+oWW10yfp`AsQ7O(&vO`2Dr*epxp3+#Q)Y>7PSqxyBu+R1p* zpkIV>*zXAjCmuA6ptUC{84jhlC-#!>3!(kg55~K-dJpf`q+jwMxXHS=379xYhIxy0 z{(9p)1%}+5Wam<+OFpdL{VA`@_CceyvKEz^aiRYW>z#o50WHhFV49s4bugf=-}g+y z@5*N_j~7+Hn+T8I{bw5aP}R|)qnF+wwOCl(4rpt${bR_r%l_l5)fkegY=^LIZdA&= zb9Pc3*3N`w$KyiPk<;vAIPfxb4B$kYExI2Fkuq>XVr_$w4fxmC)6igOoe-c*jH~5g z8Ie>>17`~^kc9o$T$|a+Wybox=39F$I2S-k>#@ei$9Hjbn>x;{GH1`{a4u2zBE8I} z`gyM(H~OR|PYV#1CC@0M7Xdt!z=pXe$3bg+ir$%Z@2^PQ-eK^m)-Tp8y{YuKT&b8} z?kdjjkemXH)z6wJCOnwn(o8;I0w_WVW=!&HETA)oaRfe)9ljTpXMd1%1-2MVY1W)i zVV1-_WCMD@`B*oDj{y^WZv7E$F9mSDQ7%E|t$s800>MIH1(2rpGE;9#v(NWwfY%j>wP7XOsfvpZr&QdLF+T8rqf{9mvr~( zs5JTP0+cM`(g$Sj3Q$gYDV&$Ccc3*>Z_`Y@$%=e#Yhdb-uCaJD`~t9QE|i9>>SWV5 zHTP7Ni98=iz>PAlOd0v31*rrc#~ge?Y(4R;r7Jwuc83tipOWyP>w~em9=w)V(MejK z!Y1b}bydNWrl(?UJBB#k7geXSap)rvlO0f|u+^=4jia2|FrF7i(3^ZReM9&GR0e<+ z&!@tf)Dh04y+?h7RHkMV_<|B;9ZOKB*1;eUZ>+p*k_F{4<-Y=9r)|18jt9#AU!?si zSs~gHpUHYA;&B9wCSE#nE;U`QQ2@Ao$_D7fy1T$a~E~2 zobr<5>bM|QhFS%WciDkD0R3o-OpeoMvn1!n8?T>A~zu=ZJEvgX?-+tUBIaYeSmfGubj&22AWgdja z9%Yf}3SA2p?|*otvgSF_8CI(e z(3v_he8ZPeddseK`d7y`zz4i3S@1JG0YMj2ui2?q@BM=57enI4e!hhQN@SWb!@#78 zmF<9|3W`>jUGA~3-*x$@5>D%BPz=?1tFdCtCM0xI=XQ#r)8w_FvL{(gHcs2YL0Fyr zo$tg-e202O>W9SYtg6x$T~t560Nhmgpj}{!$~!7~u@)zde+IO@nE^pFD>Zyy`VJ6x zGnHo_DnKX2p1tS?hxWhae2^i*b6(2rZo^fc$;S5wTU|bRf9j6x+llgO{Xv)?N*ifB zAz;3!=$*VYdpfgw7xBi4IVF>1jb_|F)%CLq%BOa$-cfQJB+lt6GA?&*3;Br@nUQ$I zao93vNeYIb(CB%pl7F$<@^A@D~+AZk)_UM z>%YvmK``tGEK?3-JP4P6*d}2+8?jFV9N6pcMSxX|MS+Mw8ZId1D57xc&6^1s@Ylb; z4g(T}5fuICIXwQR06GFxpMyF;+E2SEPt!R6Fkzcm?;wA+{&a%8E=OsKp0RH|@u8We#i58HTS;&45?aZi@9)22Nl= zCM;(%_6Nt?C-9w`tmzfT6(Pi=_k_z`glgY?Q~ZYC!i3iP6#-rHo)1~ z>sQW3%ND&(zEu8HP#yDpq(Wil_5^W^f(q8B!X8jlz#lABua{8O^ER8%V}=CyFbOL| zfY{iDz|ek~N5MMuJb=bPy-|F12<1FNw-8vV$z{G{17W;vD$5$RN3NKEC#B4#zea{E zVci9qalHh1l<50artv<*Y?knO_GqKDPEBfbnt#Gj8vOWwfaPyKf(XI6VIGQgD*>P> zm$!D^ZiMv(VQN3657!TmH*VHCyuh4{1Va5U#3;%qQK3yz9WPZdY0p!UIis-0G7H*2o-*&y)oltyjr z(P;xMC$}930MZ&BbrhB|{}432ubO?Xbt9`nj}iN#|bD30ZW3^Zet|( z4b#jNk2~3&IspAwNuLEr#}Zs1QA`q&`cP6xDg7c6^jPL7G&y<4I1W8CIFe#YU(^NRuT&R5zUc=&F}dH%(n^6s;blAgNAl6k zgfK+YePOHY=&(DH^TRf&Lqmf7SyQGPOz)i zW$CNZCMav9J&VMa%3$P<)=HyXw>(T2PECkBI$G0-FIO0AgTyO>G`6 zbqT}ogjgUe5YYhm6`BF)r;#xa7Y}BnI$wlIMQhjkwm_;p+CsI@lLS)h!l)_D4qo>F zo`vcF~k2A9-j}O zwkW?_;&gsq49f92SzVU2h2yox9c9~p7kSjMO&PDQWQ5GQgM=_0VVVvT=+CAIcZ~c^ zcBe!%?-Y?+kfj&l09mOghN)`o{Ut4WsIY@wdX_pjBM&x``qQz?PNSdaQ8yYu;Xumw zIGz=`-DkX}Eu;69V%);?wEjEpBiIYLY_R)|2lA{)#En9QSlseIj_4c(?Bh08Gj;qh z+x*3PPQbe$g~cj=q`HO>2hPDaNlj$+Ay-RLXD+}J zR^Wd8jwRB@e@0RRDF{t0LQWK8haJg-hzYIm(m3s`kPR*z+?&@V5qX4fDTd_%3dJqz zuO6Z{X-$AJZ+)bM?U13*vg};oj-iqjvsh9DUO?-?Pl)jvb*&<&QXsKhKz1X_Hmn8^ zW*rjI9EI?>*2x!$oF*!}%8(9n0#FRrWy|>^2Q>FwqgUORz8ul!3m){<5MO4Y-`ND! zUlk8pS=5mPV{_*7$>xH(i=7a~UM4W0mZEH+cz@B4)YL(V9y{O@&$YPClSK-;gUknZ zZY-DiUy2Ob;(q(_&SYo6nFRioBaf{{O)^Sjae6+=y&5TBRx6qyLl1CanlcU}ynRS0 z0wv;P0IXDymZI9_fP>`6_L=&FsG}iWM1*m1%<5Cvh2SQ#u5cjU>BURj^awX%0EM;6 z37D6`CUhf6eZz~@$0voP*|g@A%JH6 zUb5Q5!%MaYv-TnKCous7hWE=0&JFpYvluBX-47l#Ek&{~Mxoz@$c2ta0P{o_bF6Ag z%+T6s$j^D6o_bS7rmsv(CcvPVOe~Q4`80n5+{7O(>CKQh^wi{p zl(9NOXNYWX#9VLKGYspJq-t_4(32#JkGNG}P4pz`HG?f29=P1j8TgV%IoVvz#9`;J z1vk$iWS*ftCP>;N{ZPi*Q_b>}>Sbex9J<*AuK$zBYJ(!u=m^BV( z%0&F8j%c|{iC_*bYSc!W5xokYn?wMo20sJFJS_yjVwxZbX{OG6i^XNG5d`^b!T;{M z0eH5Z--39}=i_EbROmEL#Xi3*U$*kk9k1iA6Jq0sDiVjulsh}6ws^H}J<+t-Gi717 zW)jN_GH_08Ii7*8!+lkU2Vrf~kJtqS!ueaeI@a*J7o-<;C&YT4uOmsA89TU3`z>&M zu#umhJ`Pq*BL&-rfJiLY8@96X!~l9*yaZuzfaN%zkKsJqhRA8c$Vrh}s!9T$x{lot zqOSt{Itb{1`T52%rmTi!T3>G{kd&~GkaNqrD`P58Vy?h`?Ad|*Rg5)VA1vAbcZfUc z?3`}2N?Azw;WZQ6lcPb@@;}k9a^Jqg#YRpIMO*B1N~-UZtz8m_oWw?mMX^f%)Bg!& z#U3e)LWNbN5f&1RW4Dmj0{kny_5c1q5acS|urYRPUNsKA0r@a8HWQ{dXq$#C_6e*r z;DB@_fh>ezuzot~k3rnTC{eHIkOtgk=~_|wXE+49ONRe161EJ>@97g9gs14LPe|)+E|TqG2AJcNV1oO^h-}Dn6POcjzSfmAc=e|Z4gO7u}8hkw`kmx z3J2q9sENg14lE+q=JIhEL<(+#=&lj#NjEyaP{9Ix>)>cjpARV)*dU6rkpKL%`|XT$ zc#ursV9bO?-Wm=|()d6g0MIfcRDH z`KQ;o7v*e}#r@$0!VE#bV`KepZ@ae7_qJ=OP{<_-g8_#!pSDlDu(-&8_lau*AJiU6<679$TL1P`9w^SSLIXiY9i09DXrZ2-T5re0;FM) z)>*Uu**3R|ACByKp)YXAQ-z9<+-a+iLW8=b{obyQbRkp26%xd2K>U_zhELQ?EZHQC=}i)u|N$L(r!*Ihp4pvs8V6*B+0c-`Nm z)-`k|uNV4~e~#AE z1k1uky&g@kVjO@VXxX{z)uyE#Gt}~z%I*dD!HyE&&F*72$_jeLA>WHW9{TQlV(Mw* zfM*VgMZ20r+qEJ(?2a@Ocm&3l3WqS)KkaS|Q$*K%1%*%0hPHku9oT^X2zySP%In#j zSofdnbGuGPQv08GCsBi8@2+fo`jz#2{@-#X=i+GqG3UoO1khhqunt@1EG-!dAJ%(x zPpAI1>ILHK8w0rxPRE`JytXz@HdUp>PP7Yce((+h$XSaQ$gvvHeM)pR9^S#$nkm%! z(pIBirDPwFaKnRmDSYxGK`_=nlk23}HM?x*#Q6EDk5o?U9d=u(&bdS3VcREavmLG^ zHn$uQc4=lSC7C%3Pm=fY==Lks&138)N42b}p>`c!GlC&s*gpL{e}%G|+O15J^0pd1 zm5jsy=GbO|Gruc|?|;m5!v9pV+U?$m*+?i&gCM*a+$R2QQ}k_wN%5n2B73LOwGPhf z1jdZOmNhT01w9#Qdbv7^y_(`bTR;r13VR&Yxa+duKal)874o^}XP#oLWnj`>GXCbZ z!jezy{}lC%GU%r9%7jUlg753zRr~WdF#2W#F|Clg<2nOPp@p<`#q(6?EeL4Rs zV}%oGVuOzUB$5An(PLZNg8XLBVTUD0VD7jZUY4<<=enJFO{;eRjm{&w_KQ95P3gRQ zd1ib0!wZ~)qz1J^?{pR444`F&XjjWcfGUYpkb*n8TZ?_gaKb*i?EQxwv4dlfo)P<%aw zU9f%ks(XIM`0T-n>8`FP^bs4jj+#MHNhkut zpC1BDQ|8yM*MY`=Zk*!>zuIV4TB4$~xJS_s&q!(i*jXNGMd4HO9IlfF))k+-~Bxy}l0+9s!U-kXPx zH+p&W$d`U3R|WI}*O51<_N4Qw7w5=|z9neoQ1E0>P20}zl5=S9NBfD5YjzTF|Nf0SSC zg?B^-@KFAxThzSICfLL^4;ja!T_-ZKk@OxDUoTEHDsNx5JL7l6_2IUQ5keu{W9yJ# zes3CjI!}1RM(I&qfL}5B!DHch@HaY~uAybrR@pRret*7%Nm`mc9YBw#p2+q7g#Xf7 z$mV8$M=}Y6U{BkVzU>>b1N`%hGVKHJ@7aO=dZ{RT_vs!OPW@d_Y^+Mqs>m>$)Y(fm*>Qa=BvNS!gxaw1f1JOf zVAyoaoPkvi0L{= z88I2k$gi{Q@0%U5y}CXiJ|Kh85U-QgteF_8KN6C{x@WJnb0fU!F3#ZmzC<5!G|LK3 zJYFU=8{!7m)FAf3zkKbRY8I!q6yOT3Yz#~F*c(pcMAqJTj%P z->$F~ZQfiFgBI~iI`d37tuh@BOw&_Nbc@$H(@qOBA=CtDjeql+hbH0uuhdsmJX@3yd3ot*KcCMW+KK>}Q>0K~}DDiEs9)By1l z62JTsby_~^jYtpq1JK_~hF *F+6fo?EO=co*w|-lvcXtT){E9fg^nSE+;^0>Eqj z6Yot^JIy2nv4)|WA$b0{=-MaRsNGjru%1F{YLOtX;F53GW7lFQN!Sf^BQZUSEoh5l zL}ZahI%fzT@1%_pbU6H7cxGTJ2yzPKm~bP2LEH`o2RD7efU#622}mjE>#CNU1T=C) z1(53nBL2D#aE@qDdD+3@ZDL~@nAv1R>F&Pr`RvcO-?b~m`A!J$MKt8mw)b(o!~TOz zPNWP*e(p2;;4@^Z7|$~tL3w=Zr~S2!b9_DI&)hW|aT*umyLnFyd3>9@vb3v}K0zPk z`)(~KdycL)8Ij@>&1|F+LZJQ~%E2m+&KvNNx5z5SN+45~||?r6u74!&zF*Lc#gOIxi* zl}jnRt;*8{cYAE9>5Jyq#)C&%%yfFGb9`Oo&yrsXr$k@;OvH0nj+syghox=1`!xIE z;wIcwBB{hi3LY5o_tNa|O{&Dv$Pgg*L=sAQRa+vezA_INM;aE$j1FI>^^nFA*4G?e z2MDMEH$)nTi{zLADoq{$zb5c<=H%*qz(<5*?U}`Ans=ez*7B2++3ZyjeH(IDnymQ{ zQUq?DTp$s#gFT=MU~iG6w`NNj7wM7AH}XtLHA&-b<6uIX`@(!h;;6xWMnE46|2E?e zqK?DN%NG3{&3ILskgWxXytbEiUS4)he|2D0$Fb$6DzNE);5FK}wp~?)X6MFQS2KLf zuEEM{IX}S!uFPom7@3`ikRy&|PoP9ZYN#%4aXYFXLev` zIg|oBbTWlMVf;@U%DLlP;Q`VYxt}Aupxbd@H_8#D$ZWpU1|Tzf;cp=HErNC_W+6G) zLnx~xFA5fDR~SVMlu!T`B(Iu8f?T@(;j6`odP!8}k2w-d$@;;IAw(HjDn&!VYQu%d z$w(RYk@Ed0VcUcmmnJ+vCpBEA2lFffDfvi$%3?M_@EL#LlIAYz>of>}l6mnhB)-s! z<@Y4*t7d;}JRZ=XP)s!9=K@Z}O%HP7Xq8LRx78jy6JQTh zu1^D_%SGvBYp?6Zi>8QgZ=XLT1-ER^g`JP1HGn)v*3+kMip)@FaeSVF+@o?RXV!|C z90-#{$h*57cNRYM7I{dpQj>(?XU0i9 zY@GjqQLJqm=QB&frK_Y0gRUfjKKmyrI7l!EV7pE7@{IYQDxK_28VI+_Of3i z`UJgy%Pj|ot;B;Sby*Vbt;SQ;NH62fWETFzAkv6MNu)_Y?yzW1z{i4KC>D)C0AED+ zfhk@t0oys#;&{H~CXT=P!jK|csSOUECNSBwVpUDjc(Row*B^zH(I7x`)ly=flR||w zxCQWKG5bPX02OTe!%+vfa$E(6s@`8M_&*foeZ4+%9Vv-_9{%Sg(nn49z+oaApbS9- zPsdiSZ*5Bw>_|@X>pwOUcs}HSjsk{Gn!P3EF6RA1V^~KBi)Qj{h*5(>Z>JdZ{PF*eEz43W6$9uxc>!l#+^kkc9HLL3(!x+0TW zDP4$ODsK~Og2iNZt|tZ33DQ z>1PB3g@##?C1mphVfs7E>BVi4JEV)2D z_?Gh=<@SKsqSS(i?+SqSWH4Y=2Vk@jJ!mXhJEZNTQe6TF9W`k2Tc@MoBL2@ZcI#H!v@TK@NV(&2BI=DM zTJ4LKMZ#YP`d`BHP{9Q=A^>A#jgH}#i8IND^9>qeWpNaf2aI}BfG=2`J|+I>7o?4mAaZ)Wtcf5$(P9NqPc2Ra z^^_NFM{|HU3Rv&|T$B<763-cZFC1r*Ap~MBpiKqJ>TqFuSCgq7M=88;HMxIWcYh4P ztmdnH!z)#ji8Rs;qrW98$VG!FX#uNQ+()T~ouWXR8R6_YH=;t|E4xXEz?A+TyaKc$guqx@Xwi2u2uTqE7Kaz{sWnU1vL zo5GV#_A!Bazb|r9sw*=vm&irb10{Mc^96}8Hv@618Y7=L>_y5Pb>xs4ghc)T5MA(z zF>$Yjk?&25I>5&q_t0NyTYTGst^kXSMbTm}wy~%VFo-Wp{bG~tVz@{n?8#$RZqo0U ztN$_t(`&(J!kU{m2`HG)F=2CKyf95j4~iaLQ3)}uphwy%^^nF(SBb=8*eP;ExnyXZ z+`=3wm9p*@NmJyf0cgMay`@q~6Bs66vGBBsvqP7Uf8+8r8ApeyZ;|S6d8=OZ)jx-* zNxyOaiO&x#q|-u(+Dem!Yvk1tjcjyQ;Us8y3xkCni&bD|kcgxd;I9RBQ?!O=Fij@h z_C+uLZD{=Vd+gFmUty4|SV&@Q|F2i~-?Y^KJPp`f^Z)mcB$yx?bY$3I@!u#_hysyT zr{+Z-mdqt^c$PE*OLrqYwj0GfUHGlS*>3-8*{mWR8U7&{Z*G${f`o_sPeHYm5X=5D zUow)V%Z|yBSnlzEwO%2t18)9_-vaZ)qR9J-AABF_^DC#!{5FoHpF4E5SUx48HSI#OAQpv(uCfp34d#x#nURH;WgnU18@@{;>BK-O`}o0<9D6zfK4BUhR4x zLg%1exuS?h-i@v%yJ**9VduISe4g6X4S5GM(i2@jUSQuJsPwNKIJYBdjrTNxosP0U zIVkU_z82cNt(mu2=Y*%5f#%oA7LMIbr3ow4aNkD_ROe;j!tYzNyqy!Os~t$s;cjfQ zzHzno^+T7ByC@vG-)!#F22P*=f1)k!8ElzI*1+x*t^5+4t4}_epcYw7XSDpE~$vXA@zn zNT=GXT*mg^qa z|Pxj1!e@hHO-pff&k*cSy!DR44CKnBU3dhG~oSzebTg zOvUV=nBp)OTCw|{nTgGY-IuCaC}$(9=8S(1KVwq6-_Um$4i$&=d##M}-K_}2HII>N zC96obxQQHZYtZEmRK5&f)yaPtHt4VVEaYt$;p%FYgP$L%T8aCt?N3EA{JPl|H59WC zRu&29wVImP%kLX?X9KRG6K?LjNTbx=QN8n*qh1Y@ei|fIIJq4vTM|f&%K5S}B1F4j z*^9^>vG-U_d+P6Ao$HFeP61MKx+v)Ng~+$0JtI}ljOv5OuT1>B;($^A77}wt$Y3z- zKZytfIe$^ipWUImRVKcRJQ?(;9{RariNWR#9_M-p@snGxyh$c$T?=AY)Lv0u(aPAc z?fLUZmk3opyu3&KyGso%e}Pj0Gh)Ok?W_FZwz8eSQeKB!Yzg*zAPP2&AH>D@?k2cu zxI(zoUEuU8#8h+0c^U z@ygxVT{G^|gG?WGPvz!iR^exyBP_}jR>79xbUeHpUN2_BqV-Gs)fgU)838-nuGAiA zn4TVPrzO!RAJ@$nFV>0A|I!m%v@0PzW2=#d_=5${)AL2af?|h{^}1TT#LdXktb&h@ zjXC7Yi!RKzXgvPR?LbRtfh$k7zd|l{1-1H+m3rrrH97(er>|cz&h7qqr1sEM!sn|> zJC(nucc=am#n*LJ=&m`jtu(TKLwwYv2y{p@DQ&e`cRo8L9m@;!&-a}2;4q70fF}%W z<=Y1KoTy@PqZLh;|AcI^nApu{g_LBTZc+@+pLM7#R}A)Fv?&J#tkmflsjx8Z>GOwA ztd=1;y{ejXkmS{xwiH>_;u*L5N}Gpb#l9v3%1rzELM>Z|D{*fXcO{EG^DEnnbe5z& zuLvRTI3Dn>=6Yj={gBJ?HR2uUW=40)oPqYBvti@VG)m`?5e7@g%KQiWzQj=N)0|gZ zKCc=HiJZhT3L@MLsQh&vv8e`3fmg)K+wZx1H)uzI6Pltqd7iT+^eatIqGGm5G+C88 zoooP30*kqQ+Zkyj=No!Oq8V@G#4O)yfEvqS=r_hKW;j;+p*40<^{(orm(Rrbo$4)4 zm|3A^Q&CrQ|7_n+XYlu|)yMaZx2_NGTSm5_tnq70F`pZ3mu z_|6&e@Ox`&B%X|^&ULfo-&jUc7}L#lGO1{sUl>5~>i=7A%jr1AmW|O)7gm*CL&Hu( zVVwTg-=ewdj{PHF-7cR!WdIH2CHL+0vqwgB;(DBHnOyuql8l6P{NTM!b-a0zop?0E z58)uWgPHyPwHn?g${)TzvEuBzI|HMR+jvjmfWWpneno8m){*Ic=QNBgl=z6w%orBr z);K)9;jE)zrHznRQUB6WEi6}BR^xQ`a=A0%uUU~tO}S$6i{U5Bk?N{^$FzQ^;*@{+ z>|9mlEuMJyW-IYrZ}N{MzA|BPuZOES|F+Icca#vjZoeL`Q0pwccl_$1W5-K{#GTIQ z?r!=Q^$t6`-?EyOv~Ii+;9_?^Jl*S_r}_{F0>AnZV}rxBx~V;T9&Xami$4gW52r^W zjhN~VThiXiOgL**iFF~Wxp+XS`jz(Oe3g9UMRn20nW@0oy~=zZN!Taud0l9-_S_O2 zGd>ZW2kUv(QL^tYi^k#o-Aq^esFn9vw0-FaqdQ7q^PIHY9E4-Q3YcHAX`d+l0(|Q@ zO_ir*cg{Egx7~QqdSdxemj5Nlq+_jKJ+Y<_Gqvt4Yn*gp9@DU<@{60XVzZz(vKzT9;xUpux%9o zB=#U47$xuply{=Nrpxc`G9*RM18mUydZg3)@$~C;be(o%2o+CgYk!@G3^l4;7P9Lv zez_6-gXxe5F4^;Z8~S2T*gIuQq-2OS*Qu&QLe9&t+s4%Exu6s$yj2M*@ov?*J6A+n zj(eVDJES`iqlwInb8xc6&)vd-N^-f|&*pv2)yFF1XveSlyXx3aGIT@VnQ$h`Gdp%` z?sA!q9dBZn;5zePK4?^o=K9z2z+R@Og)hMgS*ReZKC?+tRO}4e-U6jxJm1v3Ng3hY zxnXVR$GjNVY@#Qe0aj99;lMs)%}4Ft++MhYdH1fBuQ6F$s#H@q-c8&6!$Z=7!j>i|Ep!__+0gziY!p%{L-H+xk9laAWX4XjJ1E0!pTa$C~?k2cT{KXK!aAp9Jdmu)c5u zI=s~Ryb*iOc?sT`Gx`%5@zJyMD+~UOb&ql^7*KUnagwu=L2_b_OV~hMb3@=LBJZg+ z!XtW>-foT(_Le}L-++1C_-dF?W=GBe1!i-H;czdgm26O;$_xGQ+?k97 zgLs7XWA#wxbD#D_Hl223*%{Bt2^;qZJU0pR4DLyL`r23@7C=+uxfO*dn8seRw4{ zJ4(s^5BWA5VnmHXln545YJvpCDR_5JvFU!J`0XtjV|PW)LVgZE7ZDqxrvEyGrn#q% z=If_`pYZNS*Jcf{mzktqcM#exc~7VOwILs0db~|seta8=dkwLt29mW#DBZ`y3h=LDi|`u9FnTgk`x z%EuzL)jEsPUNdYe*mlBxb;z#*<-8%XG-C3_kntVE`VFa0(;7H59v;?FS2r*`u3DC2!-FIN6EgEEZ^5U3Hx52UFah-C!t!%~0 zHy=;kx*}|i7!Y|XPclvqaCSd#oVfqNq$qa%;OU9821xzztZ1(zWV`5N;)SeOZlu6` zPv8iL0A|CT6;n9{D#q2~K7SwtLJG)Au#Q!&G(59}m&)-bbF+ubZrKW9<*^AnCcjT} zw*#q89Zz-VUI%mb-*SYua@M^qrhe`}h0=~k`+KLukcrxn*f!12w?z>+y9?e03zIM6 zGc2HDRosf7yIBMrb-kS|UCgeq^q?7@5M_QLQs-2^@5!>Ku^i*5vv<(}yzmREuqnC4 zNO~^I{R(>Pf>#@xc%OUuc1OO$4(miW_yu7C9P7db$G`ZkT%MmnMe|Va&I{@#)Xzl4xI;0t#XC>96aFIwp$EIqG!>_ zpXz%Rw!!Xm97>N+`iaO@a8=vhQokO6O-nP{t$yVU~6U8pm4NWY^U%J((3qGM;Y zV*OVVJZKF#c0<(WzqY&fEU-D2D0GFzEHd zQxg?sj}=+M(zh4))b&P`is}-Px!GcaDeW2YiR-CCgRH%jPqZTn&+0`dkiWF!08X4r zy}kOMe7qe=95LKIDNc1oSYy60+q-RNK;L)tlkJxTam`05H=bxmC{C@hD>XhoS?64J z`*B%zT0;=+$38r@eI`AWce%k~7@m0;IH&ULhw5u?Adf=cF`nl>XkPv>@JT#OoVDdQ zb-Yx?U5Og9O5%_=^3S04oD@8?#2sIim^iFb6{6o;0C!)sf-A*)U{+ z0^@$h9i_yG+4tTeH-NW6d2em#0KUo(MHrqz!3;HkfDomk=QKkmouagHkp#IQgEUDhirH*=tpFON1fhoL_hptaq zWN)-nIGTlv&Wp2&on11#IwdzNXeMdqAgNh5nDe2o;@rB z&RV5x-X&QW0CY}bPP|Yaf(*m*3YCh{fazeC7a1;wWkPKck3;&Thcw(OXR8!i&mKLV z#W9@o61m9y6V2<)Zy!QY%ar(ma+l&&6LtvX5Vm_m8l-yq9Cn4OpyX8%lR z41vXB%rPM=m11TnRBs1{C~vm^@REP8o2{VF{}#R+E!WQmg+K)BRD&QO+Yo`Ex)J#I zVAjcRjHSe=BmsJJ0&)uK2J(Q-C^k`?gngEV3H`t4xK3Bev(2HFjxtVh@8%%BFn&s3 zR}uw3c73LPZBWZGgGh~5BTeuT#DwE(jZ5O6GIb%sg(gi&S^qilt-n{Ec%s8Akw>W| zGLYgn%C6z#0qNyEOh!NOlM-XJnh=yl5$GIHSrWSG=YWqj%sqP>&0^Ya%1DmfY9-!d zNU!wh`$^J-MJYzejdS$2n>(da1=n@c0xG<2Qd`14hMrr^$F87*dd-K|&^Q8rL!uTG zaygJtynuzht*+(PLc^w*9E0qjzvX%i5%xsN0@M8@=*sR(9``xKk?hM;}Bf=nL7ri(cqS`2_rcbT{sC5C=r2WnlXB8+yt(gP$dkS-KPX)b`C7i zTU0+6YPa_0lKwFK+M;K^P(lTc zqg_-AiY#zSVNQ1@`5?fNmXCVIb`E<*2_MnNOlzT6@Gbsid+L94EHT6Q8%b)fnyfzz z3`dbKRFG?ZdV^;DxR~L<#IOBUJ;wUMH2?b*JUGJ7ju^;tv**!QB^k06fPTZPVxlUE zk3>o{>JPv&E=f;>9fh6*alyui!I6~p?m@jmBsu@uweHSDZIa^G;ak6l@`%-AI373< zp^p|9H-zi|hOd)m5Llvj!)GMOlt?DSTaj#R48HvaJ@mJnDsSo(Lt1ja*uYpuVMv!E z^Yf}$y5CcjOFEcUCNmnxHkOrl|=75brX|lfFo10}@-9x2E zweq4x`6ovRBXmjL<*nXAr&CBO2Ed_kSQ3T?T`bt}l#t}!2Ge{B}=)WXH zx5xv>z@8KLjzV!6NKZiY_I=P}S{`^2*niu+p>c>MpzU`8dPhtC~RH>9I z?Rn0g7a?TM!mL_dzt4T+nEJ9>~Hh${)rJ-Bkj9U04F9478u25YY?f84Jj8w-`?7tETAcn5+NoeCd=UD z&!GB*xf~Nv#a@Udeu9fWKRy4{%{X4%b3*hDs|EMBN8&Zbt^qSy(p-D2QSlua(JXY+ zZ;%QHz7>|@5V0S`Ye?%7pobgyQR!zsazuJ3L+~Sj z3e|zP9tD@>IZ!Gxf1Pf@vmi6VkTSe2qOY-*TKia%7A|3n;hm*18+ktFX4~rFUUK*! zs^Hjd7rfeW2wG?*8(Wv~6!$CFzwa z{Ehn%zFJEP*kG{O+K&u9ha+8a(XWDHtPISywW*m)eCbifw`a9!@Y zj%`|e^H$n~eMi)5t$OEkDsIASezWD}|AtX8h>@xaF>9hGsE{ z`zH2w&otWSW%rw`ZWKO#7C79^%Pagk^v54)Xuu}9J6#K>m~GFK?1eV_d$#QtGCGt4 zH#F33f9tUNRc6bT>VA6`&!-VBx<8!f-np)`PC2|&sc83Su}~M5*FSHdWohopCAnXw z+}jSX+llWC5Hyb1h)$I!sFY916$vStZkK|zHyZg~ay{zeh3k`aFy)Jlo+Kbo|8(#` z3J=z?_+2LHJw@Cr38CrLR+G1J46$d#RA`vq>vg|%zYPfOuBi^ae?^keQ`z!Pd(hXA z?a;QOmKFTrvz7_Z>g^z>09o6g=%q+#&%fYGP;GWuao_(#ca4@#Ol?wokFjdJ{G(SF z<=b^a2d*=ex((v@J%m`9L+ejyeeC9w5S`%PuNJNGXjiXyU1h4`^4Yo8$7xsBu8fLE z2jg(NsHcMpw4)9ml)Go%45>u|2rVbd`kJp_+PlnM)P3HCILW5@B!{nA9=n~M(rbDJ zAGjV%oO%{qZ{=>dzU0|yQT6LY|Km8e{N&(TI^9_}=1^|`FLgHEg6*~~%hipyO z>n^1P`GF%z%wUg|`A!>b`FDEkYlqV}j;=h(a6ar%RF2GjXyeaK4q6qLv~Mf>3(Cbo0=*Wm8b zo!cUZ@q5$IwTIk3lsR}N?%ZSlS6ZX!pjHCS@z@Ia*Oq5?F)x!eQJ9jdjEOc6I|J822j==SZr2p` z6ql`dd~m_VfE7$egT_`39yzV{maz+=rhT)J8lx4r@J^8hBUh!X;Hsko?}kzByNA86 zhQ=%O``O7K8a!`o?o7%E%oI)H`x-Q=TXhs~PvBgs37T_5^}a!e$dkJ(55~P;YMghp zukX|3(9>xHrh%*QdA`{(V6c4<2lp=?= zd*OQ&L`e^bqnWzfB z?y*Tk)%{w${d5w+{CVa7!_}JyLbd<@)aOEn?hup$ui?w2ZNb0%l$pi%((B*_xH!`-b=>JIj{3t9;_bXGhL`ZKhW{*Azt* z9L;N%Nv!uRRX*he$ymHcA?*R~G7VgHQvp|iyJO`NvevYsrN5y|Bj)mMuOL1%(Kr4M zqml5w_|W8*9@#@jDlbD0=HZBulIAM*)KKZ|1~n4J51cT{PbU}tn({L8^YxT@64A&g zJDPL^@BV?d_ca}2Kl|Hm7zYcwCe?Zsh!e4AtPOkddCu5Rw;o@@y7>QWZC59gM30>; zeRgzMEo{1(?Na6(Pq|c`e=CIkCgAF|y|W1`sNO%7YL1U{Qw3+yjXE-Mys_t|RnIgG z%JBU|Q{T9OWktS)J#vMMzxCm@qHLs&Oc|94R-7zAax`5oF%CJk#01u_n9d!jynUJ1 z&qnoU8SiYA_B$}yP3??4=ZFelAFeKKm((1idRW)NqtF|tzNvA=xsRBBC+}CJ$F4YU zD7$hUiYQx+3xPeDI)mxFvxVIK`xZ`_JHbN*GyaS>h9wEApg_HS>PTHHDSJ|L6XQ#)L zW5?Q0FN_d~LIfkPGp04r)z5p+A$(uXU~v$*hLWihSEYB+4io}rHPp)@X6i3oCRLnG z#zYJ953|4c-GAVdDsNw3|7_9n)Y&h5&Jn?Mul0^U2&S{VNQZg+WB~s_W?fBP7w_(7 z{126gjsC@jGm?Lx-Y2El0a~aVQ2|<2$dzugK6@LtyCT-3ak8z*>x!V&_Of2_V%bq& zY}6nwwD&J;iKLW5L~t?6wUT3wA2qS-sW!!xMQ^XU)#`a(B`cxT9kbr*IH?+vsR+|s$h=H=9ehLxkrEf za)>xS-*v`Wb|RoVnty+i2$@>B+$(PZ65fvg+ha1jStepYy_3>)1ThsPBB|w#>!JQ?L`;X1$dCE78H)KQ1TOG_TJYpa8BXPCdP91}ukFxt18Hz1-^NoIlGg?)gZApqw+hDkYg{_BBo z**t1@J3jj2{OTVL^8~tJd*1Q!fsO@bFmPf6t|&b1eRJcJzX=1Y_<8nz`%#abxq~G! zz2Pv8$z%f2<&QMS+exAnB4YQzjMH4!liR{LSnK;lU+0K*@ZG*GA82RFSzE1@t`hl| z&DouOGYYpOAaDdBTZPb`g@0UDv?G)lF&T+P&Rzge<^_1+ai!mo{+m-rVRz=RM}wQ~ zExG++)~T4reGA_Frz1;2o)2E58>_R8)4AOsKgV1@52V_VNGF()cISD4B_dT7N@Fb+ z{IYp-d?n2f{56=ael>hA8bI+Io!7~NSGhb7gFjP!W4fIQT>kz;lhxnR-}lT)u|i=w zKY2nrGg=|C*=>PKFLEjIB+Regkw5OwaTqhlJT`f1+Y1}H3ixB$>j?Rwj)9M-thOkj zBM-jkVqfn^k9$7l)oX*v9se{18(SXco^Z_u7jAoc=Jjn;x#2g&@)rD$qbqIU6Z5I# zmM=WZ1%glf1G2v}b%!Q-N}6yFY4_u(+;8i(9uIp9BAKczfAiC1UZ{XEpE3J&H~;;F zux!FJbeXDsF=!M7?k zl?4H>9ZR!Q%2^j?9CI?F5JAc<`(}-wZaL?uOC!BSj13k2mQ&uXIZ>Sf z(P;8u&`bnD1RLEx>wE+ec&UQhLjd@Yu$T+Om*IkJ|17?y^w|iK#|L41f}s`$J^YC^ zlUq+e%)|3NV%#)D#+d%UlKSs! z;AAr*kzJBM7&adIuSB^&kd?@}-s(n}bMeFKxUwzWD*SwoW|Mq&0v{|3w<(EH z4vLYh)u(9JLd>XzbO5)y9bJzKM~ACRK%(BQgBRI8ZK|9rMD|vbJmFzjIdw!a;*xVh zbKeK5_YvWvnCrcR)yh2MIajyogFh0|J#><+>hA|@3=)WAbO5c!@gY6lE2-(&PUrHz zns$>B`PDDhtg~f2p_fbRXgkr_1X(Y5f*komBUmcy)IN6!mb~OMz zsSuHFka`5*!gL_y(Y&w@_)VXE!d^^GhPOW3QisMUtc~ z&(1e^eeieChb4vL|IU#BILKeM2Refm|GA+#c<-{p4Y>WiI?C&?cN)2jMa#(alQKyP zNjgtl?mQ=c(D>{0pwWYrLukg4og~AFLqGnNIK$=FDBrvH{!&SIXGlz zq3;yq6?2m1_uRzqdDoxwTL}*3?LJpwLhkJ+wpH1kRlnDBd@|#M!A7#u#{1mv0<{Pn z{Azhy8`~GY3q=ug$L(lsSF4{7UIL8x3oy1_Bp16dbK{EX#nP_ZzK0!d26C^K1W&xd z5&gg!&eV9Xa@}ElP>=Sr3;zsr8*F?$M;p$^S~p&g=94QcxE=Dk`?hS0-%fEDs1v2L z<_<(7UP&C!^EnMFZ-nj?x8>d#V8P60G1YEq4wTb0#U^}>2sYj|ARMS2q64<&?mI19 zh&N7MD@0}uF3eK7Sp;CfE8Uh+^Z~0CTZh^QAJah3qBG{Jf~izmCN@nLiipS@mI&wc z_z01YOVO`lW1A=xPa+=nJ%I0K-@`cy4hp2n2!|yiH$_2G;7=(4cx2%czT$lRDrg{X zzRIY~7Pc&98>6>0(}&a3r@6_&{G+laklMxwYU)J=-Nl_TW%b%Q+TI zI9S95?Ut#lVA@i4(ev5O5|AP)$=kfY_D9V8r%6F*U_p%vr&|;DfUUmqJ&Kv?z|( zv=u-pBm*ExBbFr3PF(@J3lNmU{%ny!oLd6zZ>?Plr3Os^0AaAH5xVmvq!Abzc~TKz zXeHc~H0Zn$B>?Dc!z(TlV&`+kWv9#Jwg{M0p(Tn!mk@$Zisks(zIe_sUMoOkvHh}g z3z;7F5!!D>7u>G-%mE?6Z3TM3r1>u9FtSj3-u!ab4tlrGVxxA^0Npnxy(iWny-vXKT1^ZuKkd5y8t3!n znli^ES|z@~j+l=H*sGd2AehIr=->aGq9f?813#ig?yqIDFJ{YN+=WO(7$o!n5W%ON zV==r;6n$RexLp`(X2Or=V`+9utGGG^Jb@H<2vM{xMCrB;gA=oje_Pd|jYx)wJEMmH&0!t(wzUN5nqFg2ffcZcenmn)$LTYCqRZV9Y< z?wzB)ju0Vlslc76sIdw^6rVY1Q`^Z~41wt4?H zPYm@DQExCFCYBM!=1DVSNpW3j>dJ`spE0VT=P3@{F!!Wi@$-ivUCr`9HT55aDzKJV z>gvoU@or{R-Vxw1T@k+V0&HwC-)-4!Mw-Jh152#;)2!A5jVzU62lfW7{qiU_hk$yl z2pqy_HIelV+^~>83F)o-Yi$5_>VU6jUB^?U#oX^+iwgY zeHw6f7NWbCh7572Q9X8sI35=(Se3XUWj9jSeO?4R!rbC0+{l@6t^-?f5pKx-Ig4AT zLUVZ(=&%>Un`UOXHzUn`6j#=#Y&Qe}i#&5# z$OwiE)4l#2?A)S&mx7lC+ugqM&HQFa%sGHCiillNSNf8G-G5@MOZ4FV zMr7YhY-p)9I~F#BO;k@{`xzHU=abql!#1brxydrRMluZ-iVsnbb^qjWQ(ESJ6Z{8oWpe-5H@7UqR)bgQFo$1&+!1rERu|FI+qgI%}a(u6>Y1A>4~cqnq=p}>3> zNKHZH-=0(@vFVW@#FU{Y*&ua`5k1{|p$_Hz4AYTC?#wtng3pOC7t96a2)=2s#;-nC z`GtGF*5(*F)a7BeP*$iN@-VdH5y7k7s=*h8d+PRWIYi?BrBY#%;x#D#}GO0 z&=w&fCdkA+xc@8HRY+j{#c7=7BJ52LvQi-_s?zvE9=}W`59l6 zh)gk-0v{TkcewEq9V2Qs@5|6-E-fFT_Sk}A|OgB5G5*(J4q=t>F_y zfH@0TKv21wc?nfs!oKc`&rw0q9Vy60Ur133xwBQ9KfvRUOnIU(ikyKEc%AAAE|_vt z+}hyaVNi$}$>F`k9L7}d7d3%P>_j1Xvq?gw;S-Z(OWs#$(c6OEBGQH_4UG#3zy043 z1TqiC2s{I}KM>gg@{jP~h-`(of`i&wU(tSJ6YtkywB5aY zF#TlTf>A^BN58f#Xk=cv=%J+VUmvdHxyJvs!A^scJ@=ca&I@PP4d|Ys*TGah;MFp;UjK=dk~Jz5Nqp>hZu; z2RF*<K%J$+%UNc4s<2HphW2Tq+FL$p)RX!VMcZNT7 z*J&m9_&8IjQ(ey4?9h{P)n&i^^(Ktg59gQncWiBkR^_(q4=9r*dD`*&8RuCtYK-^w zkJGs|7s5+EjxPRI`L?E__R)KW;I&F`A$!LD?jU3H_G0zIHgfWAysQp>?TcE!Mc9}%d~WcEvG|sx6Q>3*AGO{v%dV^QPMEnQMQs$ z>Yq608R>*wW!L++VZ$^hKhxb8*da5(d~P-EYN z_j;Po;{rz*mzUpmAsg@i>zlpJ#*=8zr z?q$~-_xABv_pZdqZSD1owua@XJAu4B&=6B($1Zjp%~folWB;h>3ni-DqomBYj%jRZ zn!Rk~doPJ^TUCBkeev^skMQ1Cw#VYu;$Lyi@OiuIpA%#O&XSMhAtvGL+&_RXC9&FAesb9KMJt{=#M z6nyqEDq{Wx>k8HD=_maxSX7-K=4%v=?6@1BJ)mPb+j}|LLV;ra8s{5o6)=r|-{w*I z*4Exe>GuB6_v@Va5F(uQ*nbH1v(%QLh;cPs@9>E7TwU<%WN+Uzw}j}u?Kax{Zi&xS zTH;4~omI?->j9dCZAm)%buv5V^NRg&lq;u5Ssq>Yri!&aWe@qv^;jgec&BUE-PfE&4_Q5}I~TGTemr&RgW<9LJs#KJ(Qyag;)oMNTWxIjxp0rv)w7uV{rI=O z=c%rRzdH&3$&VyS)pTH7sinTg&r-A9rarF_Yz|r1kP)Lst(qLSebuS;Xx%gK6S{W+ zHuy#zJJz+Wz|VR4E7z`@o$cRd zG~+n9k!xYJ99dH1!VaQMo3ye2T+x+_)+mOdM)>oK`B<}U+w9LLu8M?gDJR8w^4u%q zJ=XWSnk#0X>BfzTm-*J06AlzFRx@ z1cj3K%4}a~_3MP?(fxjF;CFo0qpmB1A?FUMsxWR8_e;gn=7E$?#P4QI4L}Ip#z*Rd z-fY$PGP-INu}v}S98?A23ZFcm&+Ll~7+`ERSbrBf%v|g>!gqDPj?D9L@Q5*1Qpi=j zxeF-6g5nU@bG)44LquGkYk=$IOWCHXKRNazDR%jFO*mgyHC0xrOL7L=j54)Gp?u{L ze*2@zf-@CwzW5Y3422jyb&f5H(sR|z{20T(*IkA$DMMvip)U}8n!d1I7Q0Tx(03Lb zR4jAoe3Lc(D8V!=&6Bhn!*C6p-#WUZ?!&dj+3kEZz7= zpp%{aT5nVu0@#;#2ldz}XS5)zQLbYw zv;sp=bmRG11Rivp!r4)hSLJsQEBZlS_VVq!gXSEYH7LsKNZa=8sLOjV1>%xx4rF{% zY!9vfQ+d%Dgq%#pw4rMd9k~;N?^;@m4`pWQ1;m2WR z8~^t7Dil{_Vb-PuJs%RapoinT@EQ23S^Q&U%l_;LJdQaJ>e=+Dpjh%|#>ZMOlxk6E z$C7>=2fETKh+*FtbT@lZWjFy7WWbXpY(gLS8CRW{;W@)yw{s zFsbrTo%l+_CA`sS53@=9N}JLjFxAr_$|U#)62nORV9miRF3#c8VdEy49~X1o{hb44 zl6)TN-0nP-uR~6vouNK}E&30^oo!_r-+kz};bGQM0O76Xp#QGw%8L=F>Gl2!7M!h3sO`*1_?<)F~- zU>HnHCKVhd6_tdR9m`FWe-k;RB)Sa(z;_ENnMZ z!+H~&s_y$vd%upc)51;Fp7TU~9BE8wI!D^ob#{lJ`o)*)i*KtcH&b-Wv!}y{e-Ru# zwX-a;6u+aw{Z)6b5lxH1>W^rTIs2zp@q^v7iRch|MZXV)ayj(-v-_o<4j;9iKwxCJ zL;cZ6m70(Eu(;Z+O1HBneQVool;H&N>?+?wC6=^!A4IzOVG(WTLK{^wqDfWHEvkOU z&M$*nki?SXed`H_1>Fn2ny>`mUFWhtT=n>Bpwbuakno+ENTS(#b0e#urBpVRS>5cO zv=cF=C__!QCb`uU?eH!!3Pq`-&kbc71?qIMNQM!pemz>iQ2q_(M<#8`N56EPFUg%G zSVb7UnDwRzpS+`Za3j>D5-xAlDd9U zma*w)i|WVK1sbTk3g80EyWS8}M?>(OlcEN){E9*UC*kj|0iL#;G!)_0`KFF1oA3L^ zCVTM~^zP<>Clv`=j0$@0>{;INf!1pf%bzg6F%p!<+^e~e!`K6J3NSJs)MGC=MKFs^Y*)d{@yy4 zeNnaK)ZW>i5qUNUnLoSf<&aTRR62qCEZalon8wez_^1|4Rb@^yCIP9DJb%CW4&OG2 z=QbRTo<<%Ai_1)q*gDo5U`Db5Vvm`+M3~XwGm(&}ai5@Djb>2FlX5XOrRuQh#b;2+#=IF# zHjES=AYN)V0La#l&mRH3z3|krjb6cxej6^EnL4yqEZH3E`})d^WA73+V<%yemnhfV z+aH#3pn$7zQDp{?BLP3$J~eY<8`^!ypxX4{_10b2IAPKLMB3%v-GXswXlcy?``@FQ}f|16^=8d$?%!PZ85 zz17@^4}w)TwHx^4MWbf6=iTdr6?e5g=`m{E(T5Xf!b3QZQI39ztF?4a#<$V6qg9_Y{&OU8F>pXo zhFpXiPxunv2zFVqlU~ zGHuEWAL1=jg&9z^dT9N!~C=0(r0xXKYq z`h#1H>>hWOkw|#I|Hs_jJ0@}0(KdeajcUXEC-Cm0jUKHfg5*K2pa>0A^)!gk;_gO z2Wc4)7Tp#3ED=kKLs0%VD@BuehL8tMdl2I$;MtIN2=Oc-W+%F6@W3K?_$!l@;r{_S z26_@gjKow7$Y2qPeX8c5v{)sN?N()32^OLPtxA}kVofK%GR|Iwi$<3lRc3Z_LU9bUK&((hXf z*A>JTFnwgC=o7gNZP$UsnA>aT5RJSL*k0x;QhvmU^K7N$9r)@c^O3^v?+1mciiHaVN>|}t?~_| zN=V8!7?~h8-90A7a8(yOAX1l8ItQha)2S#HH8JK9HLOLz;65=~buI&k`D(92n(M-} zWpQ_Dau3=}jz)r*+WOhHUUm8n>@GUu6d|1e!p*4a%j7X3G)`ey zC|OW)THRN5I@;WoRqGlj(+;TjIFxWAQSK7rn02DmGgV&YA7F;bviPfB3!ht#oO@DN zGB*PqUk{6=+I@oo*kN;CZeaKLf2<*9D+c0ab-8MwboYmqqQ06Ws8Ja%k}7Y){7@%Cpd$=*3Itt% zQ(ZV$n)Ad~$|FKZUF5|1Lk=EHymA>NKntP#JxQCJX^O23xKD8NKyL9t`m{up!%?W- zJ%Ff2bo6N_Hi^m{kha$`Sa5{vmH|w0z%U&+A$E9zNf%^r9kBOwo~`un1Q(Nmx;dNz zRAphHeZ%izQ7#~W})NGjDMIxCU-bA-Gu{iRQ%!n;Rj|LUX$0C+pDv;rvks zyN0(k%rw&!LTZ@$Pbt0$G=XNGNg71!d1= z{M2U%%>xd4m-#~TNtcKu z>6-XQKjS^9&W&whecn%XEfWmE#0^2$IBlEpN&+WLq@0sjK?KRFpjb2@a(10opfcObKrA3^n3K@> z0Jk4+h@p|b$x>+mPA*EN~y06 zR$@j`W4a#VW%)iI#n*2>IRZ0u7CzrUX#>C)%npIY6(gem!L$kSUII-xd74TR zK{kPGi1?%-0Uz>+h2irC`sfSSz@zw0u{%QmmxOBZR~kI?Dkpgdq>A)~PT;(sQ)9&4 zrc^23JYPvX+ag3UL#Vfs2oMrv2uWrxB7-HggoZbfbOy#5xZgxuV)#42HiN@Y6uFLU zYovtP5HP-w4dD{uDf+iaD3P8k+D^gbt^dEB<`dBY_rd0l6huZPToZ(Uz~l*i$wDQC zmU!D7b3$JffFNX_AA;oBMx%EuHm|E25lLa*rVsU~6 z1zC*r%|Glm;$3=$Q}AxxEc{H!r~QZMMumt5RA+jw)*WY35}kv&=Yf6O%5M@aox3X& zv9(;cWh5s&XA1I0ZXDqhacq8r9^#n1_voC^*V)`}wlT}S8F0jb)QBacDbfUqTyZF) zMeYIoQz_h{F9``u^|0s@Vk&$q@&6hYkzCegdg5hGw3&u4!BY$a^e^FQMJ!`66o+?j z6Nxv)@gT6mnhK|RIOA)DWt{s*d=?XZnvZm;r3>B^{#94SSBkxE;+X%+B$L74cK)TK zGhW2iWkro`>nmWDf9m0kETqkNjE5g6Z!6}IhB|!@b*g>8$ZB z6Ea}w7gC1Kmd4Lsq=@&Sv-1jR^o|NpF|@Bwv->%Pl@;WRhKeb5lF2W5ll58qLk(_DdM|jIMhm{SZi|JM<~pyL zw)F1{ZJfY$q`n#3k?Q!nRYqS2s8V&V?hQ!--LrlW_d}?*RJ*4YVSC#u&KWREO+Kpg_z)q_A427 zK>}u-miL!#b$Od@wmz(JyR#J@Z#+tGuG0WT-uNa}!(p^Wn(6~y$grJ<-zE2Jr>nte zG$Py=5RS!~WVbA6d+*b)E-aUl|5dPn&$RxczhGu-r&G?=h%Izd>7q(EUw)-tV>YB` z6qnf1`sOon0Cq7s)wt}S*t3O&^AUmRolvKeI&D(GG8zxcQk;H_5L#hLZl=eJ{36^ z7rnjH{D71FWxb`9{!)1&AFkyL&bd^q#DB18MuXW>?zQaD| zrkwgF9W5*TK*7Cfk6;D=vzJ@`V)W=^eM+QmhbmwF=`LR3qrorL3H}qGj)m)t?KQmC zzoO-rl#z&_Xe+&#bFZDu8oRO9=w$;n`kyKbll0owPjh;={p9sCqb+t&?unXb;Fi$^ zwF8Qa1AD^A?THKc?t`2SG$EW8u`a8bT zYxuaKLw|m*0A1zgy*p}f_Uvps<2WX`*e@exAeo;%CI;0snDHr1TYcW=lOrC zYkzX^8Tk_1M|&N6YEQ?Z9mkLAKX`U+YZf7o4;v; zu#OruTd%?~jqp9ywI$`hd2w^gZZhh3-#HJlu&>$91FL-?o=a6>H74OyhD80KKj*rskc%JV=eSb7h9`>dcUfl~QT+oWg#36?>LH|m8>9@$Y08^DjJ9{@7-l z-cJ$)Zi*FL$g2Lc6r-e_k@#)vWCw_X%VhbRfP4FQhNIsW`4)8_|M>&Ukxb=yLo2R>{=IVK@ENRX8b3L&L$rM`!D&p29M6zetbrqtiyu-(YDOHllkB3 zo}|iR?p`d4sBs^dv%|Dj*rKK;TYoC{IJM5wgLiy7@ctZQHQuAuPD)s;RyPbF)qZ{^UIG=&(~|v z-r1un{t9XM>X$dY1h{;IP)ts*vE1x?Q>V#CgC)Gv)1QXSzmZDw)rX`Pra2K>BfX`{ zuKM5RjB3_lBdt4rEsY=HTicd)@^UmTWvfK_$Z4CF7gL90Rr9Y|=SQ`gVUK&w@{1A| zF5h9>8ieGWu|CedDqHaDx00jrQ${bm=?sA!yMFZKKeSA5-LcQ8+WCsS&>L0N(}_W5 z-s`Z3Q{m(@Jd0mTcPS|gAr<+2$B??_UU{GWwuk$$Y6kXpAt=l9=?mWrsvCyNHz?rW zmF2s7^xWX^yZS-%@B3CI;py}~g;+L!e~}$oh7#7{|8e5Qn$!LwD@M+!OGQPbz!r8q zD_};qaBwHIa&L43dumX|G5wi%^K4!auEASFzb~HU@~Y zoVuO8t-~L{a0A(pK-63wnaK!3=T?sr<(rP2M90V-E>95HAg2NR{8-3wsvtN#ZtLZH z8j);+*Z%R?SU;w?O#}%Ql!mP<91%Tmuh0b?PbgWGR4jgoZTKkg)RkoUnZl}>9!4h4 zf zn7x97_`f}kRkrO!|LrG$Y`palL`r`)GKkt6E6+(CayIae-x_tcw-8nk9cB5c)$vjq z4L!j6_kKN898}`=xnS9`=*333cZ^L~&wZJZ+o+Pfxn1E;UuV zbOz(EP)M|h-FhhetLiiG%67L3ceYrxF~>36<;(lRM}ec$PYms&Uf5MdO+izQY$7>@-Ih2e;gEP zh1a{ob@OK@Y7Xv-QJYI3wlbYDcdrD!Rd)~l-W6DK1LgZx2Jdp}6zmmHZAF4ZY=86V z`9Fgr_1n|kOy~S_*9GCCs9q1|3qj2#sgzH%ywV8Op@_YE1pYpxct`OI$TGV4)iue!}7V_TzCvb!|%aPiC*=I#X80oL4DFj~_tvAO za%Znj=aq%l(G<}EuO`o~apFF*ntlo8ebx~>lb%-hx@mA-@;rK++oY(gPLEa{^)>wF z{I-24+I#~v*JZkRs3@?A?kSd6_$^Tnndgd4y=BwL@DJFYJxzZ3^|hvj&K7& zA77v@?>jRz4bug#@9nnj-K3!&tTGH@8i{4R0*8yh5C0>B)4yT555;8s1c6SOGxfp ztYA?@S0Gu+at2#xfcu)%2m#c4TnO-+2MQ6%T{F;WxKiyPK3s?TSYJ7lv^Z2Mv~5wVltfl;^ouDO zCjFg0osoW{Q8f--1u^pu$aLVgeBy8h>HVsU@ltO;&?%&NYX2vX*7PO}EfG$%@hOqbf$NZdV`cZ0dA6Tuu5AA+a~! zhA_V#CeUSR?ge}9G@XkhuvlMdZ%@n9aQYQ|8I!?7OHXDU8ZgY@~ox{%wawhMVwOY~#nI1xArD0T|z zNC^khtP#Lu331`bzLFFwKiC@}+1J@HMcH%nfX^0k2X9*aRYSw2g{NMu>ux+5(+O&T z;gY}!D;1fZ-Gm!>%18(>?@TvkWH%(#9d$mF77IumIqcvu(m@CIylMNS{PB#(Q4G~ute<2iZ#zS?y-5?DBe$lgW+ml4m@a+?xetVf4Ko`al?DTI^~51ifLq4L zFLvy5O|{l?7~TBz0*f$TV8VY=@cL)#Z|X@wFV-ngoI=?3B|+pnB@g8BMW4710^EFo zAzLNPeSrF)v~BFCvk^m#?Vpv77ycGKx3fp5e7u5G`Qa$U7Enhm=cP?^!_vOv-rz$r zPuh+=RQ11x*y=h`5F|u6iAZYh&0fS(LD3y!1K@K!8CggOqo#A*YKv%F9KOn_AH@P{ zPGXa53kNVhZa}WmE*#W3%w;JMG!`E)1u*ciw9_LvsvGZEJ0<|2P>dnb2?I`#++(L9 z`2f$J2du~y?25E0TOhW*RpSwnI|CUj*_0G)GEFMW^LJUPDa^MSBUb>|ZxV-LNH;-J zR3TtB0Fyyd6m&TKvQ zc&6Fg+@}Q#sv6mHi2VinW%WdXWTxUur=$g#tWzW*WRxTfxfSur24aBfi%4$&KLhAn zD#*;ifWYV!`78jK6wx>lu}w*WaLh~(v}Nwr64Sm*gs)G~U5qFlasS)&dg+*hnWbT0 z2{k7|&PC;IMZpC}{AdW%jLDZgEOf~U5kfOSJ0*##2taRoVkLDHH7@XQ6;mMD?YufT zim6VwN~kSCqwCbo^;uhO1gj#>k#H=5HTKX>Jm(6Ucn^{ghbe}kwE)eF08nr*LecgC z;8P7U3}hyi<=go!NTaO4o;Aa7F{Q!q#-P}fVByq9yh=;PS?iMvsFN(>s7eXGGF|$>_2sH>-xJ42U}Tr#1SJP8-!!t?y^ZL={Ec@z zKmmgRE^=~9_!Nc_@cPN}uoR1g=6lTF=evpH=Y!DHc`}i8c7JFoe7+;obr``Vp*Hcv z28u9?GeG^=S*2=fk!x?jzBvd8QgY>1WF`&@l&bS9AYj~T(^ceo?hLt+x9NU_>XAnC zN!UN;o3`OhKgs<=!9u=iK91sg4&I1H39@+ma z$Ls;0iU9htV^EY2!*i^(NG%CWyIU?gmozf;uY_Fo2RC>Eo57&(w!8rL8cB3NexJE$ zK^3YCVBr5B96E0AGV&H-X_oMt$#CKsiRO)V0D;L9H@#Xwj5QWwti=%bij+O_J+o~N z$JQkL!)u@6QV+10z>;SQwxOmE7(@MyFU%te%yG zC*@ARquHGe9Kp?v8f@@KMt5)LEr4&skfae~q3%_*)520x0%K`X zuicjY7d4zNw974-=ACl7YN`vmcn5+*+YS=1>yRD`M>IUo@a7rSlmANGQ$lD{oc5$I0OV2-agB_hqpEP0-1=ZWI4{YJA48BNsDk;1v+s9%_9IeY z;|k@Ee?hn==tGv^a^cJ+QcZxhQd8uX6wbRc*TPv%RO;*fp`U^v@B|pN&&t1}M3XVh zdY}?A6}>A+>0Gr*Fy^39ptC=;3_rgPf_i5Ukw*UDjsGqucBm7j3s+Zv@6K50jj)>r zZ!L5V#>=`~U2v1VI>PX7mC#z$gOZkWuznoR5}BARVo%aJcwG(tlr94agyka4D&&Z4 z=9A~CisO29_d!kB1^iPeB&jqw3-%?z!yPXxqx5dy>lVxPf~SKBQ}AOAQGk|(+U<666ovk zAzLMh?t(7<{K2q_H+n{z&T$NsAkegxG@UN=Byh`h)CU6WYf z0~cN76cqV0C6;=H5mNK5s^aG!s&R*q*b_@pShNg}R$_}fS;hRKRwAE!k&F^@navCw!XfiQB&)kauYq!-zIHoO=R z8LEev_!aV{>4IeZtSZ1_x7G@^1^*c>g<}_3C!l~4k%`Dy2@Q?ozpvGJPaJcGVCe{C zzl0_M7JKXTHRpsDuC7iIOwnnwj~r%MkXEBByJbJGKy?qdJIm|-=O~OaEtXx>mh<1F zE6&<*1IpHZE=+v$s5KD=T3(aFCTHXwg9CK`+p0#^Ze@0R?rfz#Yxf3lk;W*w3S(@2 z>gN*#eE8APj#E&o<^BughIQ5-UkXsYzqy%CYMnG2R8?;Id(px|Likj{eg_ifb+Ops z4dTK~VT=VPAMR!&-}8BDx|1$UZsD3DF8T>5T>j^$R!;qhgxP4V5R&wAj!bhRT+pUS z?3`P5DTlv!@%p)xpoa?~7|stW3pQHTu_3_P?%}lOa@z$ssm~eYnA3;KECgB^*HzpEgg3{^S3t8b7Fpe#6nV z*I}NJ5hfpn!E{kf9>Avkk>8A}6Oc16ink>mX_ORv z91Uisqn`uw?*bkw%J|!Qj%ZC!#m?AI}XS7hGNw(Q+c55cEXJu}gKE%JK$C8|Is(wNW%X zh|Q_Th2~VDZqnijd`EoE|AIkxo1>s@TIH?$o+YEQ8aox!uRKiJVc#1RTW{*l!lhY=P0anbby`G9hA0h(eT?9E~oB!%jDlqJq3J};$)dq>7XVMN}D|H0EE zw6XBazb^&7(kKf_C~jM^_ep>^m&L& zwQdkG?@FM8Xu7+I4w|~*Hj#-{b)|TX1UN)MrbW?vZU$i0Qac)5kA7)OG2N5$!?94y za%5G8^*hf1peT5LM)^I(Wco=|EB~wY&3J9!;fL&5?ohsG(CH8Dp23ele#ajAdG^r= zo^7Q$cA+>|HHevSqu=2b5Y)z5`%XdEUIcKqou!`4|c z{cirT%k2YEpFaOMD1Q}8J>Dxg`VVR2go?L1Y4W>w+1r05UY_zRnqk=L-%qzGl1pEY zBi40o9n-o=&>`FlZ_U~_Gt5a^4BwuAjTxcoZ#}hTIDSNXusNgnE#V?gG6nQ<{KWTP zi$mFG2Xzb^UGUoYr~I36pv((B)fL|FC-?Z#^|$)Glc1!|A86)yS8w*q@o0W>1m__W zuglJs{baL=->0G6*U$DGFkN?!{L@^F_fTs>c8M>WWJI)?UMNi*L@Q@@)0esKJ0%!z z8C|r}aO^U5L%COwqNQfM&tIK?8NP*`k9*M@L^P@_wP`Hz#LgX!A0oxkhF*{hMs3XU zFT}^*L_gBo@is8^wSVo-FY>DE^gZQxTPOzPl>8XU*AT!jf9J=Bu#$@1-{#^T{ya5k z$0eOKJrgr&&)?ZknY}dM324sUV5!JZ zBiw+F%H*E52US}*em{dm4(~CGmko+p#IG6KmTq$0wF=N~KOTk$zx?_sLStsatpo?B}%+L=C)T{uqJI3ri~U~XrD z+FVl?eq7=3jm7md7p7Tv>;wNc?CcW%|y?xo#lYz6W=3QvKZE+Y7PHer1?@)Ny7y;MT^ zzBR1YkZno%J#sFXjXQHQxK>F|X*;WHQus`^RrKhwO7G0pq0B6oC|}i9xkzBW^vsu<^lLmA zJ)<}sY=u!EtTTWRjBQ}lC1#OjK1iva3oqg-~RN$01;8GUt%;|Cuubo}iS zT)Zi(;JQtzcmgB|MgEa)lE(L1E4+22P?1} zauOGHv{olxMi}cH-T51fcjL>}7rI$#onhmvZ}NMj)~P)S%T_hZ`v2H^^LVKC_FsR@N?+Eloli%S^XT5mST^BTGm~wybkoBv~f=ZnAGN znZX#d-0$n0Gw$c}dHjBV;PJ4{oO9mG>-~CexoKlpNA_7O0|6P*G_#IhxHW7E5$ndI z!Wv7`%dw^Z7^aMw?mR*J?xe{WX|cLoXz%joVF45*iWzZ$)E^%4i;W0BUfwLI+@bp! zK+qet=f_&?Wcav_E19XA?7Q0Sf-?nLQ`_dNRPPx?$UI`LQ~Ty;`Dk3*$4zT5tDDIm zXN(?czWI^wGpY0JQfg{j>izy^2j)gbX!8K85FTFddzmhY6NzlUb$_{o#f5njaAFO7DmT#0_nFB#sDo9awZ zVYO%faYrhj!`TTGqxI|dZ_ zysPv$U)2xHmCjbBTJzo^#VhE=)|#<>aqIe^gx}jdaGPJvrq!j-IYNK_4*Bd%Cf>IS zC&*>CPQ40v!k5t;Yl@eSxZ;gplNj~E_fD9sliP>EH+T5uGkphV65Y!88lRc(QvM2X z%1d4A2+GQbt#m6RO8*fLl7H1s2KO2r90_Vl9e>2nt8TXLvifkJ&i{EI((ckNr`@Aul6s0>82?gzC^r4B-uQ5&Kq-GJ^tSQ~ zLf)9iU24HWQGQ)TFKgL@I$G{kgP?okmBvAIpJP>r+&!|zD2AZj>5{@UxeNKEXZV@?0C(g3}yiobWO$xPE{Ur^((iV_53N|_d(X4zTf~HEj%pmq`a=7 zG)lH4<45t70pNK!ik_15Y_YJ1@GsU8yF&(pzjPREbKUQ5MJT^6+K_}qc4>X@pVX&* zx}9^`JQ||ivK6-b4kb4jzvc5F49^X48~-+Y`ti*vF>GhOZ}wkqYV6L9S)Tm%QCcXz zzq=)O=rJpW-GEnR9!SsB+#GXYXKBHu{P4Nb9$Jb;lzBl;RLwEYq1+&l++GZ@{?q)n zw!}-=szH%y@*!1YmqUku*4=GY9#D|)_G~guF3gWI!&H%(IuH=C@5$#kM~8pwoG@H< z_hZ=;vgSc6VIppKRSe;}F}>d+Z~O#udrpD=^}grFbeGhB^aT4e{*^eNzm}A~_fSDo z?$IH;fQa?ZaOQ=BLu$;}7lqR79K_*wCq%m9bx-X{h_wHPVhBYWnV+)8 zz8>qbt6)U2TGDMxTj=aopWdKxYncHL!XSYBa}CBjk-<9!t&Uixh=i+yG|@VJZPrRZ zJrf+!DjUh`K<#sqUIHT*sUw}*t-o-z$(z&B_&oD$HUDfx&3Gg*=Yi>jgwbH~W^Om! z|CD>$2-H>b(mi;H_f)XQeXn-y;89{3rR*665N`QRvO%6=0)zjFNDk@ukn-sfP<67W zsJ?saC$4Y`eISpRfR8+8hqUq69~q=8K)Yn7WRU?r-)MsL2-3Kvw^vLjzW1*9jd}`e90iGttTN%0@f0W#zRE!&8DJy3J!pv`f30w|0t6Aoww~Sa? zmIxi#eeM=R3dzCMjA-VeJv~;AlLBVXi&MvPNbwejUc!yt!C1RH_*A4M(DOvI^I==o zsq&m?pOR`*xzg;Em#c4t^W26>Z~OWAGG~-egTK2qm>avMKJ8~!!g8~gFNcb1%w^Si zmQUN`16=f~H}UP`R2Q(7mck#z6KH%5mQxd18`mzn7@0S3Z*v6W66? zxGk?17u`)bS(EoT=A3L1RqsO2CH2ORn=18aOvx$i-03xs5^tdspdtX{dofZ%$k!hr z*=s?U8qI^c{VXg<0W-rqybO8`2;1ERbJ>%UFpKm=bQ4Wpi2K*-YXf_To9O+PDzbkU zJyxG+Cq;l?|? zz13u=I&2Q=7vot&Z!_g<)dS+L*#z{C+S)yMS*;rFq~ve_e`hvfD3kU5=!Q?#=_(0@ z9Pm2sen^f`NhS{S?3}htr=AKXde1&x%A9q1#2B=5yhNHwaBlk1^L#tin>}GayMbe6 z)v-54`P#c>^L6c`Hg!=hZZ2sC0Wy~p3A{^;XJpM*eO=3V`$zGXQLJbGN+_=07jm+a9Y-kgI9n6tQ%J zP)L2bG&$_7!S0y&i=~fP7OUpX>d#fHdP9$Yn7nZ;vLo;VUujBDugd9x3PXqS!jEj; zUfYMD_hJCYC_KN(2RW=fwxK zzotLAo($WpSC5ZXpTu=n01q4ty-2o@g^FQMsLWH&0R!3~))GYwj;Xy<6_h%Pmi*Zx z@`)S7y>Lrd%7a>vQ7O_v(S*!^NZh0;;wzi5T!C-Mh75@6=?ZbAqM(asXOFYZlVJU4 zpNbjL{%>_BqF|&#$Pk>HKS-gh0SWA_NQp(HhS0lR=szb0t-cDaG_XGlJF827%cuV$ z5egV5%1<(B#^6Q?%z#zw7TokZv`lXPl4Fi37_1T&6G4CrFll8S;=G5t!=e8W1CkP! zi@y3Z)xQI};T*^{_#StI-;9q0&^RwZQ4&@Ef-zsV@4~BxVsTh0L57a}DQItxEGj{p zgjir_+7Nh$q3W*Fi=;$OOk;Wo3LdO`Hxy9WIVaYvn!uDnwZ(r8aYYki{5HdHy~vmP z<)zuzq-yy234y|*a%7`0ICB&qE7&Y>MnEEk{U6XLoYp7oG;n2VF-n;|DSSs>kA1sb zr*I;V#_oiRTwEz`03!ia3%x)DoIqh?aSLyqq_)s#SlAmxclu2d8FWb*;+{K(d*(h} z`(Bq$SB6^B9Y5d1Qh`uGmwD5No7p1DOnkQ@6JhBAzPK&yUkPtW{r3Ng<~~y2R1EN# z+Kn0$EL5mqtFT;92w|A1}Y;j zkA1niDNh>T-!6cRJpS9Syx8MDq9=J0j5pOH#xNm5K4{$k^bnpkDSAR1p-(haXvx+1 zkS~LQ-zB`w*G16n9%)wxav)(fpJLOOTzVTWEc{D3$Fr0Kzk_eNxk)@jVAYSMQz!e9 z6A`d%aGWB1Q#q69JRBvmQv-euxFgbIJNbFy5vg5GnT(0S0&Nx1A&ZSz>`{a5I?{YK z18)N%w|fp{MVVbC>jfQ~wT9otv@Nie#l`aPyEo#Y+h0f_hAn1z0)VP5OoRt#-m@Uj*7ba@#`1NNB zVmp|RlA0D=S;LU)MyJuA9~130QviYUK^yemT|`f=ROWKyhl-*Q=j{~ z2vLKdfz#tk^qzy7e^_+r?aV^LrBZVcCSAe8iUI6f+1AG2i~nZE!xVtlcQ>B%uW{^1 zNC#w*O{e1Tu6Y+-S`x($mR)i+1Bj1IJCi$BIbg{QV_I zhsfVRNt~6xlvfUeiA|ww09k4J$}N$GT2=fy6e7iL7h>Tt0S8$M%%f{3ei0$l+Hk?7 z5NTa!UjCVp2e&TUoHDms@$GFrP&j6HW)cPyDcRw`!n$c|eLp#>2Jw^!^3$UgW%%Jo zf>Hl83W{^5jeYt|=mw`GKrBlEn@GB_$%It?XU{<=O)zz&S5n_6-R622Zn9VQp2@rk zB4?lcRzr90=9db;n+oy?(`ic?^A}rZQlhx@l2KIiXMCTvkM=uIq5hYm zzem4S@B<&WTP$hRsW}$RU5kV(4C6D`F_f?J(mJVJ7aF@7$8Hq)LP|g|*OQdZk(~Ux znB+dhv%g@U9Ss$A`d_!f{gxT;>+novqPEGYn%G0hZ-_(U4kL*yVZD$!1um?$!(Ou( zbO)d+MJWx;PxWJg^k8xa0e;1zK8u~JlJ7fktdLF{;u+w& zYnL5np_)7e{v+O3K|ILZv9B&A%qq5j(o8J~__jp*)70_WnPPecK{;5)&dQ*KsZoV` zfG!C;CO(}sVHuSPY;Ewk41kfW_?Po=jgTi~@(($$He7vgiHx0TIm_R4DDMP)upGkD zs|`hso1RwZb{GoQY4Wu=83*Hsj6LDxlQfxI^(PCfC`Wk-(1LZ28a~_QLvB>s7L=VAbbv_fQVHs zpxqPt>NQ@$wK0d{+iUfJo`iUNP^u-B=-jl#9YY^v%XG_o@+8C@_y0-;WD@;`Cbj406!Z6Ws@`$S{>E?Bn_Z(;on5ZuX=UAPyZfi@MMdthX7i6cK+oPNj`VB#Jo$tnh7QvjE$9WXlj+^~)isB8nZecEBuXRHzhc zgp2s;5+GLw-T>wTg627(P$|53@<%v2Cb1JX4s?po!`A` zzbIgP;6fRSecd2~QLB^FdNJuEXl@bJG5Uc5#R}j|19<8Q3@bg9ENe8_3tZc?1zn8Qb=I5l zJy}V!51QqgN{#OLR|o||a7k-EMQ$}O%YWJiCb=t)pN+qm!&hC^OHPlx)@izXQp@E+ zlyqrb&y)6~6ZAVdTB$1ibjJ8fpV3bpwTl8`lwpwawflSZq!!4OCi(arz`J? z+2l?_Y8tVfsgU67j?Dq@_KhoxmBIk=!;HU;UoE_QbbmNxFJx zb$tC=m-M=?9=Fe@em8K6685$fmhW*H;u}A7-I`hIpx5mwNH_UYC;s|g)wLmp@ouSB zcH`Rim(HaGi(**a^B#7&2_y418?|N?kJb+H!mij*UnqtA_;~$+YGjBsneR|VPZQ0m z*@YyIUV2|=RGmim7cM1F*ls?ldS*&~`npLu zU%K;v$@5sU`ZLMNR{K{~n@jlWmAk03f4UyX)9FmV5HvgQqAhGJk#@1T^J0462g5a0 zBx%7>I@O@Dvfcd^ki4}g@bOAM@9FDf7#6opO7`iy9@kz}{Fc6a?!DxuIRHlShH)m} z*pm+adx2Oro9V}pbP1slU7j;fy3>cFhi5YP)aB#ue92cdwaH(uiQhc7eQ)!O@yR}y zbL7@w`yLP6Xsv_mK{L;@pSQ2@G1QJOsUDpGUs0D{X(QY9-!0Bu@zk1yWSfATm=vF0 zKJ|@|2Pdh(E$TLh5;9NpYd{0y&a`{oAUqTXnS_@7$enbdoWM=Up8 zIcu;CPUD~kR()0F)LVkj(Fac${)Tsor|%7?2G+!MN0dv?`I+7l^)TFmO}UATr6ocG z;}i7ey_8>^?wxP{xvb)u!!cMOo>q=pdz`JppWp_#;hlEX)Tp`MDBEJi_)x6zhhMMD zmZDpGWNhYH6I5d64TGpzH{qq9;1Uy%z)MI8X4OZ$E0(2uGrCDx_4;% zT=q`DQu8u|dks}5YRldLQDI=xXLcfp5hHJ5azA2^o@d>qf(PeAEgxidfV;1p`Vd9g zjN7q0!+qsrYjvYRuJi2hjAp0cWMM;cIHB=-|B0ajyyD++`GI@$)5orzSi8}TI71DI z#jo}}i!)tm9jE7=(+GmXist%FkN|I0v$|1y^(cdxKN?fbOO(+jt%|NOmc zd8{iB7=jFZoO(r6&yF9IJDDsm>whx6MQ=$@RnGd#f$@N@v~1!h?e)e(wR>>thQjY( zNwvDh6xW&iG}rt*Tzlb)EpXPXHWlCleNSXA6mE^2_kK$+3Cq)s}0>zS+z0>r((Wy`F|4IbO_v1q@3yk@{{fHd(xetr< zQ@GBob2!zI^n-2Hm8OrHEePvNV{TQKTYg`5^7)G!p_E?@2VFK=&qD2~tT#MbYyTrQ z<)O&^T?f9{)%{xcbM~Zr=F|{nIWLZ4PT9}o4eLJVT6kneW^M!(3-vH`&m(K%j9(SfyrIJLRaEC#Rg=%?;59n)h-I% zV|n2kWvyLA_UfXQ8#@QU*rTV`Yg32qou0M1rMD!uh^7d!?n~EY?7#9?hs!+m4l5~S z@6paG!TwM9xGSUk&dr5({PE^nYBxbuhrac)mGZ$z>bCq|;y!%8JUCUludBBB%G$+l zlOmL{9lbr{9X_5Mj_|h)K2=Z})^_e=u>91W(K*YX({#C)dWMz^{jDAOd4FY@FYTj> z4xzZ1ua&Z!Iqz18Pa%nD$#yQehK?%b8-_R=j|%kK68NQWO3O2=n)+vIclYh_@4m^u z^juU!RQkfdI?n^8*myogvci)a5k8aUFxfEK`j%J;F;<73Y3(%7PCLtc!wWbxb++)T z(JbSwhj471xXT_Y>|V3{UpFV_R{jKqB;@t3uRnV`nlEKdY1vCDF`FsDfg!S7H>;pf zgC*H9rcH5q_&ctv&NVXAQKhhlCrScJ?}Yh|uA$wH{ovK9CUCve>Mo;adZdV6R$Mrg z!;xdh+m`b^&vFquZTnpFHk+wE)o!{@6DMNn?P-06gHd1D9;}%BHN_w^8&F{d-BMuf zEU9B#wz?B^kH*VhtyoD3I7k0>HbYI>gHkPrzYG4*;OA1fiW(~!JM;|@Fn7c|H|!|J ztZzlyR7|wp3i>s??n$z^hoD&bAHVomBhUUF?NjKDKwZN^RLpi1)>-GMFMs>^4c&J_ zwXg;&n5wbMCWXZ<@~Sxxt~IpyF=ku{B_X;@B_cVmEwvaw^}WY(T)(THR8FMf>|(*~ z=8~nKL6p!#URWQLiW+%Yms^qauGde$q+Eb}$j|c*p*Q}fX%OX7+a8}x(O>n@(`ZnG z65`B*AfGcXvkx>|Cp+znOrNf` zvi|T+oTtk8v5LXTB+(M9!s}}(3^=^_w|Euge$={LL+hU4wa6c!X4oF_RDD9tWoBL) z``zj!xv~5(z9^_A_RqpNa^tcZN>NZ!TI~Hk_SX!*iOeVco(aRc1dZ1fvPSrMC_ZfV zCONaYmA$rs555(LS9jg@XEYQwMN0Gh)A}!Ry(|3opHo`&K*O^FbP#*!4iS@d3P%)? z>2qEb%xhlsVr(DG)ugUX#KSb9FEjo(y@e24tcT6_R0d|l{nBX4aBPfK!mhmA+;R(% z(8JW8lv^=a70I=agfrFlHzfLv;mPStr4u{VNLjqu3OKYYbjVdk$ec+RGCh03Cm!AR zk6L+X%39rkC~N1SVDI`*bGF<%p0ku+-|a2tsjJ$JOtxPfu3|qFc03;}0T#0@=7$I>%4tY7FhGQSs~ZQBC{=s zYbmVUV?w`^_1s1Ew;;k!y%Ft8#VwV(WwkX#tRazQJDTb4wyp2xRF%M1%}CvNrJdW- zdH3X}HkGLcSGYQ1c%D;X_V}sF$m6(3MCud{zS7yo0Mh@~d=6`Exii)ow0EZpen1Z| zy#z%-WOfLg-3uxC_ULv02)uL=?=}YjxbdeS2#;>w^@{HNJ6idKk?I4`|m4LAu6mQm`UGnt~_B?m$dz#Nr<&=ohe*WA`&n=0Q#$GalFqJ?4qOu+}o%$>4?FM@cf;7oN zxwmN`w>6EdO)lo|v;C4G6fzrcJDVpl8j%pZVfLQSIuZt$b{>DR%Jp{ulGsCFcZ8C8vIq z5B!XC%WQ8C{wv{Hut*%#p;q0IRBnggTIRB{P>Zgs zgO_RYJdtDTSk2JOTLiMw@L^KMbh`m2bHI$9!f-Sp1@K_2PN0pWg($D|n7-?9)dU zb)rZ`tk@X|G!Zy$vLvYKn;vzMOnD{O@Fg*!fXEuIyoMkUW|V*S_|MVH;Mi>W!)Yot zAhETMP5z_@gMYqha|rg*Cn)vcl<1!?rVKkT{`eKOfKE!^jUn6H5aRK&11OK*LPcK4 zipZtgfC(Lfe+6>S@XAP|5Sy~f7y~agomu}B05vNOe>zvd*9+jDhCM?%oIEz%oXgEf{Snu&#XmJ%Z>G++ z6Rxh9DS9pJ@=EuY_hNQzw~S^}OM9Nk1U5l{T0zwIl$5bz=VJSAkw|j}*9AqFD59sr z8J3Zx1AzWJl-{28YgXAc6&T^}A?xBH8C94SFy3*6b-pO7_QW{|@2w0o`1`?~+lZBJ zxG#mVgV$t@u0Ai=c<(rHYJ!8t4N=~;6GJ=ryXI+MUfh#ZR8=D17!N}7eo~17-`Fyk z&yNKo1CN#hjV9iJlWK$#w?$SIFH=k?_GnkTwty{k6`Nt1^o{d}*%>;Gb!`9VumDTE zem7J-NUS{MapsRHeZeVcqkg{NOzrS5<#0XE^!0a&c)(>^V!FWk#xjwHZ`TIaMRUF< zZ4mnkLI+}>)Gbw>BkdEdOqV791!vRvKi%`!KVdD5R5aDhT53&z8?>h3 zB93xi{MX(P2TsaC#}dKhz$#4vF)0R=tc2nlBsOn6Ei;AKc@5tOkK4_64K+|GKj|}q z?j~ZEgooEQ)$3Gp`&8x{$<^B21%=$BRF34q5_!v9x7 z)4*jZtDGlLkU*qZt|u3q=uR&>PVpSzT?@EwawdSr?IFhWaMkp=fI|U)g?w?1j9sCt_Ie z@TwSCessu(z>-sZ7naXHhD*Nn0o5}|q3rr3JZS8_Ray8^8lmOZq2!}tQ4Z#x>9s8H zCb~|f@1T7VtrQ{FtZX0Ae(W|S97+Z;JF?-4O+TN=e6+J7ZMWcQQ~W5fx<+H}#OC}1 z5+tpAWU0`d30S^S{4)ag6GL zk6qpcj0){)A#5h)&Wm@ssj)?b8A_(%8Imalf!)!~;qcmOej2&9g)k|0fK`4?qL+3* zz#~#>6q3r52!Nh56F<7pr9=9ri)2odC;?nDfIVscg`&}fWxQ+9_l53x`5wLM#gueo z8uCKOlUXE#wC5dQvO*shPI)gtDuF#zkQ2kWhZUQOe7d@gc%nZTWdfIW6q( z#8?3%wtcWu9}{VeNrXD1wghKTNbyXC)EkJIpw4ZOlQ39#1nIkBRPNO}%ZOX;e>VbKl)`SVuf8Lhzw*t*+8vzUPUIUY7tBr0@!9~igdrg)+(Y%Uoy z1b$@J3?O2BvzubWLU>F~0;od~;4zqcmB--to%zD97jXbRiHBeVm!!?cZFfIAXmB*ioh=9d%X=b|q9)N{1L2{v5zTk`y7WQ2QC|R{7}d zG*RAuq35I?(;sm)mqC^K>7vgxPlhyD4Moo)soUicH5r#90{r4Sz@o3$K>s6BXC8U} zY%ZYg4nY6denmtn36i*xcVvhxAajHeoRF&;KC61j2Y76v{Jj|4DTS?E%2fYMOoh#| zC1Gi`G<7Bn04f-}#6gmseLqA*N_v&F7qVB|KsxKIqj_{WWHDTI2 zOE84$;i9=j83WZAEXLH3UIw=5*Z^A31ANE#C2!9V&rf>1m?p(fVZShr=hYI1_Ubjm z?Ec5|@CMXL(-kmSjQ(P@ESif07jJaAOOWtTzD8l__2lDd0%<+UM(LPgY+; zbXm`$2FJQYQ+7el3_lx6iR_ui%#fuYJ;`G@`Rp=0y2m%}7Jlymb)MdLks0$hu;Yf+ zU@qx|$ROMc{Ir#SC*_bzrYcXF>U{zf9Pylm9@Na}Y#3pX1er{&-iY#A4CcH#!e^)# zM$f|#$AnsnggMwk-KG-~@w?WNPPB0=49mTtvesh055jy&!FXoo4ubKawgP_36rkYq zXR6i+JkHc`kWw~h&YYkE{9xk@{&=1l+ze`^B3(}SkWmb z6@ZjwFwUWr)QW|5i0#WxloksO*mM5bWemF~692h6??Z#2F=$BQw6!8fd7?+J7Yy zN%20gY4|*Y`T^%TOdFu`h4B7h4UvUT7Nu)+{_7&Npa*55WL#^m*K$&CyX%~XCqnQl=bYXBiW@nE;tC+*X zk{yo+W9!&c6EXhyvZzCFii(FG(|;O9F#Y(tXhF2Ppyo_ihOM|)s{C0xu0riIrao3A zJ153;!ZJ(%yzq-+SQylu`vQeEb{7PGsv1UodJ*R|tthzqHud^Y(TDgQ++NNLp0HS~ za)gaWzN&+6+Z0D;X?}Hpk^x-q6O*u$(c(WJ@l-0yq-_6{NQM~UnMBZ06haPx*hv{D zdMaSD{PT05z`_u5rLZIV(tw1;+h>C8CIr$77AR<#N*;0?0gD8ja{c)zF2!>w>4JSX zkw1n70b|%~*6n8sfxxvyM6h6Z`~nK2uZ#3$-&AaxQUjo_0W3zc4B;sXeZWJ&0NaF~ zcz$dH9zRy|j<4AU5E@|l;ARn(2nsoVlg9eB}9rHe|2 z5jvk(QJS#WSzPNOg<8VmBtGamG1^ZGAGV)(I0Oc??w3TYrm=AXP(F|#reRv?^I5); zG?Y^_vZ&k)oXwGV+#ljIbnutJT8>s>qqj1wFbSRUvLM8V-5LK&7y?;ulbPDJa1!=L zm`jkHA!m3517TquLWm9Gn}s^-6(QY;!{$IotSv|xb zo>29IZ+1F2CFSnNtakPe4pg+A9SPX0nF)vi!pGmb{}^VGhpX#6<{D^4tDSO~H&YJg zOF880uK#2al06r$#wb>uoM4Qf)7xE^A9Xquuf7=eom?rAGyl|zd-FrDv!*XLs}4NL zA*dFp(+2G~1U>IFh+9Hy;84i3|c&38h*xcWkcnzwfyp) z{0HNIgdF&OVXiw{^F(R2PFd9USWlrVLvYLM?8G!9#w@Vr{Qm9Wzbx)iaDyJ6v%X@U&WwT&VmpjLTfwcCOj&m|Lf@N5UxH z{aVQ>J%#rU$&nT);?{DemCt+u?YvU=8gJxh8r z<`#c2By9M!_qjtsB&j1aY`Aq>nnujk=M(lHufLx2)BkZLWNFIFj>CTy209uSN<%t9 zosysao3~SkK3b(jw2bfB(If2y*0c`9CGYx5aNms&f1YJdQf9`Fn!X`Q+;HzSxNqq0 z=)(WF4419hy8NUcPR)Aji-}F?RX3=onl;KSMRM1jtA#_Fk6k={@X?;v#y3R?4%E37 z6hGFzn0j`Myg4t@IXjWF@dU5^yZ*(o_|^FhKSjobEs~9kC^p;rmxpoFz}Pjr&0jT= zn>AOFjSqHK(PqVojIM&?l zzwu@5oq|5uD+`-Xl!-(1qTccOhmWsg~R z5c^l_V5M~3lCpzC4a0MHyKI$418fZBGP{fJ9zMd(T{$UGdQVn0_FP&<8vb-Fd+A*D zwWQAdS<1&`eWUeun!mhe^*TnA)!S5#B{clWh0|icf{r6P{^hmmqDr8nnkWZdX{pr;@96f zrF2yy&bGPZhTHo;hTp9p^999=^Og^?ekanrgwm!zn0MW0Vmmhv=Dm%r3HjBX@`ChY zM(z3iq+1`ZnfliZO;c96T%&jVQb-$_>MIUd|3~5d$`5Phi6_6bY8HJf3%76j{_*Nj zt*XR4omnc&X>m#I`aePkI?FQa<|xD2Q9|`U)jZpVD*9}+RjhAlw6~LruW48?yZ=UXM#ROk^5_<2*mPfv%5nbp3>y zZoRL}GQlqHMh=f~)s|8Q;e>gY;%A+cLOu(b0dZ~@j%=9TTyPkKE{Evz%~tN3iNO=n zo;dE5fov$$(n$wfDktb8jh>>GJ>Ks7ruBzp6%Q4(l%=GUz3L&x6;Zk?tV;^s0`7eL zOE&*pE#65@=v?t*DD)#)bci__lHY7Q@;YSK&}ZNL-Z;fQyEwLs`D?AnJ90kwmr9

L0>9AKZvmPcM93^8}7Qibk1D!p7VEXDM|F$>CNer ztQo!=efOC84f=3F7a5S1_!|*cuZDBTXXSz@S_=E16C7-DeIsqhyO|2(wdJAsJZ*@8 z7%OY%o7vfSbH~p<%kZo#-{E4>nk{?j6ade`+uXvV8xH7Z zyYY=9`uh3t_A|Af+AcpgQqnsPsmBcKrT)6VZHV<7>TH&M`C?=%_hvRf$=g2Wecy?s zh+EmwojxZ6%DRf(pIzFfdBFvL_XP;(4~%#otIcZ|3>N-DvcKngFZ03`2acUee79S6 z1JTm9IJD!??h`)3A$}(%!gNaO*(3egYijab5A&*>=01$-CeErqB=?Q~!l(R_B8*UJ zD!#3jE z%{gC(2a`P?LY|=>4=@qV$Iu%O`OD_*+Xc)hKJ+OJG+#|omb(4ntee8;5pB$ka^O&O z2z<{D^f~CGtu3EO+;l4pp$uzOSrAKtqU-rD8fSPn7-f1`Ys_L6Vp|-4psk^#_+2^A zU4F7>;vx(*lvD*Dp5sC=t&w77JFu?bLJpFHXIn7ZSJD3dV1TE0V}2sNlnQ6LlbLFe zsP`(2-jkqw1Z!5NYgNgVUwnO5SP0I#6@H@Ux#psehj60#gwPwY?02)l#<}GqV1-2; zHyN$^$%k^;xlp?$vtp~EOCSX1neCu`4l8I5ikYR5)OUON}M`X=p z9qz7s!z&EVZR6N_Db`8HT>Ct3l_b4Y5~mO1bm3E|I}*3l~Zy8tGegHB+WE~>Ej3k!s1atc1;@qWy0cm%m_AJ zaA@l}VeZ5=N2l(7mZgEN;U3#!iZibWG;zc~AEyL=sXH{j_bgeA1cbvPY$+w@(k`!`mfA#cT^F({!rN^315TC8t^V~giml9*zdOC#7!5~f^HGNQZF8$E2R)HZZn#)Tw++?@& zVd*7I;9N`Eb72W&C=buP8uc0kd^J#sW%-w|iS9fu&wcqRh#O&-CC)>lBLLbFs_gx- z+T>Oh?l7a*Bp{MFX%^}I^;#ACit{Mf;_8Q?+=syNj2rHtmA|3!onV)H$!xzjL{wW0 z!bhL$uC49v5Ma4asLK%LFWWq-GQq6b__>}$nQ+ov78!z%_$3e`#|WEbNwl3JjxdXs zqD6?5RD^WVp}yPCGSfai?(o0uNx7a&l%?>7=UY-E*@U6v<>y7iC$oLfd1WaZO7e@1 z_Co=Yq!@#!xg1M>P*S}tM>>y}J;Q2Oo_RM&x~a|0F(az1Z=FsrtrShA61L{qU1ndN z{RECY&juwIf;GTuEx2~P@+2G-kuzFGEx(#T8+>`-3tPk{%IVqr-m)$XvRD1gC?BJt zEePeWN0rNoPnQ)aB@8PcP0lQjn2n7v!*_L`d(>s!Rz1Xf7*H_z!sealjW7eb@xO6I z`<7^jsOz|~XWINK+Bo2JoU&)1hgy>3$GJOVeL`<*x!*#(aAMX96*!rs#zluBm* z;jJ6JpF4+lA06Q71aM-~U0i$2!kmQmm%UR*J04CM4SEk}f){dp+}7lB)2b5YhUmU+ z`;3&zkN!Nf%EIAse^iZWNk>wXeDT{xa{aYVa6~;)XDE#DmWx-3s?mE}q4Q^HkRiFQ zHfM@t?bc}zNdnOo;tTDVEgmiXXEIR6{vq(r`f@DOyZ^AF^i1=q$0ly`cb#u}QH zVfsbMXI~BEpZy>u(wz|~ZYp&)olso~kFMLilMu`$Ujpe&jZhJc{hT;Hh&O`t)^JzT z+SJMmsZdr7zVH=E6-3ns+w8C0o^M2W%bpa~jJ~B%G*dOS-slg?N(XRY)`u_`=dCp! zlrGn$sje;(_%`|2PboKD$g%GU+wVKKiy3$IZ2v6!~!Ure+d`2Ob^e zx0Uuh^U5?{-S1w{HU;y~nzq^3LfVy4d`ZJAmsWS8YV1nsfWp&;BacSjjBTYn4Od|6 zc!ORvuy=H89lP(7DTg8XG>dY(bKCgyA8xPe4IO4{`HVh}FM0DT%9_zl8};Nw^%myA zRft^kc1_qo*R15tj?7x>{ppPH*a&-7p~J-ZTx(-P-s5UnSM!2^#zL0mXWWX(XJfR| z@auZ4%a*Mzp_%SZ=}y5ywAGa<)db!s>}BGSr~4IkSQH(F zg>wahHvynTV!qtKed%xMo~W{_s}?>$zBW|s$WDgfzv0xZ93N_-k%^U zDil3|W*Fk%7$JjE7sTyx5I_VX|7fl7C6NWbvk{U40i{1*4!SVNTYv`Luch|v&$Lef zY{CB%oL*ivJ>**shLa|Dj@-nDplCV@!);qW|+< zaYG$CfMTapF`U_N{kZ;&5iC?VpI-DX9ci4{X>cZhCbVY_f_^B#8GW(J$9$R&II6C8 zI4FJ<@LvNB9g1z)=RR87@I8v3fkZe6(83M$@Q;TC?cjke5$K01LYjJz4&7!@ec&?J zb@D*X#iw#TES`f(RJ{CRUA^mEwr;k`x+b|q31WqRssA}xWP3@|@iS7=rwljP^PwHyl8Sj+ z{*n6I4$L@nj53`)t4IVC11t`rXV20*1mh_Z*w~Ij{}#JRDKW>P?5<`MI4`Q-X$>@2 zY1ajLDh-8|4bZ9V2uikSWF77BVUJflv2T>$*WUFTzB3)X$uhN89$>)pP1 z&)Ot-BR?fByD5=WQOZ%#6R=# GLs&ARo;_U~8i!>0I=`(mgXIT2vO)Sof(O%#{r zIf&qPoGuE8l2BxKJqG6N>4AIa)reL05Yz(Z3nHx(`tAj

o}sZ1HJr?cRGS;b<#=|wnk>l-{xQ4@^No6G4wlFtxC zO{_|)FB8z@zu;|}N}vJV^bt2z0V*5N_CQ7@>N{_1296!y2v{MQxx%v^SI z*0|4R-UQXWZkld458qv(UhOZ0^Hwt+>y(4|axG69-vm3l=6~fEh>pi#|8ig#_J_XB zys1n5#)KNdQ_9HgC+7HXVTAdy*p~!2BvJX(0>B{|)3i7Vbx|CCYAjTVqP0biNmVU4 z*c1jCmU-E5%1+C}kE>9IpYwYFlE;>l$OZ>2pfMhk|8g8y+Mb*-VXGe`;(s+K+!576`uc8k}jh{LCfHD}0-zouK0DzCH z*SIrX*mI&GN~T^awz}Vh^YsEh4uxfH7S4|F~nougBCg{fnec8 zRi?pQ<`jpKayD)9TPGKXyDPEg-Jt#;k+IF=Ou%gsAg2A`B#bOB5|$un&H_QR>q$Le zld(YJd~+3wGy@lAxkE^cz1Bt5i3XUK2vAADwi~oYmN3xAaH-E?+JOY^vrsJMj^1aa z`XlXsEJ`q|59Ccyz-cA)$|bft-G|j8&Xxzi10>@FP|#iIRKRq7Qo;wrq#kF;aMCke ziR~+f8mFs85&40_eceKr>&Eb7%ib8VJY0G$((!+id#+Zh+(a>eLm20hl4JH?TxTdcoHl(M^vff#ih%%%M-J{JuzdElDwpXrYl{8t@q4 zpCX`Xxa6&qLTqXj#82h4*D83VoX;Y!Q3^@&84W;r>il2-g{=b5BES}`F@J;`r;A#^ z@p(WpYwVUdIl^z*a^t6c9MhrML~Q4|@44-T{ZJxnkpB#TJ`4y;Sb~Z0a4@c@s@+fv zb0~LV%=s|pO%Q#(L?W@LdT4q$MG7e>8i)7-xKe~_fOwG4~GIaV66OD`1v+j;|qoZCoIJ>GbtUCh!B$+ z^=A*`jC7#%dV~P?(>3%U1*x#jM@9YA;Y!(O>ma?}o91+}TiF7!hAeBFc zCT2cZ6;6S4;v}{fDcHic^__YEOX^^&fb)0HUE^4aIicC|n8V1whXA z!d}e;E0_!x$a)5tcJx0ZfwlkFS|EKPZ~?1B(iH*c(}%SLI&ubs))HS9NOmTNH;h%F zN_vh+LVOT0`?*A$o!|4_b9ANFmEJ0bQ+;#|Lv5&Z@|xH#trjeMaL}0DQcV{n@q{ zNqPPM`;e*H1dF(8zLatFPImYps;L$~LgZ;79|!{j!8;KRXAvtj$}rV`Pia(5wc(sY z@n-m$&9Vt|uX)M`%J-&6xn0INejq%l-lXthMf&8x?@y+_D0$ZUAKJP1t}x?T+g;fD z<4gXPc%Rbv-M@VxJP|+Tm|{||a;109Peyx`AFdtsE1Y!cWnPz>z33}=y@evqC4u8q z_MMMk(Cr&rhvT^Ts)&3-}I`XOe5n8FUjsm7P&S{Dbr;xLBpSIdW&J)ND+@@-gg zhkbJ7k4k3CTLN)~0>c5C>xdb|d7}WCGdw#Ny_qtSpWAFYP)KIL)9UP|>#qYcCVdjP zzjv-~%#(I*IKav}7xU+Um(Y!1P#8>zBovp)A>&;W<;N$7diukV*^rxxNqZNKlU`|m!uiZlMpevDa-S#IA$Kh0gLs!Y_y z`CiXl)&abk9G%`&7PUmA**<5a-`ICgMv<9G+b53K{b*CKvh~ z2xSuTad<)Avee$75wGkOZFVl6qaDwVUw9jMp)*w5g8Vr7MY!7%rCUiNE5{AZ2HxB9 zv!6Yl{AXmfQvLh)&T}^IPPpC~J+~>!-=g0I3RC8_t?%h;l6@~f`hM^NQ>$?-Gaj$_ z{0F};>LY8-_%#@pB1HF`1UPFn=lNmHQJ{Vf~3otr1tkw zDanM=SDqyTr5bN~-j~X*Oacp=7zixDGr|`B(sr5O^{n~%4q}b&$LxS52U!8a`Jud znYQ_rvUb}83#{XoeV{I{>}9RCZa-GFWxmyCX<2XglJ{4LXKn1i;IFQ6^yxPpDYViM zN%LijU4u*h;@1zdl^)&hZ4D{?E9->!qaBUCz)_L;H7~nArmhyJ{@{}Voc8>A$^dYM zT8zKf+uNdFSJJ1=9)S9T>K($FZz^7PZ(8`PQ8@qP63tQyyFPS<$a^e?e(!mHBe_O7 zc@e&Oo#&gP3F1HVF*h7eY&-e@e{-ed)XQ1V@7{mqylyld!f$jgC@hy-cXYENrDY9` z9Gx~wX`g)@>*k*qSL{$i+dO)Z96^yQ|7iHvzY;b@2qJIcen9SR=(*lSyCzyS2fsY& zEynA=cig4B{UIaDz9A2{8k!1YK2<{ms6PH5VQ(G}<^KNX*tqF5=PN#%$5JHUX5|S<3%qby>$)0txXB}oRGiEvW>zbK9-_P%R{~q`K zM|!jv*YaLo@7Hr%as};UdUlr$^I`SfdRryk;l`2fkG;Pza>~aZsTB8Iqq@DmL6@B_ z%sm{MF}V{TG3pd7aSlgUgc!oD8gJ#*_9@5N)O|2AOt0r=uJq_>*w_#n-Udikx-Wip z%QU`bPjuyOh1sM9=YdrTu0wtyeQReeQgpX5{aZu_g2z*5_jd-mYjz(V+>8{E7xK*> zZ9nJa-((wCYTiXLk(O7|XZv|wst0RnB;B}Eb1Shpw|;lWp*_R7<(;XXlnV+n`5QeNZYehpe%w!0&WCdc&omr?Y7Ozgi z8OF=0tc$0XJGe)bt#B#K@!}B;P1}9fT^Aeu?vy0Vx_>Nx|1N&}({Nsi?yl3YnC5ff z3IDZXWa?zqFPR`2XEr#WsM0#|p z^}{62dwbY89;u=Sbbo6GH_-Q)^=>128$ySH%1?RVOz2;8W~!VZqd1yev9s7+g)_X>vZm^i&NgGN1kVLiq<^k z?MVH2+MBVZm-$h%e!7Ui>J-oCrn_<8Ce;jcizy|hr*s`7vDx^+A+$cLQWNs z&Xqo&d2zr@<x1=j?Ou&dFZX^e84(SeJ&#>c6`k{NrXC@9?`ktnvBB^l3$s3Im!Y;Tnd; zAEs<4IwI^&UrTlGV{GV%z1;fxPFrf2`7+rVY%VrV{D6Olb$>e?5X1|_;)ypji^+{! zDw5)X42HNUL0$EJ%jy}u=iP?iC&O!t{I8}l zQ*?h_a@;IdX;Bc~Qa(jKdNqB$P39A3;50qx#F4nuLnmHg?BCHUDa!bSobtJt@Rb-Z z*ir1to_HlVVww}<0_a?&s3e89n+4(pA&x6cF#&7w#mfDE$CMCoaaEqdy|H>U{9thK zwD^RIk{bnIdXID%@g(lyM{#N)S%KKsL+j4f3k z;2-{?EMLFDS-qk3*TB`G6)_4dpgdd9lXmP#T-;@Kju*NIHJB@B>%_82)b^mHbt)|9 zQ68+iI|&VjN>gmam;3qTpl86%xKn{auCP2`IPcDQWgvecQgN^>{7V-j%%ePJ$lzVu z9lCO^@u^qiE(#}lZxQY$LA`H=R4 zwr&}mPgdSDJb3ZT0XC7n1Pb(ffu_ao3y;D%qFYOk8EcQ?=fdN7oq8{GtaQ9N2T8NL zJF2Z-FbTZ-nUmK?QxEpmjHZ_%ZqdyXUu|bJF>9+UO52UN@oSnT*=$eiO^p7)``?0h zX-9D_dYE=^-R7NlVZYmPJgmOf2B}1^PqHT-xuUjRF2olY+i1Dmpr5ak1+Pf;0+zYrFi=CsqQKz($RTnGM zb_L8?MOhgtAsIb45MMPj2XH^mO{RA|I?>zEWqPuwLYjrcFq#e~x=(CQGsXcle*--N zPFwAceEPCXngHEgG=>pQLJ)mnsyg<>Ep9(8GuED{s-3t4B*_@?e*J?N%Fy)3Z~gka z`E~Zr`vEUZYDVt_mN=0kc#nVptY`ORkt!K`XyDAJM$5;?V!4uE{^9X3>7x4xzETOU zjD66T_+x0v^f0D{G&B~@xR<-b_0SOe)2_MKxfcQ;67M+g7(_G1I1T~?`{+1(^lcuT zor{(Gdbl~;sgqYnLxC_9^PHoUJo+(cjK{9;VvZ)q-%DkOR?Mb2?mSsJ9PV4Q zj-AQ|t)!hpiwf-8Xp^K3T^ArUY2)M7flH-yWIgyFe@c^f*4=@0HCn_|V2(~4J`O~E zCkzIZKIFP%>muV!`V@)+KbQK5vYR|jtb6-0OY^>vrgnl?Y`48-i2K{)^hmXYFRWoH zEIRdtg^(jRKpE$#D8|;j?0DY!r#Qf#maGIKmXBc93<(H=+Z| z#TYVK7hY67`f3W(*W`pjM@RAgv|ag_yX-oKM=;pt*6-!x0n*CIP=j;@Z^7UY58DgO zeI00iCPD(^mEDnR45L&MT`>E6>zJl%71=OujBM*=N`<3TfD434yaUws`$0@qJSyq4T}b|$519(ZszOe0 z@*2gIIM}>VrQfo7tvIo60*oR%P?|tqp8S^xoGy+1)${2$Xmz#mt}qdAm4wcuXTn4A zgHuYngS0#u7-)sAnmGS&n)v%(S%aj_xEEtz7~^J#zIX9ng5UP@bsE#Cc9}No!X}*5o!C}W) zcj?ovn0}cep)a)F0dd3f9;e1c5r^OY){^Y%knRJfg4#>TdS1Vh3jgWmZaAgeO6{3J z--TzqpZFw%nj2CVxChTU#8AAboFkW71LGv)TD$YK1t|BqCO4+~)$+%aqfXt-hz#v* z>CdB|iq1T^-+3~PQ;JpC`i&WXuUJLQ!sSvWGAVt=L4$K%PT_r{)wJb{IP2Wq{Wj|M znhT3Me`T`W|2#JNmuvGA|B~2N%GTMj_YEb5bFX7s!Xx#`RY}o{r1$Sh1C>68hZk^_ zM1m)!|6xNNyJuIlAW!l@51$7C?4GbwY4Ktz4d2+6WH$k3cMJGQDxNc4(;WpOkHwUj zbmo=6IWRk+F%>M-v0tsZLunt5J{tl zHZ=su#@7`NLbUXMwQUqdvIMADWU$D8Fgm_+4dOA;CMWox@uxvsq(CKRwDRxtkI*8x z96-N1@iE_HH_^$Uq6d;BfsJx@-RMbn1caaquK#L+*49!Ae2QNFv2aOFA$jPm7%H@& zq@xmN!TJtHa)^2p{{@@s z|F%iv+lis7VnMT1U?s};1BR7J;^;8~9^wKGQNYKvZ`+u(9h5eH!ImKB_?Z|b^Y9B@ zU9*qi8(sw}J~D~+RjOs5_EwdyI?jS6$zJw(zC`G5qE#K3CN3p8+zH02ej=ARZD?Ks zU`eO&Ip_=)GXrt~75ap+Zu`4-POKt^H*K)q(S!F=%P9NPs`N+0t+=n}GrI^9EVQp! zWP$})qyd_cN{NT4{(#y5WC_P@&@F5bYw%Zbk{8?<>`6u z_DT;}wikJWkF#!t4ka{tqx#H2uwA8A%x1x;T7u?8T~7KkDbx-W1(o$dLuke+GFoL>`snjRaI^Q}(qliLMZt**p0wn+0ZyH^k zyoYRngp6>6fJL4J&I2KvK}Y_ee*V%OW(zfz-2knIugkS=jskh%I>ZAW@qkS+mWN)$ zl{bCbl8o>HUB}(#82mOvrW9D;si1SocqSCn7Gua<6kbk*5fRqpLILm{)$S zsDY^&#BpkaF_K1WWv4vY_7f2ZjX=Bs5ft(5lIHml*rT!h@YgHFV!}T^5YwcvGie!Z z?+le1*_K=l60b@d7M8X9xx<=w{fTX8Ew-QAtEuPTKY#b)n&T6i?g=9*txy11!W0u| z_yS(A`VHeLmXTXVtA|-8;Q`OUHfNcrbqres6SO5W8iW@|jQi}Zd zVIF36uaY9ao%_rUMCaFb6_=_ys`^UHA$rO zYp{BOCdRnYdA%{NLbXm0fJo?UcX2@z3F6|owfd5?XkBq-j_)D7%ti=ZseTFp*P35( zCaelN`I6~z^@Ne3x#X2a(8-F!k8x<2q4StSL>CO^%&DtEUWad^%#lJ%0R9&d9YJoMaW!m_9Wb*>u-yCT9w3YoP7e3W@a=S8OkFFa}`Q)S#w1O?9kpN@Z| zrw~!bfx^?Uh8=psal$HZ;rb_F);Cva5Ke!VzVvm)acyn~tae$+b5P8-8lu8!C}%|k zb#T@X^n#3*36ks4nIh2h@u3roh*b7NZ+TSmFm76XuYk*h?Fx9XD=D`^hz=rrzYB*9 zR;SJUgvd`{(%}EnyuHE=F5|Pj0LM8iWKp!fgg5u|If`U!HYzY;c}rOnm_i*cj9X&X zay@ikS69cJm;!L865WZi6a}k2{O;fX zVjiup<}O6C2$glQ(;a`7$!QA)YIcVgU&_fJ>)>XKLViO1UP{*3hQGxswqbTErer4V zzk3nPH97dXaqTVs0D2)U!70D-XVz1s_2R!G)N*A_cT(dHpeDhlL6LxR30z`?qgG&E z&KIxZUitz6|clViQ zaHRz)W^gOG?);Q~rWE|lD(7qO__{m6Ac0To_l?(l4%Qt{(IpM6+-3CFx3a#fO50sG z`BwzRz>||vm<|3Fm2fu;#g@S$Lh#xlouUHogO!%NT(I98pIs$T;{v1y`o90c-~_g= z1$?#;z}P>16qfDTs=(hIL1ZdGKv5J=AmW7Ig|I3Af3yi`7dj~Rmm)xcb0FQ5$UlS# zzS(TAuJ%?w6VLiT{y{#1BX&l$z)MX}2=WQRz7Pfh8A%buuZsw*4N`<_O&wSl@S_M) zg8xQ&t!r!!Xb!>I>Yx9*>CnGn+l*$8KJkzvnsSKq0j$a&c*G6mG6^Yb{Hqhx_Q5OS z_XjJclr<&8`opp&Rl)16{!Hag2rfzBfSVF505;L5{=a_-v1xC?bojbL09OW!2*|i; z!n$7s%3dL+d|dJ#X(R#@in=sY(*Nt*mbvLVuF@b8sBEwNL)={JnZxr{$5)h@8ql|k z4I+9A`ntPW2*8%+)Q-xWK?cV_bKiKkCue3KMdn|T-w~h6FP}ZGx!Gj4{@#0so7wnv zIQ>mkV0FoTHOf+%JmrPgVyv~A}17pmT> z4+V=Aei+}QZ~6MvgUYV-rj#BYUKGn7LoIs=Ph7X=rIz+pe3>4%zSLsXQ@Z#dOdETB zj%BVos@54ie6+4u`3#-J!B}p5aU|cjmar#M`=NsWe6GIXJnnX>VbVEX`h6T>;L6hw z@7$i6k!kFu(OgS|twp7h8*7HK3o1{cKHj-va~Z;x@(cZ~UpJw_m=U|9##?h;b83e- zPQ{?%oEyQrpu(5-8Y5PhJ>jc<;9kyd0<94o*!GAv7BmkPma<9$rU^CHq9Ngo_K2^O z<>~Y0wwH>(MErL000%o@)q3xNMOQ7s4hIa|Z64uYJCH|3_Qz>9AABka|5W>6c22A9 zQ8jF-)Qmk9v^JY_L9=*#;HHA-(2;AwN8%qXoIE4GIO^ag`!rJLNijJTrkmMY)(WZ&c29v&H%H}S;{oD*-NU2h++#{Vnga;wVkpZ?vY ztzPd>ztK`xP-^8kuRpZk^AdfRMCnOa`Oj88NU4Se4z@|~ry1(cY?)ShjTEn6bp5M> z+r6*J`E|R7hRbwnSxfsx!|&tq14W6lA9<2hkz-fm9mRDkYq6&>fqEgS@6&_5d8>Zc zJmQCJnyeVGa2j)uEFK(F=xxPoCQ#D>{%{(%yhxazvg~a_Ux3G zq)D7CZ6s7~f#qSLxwoWvo1aEu*z*mRzprxeuN&$vRQrnmcw3oiZrDC#t8vsL(p+Q9 zkmQGTjqYXy^HT59N@SsMRj)K}MJ?vk_jzX(iSqh%r=7NOw!FQb@eH!KgRFHS;Yw+{ z@uo%l{H=2rJBzqT{Vd0GhCsb^r-@v>o98`h{&L3@J}zo#>veTGn~a6bM{0hh{U!Tu z@Yn|2BiE9SwAjUJy$do5R3{vKJZTxfF?aByE$>rjhOI)>sH>>MlHHri-{Sk)e3rVZ z)@@Gs;!yAcm9pesrzO8E`*CyZ>Y2-;HH{wM^S4?!b>2^J``xm7q<-<1P6GA)&_cfc zSO5DqlT*Ixs?WK5>htb9pUbvjb*BgShM#VrJs}UpPx{La{E%(a=&88sF)`%M?1-^@ z_piv!Q~sYd=XxCEPrt)|-9EM8U2|*V{$D{jZhWjZl*<|Jf3Qp;T(bCdz{Q@4Eh`^z z+e1&aDO4{V&PNu1cq@dB@o){>Fu7i|i?dZvjS~j^OucA2uP7{e%%EvUmGpi&_cx<_+9>@{qO8qvF(L$GI+T@ggJOEBRoK zwp`dgC6lWkzljlkdCu_9I~C#6)H}O>4@;MZhhpx(VPIgliMD5E{gc72cMQfWo8?SO zYG1YV<4OZ}$;3m%>-)+vL=S7iWBF#atCcNZT6**dUaX2z{GV0ZYyVof#5{zTm3|yf zwY`lC{D_EkjhFJ=LYEo$j;X!B_g8Td;y{c`lD2-%E1ut$o%(~Zji$3#Oa=+?h-Xm- z^Peh@RPbtBzi;PF`WH7m{XnaZ%N6sL`^ddXE4eWl6Z8wA`}U1p?C2o0w?ybD0=MKK zH7iGzog;EVXO=qb37aA>Ccd$qp2R9E>g^(UJbGai>lq~(A4G6z576vbt+{md$qgmv zYUuJA_dz#-mNL$rpr^0V@_bEFi+>MClU7J_LUm4&Nw}1zQQ)XrFFVvQ0(~Xc0#MS>D>9q`hkWZ zU&Y0q>tmO0#edzzAp>EcY?9S~DQ&M{u-tbt>J)k&SEvFZnDMeP}g)-w2xK|Yc1=q)}Hs=hi3m5yb zMd$F_^bX9x)k({4Mwn}Ht_=NBVr3mtaWt=w6bM;-Wh+UKkI*iu{%DA|OU=@iFwwvL zfR$KvkejO&T(a@m-0lg>-KC*@cN%)6kE(J4l%nJMb^Y6(EotnL^!|iTl zW27f|?lWytGdn#B^z15Qd6p4O6BOEuPUk)+P^1}+CUH9{4fMzGh|^bW*x1rUF`~t| z=5VKrljEJAQPe(qgfDg{a-WiJvoAT4TZ;T;H>M8sVcB}@i;pK-dG~scBi9kVqF_h$GB&_dIpsc#uad z3eVmOY3L#N8pLDI+Sp04v%7r=H+?lCSOzk696yhE3xkj5)>oj$a=y^C zxbDR5;BLcs!$Dd(3cKPv?wm+wYY<&FPcLRpsMag=x)!G{ifB|-OWBv|WvT@xK$!(K z^|IHO@%Nq`fd@IqvnM@hXxlZ;GQ|o=&pH9 zbP>nl`<3^=x#ITlfEU7$rQXTJx*|}Y~N?gtzkI@a^yOEmY+^=Tr48IAHT-9 zAM%=}Gj<_I+FdChsLbg)SvGZVJDqfR9iHdDx*PZu){NrzvfW-GbE=H&B4+n#>~ptF z1C0V?8vwhW&t&3x(j?5d<Fv?2kN*l=D6qV?zclvCAHnQf$rYy+z*I$)alhkf=cWeEUcB1nnSO!;~c@4f=1?b(f(52a_u=GeAAMMuw zJb?Tn-f3iM;Wt|^Xv8aDLS_$rUE$Bday{L-O2?E_wc+dJxAuK*1sz=4+M;RXJMR^@~JaE?}%z^joPchzuC<{P+pnll!~gQKRPYg@=_ z^Mn&}+&}7QB!*0cEPD#_MZ?E%r^7qaXl}qj9h3H)*Rl#!-sR}hJlPX^Ft8YN9#@(+62^;``x=nS)t})g z#)ut8)TDn0{3L9L-mks4kB1&2+9!s)APW=L@qtujJeDw1LaDeM$S}p72^eQa@2A_B zcTJZ<(%RG1Gr@j zGb?+?0gk~qv{X^6DqTKN+0@+-#cpy;a7=bRiretz1!?q60rcQ&HtWI{#Ai4MOH$Vv zh$Xm@2{V5Im0J5bP-6vpphNs7k@!&w95RwE{m8M40?H3s8T2VK?N^Z(r~n=6Iz+K} z?$I7nMTSH zoGy8flfJ$^y|S;NGh^an%=SULl$pD0^e>5T5gfG3*Bo+EH?3EH7G8{0U5f5EWxP3a z@XUn^PXZDe>UlgzI(`DHlz+xyS!6{$Lq>_&ngUq=R3kk9Osbapvxwz@{Q5@yuE~iA z|L|akqb#f6Y=-M>BasAq=wQ#N-~8)~xBYOHT|+(OOC5cwWWNy>ZLDE&QV=|PEzmgW zfLwUCVYw>fmEx^ycWB*%DtSv`_ScFVaGuXF_wIl7OgtRz!3$!XgmUIzk?uBwk)z)c zKeaf(dGk;|t6ZlJm=U)FG7jL8nMcWEQNz98T4y3p`nT_@Dzru(yY4c6^R6Jq1tA~r zA+%IN7Fts+_CS+Ie+6~bM9Ei5PnW7U`->s=!^6nJG0PqCUee$2GIZm&{?n{l%v!eW zq7RYRJgfcDVqE=s=E=HZf83y#gNBZrpT`JeAwLU#w1Jn+J-5Mz&kfuCYfdk*y^`k& zbY2I<3^l@Zt_{=3SFHy-)+4?8Vm-vxhk$=9!gxMR?n@lE1=gW z@a=pcnJ-j3LSirKPAZ`#+ATx33+VL%#oaogU0Sor|5%j* zddQ}h&L*!O1sw+;qC#;KK6s)i!iP{$co*7e5a@L!NoX9XxEy#9@E68{>RKRu6%lx2 zfcGcBN;*QZr0@OBH9AEV2qf`j3)I%ny|YCMzj}EFV$2u6tWoQ zm9&nAofx4-X!*6XPl^`(fj0NEA(Q1thA@QS;~N~}0k`RQGO#Gf00SizEMN+30g8J5 zD~^sksu?h6f^G_DUNqf0d4sM|m0xP#KSTwuP5a&e(xYpF`vpVPq+9^d0P$Qq`+hNe zL9AjnJH4<~mYtq3m;Shakiz}#AiT*}*CJQ*<{^+`{H)F~Ptww4xC&S;M9N1R5qGesUKkO4o7iYr#2)?tlf$OM| zV5sy3D&x;M8s0qlfXud-v;ZAVtG9D6qf1 z{7xfvYGV3!02K!$yfD%MMS@~Jv)<@Lie!`i%Wx+_n*+r_kOcUVCA7m(+D4;ADIp=;7e8|q_C6# z3S2TIW3hG33 zk%0AqpKNR_=bRxx^%S(n{AFl9dG(?eEXE{D%*Zoa<`yc3 zL0T&a2~PdQ+W<3`0qKLrJj&d^B6w&qgZ8lFAp;~K)7c3E!N0Y1$!MjQ0lRha@{528!4<(Sg9i5*t(5u=_Jrer=Dfn;kuoyZgGKs4 z;F}qaCnDcq3SgB*B>ojS;!jb9+&N_U>aF|Hctn(l zv52vZ-_EH2NVq!z|h))Am(Qxwzr8_VAzo?)&GG4zz${-VI&6(O`F(&nAaXi*lW1( zD;jkwDtK*0=mr9hh%cz+`>_K16bQF?x z7U|!f7PmJNqb$=yjXx~q!x~BOM=0k?=aX0A+@KLKjQ&``77AvY6kfVI??m?46LG1V zVeYK4o(>qE{vhJ(8N zI1GTGRuEKqal9gZ_B;PJRye~UmLX-2sLsPS-Z)D7f~l|qh7HX3@UhF8AIB9aPgI63 z1&ZsBU>d0Zd7{QY=c5E{TAEm@si>wsSC#9Kg6+j6a>y;<*n*63Ghy}DgGCu)Hr1&b zRRopTrDmcuYjTNSZY^W%rDd?h80G)*d^K8jDOeq1@5hg`lKKvy$X6T@!K28RU?QOV zBLIqg2uKgDJgzl&u=;SSK)o#&cha(E_+AT9WRMd}H)!6)9uf_(?TPQgD; zevs}vH@bl70|}-3=pt`gLc~3U-3XO~U`ng{llrmjQNkG9zexgr(f2|Jk0JQgy zV@i=)b~+H?6vZ^nLMM=si@*iAnV+nAo>ZAoZy4KOGWO&biJ#B#^;r~C;}gaB6&JMy z77@4;6nh2P_{dM3oysRS0dI7ljYS63j-Qdf>n8l9+7u8fq5?v}&_F*4uug3e0k-MQ zKU^6D)(NWt@G$sxI$0BjK-2~9+OcJOB#a45_6poNkRR6H_Cpq$(dmvt#9&GyfMMZE z%np+HYF;5Tbp=`ILHj_C&rx+c^}nT$!qxVNh!AG>=3k8w-`l{LKd}W?5Fi{Cuo@5Q zDFJdTTsZz04k;*5`Ka|zbT{w!wFmx#vjb}BCfLf8MGb-u&k~g3B#bT@pt=OKJ^#MQ z&j%_XI{Z?0D_9X&`oN8cXMa-82*Cfn@W}of2ZqEi_}GIFegdBVU%!{y%i!q@WP-cY z`S-oY0hCR}))%h!&S{O*cCm>IliQ-E>FOC8z^NX#*%zVNlZCa91L*?Cxu!9okDNE~ zT<==gor1W5K^5w4+O7VG#t|wRdw*(P-|v7j{!2|PMeUF8$_k%mKStIvV{}Wo{YmW| z*Xnj8OpV{rU%KFnoWrhp(#yVXaLreh^BX-s5ML+SP}g7UT+6|oAlMAhZF&Y}H@%6J z-@b;@K}6=x^z5+*S2Co;gLllB;j8&DP^Oj}eYl9Vzo}JvcIafDd$r?wO&Q%8nU9W~ z+Y5QQH{+YS{c@tVW0m}5h`%Eh@nindH@weWJ{=o-d`9z54`f!m2R;i(h@L;}>=87@ zsc?Ss86hi^I?6~>?`HD6*FDd?>VCRmoKRa_tlny6UHP^!?A)IGzKAdla&_sXu|vG_ zUv}RMx3SOn-=ep6Kg?hE`eaOvGd9X9OrOyAWRynl)Lh8V3{^2Lt1Kz(tv%t(?I*S; zD(5uOwD&dWKs@B0fSe2_F7r2{N>bNqO`jKXHw$N^XV}fs0YC0M&0TFe=C_%f1W|5Z zdQx`DPjBUKJ|StH0(DD&!%P zv{?-I;~q0xJGE0oF5*mU_l2w)(Y_53Cu6>6WFew7!MUdY*CW>n)u*pb-=ZIO+mV+! znFN(ZsL!EN-(OjG-(fv%YQ#L8@W|7U$Ey7&11#HJT;}$CZHi50%TFbQCzPBj6$;=GTzHjbpCmOvcacmZm&L6n^ z*PRXI*rtMw?&mh7sc$3MU5q=^VD~Cq%&KRcGJg4vY|gK_&j&9uW!20-ZbS-?xJv0t zz02Kt&|&U?xks#=`bq0^Sp9IX8#ruV)F{_6{tCg3$i9B)BJ4-^c*Ra1MmIR?vG-5B zZJ@}i;ajQ=TT&_FPyMb+CFC62-^cOszc2+1+fDv){d$0{ zt-?E^28eGvlVsjC$4p(n-h<@DNXKd4#}^J<^gbi5A6V)g|Lyisk5L4es_ldKw?D1L z-Hr^=^zW$LFr-h|RPX4Qy3W}vuwl^PBYSLl*#@_!7O~BB-@8_bUi7cIG~3^mpQfhd zm_O~N7E<4C&Rp@-uF`NYEK~x&?``K)ZuN)1S$8+T*JS0nT$WNGMcMWD`F;7)T6vV} zxJcMoGm2+_p`~G4)e3C~dRWJv&v?lb5$UG&;D3)U`i--L$UO*KDWDaauRIsG1RU9X7AE zk4&vv56iW`_97MkM)0}n{%a{mGf|H=#$rApb@KWnWWxV~+y z|Gv%xFVYqV`s1i3pOE{#hparOp)j{(Q5PRG_m-PQ%aC9E(T?~|_pe`@q#2i=THE=){R8cT zx&5exyy&qH_AjEZrXrKJ4Fm1BTO$n)uO94Cm~|Pyo>7`<^**Qf`V%}HG{43(_0#J; zS_U;)lb2ijTSj-t5}%#c<(#+OI@>cTxCmo2x~5EYZ{0(*G;Vu%5SI zRS~K5p7`r=#7~rS@Gn)CFTqZ`7eji1kA5F?Mn1mxD+K)6xbgaqSad&5#bq^&Sb0z_ z%_2_}SL=kGaSC5LXw4f1Fc);5@|sPy$Jkya;7J@O%J4L122KAfGO(>GB6Hc6nG(%~ z>oFaEfAIcHfFsg|yF~S{1s6PdbZa0pwa)Tm>$?4@@vE!l@gq97Ckd*|D$LrPZGo<0 zy&?0tq1xN9hbOnw>iQN^wzpshmOq?%Lwl;eD{b8PP`{CD2zR1M{JEFvJmeSoa?R1@n(E-`|JS~!H2i=L(W?%RM)qpK4oyo%07LQI_2ppQ}4^3?^$>yb&=Oh z_wgt7)%YZ-Km4sAwsF1Wx3wdSy+|Fo>)t$^PCJsRFHcl^Vx|E#$EU&IWn{a@TaWd2_@KTTx?K#MXYjJ?8jr-rRQX!92BHc`wbk;9TkUQVWS$j@^%HD2evS zX|!kH3o|b1-iu+PeDn8fX~ZIT)i1nFuw19l)^*zke#WUwQ*R~YIq9#8rhU{xd{5@| z@1SM268p9{AY=@$YWJLa`7us-q4r3MJ7fH+d_g&=Q#V`w-n*U0B~qDN<@#N)v{p0= z)br!}Hm|HADs%BVZ+nheL^`=Q3fRqsS%x>oIQ~td0~Rr1i_N&K9&Wk$x$~VzwUjew zc7)u2d+QBVqggV(ORD{AnIGFigY^uD^j)%F$n&*oYdN=gr=*A1@vjK4c$rwhj~hH*MJSTkM8=eEjM+^g8aY`3tF!4B2vt58$IE+^T$ z>t$E*TfdX(Zoiz-?#B+++`&nc=)QBH=B^n=TMPgm^IeJaMiisSTaG%L#b8x@&dixu z)Pg5u?txsh1OAO&i#6vQd8MCe$nIwH7I7s*b#MbJez{%@bThy+S)ReoYwBeB;-68D z$}(=*LQHmQ(q0Fg^CG4g_Yl)txA2;k8KW}TXy7Q@TQ{oef}L5&;(cORFM_^2Nsj(8 z*uB4c%xm$LWk|oSGp(_7Hdp#*chPd@{AL{5F-tEa+ju0^DP+~GCw-`vo;LGuP9s$Z zH@D#-oC`BVI$P0sSvjdXXA!hn<*6eTpCako>g)-)f^+y z=D-q1$8wv`UPo-V!SP)@ob3{vi3!QyNY5$=W6vg2cM{{ zE$N%C06L4|<7zm=#XwWUF{K41J#7(?8=Zs zpS$4#auloTcBb&^Xc!<$L(;Q5dy*73-23b&K_nj#ce`b2I9R7=S7YHx~|CC{hc~58V{Wb6PyE{+2~5%5)!F>@~@VFf6NlMR3dm8N~i^@xCmFqlplN; zseJ&QkjDr&C}N4nD!3(sdDgGTW-RS_zrYHV;o|<*wPq!t!MW>~8xj*7BfABA3@kL43vGjHl zH8_(DyU=J2+R^Y17X{TPhJw*lsR_6BTs_Pi$CC=PhRfO z^61$|uOukVHSX*oOW52t8T$r|aX9{8@iZI**#_@qa@(Iu?Ceom=9^wpKOM8wZa-y4 z>fgGI?s|h(*;vQ^R%0;IILg91aeaY~yiES#RDg6N=Lk>jcpQ*IRidRnGHeaHP7*#l z+yyi(FVuaM_4Zg`m!@T^l&&~4>bH<;U*lbIDN#s&H&_~{6sEo7HJ8Qc-ief_Ftjp! zctw*ZDC{Cpxe!9xVpev7H19s)GvVAmM{?{afDhv!bV%Ub;~b!CnBe?t7jK36#b z2t#K`dLqq5bMk?%10kU&r^oz)c;hsWLXPuJXqpU@8mXo;1mC!^M5x|tR6lVUDBKLB zf7$H)B~d2OJWz9g@9@*xZ8SQ&&%)~<=KC>m^GAD2AqsCBr8aN8JJ`Z-v|*R|kvyh$ zh=+i~6=vL+`^wL7sBg)1YS@r^MG~2)q_{<2k6dBWgg|q!z#ao4OM5Lp=iNlhhKl|? zi%*39+%Ql3>5UOM&rjLZQZ*atd1Qg2Y)*$wHa`j(H$xjx;76A{CSP%!QHc9{n`V#G z8f&=Gqo<^JwXVu#UhaR?0L-1uxlI9ko$AaW?J%lf5~n3yrHtufDy!^&P{zxBy~@#A_2N#X_5*@y36GGjs>?Pc z?6$#fUL;$tzf9I8KzM2?=i?EUp_F;6T9X4(71;O#sj&K6m}ax_y6&o|yD{;91hogl z^}2fdgm1`vqP&==!zjGYc})|0_KIL#oH}~4tn^!#JJslJA}0JheSti^OQLV(ES0`% zq-UCHGr-<3@nH~|VqEV#$BrpA`{cpp_7R_RoIfSCds8+}mAO3MM_bH#?~XXlOMm;5 z%@!S0STrDgF5UDwq;hx!6Ec@f^Dv570N3P@rU^`$Va)kI6^F~hrR-hiGYE=Zw@w}n zSY0DW8sDt58)SO1l#Yir%RbVyyA22DQ6!Tcu7qIvYjn?w@|%eKrU>-Q_}wA^FZswG zeEDf1tE&wzoSRaDC*i`^|Drg3Gu~Qt7LGli(iJqE%i!ARiZ|4{_tGXM9~j9 zyOAwHqIZk2**X!v-W-p-GleGHq`F4hTY@!kXu=dzLYUWJ=w>a)qOImsA;@}mb7Q7~ zO6h7{I--S3QJ~ZEw8%9kEjxo z+T#`bF~sz&<)j$cVb$z}TBca(`RawAup3$@!6&@5MKqtMI0(#ve=27cm*=D3c&xir zfGYg`4Oc>Nr2meTmE?DQ3U?IW@H7eg;y8{Fb^!hHIR)oHY2JN<982v$w*|#3!=A$2 z#oH%BIi0KocF~(!sLnPXG<Cw?0Eb|SGy3Vs+qvnQR)hXJ_oSkSAHkX)WZX+d;k9dR5k}BQP`pA*5NMzuY*PY z+$VvVqswE~CJ_?sfvM(7;{6hQM?^=6u)-jL98gjPo1g}^Z~zK12Or3${PgnRV;Jz~ z#1Rt1K*|cho3G1Rz7+Zri7^Q)2Op!k@;7f0tCV>L>|^=%Vc`527JN{6S77v)BqLyk z@CVCJ_2Ab1x5e)#7|e=)>kf8(DkPrqa0RYO;8v$ zB0}L8I8U5~(AWf6?$HjRz?iaFXjco-T6-Xx%SH1t$8V5-4~at8;(}$w3$Uk2LKB8k z3qZOrFf%C(5Av8HTRR@fPK@F$BTVY^D6}L-!}E%r@$*mAlJ&s_tjc9sG>_c<+o zg`ABEWKaT%y0SkBi5^*x_#La~cm4$ty6tj8Zo%D)tA9Z8qzjaWNr8S3g}Okaxm;0$ zL}ZiJh5=O4&Tq{|^WOx>G|}j2V*%#DceQl+MZZk?Gp&-?=V{7~QCTFN5>$hhAU{F> z>)A~#cgFEVLT`39Tj02Tb~A`(R1!!U^ELdDC6uuUbY()l4qr42$R6~#g^I1VeiS#t zpkS?~NIrZIr=QaIKalwU8S?v#583UPSY~F9(;U`Db{6ZRON6qBkn3UUO9EYFqKNt@ zeqf5wc+hAC%1p?589Er-*ll>`pp(#>5QI#B<97WI#%8p8ZkM2zAO8tcUJ?Lu%h0t{ z@d)U89=hFo1nVt~OC?0QzlJ{|0i8c3klD8)e3<%{gcZaXB3Xdlmt_m6BtPjKkf!D( zfXevax@sT4q2A{GQ`U424MNQzq)QC#BB+dDD!2~Iij0Y(p!tNhWHsObC>EeCAqWce z`b9_Z?FsTQu(7#9io!E=RvbK;Nk<+^hl*(4w$S|E12S3g<<vVupG~7EqxVCU-NQO69bH|A_$EIQPa_`!q`Y*yvFt%(Gp#po4l->Y!T&_(r0t zB|p&<3H;EN=QvpD#tD-@!HLOmB5Wr z!j=~grUn7cSC+kXXN@d1OU?}RQjUR<53DshORj?MKbRE=wXFhgUWhGxa6mZxv|Z3= zv%&`h`SK8sCL z{7^;sVY|lKn+A3LGg$#YL@1Js1RhIBf{7I%NQsIi;WO7~_`;EeAIqYEAqTkaaz4x# zR3Jxm0u8mXDS4CFe=~7{lh?}Yro%zz8pk2kA9Z&XYOZ$eaw3fdMvw z2Q?rVqY@y>@qwV&Hr6)OIIG^nBAO>M2T|PU0=OqM(vbBiOg5>h3vk*`_fpWO0ujhX z$hNTBLA$7!>^X|GVmgBBA}_p{Tl4c$1sRHhZNTN01UhyUQT$7!Vn1&JmdYza4EH}) z_LrjcVVx*oqDBfo_NepK_q-(P#`0!2zi9J190!gF@?tI^Cb0djekYp zrUc44U`k-_qp9hwifB4$HmtOIpr8|4fad8!9V7Tei7U?pADR-(|63FzLVQi12P zNgx;1z#;Rk^TH=g2gDujH|-_;LIi|G#4&1)#vOB4;S|*k82$vK{@arko-z2nG^5a6 zPd(Q`ps)k+9f;v1L|C`=!xx%oIF5qdHB9KKy!;es{ZPH%|LxaLgzZ0wS~lD`ROw-G z10w&m3h2W6D}c`gZmWVCMer4M*Iq=qXg~8JMHGJ$0RS=nFS6JFOV}61pUlEXhF+gQ zvAr6G4T_DUf#qPK)3ofUw&ZL9umg-Y-VSL*VnOTbH+{Yfc)O6NBJ!WvB?5uL?x!=0 zQ7J2bgZSm{Md3#VuMys;9<1MoG+=-2gM9Kv-g^A8Ju9CXYm1)!zdEe~iU%{+>>m{H z{Qvv4#lJ%w8_<|&-Z_`QeQtf&`Gm`sm;0lqmP#82dV(~6Eab}uUJTIrh_Q2QHwqZ) z-1^=As$-Am^a%@^R@{`A)SA|@<401oqoYdGZD#kxhAJ1a{}{p^$bZSsx%IpB+_VW3 ze{!hH)vaeMCVWSuWA588f2u_cnKCL%-eW@jwH+@QYW;9#leM1DMdf_Eu0J-#ZGv>4 z{FTw);Y{%*%(eep{!M0rS4YfSt3z@0eKdNj)WWb?%$s9G=Zt_?5sSTN)?6T{e(Rh& zw77)%;8%9uSjXGb%cnwKPwbK}S4na}B``s}&$|Dh6{hzWJazIC^HEkV<&o_c{UUKKPTBK0%6bzC!gp2Ynt@>J7!2N;^IT$ob zkq$rGV&9m%JzA;nUyg&{0xv6^<;asT;`UT%RJI(xXK3H{c;;h`_x`FEf8HQ?Kk2Dp zwKuNn*kgF5uvO*u$0Ok{T$`MqwW<_ARL?6_keVdtYcD$2yVcQ`I9=KvTGGlmD-4gw*9d|;BbA6#C^09Ln)xUo9 z04j%znNP(^eOtMs7s($KH zwZ$hJE@1SF-nAc2D-ApIict4rZ1z>Z!nNc=+Q2{mfC$dIZqXp2x5Cwadi**y*nQj0 z;qz=#cQS1@;@RAN7f+~f)@8Eh^m0`4K3j{=%*peb+V}0}>So=%DBpJbE$vMZ$FIEe z++VcyQO{(xK+ z!npNqk6$T5EQ9)2t22qLvkwHX6 zjEWRN$OuX=0s3Jz+IR{6+ObGobK)?QusEg~c=ZqDu091@X6bzGam3-k+EtSmVaptdcC zJ}|VNr*h(9o2~J4LamtjU~MtWF~W{xiR{us((c8XYVf8jjnqxQ+6s~>F1*q3 z-w$1a<=5zYEu|&v5<)fcXjIXCPq}_lD(w@juPl+IWIWyh+tsO(*QYBw=ig6llK3<8 zdzi!5s(mD<(f2T;dij3Hk}bs;KdiEJ_3*JD=iP3=4Su(3GO0hvA(k6ruO{>ukn&IW zTJ#x*NS|_-o1HV7jZ*8k@@k%_^86Iz?7S3VQAU1W@Ghj=CSU!DamFFC!qPK;72Mj% zi^)Y7Oxc9W5%oz-*+ zNL0*lV1aK>=dlJWchW}<={-isuLe8xC0_&=52PC;pO5g(yCTWlXubXhD z)0*X{rs6jHr`4@xH+I)A9k0IZuh;N=TkF>xC*`5ju^~<@?||*Om!mdH*NPpkdJw|< zrex(auj1!aw{}N2#TUcPU*tO8n02*zn2pW$yrrRYf%GMMT6y1q7L%Bw)u?Mv=z2DW z@%5MIY$z?t;0%XSeFG>V(wkatJYzDNKJ&!O@hD}1hPj;qzz#?xFfO;mbr8M@o~Tkm$0v#af8BP?yB zrp=7X;egg-ZPL#92S`J|d&YijQ`$gw3^1<9bWmY>vdKC&oVcl1ZFc-UhbN{_HE24b)QdRRW$^2@Ut(b?AGY%7k1>hj;DY6%nh z4*Wu{@^CX>cY&zX*?NqkYz}B`AY^x?c6P8Wy2llO8W#xKEE0VR~Q%+ zww6;h@`*mS@^T4svOYF@h^aW~E;I=(G4^N>QMLXHh=3e&b6mI9#=90Fi$>*8UKvlD zxH_(P^c)Qfbw_A}Mw`8iCoP4(Rhi*>7Y#1(M#z^4<^0_gHARv*&su5&Sq9-1>6w#_ zw7$eBZzgRy3v;gHdhz`B^=YRYL>EDvCn*8lZ@e-s?a4y{8#52J$ina zo`9BY3`zARMaA3Cul?o21`y|fe`rb>V+a0#zN+6)&eEN_nIxVv=_6#mKZxEp5_|g3 zpP*KBKJX?oA(7#fGEo%9Hoq-%$2r@~**b9KsxPUp>WGuYQZhX&VJoflJu^&$nyoeF ziu!w9bUYlUGd5dC^v*ThsS7JQZLZULlhV8W+-P`>N3ASLf%=49Bm^4<^IIJP<8 z@6Y@~QEa%_^E>N3@6mkQ_$;}R^2o6&gnWqDjDh~V+osxI^~io|$*$zXmEZt+I+e8ZKsUM`BU#Rrf96KuBB{HEKH+^4#udu^!VORh4( z$W;a)J*Y7ml|w->y*z_5J`6dgZV%*Dxo~~% ziM*it4=(2{_3b>Zt;2DHb`#4cweBU2*0&#toj^44wOnOHg5)a-B^LY1)~YeK1x3nu zXNfg&aRY3uS5)0`Og`_SeqG<1&Mo&F%NRIK>d&--;>E3Yzb{lyp`CiP5b7C^m9$+i z<{UYH*D2>~tE$Oi4d z#wdEn?ALCzt=e0o_+oyB)v)!44N=>daDy@V-kEc8;EgJU94pp1??K`g(TV0}Hon9_ zAn`w{9JX6GSpNN1gHGZXbR!+@AZqkEHpG~}U37v9-gmnwEV-Am&djbTs_DeLftTsm zAV9Q%K0gziu+wZ$#b4)g;V_i4*BaQUV2)Nvxg)eXTIW&P`#vBV(M!hcY!loEZ$x7C zS9}!|wY8SsLJ|lRvjPRxf-FJg1z^PZd$S)|h_51#b(z(TFaS!KF1Jq@r@$2Q6uTpX zX+azS2O#X(L>XbpC=_;{*laUlf39|;DV#UdAxL2QsMG{&_jADD^Yh9n${59L_z1`s z3+7k8-XiGPH)AjqUpyfZnn$eh`+KFAk`i`Lj~JExBVs(pxs`&{!wfeN5I4cl9;+I4 ztKQ{~6PO;o{S~K0`$B*3$(wv+2Ta)r21+Kz9s@7bgZmfUkDu9VbeF$cVpX{`{cBIR zWM5A&$P`L*boQXL1!JilpFiOLkH~K55Fira8>Ydmje0+YgbRXI!Ug4aeSy?y{7IKYj<_RqmYBiDn#-%#7}mlBQe6**_^36bY{&>F$RyivN0n&Y*bVS{13OZ*0X9Q+ zjysd~kBE!T;Ywt-4=G9C9ps?3{1)bod{gjsu4 zk>p-Y=GZ;`$>feuMGx${ih~YrNU0$Rs><@hzaYr3lAJcF(T)4W@28en731hdYNNs$(I?>B3A)D7zVk6qM=b1doC%XTU zr*`cf(ff9c;cz)z)cNiSOS|bAEYq%f7YGZ);>xJRB~!_7zVocsZLFO9zkdO}0A|ZV z*$C|KOsTHQ%U^F=7YL@zkrf46jspSC-gTdQL2zWyPcI-$q!_^`04Q|13$+6 z0m-%)vC_D7%&Q|%s==Z742a0Uc^Iq`a(#RsfdUcdVg`XK2P6P}d*e_c_~4g{Mf41z zvH~esUaDx>6-RIV!tjQ+H}AW@3MULfp(uiYA8_N0-BoB7l>VCzm6KFq_vU>UGpZvr z&Aj9vg_#x9;(x*)f}NeCBe=4oaw(Jnt`$%(L7~g18`Fju^FPvbdgLbpu~n>tk5#hy zmYza~)=<0O6xe95i`D|2K$G`ZHrOETHa|R<1p4d<@b{B$yzX}<6n$hQW{UqvMnkoM zfAaZ*FrU)#$5}yK5o}qI1Skui;Q13;#%^BoBIduifI^DY-v>QwSgueO|BJZ1zy};N zP(MVq4T-BC9SvUcyyW{BW=wwK6`YC*q8$|~(?m#$Q|QqSRUGFa{UBe0S{J^I&S49c zbpKOQ10UN0Vme=oB~a=7z!yRRfDs=@$fz{QQA13TpJGLF1^B@@?x-T#>~(>Hbv;;Iz3rNK6rRVoweP+TO33;jJ18f#hkQL}7>4sRbtb`XUCRF)WOmK%@uf zhNw`?N*5GFlM`--otuC_MA3P^!Ce!AqC17bUt|kdGqjfTE0rS81S&M42<_5^Ra$k59PBHtWX?hS zOn{e^S~z|`)ZIl6oX7fdNQ(hH2W2}Ln1-E$Tv(-Oqdk;W7uOO5!)%O(^<)utf%ym7{;~82cr6KYukQ0$A>UBWC}r?#fSiO}PC~{4&)# z8u8PHJH>v}?dPq6DeJ_vk=!!-8jj;cn$Q!3+rAxn?pYZW1P}AvVaOhbSjzDy!mkFC z2w&MNKN9xDN|@=IK6z4j9Ks^F$g}@Rdxf{#ES$0Uch#?Y{~?!=vXF%t+Qh4l%qP)o ze>m5LCyUbfz61A#T{&P03Sg|Mgx<=O<1S(%j|htLLR)KLjvh+Dcw1+7g8z-pa+{DH zaq-j37omy4>8En?<^9D8g}dPH5d?Chv)YRlLX}{_zr|6Ck#N_#;8Q0>a7=6{=Y@bA z35Aki0=*N~ci{aBs1QYDup&wFZ-O2h^&ugtyvWy~3)AW3A?FiJOi_d|{`jI`4uL0& zm~i^X3FhOHGGqw3481-|LQNi@tfB#-%Mn5wjC}%aSxOKg^nG7iBzVMJ5Ez^;K66J^ z>%a)1jE@p~2eMaZ#c@o(!Pw$B-~N-YE6XV3`;)&Ci1N%mD}m8vv1L}xJB_pP4s2pY zF;t7rQANYX*zm;ik&r%A9;#4F!Vb_W%ZvNIY$2-HdyCDEAA7eN2maWWl!t=r@gsou z!b*M)UP`wj%M@8oP7>q*-j%#g0qda2@K+-+i`+(TQPE*uAh5c8V|s5tA@_vVQaaL# zi5Qu2%jpTfeD4Z8B&E9(S32t{z6k#^!1?}0N>4EsO;bg+KTsQ80=s{7L?;OzhQXA$ zS+6pKSMsyK4)mW~50g_lsUrwG$^^i666Kk&TJ~=? zWMB|~SkIB2{@4Y#--^>;h>%Gtz-`ceA$V0NALI~QsmKX5N)CQCDV0+d(bz_yByd2b zus*2flsEq#FvY`RXr8C}$)*+}7y05efqG0Jq=TeGf#1nRUAdlvP_HN>QZc+c8}x)r z>=q&8|#J3M~8A z3BhDgx!m|%eJ29* z%usaK&tCQ}~gPn8BZ9_$I>wvrc|cAm|cNeWwV>3Mmu@3cZMRvwnBLCG{O0 z_EvXFMrU17ge~}mTI35&CBDNo{Z8>=A&w}j#_|oljZij^I`s4VCEmb}yIH`^1q14S zVjjE`TqkH-kQeCR_)Qyv!6o#};19I_`|l;VZbT+-#Z~cOxlV6|nuU|$R|opIe^L6` zQ_Nz+yM)(qOcj!$Un#Ofy;*`d`o6fQ|rg>Wff16*9`F>|LyTE zW1ElSw(94&Vg1CO+-{O6jdcD6r`W)(yh&cQ$Q}Rd9jRuQq`H9pU+De3Xm4y!?y>r! z&~0>sy`ZXVRH9i0&mS4~u8<`+l~6UEB1-d~+F!6x|MbL%{gS8PN%J6j{+#0h2~n@L zH^mGNZu*zu*wt?ke#)$NtZln!UC5i30ZYAZQb*}pJtj5Sr(c&Y&(>D2vRNwC4YE(Z zJmgUR#rV7r=Ys#~EsmdWBo{GB?kgbtX0mhTKu6=KBr%4o{`tVp+pf8y%fHD|O8bn1 z<90>|4O9*Hj}Uh-%I0RYzcWs?zbp33Tw2VyauGxOJF6k$pESU|cNIjXRCnDo%~ z)~-uUpHh%Xg#$2i0eW-~H?SUp2I=b;t;*XHx( z`Si8XVIPWlZE1e{Lbt{TS+N9mm8+BfTC>IVW~Z_-+`^4d5s6^ed?$DRvh!ZitO2Wz zXt&{bqvP^tmHI-@wpyuae2%Ci{$2vUvYQ45z9V|QJDTl~jb$-Jsi6Ef4ju+tZjz#i zZy3)hF??1*^FddQ{Q1AE3qJjBqEuyl!*4I?{oL!qcTcz2!EW822P(A21lGmE_M{+} zRVwfR(6q3{>R76jW$3&7(rX_#dH<;S?Ngr;tBY`UN@@aW2Z;9LeguWx%z)Z=2cr2Pepa^Wu1?Ilw}Fpvo@RX6Z+qFW1o<= zza?*-bItvheOPn*neYM3qQU8($|kuY<%%t+{Hb)vlwbX=_jV|FU)G_Q4XPHrqGMh^nz(1CmSOtMuM- z65-);?1Dk4h*?e~GqTc-6I4dI+Nn8&*>pvbTlOAy+?lat=`5z6YUefo#w||idl*^- z*g<<^f6mRh-ZIkiI1HY*kE_D5Qk_aZiH8sWmQ0$R$!Q(j^rytkc=w|mYiSeSG%SkC zg6NlJ`k!%i9f`(No1Im^z3uaLFMZER#f_VJOfne;qZWHH1XH~s7wMrU^AD}u;r$N? z+4(y`L$7e6_D@;HwLeXJx94oXl(Dh-*$(#`m<>sIgV8=Q#Xg1wPm3mTx~0oMM=|ep z53DO6atf1tHWY>VhHMDzlC{(6>nj~VKG`P76WgDH{32$a{H*xS)zCjagyV)6xC)N; z17;djWl_$DEK3gumxYRZG-pyDx<$_+oFBdRS|MRu$MLa0)BTe!KTWpsi5XMGK6JL< z@5rRDl~HBHln9=O(h{UkO zQ_EL;1}KbUfP99lL67890Uf&Kfzel{eW^=F81Ju%9AznK&0>_5IUy zvvvV&IUA5j&1CrZhpt0_%jd(doHR)OD(#wjxU%PikByPpo)TtBbw9z_-*wX0nz4uA zw+<4fqA%n>>YYBHJG%VvZRykbBP9oTPByI;|FY}~U^OuO7J4u53mHPBI?AaP%(tEHX>p@2w2r$6*ev{w z&)_4P)OKr1eRue=Kqq~!Risn>RURrm`O`KL^{Kik zeGu$fVw{ia@;UEjU8E7_o`;v?F5Poeber>A%=oE|rQ~~tqq*D}U#!t}O!f7XbnnA>(I&{)-_@D5*WbBxk#{AEmOMJbPUl=iR5_d9bV^OriR@iLDqgPm04e564L zH}rl{QXlI*j`ppHT2hA1e|bj6#bsbV<{w+X%KxByc4zb`t2Z7gIP~{vQ*}ev{V8o4 z{cj~XWtplq;hl(?E7#pTwm)9UQQE`|l1okedlk9=z_w@@Gzl{ghBcp6$v@@y|G$lX zVh63*7nW_)f{2Q=RNCkKePN@>LDMUbeb+*9P;C5?*;LPn)^#V3H|p3wTh(~FW|dZ^ zF@G++RrLv?&d+S3ElI*yxF4VMynlVh;Tx|;7}$sGD#b4BL@si|q?lj8RS|U3{rX5X zcu7BFkLpDp=Mr`9K)$$x9Q8t>YWHD6Zh>#+D1gSJTbs6|ktDfvo@36UwC&icI-b*= zofi7U(MQ-Ff43cVs*XBS?94bZjP9w%pI}Ib(xQD=)IZ@`6~<%*&zMZjfP;&3LVBE& z>2#`k2L(=CQ#&$PciaXoZ%R6}naz8uu(rd%2 zMh_Ly{JXNiTvU~Ve|2Y_yLx?SK6FJMPgAAsqsQ(_;&eB->DY(YF#}eU`m1cGyjy#9 zM)hy(!O&cC3e{fH&wOh8qnm~C8NA>h*3{>WNO)WnC(=8KgC+$hbi--Wtwqf|IyU7> zom@>x1XB;=pSNTj`o1`Bbe8LGs-f%1G{xKQ4`t?G3ZQ-Ac%LSYlVI+?|Fo&>T?NKl z7BhD|w4O-YQOL9{jjLtE_)c{s#R6%qW#rR}pfb4usAv~Y5cXFfXqdd%Xxv3&v;|zQ z$!Wo}uZ4nA^j*Im>oo22>%?(c&QBB3wP#*%Xz5-%E&7jLXS9uB2b;|E?059wk0skN z8|B;FQrdDxN?sD)6GhX8Ek(s-g2*qLfsvSqLg1{+RBELQ-og_6!%=%ExU6VZF6Fu~p z0t^g8RP8`vsS)QOWg79?h2sYtU0{Rr+hy>x-7k<&n8vP>vT{_RIU6_3U9BF|DAUY~ z&D~^EP{(aA*3bt}N3@h>$N^90t>=lmGHG7Rm@Ul zvc7{BjU2;ua??L`gq>5u%=($=r5e#WP<%9oX)t;&PLmM}&HSA#<`M3d*VKs}MeMZ= z(3_=LO!KpjYScp(-IhMPzx^bY{!wRx)>S!-&p=6?{`SMdwZbHxrbaVCG&hTkJ7s_Ad?Hk18GSviPn%6c?;!;i=8t zpaOmJ>%KnX$CjD+Lm|CK`oM>ZSHHo1y(8WQezdhhx3PAum$dY@=sGKh4^ST2 zaqyq~F_(&nM|q)RRxw=1k>7kN)uA8nkGt;-M9Z+}1^n#RxO>gzNF;KLvgK}wV7|bNMrui<4SmrGaJghpgjevr^|R;3%yiN6Tux9Q0YtPeguXuFZ|T+HVL3697D2j?7*gv z8OA4r0_lzoOki#cf)nuC=(*pS&Fp_f7G^*ta8Zor|JIkEK`8VPRY(ynxQnuY7s3+) zUax*tBJ}_Iy26MThT)xpdGni^ zAgu6Z47TkC7E-F}3@Nu<7t#+@b1Quy2ZZIw=VN!b@&ySkd}GcZbMj@zakRbRPp?QpbDFoe?-I)I;&WAVhfg87w3z=c98 zq^(OtPjtAG+I7yuXyA@fXm%%$g@RUa&mx?_O0_@9=9ZL?BZv*q4qrBw@^A-^tkOCt z5GHz7zS%_^@wK|x#L74F-8n|L^+%SfwCsnU;rENHR_tSeYEho2YzE5)H}rNw_&eEBmSY|+MzOmh8yeg^iS9j zoIrGPs?%p9QsEt-E{rrcECkhmHqx_?BhWAdQnox#mOet1M? z@Q1C4lrI?{vDD0`_J}o`Y!|&`8-p4=3Qg6<(eG=FsyP$arb1~|Z)AN8q0J?st3h8C z2Id*TgHlBCITp5nWJk)&0n-=|wPfCkldPrBFLi)}xnd!OJBMYptMU~ieC?_-f4HLr zg-}dfti>;+p?b46GDx{3gjg}vzX=uU1I2-Q9vX=Oxv;ABJscm-Es&+6XzkL)nqT(@ zsmVsya!zXk>|2~XyXVZF^6H4NsaQU>$^1hM75hYDP)*h)LBo*<`nA(hIii3-p&U^} zo{u?x1i|itx(!Op1-`xf8%>1&6a5h>tIaH8l;_HY(p(eQlE07`eMIU}{tf<0c#7Ku z{PPl2vi}XWeiAqO0XRjpps;u-d6Ux*4-U*a=pwLqJA`t=QMhDxa>1a+SAeRde zdj}e(H+Q%HV4;GkD&fj6>V!%N=5~k#*;uX+FM>NvQ#Eqpk{!Xy83WK*9Tla3tW%Z3 za{2>^T;9m@p9Fd!ffC4hg(MAtNS&@gGG?L?4*QV*co)h8CO(QLVW+Xc=Mc^XfvQNF z9ZL|3;&2SfMh1b5EsK2)j~-BRuHuWq-Uw3ju!4pY zP*1DK_-b)ei0i!cg=*k6VDpPkD@npe1OC$6{@;%sc_seeh91DM;b~Frcvc0PNx(Q( zRNw|kj4uJVzJNEZrHKBatogsC&cv11l$VdERp?#P@zS`dr$DA1 zc2kxnx9<}1*H)yBGGxYk(3oTDaP{CtIK7YtmwR#?BOsq7z8(vpyWzMOGZCmSwwC?~ zOd@QMDV`5zmH{;Fel6}LFq8+p&gBH`z!-YfeyI4~Tu;C)ya;sTsAu2BA8}(Z5gQ|5 z=<}HsM4w%!GU^tSYEUy}_LdL^u?gQ2SK8V@M~K-X$3y0cU=JU=-=_{@7^0_&?rs+j zeySkr?XC>x90}m%SkfB-&liw9AXTU|(USg;h%b>Adxqje$HLO8kFT5+K;j~ilt=p^ zQFdJt)k{|9qK-J(wgdbHSGfI9T%k8b;>u_N8%=t2I!aNsuDZy9o_vv4Ws5#yRtVtQ zS}=b`!Qn-Cl1>K{%JIz}1rTNeY=WdIuRe(2$Zm{2U1q=d|%Zv-ihs(@R6LuN#@T69aI z>u)pt?2gtR_aBh|%qFC3fwQN`ML{+rDiY*x%|Y;pic2A*uy%$wDUTcYdsL?eVYr|C z>98J4lAin;>xZO~SRaJSJaE7OE#ApB{CG&5V9VD4i;^bAj(?|tQeokMVSP{HUqZN3 z!@i2c*x_T?=l>S-@ug*a;t0;5s9p|sP*lesF04iZmsaR6@Pd#*$iYOdQc>5B)BFz> zBowR)2M+@Dgvcj$$@3MD0t}cY_*x>h@^3Q3Pcgt)J}CU&51a_x5p`l0^4$OWufQC) z?UoSq0q{dFH6TKvOb`q{fi6P3hd^TJc>F+#4A_67SGV9l^Yc4V{uB29q@5EQ$MQv^ zpmq6c8yV#0C71M3-gqC|5vKxkH%t%d z$N!6q!Yq0}5rvQfJ@NnljQrm~Bxw23MzZwDN^B`B(aAGm(Yt{H|erBzYy!dIe7zZsQ7be*ACc% zMIQ{BXKUF-mn}4imOXrN&TZp_c}4xjDb-J#DcpQ3rKR??1k2-il{NH=xG~sa<@@Z* zv^zw8CAplpi5a>}GZWkJ0qm|5x=8+@?hrfsy?@Rti0bJ3r|~@3W5?7C((W|$r>_qt zS~PC-8>u7n_SR3OF3+AXcNB!ntZj+`jneWuufl1v(q^YQ^Kp|wBvrSz{F<-Vm=+7-8vcN~m;ZlwHdt(4;6Rm14uwBWNk4*O#V zm({)nc=-$kK29#5D=d6fZs8O59CN~{&G_1)-OfqEdAWnRiyvxcJj0iWS+(0HG7iKu zHXQgsU1FxsDyFs_<6b32bF$p81c<#O$C*T5&SMi|%0C3M{HI)7Ke38?(t5lgllJA&nwuALj$X^O?I0!n6~cKiKz>=}YVHuN&3zn0pt8r$=eK?(rbpNkyBi#CJN|vx z%dJRns}jPTq+JMOER`EP8^@=7a(8kOF)I}_Z}!xDUvr=*i(ZGk#VEVds@G&z9r@ci z_~^GYNj$6LEmSj&z?qu^N6vu%r*i({SgEfJ;h|k)J#kmlFOhG~#Ey)_V1r~oD?AN0 zG@(SP9lmk$ACamuwVZK$>u_6^)rrxoa`Pu#8{C`rIIu7_A0R_vEOF7}%a;bED)3aN z^^09IGQHf6U7C58A;}$`VR}0ZOQLD+tjtYA77i*`&uw_5##;F7pJCm3XgW73>SJ5Y z7_m-ssd@sJ@!fj&ewoH+v_+h|^}1>!zk=1nCG z3wLZ|!4V5R+4$oGr_lzA$5Xpr2`Aj$USjnp7w-P0i287xGL2b%wk}-H$-6I?$;l|# zS@$8jE}_RIm`bD+-ZDQOvoB@5aJ25P=cLHsSt>X0$_}qL25O;dS4IcwLR(5Tv8T2Y z9@(zt>Q7mk*^{f+9LT!V{M?~g?}l7t`C0!f3g!mc$9EojaF*Cje`|=DNF z#O1h2Ttn38bE2oWQ*n1&QFVmT)*X#=cG&bKQQgwGnc@xyb^Kz0BS7A`}Z)wR_EObT-ZuIgZv$V3Fm2Q7Jm}wV>o1?R@ z#a~%^V{8KL@KW#Gnh*`8t3`R=*85}5(gQw!#(Wt9Yc5@Hg~G{mdqR$o}%$K6`|7wq!Ge zhYl2u(euB?N2)REY~ovHvTxNKRMy8$N9Ny<(sVi zk54XJ9v`z%cp!>gc~!VCzBAYfdF6GGVp3(+a;rhkO6T9pVz0y@OL^eb9B&id}8h{nEjBT*4de1TY7H{ zS8%n@EgHFFUBW3Vse;z_nLo_{pT>@AeS9GMQ$~xqC64Iw-lq?w_j@Jq#)ZZC!42z| zRvGpGeW*Gx*u`bcH(Gmhx`)Om)szrQ3|qrY=nFI2`t-$G0PYuOTe@jYyrasW+pap@B-vW3ES| z0`~NE_H_S`rgPae(c2!JDpgE3bEP6|Uz&EYd5(vOdLQOySWGXFG$OJ(@fT}$`+Hi} ze{jxrfrz!^{Ac20^h3a17JY_Tb8ZU!e*O7=KM)a)*f_R!;Qt|`Wb9@D;+Y4PwMkta{% zb}z2XIO}?geLd(z>Iu&BC4(IJ8Zu#dp&YaUJ+TBtWdT z_uxXhEpM9acK(jh9=!-!t?67^+n+d&+;6_w?h(9x_`L3~&VO)LhpoR6T91eiOBjI3 zPz9yT!aMxexP~)xF!086*Wzrx)bHLueWZ0-g*eYJ*?oH}?_4jipv%M?f5J%G2Q{Fy z*!32~#r4JWIY$>YCO6;gtL;44VgJBEb-1V`vk=ezPI-091DRDkHrhWE^jGw`cM-Hl z_$7%uPBuNS*Xnq@O8QXxRJ}T8+1QU#C_7?3s|kvks)R_bH`3c}B7f(;D)xF?TG&-q z5L#d2T+OvOjuaBYPpA)R**Of-lhs-2H?~xJ(O0&MGV52-4r6x|CQqvPeDyEdTS6I6 zAZAO2%a0%XA{&KKwH3>@?(fna_q{UAY437ev;R%FqJlJ@u$bdI+YCO0SXEY@`B#?e zs^fwp$(3mL3es{_wnfB2>f~-Y-f&RH*xpZL{nZte59bxg%1@bc39@Fd(hBP3JnvjG zyhh=jBICKD$E1wdU~in-%0y^Tk3S;OEl+E*2p1q9EYlE7Z8&!8!{SQ&kwcPn3PttmMX_m3VL?k9{JXP zQex+Ffzc|C$Q@m_IIyH3&5Be(k+GRh}dgfH`igmbC5S?A;&DDWDU7v9wy zePc6_(ZES3B#;+h`(BCmP>Zg+Jx`0>py0<@E8Nh0e*_kk%P)(enRkN1kNkQKiOowQ?24`I)@JgDLxCe3A}e)tgOrhMli^iKqSf%b_2aX5n|=jgq)&@ECe{*MTM zUwpxNt41@i?kXa2X;!6#6Yk7ah6T)Sl-A!Ew9^ck>t7OhU9}1< z&9MBM=S5j4yWNjJAlSgd{xc8iv`|Zwfn$Shy>Lsvxd9ug`U->+*)E%q0blS^rk*o7 z2<;P@@9GX>y$6Z3pbfq<&IUWnt;}miIm5@|4hGwKF>8hy*Ab>Wa?BfcQZr%thvKi$ z$LMoAb1RI>?$5aE?SW*^t_e%g{T{76)3}+IoF1#Qeu_NCm9I^sy2Z9em?>r6fzel@ z>OL34X7|ujLw(4O3dg(S^c;>-W=!;d!OB5}#(3E)%jw+X^3LI&`|;6r?kt(=LF3$C zhZIRKICY^FrU&9%*|B7%;X5KBR-`QD^tdj)+k;BYD6hoc97}5<)E zQA&q*^;wa&uS1XamuF9d-K3>=1mMlM)f`@4&``nPXpi1#!-pEK*=L0hEiQjp;1@3U zE`Rlm{ny{jCmLNC&pSiKN2kOT5c6?l)fMlX&i{y9;9xg&jJ<;T#U|yXufu0j=;igC zp;AjRmPxpe(x)+3Xr?sM4`Jzhj=5us%_kSI13d%Y-Kp$gq_a?Tjm&cXw}&id+D8{m z9b0nR-!pas-ICvg90dZPIm+szyoQ3_Pov{GG@|1yszgp?_njVg7_dOqJ#PewbeRDV zb7ss@-zFibF6P-2$FuO+TeU}t(!_OnqypZTq5$>um121fF06GF?t1a1OFvUapAmNu zRBl$QhL6Tky$>9M0vyzyfA13|7ITSIVevOYXq3z^5?gAu{HlZ2fQ^{lY_ctL*%pWz zgj;bAV*7Ib*emD`2d#uak`s!SE&*k&;#b_$P;J^JbnNl56UwI0%1PudGoeAGX8d17 zko$=1?9k<^rwBh@LVfeh9@A>xZ#?@)Vh0oz0&Y(Q6{C}^pp2jMLb_!IU$Z)~FASdd zQMb8{=E6doX0qUe1!OD3_H^j%#&(Pl5+rISpW@@Hvw{jws?JV`s!tq~L`~Ih35B7* z#%qu-|I%B*5GF&d)YKAFNd z1T||wUMr-?PDM0lBh-U11x0{p02>++$G?0(KM8LSeto3>yN0hgsg6eJKu{w=YeufA)p*}G-?A9Q3GvR0|6lx;P>jPsAk zF*6_%a)R>-S2g*~GkyNWL=|+Q1S~kwWdh!S$U*G;kKj3HM4Uj)AYtX)X2G|0O!+rp zfj<{q{qP_ZR0ypHBmZq6`6I|D!<(=dQiwUgGj4x=2 zb{=qx8Ov?Gp?h-6|K6b@;Yk8oc=-j2a{zFnm?J}82wMcowFGmvmWevDh=vCF)GU8p zp)2^TlbZ$p1K=+U%RW>j7<^&XkOP>Z3rwCrq`U2gBAoxs#FWkMDrk&4Mc%r(DuP$)O z&>JdEKt;&ttKr3^0!dH1$C=6>DaJ?m>mwt;ACdZ-Y&dhXT>Fvd65Rj#etCQ-=Jo53 zm_dNTDB%-im-1ab!Q?Y2po@^J+~Ag4%ZJcKE3y3eMtLFlTfrYLZFoiQoEv90SK!c^ z%O@@0S46^_@zs@r^MN+vP*X(o9M)JfEE(13E))6jlu(NeuAQoe*-s$v75P`sr-OOb z#Sk`j_L;Lb65{NRw3|I(8|>=DD=zsvma9Euh%v-y7gp%;G>IdMNQM4UY18>}R3H40 zNE{_}cId7cA3g~#ub|ppD3zgMq^BAQzR!0qpwV+|Ka3A#M<;>1TrHioZzh3f&wnJW zJT5GqYkX!9D2nq9I6edu8089XNAhKAIop&)obgIdTt`R7>ltd@EMkvv=HGqa4O(Xz z-u|;nrO9y9OoH_yAS8+JZ-B-s)^%~pn_MH`ZLUMbh0OxPL;*e|U(H2QjfqQZp8oI&OuQj#8RE)>&2T<*h_1HWxtS}3YKwa_*H#95=^!* z8wMQ4j)y}Z29f7z7smBNa5OOJ6{^Yk2An5_5J2!|sH@-){pf&{#c(izP^ zQUecn3VQkW-w3~=(P?5FrZqLgp$0mt%dA40$oK=IRNB*v@Fa<04BHi(Yc;>etE}#x ziErM3nD%I$gYtAj;k)%rb#hIy0=vx^3mKiGpvRlELqWtz%mp26q-b2e9H%5*5Y%`P z2OY%YKq(i#+q7QYCXYAf#+wBeyI;u|0)T^B=2ccG6v^fTSuX1}i3 zf|DBvbQ?asL@B4T8n%?v{kw}0@iJlmj>z}qGjG6!g)`?71|YS8G3BzlT_ z5Sz{qyttqiqy`&r646{lsP7e!$CWt#{P@E`^zOPJw6IFV6n%p?LN#^b|13m-Q}V+C zk9`&N1+)nt*J!5*1iR`A9^zkDzxZ#C_`~@1#U5S7~JeDwKQlu8ER$0VjXV$xO z;?$U)80WxL3Y$6vzgQDW#j`HH5 z)4Xq-zmJ4v-S%AhZ;S&A>mozwDTDi#tj-(~+**)ZRywo*EEOg_&$r&o07aNpXE#A=jNm(Beph< zlHV+yADb^oU+DGS{J_7A^tsqe|61uI1HUKpGDPo#c8giWb+s^^;BShwMK%}*Paow1*1bD9HYgD%`{!AlKj{VEOZy1WyMV+z!a3-E> zbBbwYDXxSOn&{i#%1w7zepg-D(5sc$xqe8WT;s{Cv#X@ccv^*J^qGAcWRDr#6qz2JMWvfJXBneW{;`w9{Y_`7jGT*-TKCQ`>S*0VjIcI-3}_G6WaqD zO(X0ZqCc+%Il5?fh2cT^#y5&RD{f?-fw4lQEGuqvT<%l^A@tSOQ{GBd8oL?0B zC5~sX*R_Ay-Kt1B(!iOZ;_~EH9^tD7y|!`6SbyV)m4V*jhLT1-R++IT?ovT3E$Q7e z?StXoQr3lBgLLf_r}sL?hP0)kVl2l+)u)1n-Nj=oJ-={9aKlaJA1^(X{ld~D?DP!O zV4m2uulAz252+8?;Kg;yXdc(Q`hJG%m=@^#t)k^reJ^z0|9@0nc_5Ve|JSvut#qqY zvs;SfSRu(VwIw0R5k=TigqW3b2s$OnIH#oNQeHKC>}Dx0tdDUZ-1xDx zfUvjUl^(Q7xbLln>90SEFBwGhMydLXLH4@ zoYJvFi83`3pKqB>H)^u1&Mq*^!e6V|?gN)ca0Pm0;a3%yNlXvHA`jBREBeO<>{Q7QVdjHFhTeKfvveT}r+*&sEkHnp^H5&D2R(S7IfXd=s znDM#h+3|%d4bI*=d`*(spRg_7bkuIiyB=TLw@l5u0_|U~;=S*Bj|*41#=E|L+nX8a zRn=-V_Fnkwc0FpZ-S&^W)fL`Yv7bp(zWYi)ycUQ*%s-ah>xP?ovzyL7^ruW`=X5qj z%1uS(rfl2warHwJS~rW%1l=kfC%moCvZHf@4GWH1teSmTG9D}bHG*=5@2?d@8?~}=)Ebrj}!gJ+Q3ZHs2<@$e_6AIp{&Zg!;d^)j{(<4es;m?3Q(>n&L^ zr)2>BqVCLxg$F0FhzuQ^VK= z*00ALE)+U#v-gh;sD4U~=*0O8_I}W%zx-j4-A+q))w@W(BQ-G&-HkR1x2{lWJ6qe` zs_R6g4_+Dn?8f#a@0hCw$}eTkFR*z>^|e7p2|{=fG_sz)M#?^S$HO(wqIf{BqhDZFTHxG z_6<96Lf|OzW5gETl`JpHjTIFSRXjEF6yt)dOn_{v<0*8LbG%>Pdd}ZU zlysgQB^{ys9mJ;Y{6b2)ZTTafQAi~}bGzX-SF;_gjf@F{{HX?G_ASU_>Kx;AQ-)8U z_X!o9UB_*Gx<6Ytw^RAlCa@ok(~@xW6Ut zdNAeEkiDG;MRWuD$fGU98NilFW_3H?fSDh{GCuJQ7|o=2X-K;V9|dwJ9zGnp535hR zBpDKeH|)pSZRwdOegBH^ks-Z#lU5BTRdhnHji<{4qIB%g6?SLbbDye8b5EL0)$-Zg z%RlHNCjoKHhc2KNof0woa~U&qgKwSSGGP7m0uBfowB3*iwYN;mD3@phPXV< zXQz7QId{qp+PA9c;3FOr%@9FC`;d42XK>W}Ru(`F7)okUR!$;+JgUTu3!gC9?hNy) z)qf_tME3dyg?~c_%tynPh8P59dS6wEA9S8d{IOY4bfIqgeDZig zG-dxK7@!9PZpYkY(sT8`3_?Q3DU-!Ov3!FMTg<9T^0^pQ$-n;u1n0UtP7)Ocug{je zrt)>GjMr0Gbrf9A-p?})f!DN&1xmN-H96gH1bOFt!pjYO`U2hf&=Wdw6aPYa?~C9! zv#P(9ml1iwP*%oy;h3JO=?z2uJF0%U>m5v@mBPw$$?OJhdPe17F4- zwOd>p71(R3bDK|#5;_k9u(jfi7mgn6(|-bRbMEm>mhFA&V1i2}f-tP3ty4P%N3!^x zeo6c{)fbToYcS&e?WPRqsBXV*H;@*}rGV(Qgy_ZHj`V%ZduaXWQ+y z@S&*+5Uq%MbGD|fdzj0%TpjqLR>W!BU@vnTH$sc&UdZvwQ{>%^g-Ako+Nh;dnVVk_ zZH3H5(ZtBB9dT0m4U7|fFV3-pEgg5s-<%%quGSj;BDo_u1mF@d1jAiJ9PkV0p( zU$Qi^s~>wlID3cqHh~%@sb$X*N(zZI5mUpOuLhHVFr8T^87<2GM}ql0j)XabmP&|c zo;)DBpJ^Fw}Ud4zz6D73!t%Qf78ZoEC7|LGSz zi{?Q*7h;FuJf}6vGV3}ybseiPKQtc>{TfbN-psmr;6(zC;niDD0rL5K* zKU@hS?Enwcjue*9I3rcf3U=Hs9<*0JZ;jjua_5-_?Aw|mi8h({gmwBu$vk$a8R}$|Z6f<^S8Dy6cK`|K z z-UPq3RT*j^Cxu>o@F!E|@{rBj&jw^yd9?ySP@F~^IAbX5LfRL|rJ1@$84k(!1$q0- zzGRaYo#B}^B;4rBa>2_&Qx&5AzOnpXKu5?SfA)m1gX%P*$E&uCliJUcnWClWv+tFV zKk8rPe7LN|>$~3O)268>6W;A{>8LL>YJ)tPXE;}(-Er*rl&ICs;FGEHf%l#`f=z~x zZgKq3gh2K1{YnJgwrIiM0JJ>#FX3B@h8_BXvoEKNR|M@S?mGK|!rV_U5orW99=bf- zwrqXgaObx=Irj<$QI-GkPxaJ#f!f(3a%6lu^EV(!Y&`gto5uPzeA^S2HY4TG>XD7) z81}~L_9tyP-(?4bqIR0{}#VzXJ0Na6E*MsCH@+u!C3kBmX!Z>@)m z!}RjVPHr*?^+(kY=#-sd>oNDyYCZ5%OfWx<+o981uDmyY^3!&^J6Fp*rumN92w=O2 zjTg4Pqm7wrsu27J)Ev`AY<$|N{d3aX`MI{ppqj?!!#P(7k$R_fg>K)5svM?k@L@K0 zNYAP~tYf6qCcsV-1qf65l0pqcwn0QV5-FQDh@*jJ)t*oXJMm#AY=c>Rv@wdKdHHKC zkQBpxrmyAF1uuz809RZ_fJX57lo5CL5|AF*7c?BM@el3V|JU_pU_O6$PagF@9{peN z!a)^;X8vHPV2L6SZAOajSXVbNp`RZL${ouB0AMi$1qFVK(KIg;y#F_2F7nDR$Z zATbMp&oH!m!72+;peewf|50`=*eZr>DD_8yM^J{hLEH9_7y0K~HffetHkpb*q83Q+ zTr7`;6=DUk2ZS;VVUK*fCU_WJhT;b-*^cq3j9dgJ+=IZT83F&nr?g}j`z!0@YSszV z^bH5WBHOxyxQ!~jfuTuSvZgx6u}6Vkb25_Lv^I^bBA9jH_fb$UJM7Wv15QWx={0}| zL0N6olQ0PxemSV;Z17qrPPPWAUp=nXcccPkDUA?za(cW$RtIDj7~y-Ma`ggO?LUJO zgjvUAgOC~qTreRZ4;z><^Z57n%M`muSxu@ET3m104vaiO`Cg5L&k5eTgX zV^b=Hd4Zx)5lPJb3$C%~iRf{G#EB_m!3oF`?iN=&bd|2IGH_Zi!WM!b3zMm3EN$>z zQlz@Xoatw&4svR2&N}+(&>oZ2W-|3I2y+SKxzCHYqvO|T%0y~z)=V_&gam^DsJk0d zCqD(PuT5m>@y7)UM!OIN&*cR==_i7mTH%4Y(|rUkLEy+s0pn87^`1~*=!pujk$-53 z7#Y%+u!cw4V>)VnmPS#z>g-GfD+VY{U}Xj<5rKFUmmv?xfB~zOc|3k=OBCqGlEknr z#t(apX`3WWB%=j>0)+#_8+vuklnSfeplE9gfU`=laJ$2P0y*LF0yzY06ekWEV5ld& z!^J=zgE7>_(?&Lz>v9OXVY^@(eNM9rzJ-x^4jmLuT+Z^7U6erj*udwC4 z6@Eng96-E?=2FZMLMagFT-yo?R1}>;rayJMcuF{C_4gWE-m5xgfnQ)=YwU`86+yW? z@G~s>-b~-EF8cJ3L}6U``djh=OgY33Sck;B2Ef|F>Yt>#^p!;xOl3U|*Nh?JH-f_w zG9poY@ZT`TJrUsCpx1Sb0*|GF>zx5Y)pLYAYX z*vS$}CIt7@X+E%25Y_s8z~3)IMr`LNAq0qa`?q4kS(uDaPY^%K;Hhz!H2sA%`7zND zSfz?3v?G8<#T=jB= zmpNjBeS7iJiu`mC1cn<*0zbc}erSe~0{kP0Tg2qmF0^%ehRA$IvLG>ut|`7(`~i?_p8ti`2I9Y9)eVSZ=jFBk{XNTuSVQkWJyM-EN7tmM$h~T1 z#0##$7WQC#JSsiDoA3yH5F-?Mjg4=-AH4X zwy92SBb;OWc^m79M^Ni z;EU}LZ8|4pB=8k4o(+bjLBCdZU&lqW{3ve|qDhLRFVnThOXTho)MLuseooqKS~?QM z={@Uk*-`n~u6XKiyhvm47&`54)4|{$JYL@Wn2*Gy?ap3dzvY&;ocUz?cf;b08Fn_w zYQ0MxKUMizx?K&?SvK+~)^3)pd$(?8lRE#v^UpoR-`3vF>G&?%_9MDL&gMz^SYl( z?MkI$)94CBpNhBSd3aiDnQbSzC5zGWoqX+<-GiDgTu)YsUHA$+ zt#>Z@*6&IcBe+ifZ)jx2_oR4j{D%U;Hjm~f(+{mqc9xpEylYS=<#+CJn)c!O=W(s* ze`WvlY%$@wbl14u6RVtnB8BXu@tR5z)WVBr71=L*B(u#c_Oov^*)~>fmh9dOTHvKW zmc2~AW&U$rS4f}U!HRz*f*Z<)V?=>%&+;YQ1^#WXi}W0&LQU6<1%0t;YVDce*H729 z7Awuv7S!H4zMtqHUKsJc$~KMf=OyaJwHB*r+%Mde>^p9JUpUts!m!y^IMVVodu^@$ z{>r1>M5>L|mcS$%0~hz*-}=YH3&UK}lNac<2h*7$g`S@3t9f%SiWPs4@~jUttkd@Y zA-E7TQe!A;Z1zQ^4~UDMks+v+uTQLp@ZZsPM@k{Oe+6$cA5SO(7At&-!H zCi@uGaonk6!I`1X&EMM<<@d>QHti_q^qz=o%?&IwFR}QA|7GKVwP|v@E0`X)62>bo zn3&5r%zD`sJLJk~Sig^ZY*545QV|}tV&l3MvNs2$K3kLQGQd){ruot}*WTaG6;;M3 zR6MWV=kl&YZkV2ONk8{+nyaa7UDyXZ45i8Lzc~aIHe8O=`0+JInNYa& zqN(<$4hP&_AO3M!(#{cf)K!Dzv-b6#%C(JPpJ?*pu+t9K1%ldSj_te8nZ^}^wX@aN ztxO3ntHU=0X^!X+GSSzuobemgDr-tLAw_=SNNt<-PAbYnsRnHLG~@^zoZAVaJ9K^bbWd zVaN7nzUlBcICe5}&*U|)D__-mch zNxJmECX}vkSnfF;O*$Y@GvPXCSD#NTO360bV4eHK>+B8E!`TYQD)L6F-?F@e#>?iG zch_Baj@qhw>7grrxt2`tZc45}qDw1TX8X2C_Y>)y2G7caMed8t5;i+eSM3~pC`A=* z_|eMI+;Y8o@LlbPrGww7`}=1b{_N3v^`=d!$NzplM8(U5(R=Op?>dWfyi*|{sc^&N z9I;oL}=oROP0OtTzkx=1*OOlR$n0iN(RX=DU&A&P$&TLxnuQa34DMIl zPrIMHUyX44>40Or@B~SPE>N|*`?}pt+U9OpKKb4ne&$W8KNLoz&9^Pc3SHNk_rksm zY}dG?&~pP=fF{mKCZit8B}h) zZGXJy!J47U6U2Ac+5QF(#%dlE6t^^eQ^0Y#3?=E_4@4tYp!+Xha`l@3+5zpaG;SN^ z!5xyDO4R`mzoP8twXf@xm-7OPX0vRIt^;fNSV@gw5x%uT?wsw<|468PT%rkw+^5#t zHU@8~XjR$KY5(i&6W8f|Gx7eJWz?CnTawRh3VkwML?|>Lc9%}0sP)vK?&JubDWee{(6b!TaTA6Ve{#QOPU z1Sa%cIV}9Fc4&1sZEd%q`_EVC9NV0ZPjR13bRrbp@9?v1nb!I~F3DjQjs>?&C;S}R zG$ccJ<{LwAIzM@-1#`7p=}j&7BWA^U{8v0m_#Vn+lx{Vtz1Yta5JRoZ8lysZFg9hx z6<;@(>k*!pm-a!P(8Z>O;Do~j#d%2UbqP|pbH zJYMgwsbTJJ`O({0YYmaMQ>39Ybuc8dyIhm=CBvyl{_mIxc!Jimk>-J-SwKSk}hUtWBG zlNCyxtt3#5^3}su=(%numYMnC+(f$6dzTkn(>7>P^w8MUxa0Oj5pZ_6r}yLva(S01 z+znTTo2%UtS8iE6-N_|uHF&X(u>#*7W{~eC7vrXBaPIocDPxrEN^EiujutfnxLfn5 zUSGs1w}^5xKYORZsl%UbL*5RN8+D__s9V}rccYhx_}d0$G-Ulx$|EV$B@e*fa_ zQ-$qo)*1*u{#f+1pdiEz3`$IceJUGTI{)%YwG}1|wU_@RF+r`jxZKuOiEGiRH?&e6 z87_S}5|@*kZ6_?Yd~zJzQc^o|41r}xdeTEc9L+LzMD&$?@H)wo#<{|w?>ktW@!QKM z5?Zbv|JE7vy&>)V<{~1^oznE$gHbBt4BG&|h3#)be)JMNUt!|59_=W6obPA5NebFX zHIYZ9y+orQ+dFgI6ehkZUt-zoCUm>myB#HqY(Z0^_+MQ z8Z{86FS|5d+~1QiMs8#kqkkz4SW6BNB%M3gJP9oBGWn`bZ~l?e^^-r*0N19L)2P z{5Hrsf5QK?3Dy|1`;AkirTg@rjYb14cIUX^PLXa&gJs##wZbF<6Hm2CxUekwu47Fs zB#-W!F8#8^xMcgcz`UJwxv0U3M*Nmg+vA|2&JVsUnsBXZE+r<(Tq3f|X`;f@PwExz zue^#FGqIKp_e6mxgJz~Dbkp7n=QIej_7@!KbPzY%hZfbRNh<1xDGbki~>@1_FT(cs(elgsr>GvlumgeC4zQv=- zmO8tM!EgET?o?Og^4*vo+)UpC)J{v2eN$L!6xWi$H~dJ)$(Od{0{Y%Ya79MWbGH-9 zcj<=swAwt4yBKAtR&FNz)-?c~We1^acrz!$&l+D2!@1bw-8*BI13@hARP$uk4|wk+ zJfi5tXFe}Oaf@<03s$nJ9lycv*yFSI8H@4gl1Ve(UxyuP{=u8KKi<0i zL)wM%F9l|PL9jF!uXrFmwa8+VUbE(7<}0HA=9ev&4hwuoAtMbZHw%d}O9-zOa%z(lsNJI{z-f`7h4U#=7_j!9 zHW@5+cVM0loV2YURPPi{g(Q8d(ztrY-A2B=p`Bf=FE#OEx^$|wAgIk{&@bqZxxW1L zEq88(I3UQE3pf`7po!BC(#;c#it`vEMqN91C<)N$+0S0l)EmzHqfu)`Z$NK{fDRe* zi3~W%-YSvZfOaEK@g4|!P40~>Ba$|pif0}R4w%Pgg9tkptu)&3nHr3343TY8gdcSaC(k!UcWWD(h=xP%|iv@X98$SM~FUw z`z(D)-LGk5su! zSB46*UIMHt|ESlTny0L!vz-%9(( z-dAmGjPD!`;VjEn5#-hM+ppwH+ojZmXoPARkY!nZLH1t7^q$JQG4zk6k)8E4GPruT zNf8~vj}}}^5q#rcFV}lgd$}yG<>N*Nm$gb5OW?%6=8gJK5ok}c;68?JJ@rGZEL(bP z5sgBbbOTG3d+Myiyoxc+M5bDxmG(q+5L%w2x}~QK3Le;tjU>%Kg=vyM=+K%d13&^h z_c~3O4>ih=Z3IsS-(rwIo1qF6B)ko=^B3rKD)CN}l-3Ea-Vt5~gA*gVbFrwONVlXT zGK6GA+KG9d-hL8VZz|W=q4&6OUWbMN0NT)rKehP!{67)UEkHxU8tv&M3HK+@hamE| z)`?0-oso+2=FXT_U(tw1QvDl9!Gv|D9;YTlJXv_*e_T^5R3wH z9(ST*FXUQ%hOxdb4wZ7`7{bv_40&O=Xe81NJ&qxg;(rPGrqySSwIVS@6kS<&EkF~U zpyjy%Lqa}ZK<(`i3r-!?WQX^0&Y%Us+_@u$jQ)pM8ztY_oyM#R)#1vtR0CV$4RKq zB|v6q-_uo$$v{*h*0lr^LkwI6EcUgWfP>>SETD;nmIi=i{3CHs!3c#iK3~9|^c1)aILQnu26p*O%k<`eWJJS(=*-X!~^(xnKlL{U)Fv}3>n@s?fh$y0kF#ikpruD}6?gJ8UWChw1X=XS<*0gn!B6cgDs8H2=o@~$ zz1@K(%I{1$Q&3)7L~ii0X19ktWU2p&fp!?cj1Ek5IVQ1%CF0B{K^kBdyH=XjLZEO} zx*ovbKzRwISFM1@G(TtNd+-2E|LY(s?)|s)An>C0#sp&Y&nn*s_@~gmIwCxVh)QY~D#d72cyPb{UdSpzWALl-27}1)|{h38k(R$MW6-~Y%ekvOV&Ysikp7_ktrMr$0#x8 zD718HvD1nyvNkV@tBPbAqov>mS$qYh;Rn795Mq{609304J_qpJ%=LMuijjV~X$pJ~ zcpDIBiXHo#noT37=eMj zN3dHCsO-es?7?=7ROHh3{;?C2s3u*+gU2i&89o%?*zhb|eCmbpT`Mmyz;}wIm|u`5 zKxqY)=579CbY>(jf|Cu3e(gT~LbMS-setAtN|=drD-m;9vHoueu2k$O>ZN1;kHk7D z?#e3*8x)(N9M1Vn zP+Bq7GDeuf)}zl0{uhn~L*x&J_z-;WEEW*FS~TY(`{OlHDn(-7VqloI0m zn|iIHAO!}<^RB59yrkpQ!BpaMhXPpmM@~D47KPlJ|Dodfw;txmrRh>H*n$xM0dagJ z=CcY_RSPNxULekl6pLV?W=0Ug!{;KGqOcUU9>a)5q2u*y*XJ9nQ#_-I-?fsH;N@f> z!~({_B`hcY?r~^*1T>jds>s!K(d`Og?*NNpxyI?>W_1FP6f9PQsRj*Y5Mbm9+pWDA z@G4&PEof^?WC5R7baO6&>s1F;6CA?ZGts2>Qc1694F`xyG+kT^0Jp=62chXvc_M+)2xP%qfVgS(tND0VVF1}r66WkYW81hqS z{YUW?!LSV;h3XWwfdE@^Yb}Uz*>Ss`&nyE&Ta0rd&iGv-iByf)o&9^Gha}^r1hAC% ze_aIw5!_BP%F+8VwA;y#_(^biK>(ccd1VKnabXDU5^vaBY{JE->_<8~KY`+!=J6Oj z0l*GD6if(mrzsO(5JP6<`5*kBJH@D1Vo>$x4fgZi!&nVB4=sy-SEvptt6#rn+=p0^ z_d1jN^~O3=6Qef1!h}_^p=i|hSqgspjPAW+9ZuiF2ly)8-;c-dVGz2R!jAm39rfGF zuRaXo2N6>)bolPCe;3!2;qI2v)m?rAjtKk4IRg;lm%ErRsCZBeTq3ScegEJ{P(}=~ zWasnBixku2Lj%e7PP&1a5mOJ>(ozG-etQ@9I0*uo1lR7!@sw84`{LzS<`{F92UFvm zK07ZTG38J~ru}5?x35Z&iyiaj-1q3xA58zCJ@lqqyCJH|v&N@}EEQY8qPUKhAKj@* zbTZtBr-lp)G<9x}!XB~`&xmTmOBhY5di>AAy~^Hcw9nK&-x>G)t>3#kBPeC8S>fJA zR?}GpXZmmHNXG}MeI|OBQtrncvj+h#9MgL&46bcj~k1I%~2F zd*ih05t>_=aeZBo+ z8>Mpg(h5wM+xa#g$h8u*+uF=-hm*0|>?WbxR~@yhm%@9NK3Q7RbMD%Ujq#tS@^y}8 zmlR3{$H-XUYd;k6>$HIG5MUJ@OkVlv#`WF>$s!e_;`3>L=at`9p%!_X+4)}Ol^=G` z(2hCGi6WTRXpADNSjxKl)w(&}al5MKPT)K45fzEF6+{P7p(FR>84cSq|BSICn(oE~4n3r&?ob)NgDp0n2*X|1J>%rRf};nLHAS;hO@-Ln;UPG^q`@lcdy_7zGe4&>Tq=C!SSZ{a?iwiLuS*HFZAx6OD|o15bAmWWQ)u- z;jZa;=aS)W!>vPweWBY{x5x97iNCP}9A|R!kGCk<MX^8=@OX!G+qypF27NnvV==2lIl9PrAu9mgWc77Zslaja?Qu zs1*pC)zl8nHL**BFHleTf0eIoTes{g>zXn*XPedB`>`8pFN@x9c-1FhMu;@ZaO@zFb9#a(=IPp)} z|CLoqerou(PbM#g z{XxNG_gf8lYp4*zzcn6<^8SQhg3wc*%{aBUa%7}?2tmXlq^y%fh(=(HHNfA^X9 z(7NJ2_4iCyn}@W#KlX(4S;D>BTbbwh=QwHyF5<5#e5bt2e&*!nU$U@M+LBoYt>%Yw zNgd~Gxqq$ubcAzn%vrR^I6+Q-Nm7ByHlrRnYIqOBRDSvhbx1pJyeY%&#+^B*1oEZm z3#3=u+<-P|6>9b+@vESBh^H;hIV(rc_R>?mH+9WC%;}Q3R_Foz$5*Ovi_HV_yW$V? zb$de9^J^X`dkt+JdcCDuv2c&ldD6AWBNdjorl=!rvN0WMEARAP94}6>XLl;F#ZyExMl#|bAI(H6`k`7STbD32^zqYW~ZL%fI zs_y8=h25aMQ;}wVeo^u4>qZss-0xSvvMQ7Mey+Xl7G~3-b65mBr%gpD+h)?>exqM5EEdi^n#X1S<{4 z4SJ}j7YxhBRC~t~TX(J)s`R#asHt~b^1jzM#_XmshTb{feZ)WGh6E-%UB39;(%~<% z7W~dx~lkwN28q;6s+M3wb=N8$Xs~;lzUw6_^HC4AiT|K2WVF5xMo1Vbx zD*G!N>!pihDZ^Ikl;y6XoetcX4<*w94JEW*sH-m>c<+@^c5fiC7CiG^va?iPuslx5 z6~|MCZA-(#zGhXP9!PK~vFR(fNVe;wv2Cq=f<^w%MHJx;t-jSyoo1xX+OFlP#5{VS z%W<5u*33LpGw7G6&s`pvyt`vm_^WY0^J4#oz%PsKEtFx49?H|*mG{X+cAAp!cTt7I z5-L=uuBdcljav`Bj+76ez8<7j#61VV1QOjJXm&shtQ3Fb84`R|VeF#g&?5x*(qJ8f z7TeZZVVujKP=^$<4W=u&Wgl&b`L}3%HEodi z6LCw7`}X7Iv@Yj*&gf|YV%}J`?~@fN(5*sldO#$CW1vl@^VW^o{I)J?&j$q&Wyf1- zvT7&Oth+AF{xDnS;%2BZV@1*DA`fWE?z6S^Bi@&KmsjY&SvhE`eeP>#8Oxos3Qm<9 z{>pSCYnt=SR#tu=3O5AK^qvt-cj>u{r=0LikC$}UH)qPzTXHMCthH1FpFRIX>~7E; zyRSF3XfpfmE=Y+~=l?RDYWf?PFCEATqOE_#i}@r=nCM?WTA^_ffF(DInrIPe)N+P5 zSX{xga1C_x@Qtsbe>;<N)Z}>jx*n#X8(K zx^9QoGe&$v#A|$+Da&y>fcmQs@kZ=&NE-YAr;;ns)423T$fy(L&Sn|@@(mTjp59ob zQ(Ioe3w&8^kp6Z^e{iCv65a;5`~3YiiTIiqgy@V*LaO~O_w|dipPgjX0A+>zH7uExB2@^x5CiwT1_J|tK ztEwxWO*N(W4n#cjS)A#@DS}R7N^?$HWb;^aWwVea+CZ>6`X;{Na2~O&HQkl2*kg&8 z7tJmX9V%kx`z^xJyM?N!XL{E+@ytF+RqjbRJ$HDJb})(PpM9~ryX&moKN2q%>fI}q z==z$a4;9AnqMnQn;tN<*=1!lTm8marj5_tCVaR(pq!7x|?S^ZN8DMW1>)@KN)Y4qf ze$aW1(v!+G@dcC^2{!o%T zyBhfNw9Y|R`O_e+7t`vZ@!t-|gpU&EXuUN?*P3LhRZFgT#l{uBzzh6wYLnHDcBN;D zx%Z_$6sHPdG6#4zl?e5}$KdwOUD4!t+8qcC_tdHKEB_%wEH-+s$=o$f8RU$bcPt$u z*N2aO+^* zP}z~DIZ?@Juv^1nYk}Bnj3~*GFZ)NrAasnui3F?`h_GNGhKpweT@(o+7A8`Lv{!WI zI56i;z!|7Rv(J=yAv5r$3U63bIBWH5nW4Yq6gVvI|3W{(($tI}Ix;xt*pr`%fw!mm zb?f+FPo8O#FQ=*LeajBNDup!LF`_*pWQ8+og6};zP&Z5ySG3-Nv*OF`mOVb?LQmoN zSC7pC`uCcC1KEfAVq5X_mjqROxxyx*ze;p-LIE_X`9o{#WQ<8Fl?Ug6wmcsZSuwJc zh$9%~`q^$B1c9x}xQxhQ!VyxT2mt#ay+O;z)`#msLxfxnC=ifn&&;n_2XfORWA7&8 z!DQ_~=n=MNYOiZ#m4BJ3R%y3xU}*+zN_R2{71=~_Pc0f04(OQ&r18H}fS*zL{48j2 zz(-PEbs*xoGt?9K;WRPqvF+KJ#K=7+S&w*zZqOh1_kootF{X9dC6>2i2k6#HSjD1E z>7yRLvDQh;#?)#<`30!(!cgDrFWJK>(Bn|Rf4?pUu?*K$OAGP=lw6b&N7Ur1az8K8 zxkCO(8Q_eXHQ?g~n*^8n$9Mtl&BP*4^f+xNskIFyEZ@Sghs!KBF~@`w4ym$;=aD1_W`hlFxcOA65Ge2^!QP$X;)ZyzWj;`{?o$wKi`1SoH z{cWiGVGbSE$Tsg+xY-A#xh77(HkYR!qj8GA;`aNKX4xWZh%P;*6Dil6e1>`(A9k&D zchcf|*6fT%{d<0H8O!=`@8IpmaDH8%DbG295+UTYIFVkP@{oDvZJJp_{h8fnlS+EhbtRr_TLO;Efz38g2Y; z8_%Ko4}zg>J1)lbw;PV@q&JpX2Xb$ljq~!X` zLDw!gBG9Id&F+o;jfWU3Z;Qz`ky9H25{M@P?z?&-7#R6b$aM$*{J0=naQ!%t8B zdBOCwFtV~)r7E;0pTvkHsM_LF0!@4$urrjs|B=WN;f#3JY|}V4txqHFkfotqkIC2B zdr8jf=5@G_-aa&&k@q<5`(4>6u7p578844}%F)65&(iNMai{Yf19U&}DI6*Bx`Xj62Ud^t>A%JiRw|fyN=2rk7DTH z)+$?RP5=>F`YK9vp_kD=5|cWoa}jU=wegkEFCywggLO|5BSdRAub4d?LPg-rLyD^F zJ$cj-{`lyj4I&uoCd8?ZYpUg7&ua)#1xOQnj|j{dPAySEi4Z*o%yJZq`+y$~9z_F6 z0a#N*aF|d8TPs9v*$mMpgTfC(XPw7hQ|HEWxqVnacRs=|9o#iXQ1l{sUYd;l4aDct zh*>Q{5i-W^4Ot>1zF>bw9|ofP)JHB~36&39AdeVPlCBX)M5B5)L>LnMZ%@tK43U!V zx`ADUSQ9Ldh(!NYe(e1N^0)u<1t5XY>RTUpTLcH_{z5tAf6T2u?7A@z#bK>D42N5S zUWgfrs)I#0=y%cY^#6*^rP#faP>e8fsQ_(4hGP+~$WzAPHdG&?VC4Ly$25MBCF}yn zT;%1`(PuAmOq=7v_|7OFb#Rq%7UZ=Ha4hKZXJdeo93S@<#GC&Q$o^OTie=4eBCklo zeSMkCN8k)t2+&?4K*VCq0^Fgu;7hhMKTx1WIO1#O$nU>y6$|bBiw6CNR)O@$cW`rCww_ld=^w>SU$mc${nSbulL@nNi`kAg4q_oVha>aVSi$15WXA~e3!T?!@X1W*D zC9;Sl++Ab&wShIK1upO(k^*qmHbNTnAYs$LvS{cU82K#XHVsxXA}bv@ngA!HLy{-N zotm-FLC^;0J!B zBIt^fh<6Ry8Fb;bX}myc0}ew3!KAZJ&|Z01*i!(VYq{ABo%y9iCyeFy8M=p8F2HL5 zsa;Ey4NZE|33Sbf7D#Ng4DcM@ZMdczluFm7quas@BqGG=eg z2znin`VqDkF)XF) zx7PI{jLqX)Kda%W9$F&v3rnvW2ydN0Y5|e}^!1y4D(SNZJ~@{cfOjsY2}6W#fIK1c zJdsWRSfw)<(@WG6{WYI8gB%rJS56>8GBCzR35j(OJh(;}LzsXt(;{-c6`k3XhQ8%D8StEB~BX#ycq-4__BBMX*0xGFc=mOYstU_I+zy~%}u0MVEuP(`j}Iyaq!dMNTl|(0CJJmN{q!X$^<`D`3UsofxvZgjk{(B~{04#lRBq??51k ztW%3+U>&Yd5zfK((S*JKpnAfniV&8TQA>vz4p`5AUc|ae8TpMexwToL2O}H2temed zTO=N13U%?SV7h*-0Vnqe@o)Ux>=NxyM7hQ;h)V1buXbq`oEOBL`*Qtp66gB??n0n|C?xi71LPYQtfO(`~>#D9z+4ghW9nO_bqSvQYLt`#vS0$M7s!o=&49tQjq z3~K>r75I6L>Kmv-RoWs)2Fx5*M|5Edzha>Bn4(#6CcXEjflAzX85pncOrFvoGTo^A zcql1mWRKp^pg`fMUtehbt#kgb8urN#LUK+f%$&r^;fDxjXoH0prBJIo@3@$;X4dr zpKO)AY1=MKAYQqUENLPliKv5+s%Z-j-C)YPcn{E7YH(rNwLnErkYuI4V^u>Lp3k+M zq}Qul2~uAm*&!J{6iAtDz!0(~zfhn$I`qdmVUENSVh%-f277``9NB@J+!^AG!jAv~ zVR}a7$Fzs5XaFNcK3KJS5Z}Pmm%y~AVv+jEH0;bMaY?QN!UcI064}^zMZU30Y>7cp zhXjbs>0P|^7jnfA89mg^M0i7pF~|R8>Pi?z)o{*tbHBhKZVQ0oaE;ECaJV=~Z0))L zq1JTFe zJ>ac{t{I;f0Afe}I83t<%R}MYkQ+v6s9dyg9@o@c<0dVrSOg63RJjRi%an+t4)|I| zKu`G$?2!In5S1<{p?^XxGLC814<2UmXVQ3X1H4LSe0Q~)_AXHR>Xv@RNFQR-RXDEa z{lE-I?ClFx9~&03@OcPsWl7&UA}rn+JkZGMLH3vMa$=P{pE43;+8VO==?`g)+YpHw zE4$NHOQHDsWdxeZ#IM$*$#PfW#k_dL> zMXVa+8=K8u$?na0f33 zB6Oy3d3Z*;AmgzZ&pifj5sp{cT79Q}Q2jU}n?08dFY-fRo-Ry=WNe^!AXkN<@T&pv zg^>oSuH`qZo=&L^M4)xoJay$v{KBA7p(&2(1?ka%r=q_WiesP6%`hetrgasAz(kC# z0`}8lXmgp)e^6lTL1P~!A~U~+k8+b9qd$5v8ogsE zX6yh2{r@^!MYExkxAhE+}A*Ro?UTS|VzjdAup-D_oX# zZUw)!!bv|epQd?a_C&t0L1y2tqs3K8b9v%!ja4NOQl?8C#aLm^X~a&JD5$kFFw;z zSyI>FLtjS-(U(0Hkl_614Qr*|Se?ws5ZjA_mlOQpww{vocc=c3tUnKga)1BFac!s5 zs-(2x6jCH7YnDtKk|f!Rm{XG79N8M%bWW#)qzEBKWZ$xHW2UlYo$T3WvS%F(#+c=N zuUR^u*ZcSTN2d;B=Dwfz^Lbs@WAnkjv;7d=*;G-yc)+%-Ea*9khHPdKQrzWp2c{py zg=%iG37jiD1(n7V4>4I2*`wKSLJ^Neom8t!R8+28#(n!3wmNb+ptyYU`t!u%9YTUC zmxp_wqH!gtIR>NWoue1IZ%v2Qy9;_iu#Yeo89EcX%&PHf%wfkc3h|YL7G#F_#j+#w z8~cSv@~Ui_B#6yZ9j3yl4DrV`W?Evgza|a*9&pahN~C&P_MLFJ0Tbh~o96U-J|M$o zkG18%?_06n*edIgA(snHO^e+^MR(pEYopR53cPfUhZ?N)%a!O7AH!hs(=(q_K8UZE z*8D&>q&XpU{yrf(8dQhG%QW|7j5Ro&bUE}Mk0gG^`t?+OUA=no_K3&70xD9qpYK$u znpGS{i{1aUl^(i@ z)*KtxHP@sKpi)@u*k3<%a}n9wKX7;CPc5^-#bzuKd+=KSDF-1`7X2S%k; zG~32dU0AiZdu)doHzy2LdN$oTD|jF`-P%{=`P(~zW41Z6pQEN-{Ihe4gpgZ8`meNn z_PzV+B+jf`0cq9$N#J~&xmu3ivTT-pzSAA?Xr{J~`>Lt2N^#BsxzVSWUEX@$;e;U3 zqs`u1_5M;mR2gbmo!WK(LSfh(Cp@&SU5MCP)~0+*Z1`6}J*-qy=aHMGUj2WhBd{N`nf0a$~CK<)dni&7W=z%I%|Gq#JR66 z7<709#hDviI5fYIHa|CQ3BMhYr zYM#3mlDC}hOX`xyCqT4jh4$%?`|aKVXS2j^c_~zPW?VCmHjZw`VeYt%$chVo>qmd6 zUgy1mFli+c<~)(mEvj#5dTVUDY@QJ4Q#~*Kw`&m?Pdb{LIUc#}SN2}Ehg1=2X%KcB zS=jt-!yNM)488MBrLXNTR!3cO&q&pCu-jkix1S+vr`$Z1Z#i}1ShOc1wg6^F&vF%4 zdn~%YT1=7ebd>&5-<2-QZ_kFUnxib5SN-iW9BCr4gUoa1u%5yG zQ;S)6yLVHWuU7n6+x7)*iteSj-tFJdXu($A^a*@!6MRCxpuU;W0996wmBo}%Dv9X1 zHRK`01E$4L#jE$WkhkfF?$a~7k%D+K4B^0wZk~?k9leL%9eePvz%D=AOr+y((zlQK zCozjeSNK@ZA6C|=2L^e@1&#c%Xvn;Jx}-|BE_Q@eD@kxL@9i~qb!y6;Ar$3}Ms11o z!}s>bGHD8RUCyix$EuIZUAs2kk`q3nlb~Y97B+2lB4O?j5BIOSEFYRRzE&3WJ)|LY z;dw2K6gh(5XUTG}UGwm{nALQ4BI1ZWwFrZOImlYiSMq%5r1;JmOyM?>sA0L8k26e- z?sB=2XDjhL6sL;v_seJL&3RhyFE0CvJD;N!d1z0DwZ4WyN0T7)H(wuijp{g zHQwsEm|dB(DTA=tigEAH zfFVSWO2EWw&z)R{{02|ynBz$KezjcH+X++s*BEE>zlE{OY&5`P8#mSdJRz`cj5I#6 zY2+cs_gMrD%bgb8T)XPtAl^H7PXPvknjoX95PdM!>Y&+!iDQ&ys(WT?3&y<#DC)^^ z?41i&4!zPBR#`w7oXeOEe&>ynBM#gJQ=z)7%zn%2v8#U7lkM-=ortAvQTZxRtM!T= zV`%OukiU9AW+67j5|Q@1D~6W2~~Vvdv%S^3|7N;ithJEC2T#%8cd0i^4QVZ%et( z1BkL6f5s-HXa!tlm20Ku~vL)KTB=?Rh9n%|@uWM!+KQ+ZM!BCZ8p)29B8Coo8pb ziNySJs2>EmXJ44zsO*#vJK=;~6@mV(&@Tv6=HxhB$GFz1RV__1XZsvyM@prH^#w3B z5Eg1l`|hiLyHEAQf$6W^rTNnSVd^Dc|NPi;R-DT?J>icvmn?IO5DV?G$xGsHwGK^3 zgqK(cPNlMmxVSHYovD_UfrMq%Zc0AzqgUhw%G11#$VL@-v{$cr0#u5X&_t75nWr?u_?FGm z=}L}ncCYN;pYtNmR_n9A|5){S*#M?IwyD{xot{(9Dj80RIu^C4OFb>@ztD|UNrhRp z2j-)_%Gk?kUHv@zJrOXz&brwsK{Y<&?`A3|y@=>ci&-76Kw}LE^RzurFmADe(tFfP z&uE0CwA(I{E-qU;^gZ9LEf~4SmQWQOZLMyQLuTshUpafrNn_Ua4%O-brZQI0cw7E8 z2QlPuEwUp|PQK{b>{}(2_<~ETBbGC;HDGAhz9rpNk7UmcT=zIw@^`2Ibj)Vv*5n$3 zP5AWeMFq?+Brs;R4c~Hv8bwBxfD`BWlc)o9$pOM(?Pj%o-0uHcj1ZFUxOwP0M`GQf z34S<<8_e^Ar&-u)<#<-~9e9j3IRo69N%gNVyIn}fxTyvxe|3Me=8umPdpE9j5`t3g z2d7HSU+JQxaQNGpPo+6#Ma7Sr#%Gu^=;!&ErW1lJcrd2T&l4e8bcYT`c2#k;+aFFx z*TR1Vw5^ECS+Ri>Torb)V<~eb3Son7x%DTavLq{9SIx31kcM?Jqa*sJZ@t8-Ow&G$ zEaYJ1q^Jn=I0dLxPsl>_kbP$hE!W-tgq4R%-F&t?*8=mN^{+t0ays*Q?RfG;|6XZ5 zT;vF;0@%_$Rp(dgN*FJpdHl;}&18b5Ag_RE3~v!;EyN1M9ymC`KATKl?L^)SW1q!0 zmo39AvOCMiT`vujRYGKL+iN^SSX_YF&PPA;bg+OFs|yH0Q3&-`>NtCdIW;!eht@LF z@N6-UX+Octla*lfCXP)6{qhbG=@eQ%(*40!rF0KEBGHyT=j3Fn7SeRhe|o)>ZZJ}o zD4kI0lp0uB-ZFdilH}u-7dKttH+$^>@dI}RVE$7G2 z<}J9gWYA!kYkG`h!nu3(>xTRISS0-}L;%Hkx%%bFHZk4xx%A{I(F;=#hpBsMzSrHK z4AyH;@=QtRMdmB2!)6H@6OJ?6rS8L=pu3{W( zU0v>oKJqJi)bpf*CfN(oos_W@S6srF=PZG{K$gPDAZ72Pr(-#drjaVemg?i)&eI48 zAil8?bEbka_ZFFBCR9XUtIR+L9jVn!pixcV6+RtlqK2qOQ@S6W>ceoJ4pT0ly4bo+ z8fNX%rMJ6Y6zVCskF6{xP6%@N@x|8JJO!Bmm_b+lNNqW)&pcJwSf~{+ixQNj`BK4GlI1{x@MGf?7KNg&2P+$%;uYvRu+?05s_DcDA><&77@Pdm zO%iXMOZNL}$5163tkP5u8#jv& zPMR{gg9FHE#Bh~2RsEmD*BZP64rFl~cXM3vUrbsl|KWLHKWKf9Y-iB zV2mg^oHj)77+dl_TrU_iHak80_N!iWuHZ4GVqWns+zuFS*j-eOBHK`@djn=q-9 z$Bx7fu^bJwx;K|oj~SkFO>lLuPd&cj8rk^iY0=^u7Y1S2+TH99YZUM3haYrEP;W05 z-05W0>kzhI=%O&`&Vd?3=FOF*GyueX1p4M{Yik~czE#2u^qb8Ml@?Sm43dCUX6JI_ zD=@NNEPuT~M37}+A#^d+4o{p<5>z!Tjedtf`#Yu2|6oy~CrRa)oy%cuh`{cO&|P#P zdKkI*e4UriVTZl#0aPu!(kp!n%E=CYBZ}T0I6^)BBB85gQUMWnxE!|9e(0YL5nXZ& zZ2kARng=%7|0_^Q0WWpE0d^BSa1yt^4E+;CLwb{#veT>!tp{9lC7XVQIdcWxh?TH} z&SrWill+4&VAmXIgez-g(^7^?mQ*-?^&V_aS=;^QBpzVDA>1d>gr|Le?I*Dh&Jscm z#|ROlvj~fjFClE0lZ_OoNrwGy7X=0jN)*l{Ifv_z%SCWbJqiP)f2oy+9QgB)6mCMP z>Ef5I7@IhD0AT^5+qjdQ|B<}{1dIIgKhC*0fLGJ_!0WF+!wq>uGp?cU|L;E?Uf7Q5 z5bWjx&w4!dM&yT|ZoJ%NUVUwje=XCZO)lcc9WHXlYTCRs4c^~3bBY(earyXhx*x_{8Pw%4 z&eP)K^8%I7Zl9SYtfx7&R5f%U?IT^BoCuc>tpf?ki;O zT2hI(uWw*~!BPh>WF;c!2E2W?2$SX45Uwj@{|69$y6~8-0%5GM5%$>6klX99qZauA zy{(`3=C+&+Ic#mATS->J!-$2Ofx7$r7>hOtP}odh>k}1UJ0<=KN~7Cjr(a7?>+7zO z*PK8F#I>Eg!3FmfE{ZSz)X z|Bs=j49W(v1UP>1mbFNff1ME7&(wQYsygZy)oU| z4Dz^jU&+%PFNR+d)1A{Cg#F!nuQ(U)$XA^U>#KO~-w4LRT+^lN|=Un{#`?dKKSE;eKwuKchf6NLDK+ zIHs(4I~Ob{mw2h8{0X+^!n%Mac?Ke0noD-z%L0u|HXM2Np0O~?bb!ypGHh7u7=Lw- z@e2MPsktBa>@k(aMA^ElH{1fmMmf{c(4OU zHb_nZzfN!BnyCOPknO{MElxrQ7`BYD52Q1vQL0T04Sz?22`%j={-+IF2KQ; ztq4`xd>JZKn4_dCe)wfcl@Hpg5rtaIa0lU8fw`d^n1DVy`VC9%3@~Z*?iFYIVj1Ic z=XQ{Q@A-x8F&o7;j~r#Ir8Q5&I^4XFdVQTB>q zr?|ZO|58V1c|OySYE1SH>IeGI(UKR2_-4YqOawdOUx9@_o*6R$g-Ebupsu!w%(4F! zXt_kdp(4hR7RT4C@ycFq38!b&!gWqQzW(Y=NDFp!Wf~9(?jp-L&B&9XQIj4Ca=32A z7WVIg#pRv$0Jl)}4S8W1I3~^Q|M@-Vabfn_gav4}u7WxZ zFMdn}LSju`Vr6cQPDA0A&SYVyj6o>O2d%;mjV+ExNzKIg1IA1N$?lX5@1mHX?BhSOPp~QjbAav}+n;7hPXx zz`y%bSPo|i&=d|z!92vq%-u~+tWmiJkJTNJ>OZ@IhER{UzT?p>DXGPw=s%YGV|{eb z=B-VKF~;1_e--<=@u|+DM(JAT*XACl9teh|rGc zVRmAgFtVkzbWJp7sbeW|eH`qV9E*g`|F{*&?6ZhIaAoFUcw9sok#maI2dOqcnV|l0 zALCM%5<pai|kkI%bk?BtXB)ok-QVhFC z3&;^Uv?T$t%V0KA5O4_EqfR4U=Il8k|0wY}KIUb_xt#Llv(pe=4FwmFZ3JZrv~(Wv z=iEj_1yuuVt^udxut^1oN9E@_e(G1To)lJhAg(nI_6|_-ayC!!?@+CtK$B2I!7}9>lomGe<0{&$Zs-xxlY;vbjzR1_ zuey5!3l9Mlm0(3YQU!zUPr2(X^j^sSn-^P&gM9)!r3g$GXzAeJP(CJ#9cb|Y-h?;v z0~HEbFYGq(claV$7X!bP$w`X_!cvdgho6U!_Fl;DK}!l>T*TzDCMLD_2IX^4C!Y83@1mn7xVt%$f)mGMGX*^Krc#X7VyP+HVv#Nze(H z=A@}}y{FWF1(6=dO(Z^?{=qL{&_3z0m>!R!X!0jeC$jkzE@xpQBRWvv1BwJH2ahf$ zK1;*9>+Q%^0L%@%G`YkpqYD%`9izeP76*CUSXL=ZBB-o!*5|e>y@&EAS|9dNL zW-i~&`pN^gp~T?+CN%H&aoY*bmq;m25yC0Uct_be$XpNpsr~0kq?7qO((R74#WAYN zM8C2jw|ux6NHa#C^`8W#@frF3eUcUN^{@XVm1S#`>LZ5-*Ua_PKS~k8Z(+skoIP?c znuRiZA%w6CA`({p>J-Y?a!o1UyF$;nZbUA2DEpCKC11uPProRizPYRQ9)nU+gi=1F zZ?(^m^fbzlZrdRq`amwsFmg}~v7r2IB%%1P_oTDg`Awq`)Z<#`;>p}Jc7!fL5>pngt`UMfB0xW=CW zZ~qlg@98xusc+QZqq`^8_W;5dTIC(}o65t-Y2zl0{%hXmcJs-#cZXb$c4E3OJhUP} zUSw;LzqYf|f$ayR>dn+@y$Cz+J=1(ykTobx`dByhTob%~Q|&J7jPMt{X>~*65T@u7 z(}>kgYi3}TtvYL=#;!&wm8YarX{%E^%v{)3IJwGu z{nY7MbL;)NNz{$s8F$NB7Z}YI5ir3YbqQl0@iEsr<^AT=F_&+%?ZtUGe-<`%KV{;)2<>vJmEWnO1`j0Ge-gh=-7)vur8;9x)B5E<)2mk_-L32wWQUUF-rrd5djIh8rPcH4)AJ5)=(Yj+ zN=dEzSZ=_aFU#$gwczfDSq8pwm;V%}C>6NKu9;sP!36DE9r+ZR_{{TDiqLKL+>Zi1rrz6;1&|C9 zwhvD2Cp>ky$B_Q=S&lGbsg4QEyBBVRP9N5PQ%YGRbott7vLo+3l=!?t%?mon^iQbk zpnZC}&0lRi=Yo!>Z2RA|H~mPFhh7UObL95N1+mA6;t585C9}nrF}XKL4J9(3UX`kDJ)?dr6>x z7mVP72iNPfbG}Slnps%Ac$Io>_j9~Q%EFNbv93?6J*Fzh5Ies~CK|G{wm_1|zXDA2 z$(x8=rv90@X7@|6)^AhY;@dS3+(CJ#z6oC% zzIcEbz2?4l!#>K}Xw_pipb;DiK$iB8-zSXvbii*8Uh4RF4i34N^b$R$;l2yl*0RX` zMKew}a6`8)Ah&!swsb5FP-5KArEIE%k;C)1+2p|eEXSVO9ms3#s{P1K)nPGr#TD3l za~9M;U+R{qczRpcmYsXFj5u<;r9NxWVBGGU*glf$^>Id>Q%>{IXAyQ4#^JgrG) z#*l8W-t%ev_ef%SlT%|I@^4K|kJ)P)p;A$@>GbQm7>gyR`fl8v%_}cs?%5ydrHW>g zR%dTokeL!ah+Slp`B9f0Njo>w2s^uuWu@Q7mmYhUTC;$T$2A{ko;KFqE#qT+4At*+ z6cs2Y+HUybdBt0`ki%QF?pW%*$H$~kJvJ)av7u%Bgb4#b=(Oh&elcgHN)z{z((3k@ z)fY>?g)3RuF&CpDbT6>^CiX4keqO(c+Gx30s@~&!_`53PyR!0(D}pk5Cui9cgJBpF zONLx4=PGyB%YI%%pL2Gv_K1IvN9W!RUR!MIup|9Ltz4+JJyCZkA@`urZ*9sh-qwV0 zwLhdG03&Vk`fALjT?5*ZCjs!rjZl0UX3YxKiB18x@1_4v4~9rj#|dTRE-THRSJnz; z;Vz8)=f36$O~ZGeKHu;W!(^Sq6nmjJW0;G8RPE|TenRv~meg2RX7&V~n#o3K^~*fP zP|}oIHszwOCb*`DIf*Z*&qCm)zj}sHLQdKwts**Z(^KOPfH*8NTTq9M>6Y&$82wO? zcNM2!gpf_IuYZ7SOgY#pZ*y;Q7?uL|9@LtizN?tN?*-83Yl(Nt2z64w3R70A>E2~0i%?hB{T2EX`PLQJ380JfjT07rkshDY% zr&`q04i`IGk*7`rmoNE>1eEOe3vveyNpXlLhZoe-PZ=~5@~;H2axn)hS#0jw;uM&(0Ft`~~wb~D!3BJf12PIm!1bws+W1JRvrAoX?++EIs0 zF-ypfHNgHMQ>1>$B0g2_RRepaao_04OS$)}?Ub+qp2)hwRBRIf?B;_O2ghTMMIizX zSax{cH8`ZH<)PrbQs-2%=Xr_75M}0eoBXgkVa?;lrG&{?TVi1Mn&sc^Iv=*+iX-F3 z#OT=#Ib$(ePQBg?WmnQ_U0cB6hM~B@BT|B@G2?k(s{7w%$%gAHCOaJ@Jl!{3uOm?X zV<2zL#IwOQHA>AWng&kcSqCLWzFw3xZ+RNA`fO5PHS#c?!{$H8vOebh>R*9Vadb9Y zrUVj?fic>C(j49Dn40XgF&|xdy8g<}2tmYTxGpp>qin$|FDxa5tx>(y*@`F!F%m_w zzH}j@#(hwG%s&~0AAkk3b@UZ>Z|7!~C8*o9bN&@@sGwUo*Y4>wovvDkBg-5o985`X zkhr+Af80KdxwJ@KyZ<=6MAYz(SF=)BQBDDcJ!^w$n$z$xOY2KpWacn^%>2HhjnVzo z+2C#O4xD%774s{CC77(}e+5KS7?!}<+`tJd>`;&KtF#$oEGPaLeWjf8w#Pv#bMN*9 zmczuggm&bq(SS4QIL-Oq>2Q7Yp^XtGY;1~^UBWO@VFmR`2huPlZ3e&j$Ia;zz`*LU z4cKENw)Xg|dqFs#CXx1(ypY*1tUdOjzG`FSsfiZsml&;>?A1gVDR#qzwp?V6@;mu= z@rOR9{@T)*@}1%)QN2-7T&$)QcdGAA8{N6_#Uap0t zi2Csc$u9)lEFr~QGEDu)b4-WD$I41fvE3IitRrU zac1;B*YCKfsr+EKxKe!-d(J-A=}qOte3sY&-?O?_jGbx6cf&ouptxp!MbJ(G*YmYr zq)#Hy149}Nu8mz3iEetN6tLQ2zgkc|13>uA!GI|(h7Y?qX5|UPOzSIwMuAzDmC~K# zbM7Ezi?Szy8!4!7xK#76Wls*R8ZtMzTZY@V@GN1oTLHHI4YmuVm$i$oUK^ z5nCT`J{8aiEoHIUF;*wu+}fvekA$x3+sEF;o7MIX(4(7QsRm3m0a44O+w;&N!NR_0 zHJHImwNj%TEF>!Y2;%-jDv{LUad=eSFI0{Yr}WUB_SC2QpX0Eqnrt|XOzumN>-UXTPi2oU(1ig=A#dCP}z{CGr9v{yk@$+4KVZ$@h2XK?{XW}h|A${g){?{<;)Wk9!5@* z4DyOq_fS`5V*GAB+*Moppdj4ti~ znZvf93c$4UbR6rrRCAr=&Cw!n@<8Om?TV#u;?gEaj}&uVUS^z~EGqaK*AE`V}^p9Z|>&6G|vwb4P(AG3Ev z*8r4aX^M!KxFZ$eTiOgb?wJg!#|^P(lKh?l^;gt>{pW#(Ts@R;x`rLitPFO`C_N@3 z^mcalYyBiv1zY7kpQ&!?0Sa5*)5oYE&-_U5|ecw@ZBCvR&7O7w&7L+mWoI`kb;sy3|^ zuI=N;LOy|?nuNvoXsWtK?+?{%Z?mIpq+os4QtdStM5_r3z8-l|>|K2p^$S^_-J}YC zujp`n6&MqC5-OiGkRRVVDR$MUo}h*F^UQ5llJx67=2p&A%I+gjKL&WpbNNmjcjF2{ z%|PxTh&K-UAMK~#>yNzulZL|3M=|Pg+;8JTvX{5^bEjePW?HZ_U!CHjvyb-t6fpf% zp*lM9$L{=56F&;@%RMw^ECF$FJ6e)EozvyKng1is+z_sfk-)4)!MzeJmm66e|I>4H^p*HG#AF^@>ORU=X^5Pvx zuN9huva8MP2(Sm^8Xk|)(#ODKFAPqlUwLVqJd5D001GtRtSyXU%vXGb&4Bd)j$F-4 zAvVEs?T{0<^0Gu7NN~`5bl^g<(^xokI5BN$D5ejLV|9ha?8yFvC!7)m@ZD^EP-!l9 zXf=|qzMNpe?MZyad;M@O@_T%77@lA#O#H5I^c}}6D}aZqA?_7o0z0k0aw~#$#hGQ# z(}`X;mkjn)p$)RrXO~(uX2h3zpx)pDns9wg7$=dl0Uj-(2m38NiG16f$|J znYT;~qv<&0>nyOA<;=nbWm)&1NkPp4*#g$N`X} zXWRoQk*o=Z)rqBbhhL)c_v`z?oIZVpH>=E-ZW{N83=+Gc;IJ=MfMHDnf)6ovFh2|r zD_=BR)MK?0a`?)#uNzdk*qc1x>sB{Xkjcc16iab)1x?WeU-`I>gh(3};X9jkk@J!C zE+`orjL{p@n%UDf!i@4p?eV^2Wan`#D@JgR?ZqM!Ebjfje|RyT<~#fdEd1%vzt?A8 z0ry%Wz95fP0LWA)7~TkE@&jZiI1AU5$rWI!Nvq#f@qK3J}qh95~BR8-=q(!%0^t7&I&=nI<8N}p{ut<*FU7%#*8OY=xx8Pct z?vq>t&KrEi8{hs`ZTK#)N`{NuSqog7eoc%;z7Y$1Ps>BGF`=J&^O3)9Gcoz$AmHIvfhXYwOg zd9THLDkq-pu3khZ7*bT(`;&MAv`)BUyFh@Jz8(S+DaHIX*{vD=Q$(x_{RYQL*X$#x zyWHb>WCoBf5=JJlS5z)?XY<@D1ssdsEM5j{^O~1Q!Lo^4w7#-YG5G;;j%T@uAdftE zeBDnuv?_cyK#BO^3h%K>OOSfN*OqdHfS57o5k@a>*UgkPcm?QJ-=1lGzR`4A6CLmR z13{?=zE~okS#nN@H&;Jtc#^Q#qRZTgJ-3@*V)8>Ld9RlEi@w`9th1|_ewi!LP0{p1!`;98OoZIf`PEz#$59ND?_veS*XHVNMijaQJ?1y#wSGZk*{Q`Hh6de(@)} zHH}y+z)yJebpx*EGOl-c%k0|@2gZ>odD#{R|CPHjczKE{B-KJ7YG8w594Q!<0-QsN z;6u&Pbau>uyCI9i;Zr1rFgkfE#FUmi=2{qI>d^sS?G{Avo&BkCW6aY#{MduMBYZfl&oyRlRxtaRDALQ$1NdBun?W#JJP%YG07bo z17NLGI%2@JnBe&WT|*_` zhUoxz0fvW%T}7WRJ`- zkMDely$ppyj~207Nm9&N>#97~YkxA4%q~}DpLr+EFl73rDzZ9)m$vK4fD1Nzf|ID6 z%)!jS=U~$TvOE^iiX}~Ii*s$BfK3r4W{#oUs246oV`ReQS-C~86K<=t(;W}6Xm=m& z<7h--ef($gDad1SF9oBdn}}cx&QXo>*bB}v zh|b!=g%)A5I%FCEEW*WRx$vP8I7+PNbrD<(01mE;$Z=jLTP-~hjT!=QF1m4=Xrbu9 zs$;@lTR^W;gX5Q3O7+Ef!DrpTj~zDBQPGBhfiF0e0}U6X<;E-05u^0v>0nz!&2zAg zC2qlZtG>qm&R#w=45xp~QvW4nWnd%b4H%L}*+t0@8-pG?V3fpWyn&n6jr*gj#6F$Uk*-Vx*2T$vzI z?oEp1@wvb?KJPn>ktfdc5;*=dqF{3#I1U;e2AvX5EX{-Nv0M#v*4uA)E@M0yc?4QH z*pKQmt611o3;|{^0A-WdKVUcX#)ABUG6YF8-N5?|lYGJ#RP(cf`DBafJ$K05kYoOi z%56a*R)FBSIT9W$A71FK7LTEw27_sE=eFaYHUfM?k^-cH{r54so^>WXJvF$`1I7+k z1{|$0a1b6`UeE7Mg|9~F%g2q`B*Z}8vOwh=Vqyx(m2Va!8KH^hVHTe-YJni$>^%r=wm5lpz%_^*F^jfF9 z{Q@TUZ5R!7&S{J6HCLwn`pX3LIEwB^=;jg&j)Qwyb6=b{1G1t7l8Kw=EFZ(e!YM_J zXC*CB1VCx(g$2;huu~F{B#}hovDSV-tTf+6U7(TcL(U`52^i_{OdR-0%0Q*%9@@Iv z!oob_EVm`&R&VEoOnvxQ6U@^8yI&@+1^9G;Vkv%}&1<9oL!=e{pBNEd9e90{PyW9G zhM3h^^Q>L$6>$jR-tOf7vU|kg)cbFheuQhwb<;2Y9v_*R1&6M|Db-zi*`!+vd*R>- z5p`-V-d6f_>f*r>m^C6rV=kB@OuzUu59aS3&)azQRf|l`V$0a)6g+4|t-7A;8oZd@ zX$6A~_}Lc~r8OE5?ku*aV1CNNvfrY9B`fIMfD~e&e*fAiOSbK(s;bt$7mLSf+3wEH z;@!CF$wBo$^}c+^*}8oIJYa#K*;^2TIaRm(r#LV9>L zM3|_o{Pm8x+=%Al;9GK4=W{6A9cXU$3o&<^VG(Z58@Y6L;$4^IP&tScl@Bbnh%heH zjr@9t`isa?u6{kip$C7oVc54}f881?`fnfK@rt}a)uu%+!Oe0Epxz>oSn(`KK2pV3;ZsaK+! z7>BM~Uj~+YcixX<mwq+6!ekIw#Epvxj?-%T9 zbUa9G>lfq$@6MY)mnaQ3=E|sC$_6)`onS>jUS}7{uwTwb#1orZL&$thMCL9LuXhr z#Wi9^41-~31ew`$64 z`u#Lw5INT!F_1Nh=s%tN<221Of^IBt*HPs3OerV`iCDe$0rFD)3^MEAe{1;Dxao=3 zwA~o$bA^i9>kiuuGW}D#TQfHhY(-@qJXUusd5^Uk<4MK3)jeK9q{aeY?zpB-b zSdG73Q=Scc-FK~H@bSdvj+7f>h&>ihzcQa!i4(hWkT)3T`{rk^XT{@L=*)w1kIJGf zclZoW?A9KuSt#Fn_6x;AY}5SKA9&l8s+!@Aj7;i;r^i9pP*iH*qMj7d?CQvJ4smOh zqf4ctOw+XwdVgImsd?r6RBHb>e|J(Y^2fe8qduF%Zg3s1m9w1Q70=j+$P6TPcgt7eQ`tA0`&_?=8Z+p0f_kxSW>)Q3 z;dX4rmT42u9F>R8ZStpXZiXXV3ma)S)>xyZ5nuv`B?rt(T(%aZ7HddHC- zB|t6f_cl+LZ*FFT5%9ybE2m0p9(0c<&%X^^+x*Y6y!w>Qgcv9Krf08i&B3nje34#l zW}O;)dSWg>bV}3$_C=E}$%ADkH-|{|2R^Z9-TLf;JVU*2GOi_TH~Z6IjM+r1Vr6}* zACb`Ud+C$VvRyrACpzLG!z#bhe=YkY6jq;cE^x{)!P+~c|gdk2`<-|TTqix@-4R}ACLe-BTnPkHKaSoN9T3AY$|+vjKnjF^vk>}u`z zjw94d-z4Tw*?6vbPUG(s*nn@WB=N2wbojrq^y6P+&hHt5t&2Wx!84v^bEiDYMzrO@ z)nt0LNB)6i`@I*7<6U^*JEMEzi=!pOazC0rK?J>4#{R1R%M%F;7hkBr$3hYaq4#c1 z#YXft*kwArJXe76E_Nh8Oz1c6K6SJsvFtqaR8!AB(h0| z&5L`s9)YMgq0g}tvx~LvCyGkDHTn z9PGXQ#_xTY*7Dhuze*S*K$71utzZ6SYWF}tCR@jFb+1*&$juy#4}C58hwIWq`8(9K z-xx0GsRhNS(Kl{3h~Ni0eJ=-g_qYcyR7J*|`Fqmvs~PQ|od|ipWZDx>%=Ur_|BI?h$(E!$|uGIdR&WnbjaO33)oH zzGP|&B~dE=3m|c7-c3`t3xx6NIM?IpE9U82hTYV*Iq62v6O}ZZ}^3n~*TUH)M9T7}z#8$68khFG9-bcLa&UK6y_ELav zKjfG*8pN7vTi5|%0Cd7hW|4Tn)quSyUS=eW zN|h#_ktP0Qjm)7ace5>tD|)y$dlqbBU-0|z7kc|6-nRqL4ozix|E-GWp{h(@-L=lR z{uZj@s!7CtBuj4#xQ$A@Z6pgs4Vy)aVFvdzup_tJoig0zduQL?yF(pWq|bhG$Covck1xaA*xA*k9!V)zkUi!wH~M#x9{l+%o3^W`Et^U<%g!yJYB zR_c%n<% zjvg0O#E|L_Vd@H&e!wz!a{7#r^yQid57{ZoWB+H!3r$OgRo0V5qp!g+v0S*tb=?#<*+u`~Qi%R^cmRD5le5;*e!XZLXQI!xDq4SPfE ziv!%rfj)FO6B$f?^9QmKVMU0Bw*GvUYRn1t)!*(QGC!W#%yUFbtO;wzP!3`^SJSBF z+1O&lI4!AS=L3y$ntu&(#KL-4+X%j*tjSUyOYYj2D>#TEdH}+153k@veBwu1sD6}cVUt~hfUs9X2mpY|6lrNTK!*o7rz_p_xKttIYT6DKXj>KRHyYD#?`F*w?_%s|^27edrdWU< ztlayB+o6jpHEf#2F`dUvz7J0yVogZTvG=BZ50&}sZ^@RvcC?6gARTur^9K##!ft#c zg>y%{eYewE&TqBhrKh#Ge_uR8HNw>`-?HKwr3oX;{I2D zl6pP0d=NHt(=2VLhJZGACkBCng12dQEn!-iaF6N^BP}X3S3%}g@4JsT z7|wt+(T`m3a$ZQ?!hdjfx;?Ljk-UNjlDYondnJL5_TG#MVGu*sg{O>!*UiWV5qd=5 z3|{Ryi78adE9!S+x1PW{c||vuJi@wkp)5NQ5KB}T)#IEHR|pCd1HqOE!^Jom-~8Oa z+b8|PR8cLgfo}hlq-}az4|9Cnil~173aD-tA9x3m@{8 zI&d#Ut(Rs<%L<;qpt`cT&-O5*$Mk&|slSN>K%nHFsxx{lz71*kt6vI6sx}czoGRuo zJ#S|PHVv{#n3O-wunW^GSWKn$t9g=WY+zjB!{ywIkA-}7Ke~twQ%0-P=gz^*`;+E^VYuBeeorlg3d&W{bbkbH=~bcVc|-A(b6z`q`~2?m%Jv!HC122$HP^ z_XiM#ag&T+_Gwx*K=Z(lvJ`H;LEzwmhKn7Hq!i9L3go=7p?}?kkpoq!JqL8p2A`m# zM$wknFjY6I4{9^jvc$lCn-5rrqla~QU?C)-BUdo$dVf&0MrN6S68<2}c5!U6&l|+H zjm_x}fonKdr3#og#=|5~^9nCCng>v_?}8a+^y>X+7;mALz<=4~8vq2JE4EiQ9ZF%*Ft&28Htl;+)O-b6S&v-X;-Gh~ET^N6d_ksS5VX z*Os3L(ZpZZ*Nr9j`6qdj$ODsW9a>Ey3}n2al28rVTNXH#kTd;`MQ$Wl!eBV(7!4qw z|HAuxbfU!<#^3!bK%(2Pd--Vvl!}P$9qd*LJBf;6ZV>N|k|$&8iCV9TaviKoaft0- zH~LCp5 zgd^qSRJ6LvF@ycwY{*NTj4u1aFjN9S=Uv3mCm@^&MtCyTwEAI;#@N z@{#K)v{sIXw!CV<8So|!1u2mV(fR)SK(M2=dN>VOA_``@^+k)=RZ;GZ*W(G0L9ca= zccI@Wy?FT7;SXpa_5}!o3|%muAodM_-vKRmCaIG<0?s}c;;dyJGhW(ZtH@Px-lE=L z#8ovd_Zz_ira~-4?&G&dN*R}|A&`qK%(&b}=*pckVu#YS{NOc-Okd97^#ex4m~qMo zqby_m5(j47@S^OefN=AfmdR1{|3}xG2ST0y@#E9hcDFi|Ek%cNB`YMtbVDQC`M24l?8@A;k?eLkP>_xt;!-OZYr_q^Y) z$>$NeF`&-YAR!CL#qMTz<{qkC|mUCiO zM%l7e;4#3BzqD=GFU&6j+E=V@A@*Yf#9gW)l|()i#MLEfU5nhv?P5c?#}gPxfJH%v z9kkCp%*JTdN<=4WtO|N)JP?fTYy?W5x<2UN0TORVvorM)2n2w{xna9pgyg?-w0V+O zLw=8!2Yg@P&V_3uw=PQYk|oi>o&wYnellisi7H6lk-2eB*SP*bqJ9E&Ih~?F0|{<8!C)xzx)$p58T6;M5qH>08It6Xo7Ab z)P?Yof8FISE146Z|3zhXz)e&6AKQKDo4%1IDA zJ6vs)pe6ur-sQ|>*LtBiyiyTHW9JJ)ANc7{S)kD7Aqzam@D}b(M5UgnZXBfS=<0A; zy%QEj;P1hPeT4?MYUL%8JRs%{Tq4hB8a2FLkxK%F!R`03CHYHaiI_(3)nq$;!~TC4 zBNo#F@l!(z7y=6Dx#?FcitrS+D(?B3^kcDvX;6e(@t5TFoG}tgS|an~9dN=dvKrl$ zSz+tNEvqm`=3z-WN6RcBu&CGsUGzqCSJR~Rs1g;$@G!irS)Gx_M2EyziPlUUQJjnO z=pPfOsV+Fsu)TE>tL%a30AshA{R0vcr5|T_&dpQ+r*JzG}U+%Sp08CKd1L$0?ln)AZW42XWis*pV7~(UwVHPkyAa=%2@6Vd?eoJX7NqVJ?e&`d6I& zw)W>+7ne_QCr7fWD(M{)*$4&}400Op6xfxd^$Q)^#u98#J+Qmz3F1BCbU=2&rTnKy za|n(=i9?=M`9zO;u%|Bn$dHx@-qtiqAWrkMp?P4l!Dr1zi!0?^L?F2!=3p|kYTL@VYY_#F-B>&K~W6DQ3&2FH(Fw&+!5RV+jGorw_@La^L(EWE)mFU* zSuNPulfmblZnf~{XYRPijuZoFj(KrCVIa;^UjxiQhQg{Upr z548W=7M^hg4+78p2G3aCXWZPg=0{+-dUF6-c3SkYM9LH#w$xjeYznxh-14c`)DNO{ zvHE;ByGvOTd~g3OA1>)mxQ&AoXpRNLewgC_xuJ#{+(fJ*l+<>JC(2g|mC_c5`S9N^ z0x-*B^a~LoI>ZM+1;96UaO4^sq>*LmF&su}O5%Rr;56i->#xvEnXyqUbK+4rH~Cpp z^|beY3c6_it!e=X7*>g~2R{oMn%-Hm)dFXRkN$(Pm+mV;5S7Dh#-~*)-ay+Dd0_-R z?0+8yjU;IRJBO4NtL!@)qe$TBQZP%@)WGZ%=bq48PVeG6i@mDU669Eub;*x3S6sja z0pI+etxE!fqSNRT;bO16;Aage`7wN{H%~M!{ks6V6Q=U4F5tR(PTK!J|Na5R8?1@a zI~3EX(NBqc{J>*xl?#+f=nx!(v_M;KX~Er5p{H?}bCfSu-za|mBB8x;9kW_-9ePI5 zrHL}@(pNGRA|JZPG~OMfj~m9$jj-^OYM^?50u;AUo~9j;=U}=4qdzb+ty>msrL1!& zP@f7k82k9Yd#j8zz~hHE!S+G|1Dx%OFnn8x!@Png=_jtS+Jhiu0YGv#&^jQ7I+^c| zf|20&OCG~w`b?_Q^wqrJiX=V~)rF(1GA%B3fLnDuz48azLtTsAgNfutbvbK?bC=7c z)g)AcpT(~$e|)>pmAD(2qw_17?|1~_K6Ym_O-mJ2lq~j*7 zVg{SCdv&rOQR1lN-506-77W9o56nI`2nX_>ehhD4y6Lg}(x=w^W~@j0XR@$ktJQ-M z`9H_6s;mwSj#}_)-ow&6#zkyy$J~)KOw4&9l*ceAz_pYgPsG@A6oW zJvsz$Kc-0tC>-z>7q;i<#Oa(3fAl4!P*S}$X{z04C|#58LaJ)m^RL*G3rP)qSnYYy z2S!DtV6P3k)b0a`-SEXw4{(z^ZDzVg`>=aE#oO%)Lf>y)QfVB!P#CV&KI0HAHgls6 zo8Y(RNp8ZIr9;m~*0c$fKaLJQ5f?FF!MN+YuzF{^b$zC#uYD`4r~1HSQ900WUWE=( zsc+oOq><>F-oEDnS;ssHY_s}hQdekJcArj)Z{nb2PGgNLHLpBt;g!{lF)?vx#;xH*akDC4jV8 z78LFIk7MNPnB%#d6DzZYYnE-BG8u-~z99Y%urn61_8n>vdKxiJCvNm8{ifgd-q~Nj z`s)3ePq^kyfzt2Fi`7kHM!hFPX`=T_l#Klv-r6-H{yy`0dhg1OgNY0<+^&@HpFBrp z^~Rmdy?Os;%+H)Zv+A=<`wn!oHdi8hV(O|67SF8~i?p}{q^nFfM)&fSpIj`O)>CZz zHo*$-KB>>C4#!rg`b9uHo4BaJ&MRV`N1WSLKMzL=l=k|VkFgvZaGABL=@&}Ev-WLq zpYR#*s_`&aHwNKopG|?6nY>VU_tW~x;^I@kHCa6G37{X>ACY&{H_so``SvvM`MFX5 z`?V(|x6NnH7^$VPmyMm6fDviG3&&1HT^q|H&+*- zi*>7HpRFZ#mNHMuf=TIdEud59cAf7?98s@!Z7E}vLvV5Wu+4bmFN-Q_*lJH_) zzr2xORJuIAeqfz($JF$Z>3T<@H0cV403r<>7z zc>~=?`a8z2hm&Yh(bsZpCQe*SdiR|dmZd@ifPAz3qfTr2G@=0T8cmzSf( zxxG5Su4}rn*zBC}sU7jP^LW5sZYxs=?bzFVH{Gen;eq=xH}_l-rEd>?&3k7P?;)we zVYmuSwJ(XZW|KPNU1p_T5FzdSG1pw<^KD%>C@f+5(Dt^eF}G9AXKj&7_E~4*J0s)h z&F@bhX)DY;vIdSCThDdATGFpA36coaGH#d-C@r?RF-f_~lFuIaH6^6YQLov@zF6VW ziE6FT^99>aFiJRcblaX{b(`0R^|cON=oweognesv`tWy*voX=2ynTno6X6^8l4(AS zTebLVg{DeNw-IW^eZ(*DRW(*-E$R1;)`yZUkJQ{U$KIC4G`)Lw_bF$mtNp74>6)xT z>k`E6+`bsrTsOW;F~fgflPQJMa#Jw$`(LZAQ$;6h*g<$9c_%$h7Q&j>?DO*&O|fr6?cwK}VLdLvqSl zG)nPf9dcg(wz?ty$OfSdIIvk=i-a*unPUsKCAhN*CABy>B2Yv$LgXy#*?BOM0fcUhR2< zP319RW|5LmfVIg9|6DH;T_2z_x0BsbSr_K-{Ek*ZlGDR(wkfPI8xoJdz#+99+wL+M zMv7bYFKZmXsjyd9YI%0DQy|Sr?7PIdWO}KH_U`* zjqZoyKYm_k?M$89>AY|LkhbWy>>-!Rpc;!N*Aw*~zbxz@6X_T&dl=$zG}5vjFQ2a_ z-F9=Hyp8lVP*$%C@$qu`rEU#K9~bi!=GZCU%wPLm!bOwIWVNYsZy#9)U+(+I_-`fq z6H!%d<`Ef%Hskc`syA6KofX)$9;hZa40pQk_bvH%^cb#p+RD@JONgreYn+RZV#3F* zjEDP5+!v=q=LuV#g_Nv*RuS}+s&SorlE9Y!E4Z|@w2x23;L=N@rP4TnJ4-9^l%c-Y%SnsN2@@ujMG>-s#22adw1F#+v7UoB+9|L)lZrh zn+N3}ue7{;R&T+-XW8e66Y|r9!(ajjV-`bc+j zJjeVu=ohhQR#|fn+TohB?C6i1f}_|sOd%`#>ABP%Vp^Y5S?b-Xp_$Ts*o|3WDZ1>M z{95{4oc_5z18e3)vNF2O#qzbV%kK%*9L2mLVa1l4In$B-Vl)nI)gwGfQN`Exw7O+@FCT<_AA z%V&l)M(f@!=EVFZ{h*4ps#lzDYLmHi-{11suY>)Ssnczu&3BW34OV*?zgvVOd;H`g zQ@QpSoyH8?U;Ae8BGVmLi5qFn9`Rq7*IGoZ{KAX<2>TRT{rp(u> z?A{Itov2-VMV_opptD-49`2ZKZ7{v_`5B|YJA`G%VPt8Sl$-da#GbCP%}|qUzM4ni zB=183yXT~y>e47DPghmq>~G!Eoo@op`TR_zd??>R%T1q=?tt}iC+ci{7VLwf(rgPz2mW1Gn6O+Vx!gIBB2zovU;Cz5ZoT^CO!IflCkVEU zP}3Z-d7L5G@rSw@C1y zFCLuRQfd~^qAc>S^S#=X^%X=_4Qzor>6?qcR)<1)X+WjXhwxk7oai0TQ^OklO{-o# z2h*S<>tehQ(99k{ERsn_MzK~vmaqfCys!P3%*`V-*XdL{&NuDa^88S${)v&bl$p`H z$;TfqDr2`cuBmb^7jP>IlMTr|Bxx9#ll4b~H%v4m!Lu{GOo(7r9M&RQ-w zcKV@#S9kQZtSvQ&bTqhmK1bPIb<#SH3EC<+#&ufuzZSW9(0Ty$pPVAyQFEM?$)x4a z{>o2?S^f21wEgbyljF{ZEfpB`2ZK$RWKjG>YNrz5-n*n<{3T<~#*RFrG!_k36+LSc zv6nt_BTl&@Bie#&FA+L;GMUPH$ee)&X4`!rn8HcT6xzmD=#k>|SNHJh_5;8Kefoip z#T5sE4q-cZ^iLaFekd+rSb1D$PuVX| z3p4p66o^mu6hanyioG2BdK%tdVg&WO#Fm%3GlsC-V7Sh9@n3IxK5;F9OBbIdT6R06 z){fTiqPPxb-DdV(zB%)3Mpja+zVb>S`|-xvr;94Gn|mkBpWzQ_-mJUA%z7-$`Pq!_ z{S6r!*-iLjn$6k06dGs$XjiQ8oy!!>l4mRzti{|fF=AHE+p9hkL31sm()Bqz`@(=p ztRR>rK1=e^7$E_zJX_SAm~M5Hqi98gma#RB6P-Vv6B-nxPPLs&mSmBt;*mWl(*P{g zF9j)V;Iz?q*ay-q*U)M{fi&SW+eN;)lvYI?3!*UJWiyYd=AkxnS4QtP#y6AkuFz#w z9yBe$gFq@lAw+g=mClxjSwis}$kPw6Zpht?x;wq?Qyx^Oo`-e=l4IXb1k6nh>aau* z=H(^qBGWCrG!(Wsu}X4A zO$6tHVw%#)Kp=k?_td`oj=2ysMPs!hUbyV2!xdHJN0;;29Ro{*P6tW}_`pY=`(ydW zTZ^(+4dSyyX$I|LPRBS`dI$qVtMW%8)`hzJaah8`tGHCN->DT9O7wy=#HnGKhf#S0 zCH)+xeM-63M{ZE%11sO4dFU0%v{P$G+`$%L+d0Qbosq)cz{R$ksu}85upIYa4|Po* z)X9s2`u3;X4vr{Tj^DTKRI~{_0?H+o7dKhbNuxE*;&}F5Iz)^vO=kjd8`emEtTI?Q zGu7ElIj#!}O$QGP?oAX|QoY2ixVv~`2w!OkEPlvVcGlu+@b>&81vWeWBp_ws6wrAUaa+nv#(!jvm&~P>`zDH>Sl>;e`wYg) zQq;$qEZ3c^f`+Vbt9uD8H6B(Wliq!f^=yT34l~7$K*4%)3dm^~Exv5*z{dz&9u>7A z56tAtOwK^K*-BoPgjhamnidRF2TSEijfA5sQMMQ}>R~y?tH1MdgdAhAw+tY}qB9|P zJb#>g-B(A6H9x+5bxZgdPTgsF@a~JEu@Z=8WRAvizfQ+R5Vt-EO$27P!O!mM?lYpz z38)ayjCl|w2O+5!2_rq{(9prr#aj|QBTgv-+F3A`6-Q64uPg>RG=!}9>DE?Dn7-7? zWi%u%hB!{r4hFC8FmOu%3UAWbKJKTreSLJ0v$u`cc8}-kMhsS)3}2EwMF@F5#y^dR zLLgr#=J5<>5h)EM!eWti)-@-MnrnN5UhCMXq%m?1X5Fh8C&l5a8?Zc~yVsd+;$buC6TPzwaeV_*xWzMUVrlLliu3aY&nA(m5bRjG zi>A${3aP>T+t2%S#Bz?D2y6+jv@ni5Jo8fQj}rk^oT5*E*jzu9+@;_h^-tN~7yq@M z!kN^co4h6(oO(;9y8hV$(PvN5bbH*Ex_CF)*sOKG3)IHZd7fPrp5ucdy-7LowCk|50 zWsi$S_2%eie25C5h*GeGIMUw@JM(g!oH`!~THW-V4*kVb@6&foG91%%vD%5EZ)*kO zN5^{IG~C_)D1P^OooLPHY=fTOCQjH0%!#vozeJrc|4VeOmfs!MaB|MgCBGA8O!e^- zLClZi`1kJ;RCV0a5xZ-RMo=vWi+Y!b3#eSH-TSInwW{9}a-RfGCyJME7+yyz1m<(?N=IxQ zc7V&F{gz`_mLf8PhWx_R34MO_*p$RU%8oub`P~7f{bW62mwjqU)g^aIT1stnm+O$@ ze67C{w*fofS>HJ5u^V-^!5Jth_hOs|G~W3&$rZl#6cF3{=px~2;5bZ_tB^hE&4FVBuh0D85@F%CUc^W zoBszcifCsTf{FQ&ep;2P6!AvUYprmq0Hzd(?&F&$r%!L<858Ss9|7~Il}{8c(32+; z+17&IGEV&*s>|eNxqe!uH1m?`$AtTgO3+XZ=1Qg}+TVnQ7gn+L1Rpxb83c_~LO6Cs zA`kt)d)&-U)Npm}0dTVnYn-e%m*A?vKsN_cP$phclu9N}vwL{T;8&}dynZy~o=y&A zB<>gFDcoLLIeYS@pEV#qgnb~3ClO6>=frSQ0BaeXc;9Swtv#MKTolD;93n>-WtB4aN$dGO z59T|U0&WVfEGy76Pcigb?+&+QapXQ~m3&l@3GnKeJ4?=ptI^^~^my*S61?ta7OFOe zFvdJTAeLG2v$509J`#uSqnyw*u!U3w*bBN@tY9c$R?{-qsXMUc*3$KgC?ffnCRLWIvs5d3dZ7GGbs<~H|(j2M@3RaPC# zGpR*@e-Z9sz(E4(W=$Nb2jVVlSWzFtI5;IX-7wJ}u6&H#d1qr#4P1H(b%hSG7EF7q ztSPc=nid}G)PXTS zU&zYEOR~r}5XKBbi3>1(TTfps89D8;mvyXSp7Vd zx?IXR(BCcs_M3_Ip`h?4F7ZX%bsvas-?7y8Rm&!Z*prbMpZSZ{2Ck;vuo^&>oVled z5vp=*hEI^7fotGykQqbb!e|P2_2Mxxw$nHTbR@@b-O2x5F;}Y=Uku1f{ftyT1XE}@ zZ$ic{Pn3w^xrVF&Y8XkrT3Daj)j$hqM#@7mfRX~PDzHsUoJ9loX+Bm9OWAP?uYn+X z8iu=O9K*ICot_+u0{GX1ST^Mb8B*9YhItsiygvLVyc-HB>t&3K(K{C-YFB|ID6a0}x0fzt|*)BJO4d*{cS}Uqs zJD-441K1J+n9uj<=2~HMzu>1;7qH{0czVB}!RoKK!M$bGQk&##GJ0`mSdy`1q4;&A z>DD{{S7b3TIZEiSJeJry6L;UN8y-g*0l-4LCeX?Z||`+CmEVJX8tD^D2{=5sq>icg=Q| zBgOIr#^{2KqTbveJH_)M<@m2K6bWhQV)9r^7`BGw(9cOcD{a9;|0<|o+TqZDUqbmk z&?CO_wn2&H4AdrjBUr10=RIl1@4VGtkc)E{J@|QrpOVZTBQqS)WmJFC5~k=6rV;m= z@tyg@5pVea8OobocUH&^++c3lWj68z>b&82dpEdGpy>}_+FUvL;+buPe^g z!hC>U!-^CBH`FD)E1;17hDNL9;T}LOQH35c{*To|1$4^%2g5UQNBy=@?mEw(B!wA; zYeW5}lQyVwMZJ$fP>d#>K-HtdJ+h$hxl)=vzQen}j{n&(VKYU2&b0aHBoFSpBL?U% zcb`GMG=4&ln{d>>BZd;C;PYWG`@jAO^VV-8N({3JR_#`v&GwHXfQ!biK}&+>Sll5X zI$%5T+#9v_oNz67+Fapq*?XwA$z#qJf9GbUVE}af4@C@$vtTX| z-k$h_0{Jr+OE!XAyOV#=PX*_EiU0Z|gB_yG!Qk_fjqgf)`cvOX)zO1JxR~|wgrz{~ z=G=L^cG;oFJ?5{RKEQ@<0Qj+pR}k}#J!S)%?Ow@Ro>y0=H4BwOigZM@6W^z5Iqyuw?|IC z=eQPsy*!OfZW^1f4H!M-N%?~?b7m;`%$I4sxxx3u6Wz+pA?85Ghb4>ikEJix(6u)K zAeQT~+@&=qveb}@IEY#N{@c{`av+fzeyYI!5|L@q%eW-mFwjVR;crl^7O-Y$Ggi7! zC(QW7T-~?T_4H7hCHCItm^Tfn&6tqn&ndamk2;Sj3OnV=-d40XVZLgjyS$uT9_W3b zcrn+fURB*T;9d80S6sK=VVkJPPlE?bW2(u&X3SO=PEi$vk2g?DpyN5GQtSCD!~wgX znVaz|ODch*C>t7%b3Zu!_HJM`BLMH(;(=Z4VBPPY+lx?%};Sv8V}|38&s8-;}KFN*PE`YRQS5lcX&lO4Q`vdkf`wF2?Ir5;>#T z96z_p#DqXwc4D(3^TbrwbKk==(?@SqnCZ`qb{Fh}=G)kx)9y)a$e^ucq&s8c0#xj}R?_*Zu2LEo{WSejMO|J9V#HqFTo@@*azpM^3qwKLqbZ z2WO#HNayH>w$4aj*kW@9;{A4^Y4?pHhFvd8=Fztmn7N77FE8@XoX#t9yrRXzja!P} zozIq_q~6t!NWJ~7v~Gg-tI?pZ?Yl@ZNl{i|ch^Fm(URYj$!)z8acT{rm+28hlgpi- zo;u_!M|Y?P1-m(UWIDBKI;Bf6!jIOUf()_N_hW_+EPvM~-`k`d(mOUNU?ceY>U=F$#(Wy@GN-jeAqt7<>AiQV%^ z;l}+&9ay{Y$n5=Q?4H)*%GRO-?6^+nHXrbanoTWh9eR_`-15@_ai-Ae4n@yJHSxKmzV@k?^1aj%?z*Yvn9m95z|JZZ6fG$;F< zNYWh6C}Va^nRE5$K^NV1pS7V0D~xQ+xAmPRM97jkGfq^?^P)|r1(Z1bDVnG3i^yq& z;znGzlMy|!`uYt3S966;&1u)*BV@6BaswR?2)Lto5rOtY|s`=)buPnR(Et(|gbpL|^|QDk|n zWvu+gF0%^KC8p8Sp?$xmtW)mIvM?-Zv9Rpk?&_;PU+!t8GZ$QE*lYDjs#eL~E%LnR zs&IOliP@m&o@?WNX+sqjLuJf@+f;~5|Bgw+<(B@Oo94yN`GXPFOoMMchg+08_r;;Z z&Q}-l(@dENN%#G0=#tUS9qfy~bg7_ylkR3qS%X^yv`an}3J*zMXO1^Q_M^Xv=&_d` zwu74<>3w^Bm-!ClF9+siYE&wG-kz>HkJRW}w^oT1Usu6yQXBu`WV{r%EE1O6lx4A0 z+&R2P2rt*{VX67J^J~$jz_YEUw*J|w*D&~iWv^V5f84UnCQMX3wcAXQyoU~=vm)Oy z3C_N8$_Fc2r_xPil_)8UvhW$SsS|fHR`=WW2PvyF_sr)dIr;7N*>o7XM6Uzik-lF0 zj)~4NQ;mAPMhDza4k@#Mk(J=%FOx&=b+zv!SNHC8sUFy0+*#NlGm>CFsL!xBNz)VG zIciiby=^JD#xHGp$|q)fzT9`rdc7heH(8y^l5q3;n6~;JtA(u0qH3KHl4};_VdvGF z%c~b%VvGn=2}MNW4(+W^MbgDXj>T4(93DOu0Y^ zyS4m=DG@o6HekD0=lmV>cTeCM+s=$Kv+IxDKxqJUjhBK zy|%Pm=fP&_O($pUV-Ml5_kNkW^2bslMXxAQ2D@EXthYx?nSqpFK66X$>}!vmRe~CG zGgsh+7xRms^n{mveEw5Kkjn;#Q({k_=VF81`$0&(K-zcBsb}nNPy;SYYYZVBcAvEJ z33Pns{cO=OpaJRL*yc^^;c0dok>~>VW*S={IQ4FbV zQ_N0x+O*k7yX~lzHLKuy*Ied;%daoXsITp4wlJq&dVo_xq?)^n!CmVx|AuVsSNjo4 zm7FP|C+~25tefv!@r;=8Nee557LCqm_9H}o*Swze`XxWNV{KEP?tl`t*3%og)XA>m0@4Z9`+(Cv$yj8rHt_+1IFXFMMfLvok3wSI3K)7HORVwxTC*;0{L) z_%1ex&hF@pRGH1bEJd9dtYOEUh9Vr>8k@1cez_bk5B|nQ*>#Ef(SFLD1p{W_W4vti z2rh_OaAxvfPnk`{9t74rTZ(mJ)GQ!RF<8wxF#!($?S*&QvK0D!uDipb&02ZvJhehA zql1JhIfGt?$yVI6b1sO-yUd(c@5GZScizE1RU&B7b4>>udrJ>nma=ml-d;n2sv{I# zF0w$H=wZc~F)w|NKuQaCw&*xd(*6OgoMlt$E+PYpO%FE_rpqM-F~48+5!vh_WZ*OXIE?hj^#_1oqgNy;JbSv)R zW9(25R(`JbwxP9IQhhatb|bC@%q^-!2^Xd5k8eaeeZk zadC)aSq;GfSCNG+FY#Hrd~-Unvu%p(u$fE-r;Kndr`I=G4I5uWc(<3K#@iEz8IQf8J)ZT(Qy1^ ziKM;=5{x=yCp0%Un|n`H?Kh`mMOSZZpnKPsd)Z9IS#s!&flhb@Fq+vtP|c3HdsVa^0FbFbpX!p1`O^?|Jm&D#g0GRbP~1IGQ??3p6$ zXLXp#GN5Yv$J2OP#;Awfc)6185&F62=7Vte(ZGCfMq(xS0P_;xf=pXXMumIZ((E)m zqzCLRZsCoS#*X!vi^q>L*voEN$AJ!YR?EPW0x7loSo>0PJhx@y;xBgDS7Cz zzWhhZLz{vk7wItfGBZ%rT;bdjUk+Ie+MV7{l4R1xW|L}07FHjPWN7Uo2XxB3EGuR* zkk&FrBnX51R!y6>37eMNmr3}?zZhAcYdOZe@DnyyUa#GJ)UhJ` zN^>Nvnz2<%#(i^g!>L#PzIIXl{ijzA<5lrEEXslOdSi(TaXybH(;yPX3hdpk?r!wB zKN?_N4m|@!@#oJ7wdM1Cq0~#9FpNU|*0NuMeB-b^t&n6mw?NcX?9urwP?vQn9Ng05 z^!LdbGQ!zIma9lR^xQ7^BlC`Sy_^mQVh92Z=8ZSrs=j5!))#5!b5*UG7Qw3LwODxz zlHv5n&ax#XEVdGR4$dg4v>nX%*`q_;_CEX_@X@nsy>QqUSq-HL80e26(nO{?TdJHx zXZv7h<_s-+yV>D0#YlNoUgYVc$}t=Z+u8K@A%a!zaRO>`2GtJ;BgM!@2dMK06j9A~ zq~m}!!zuiIGI@3F=$T`%_Sx{DJf}pdI>B%o|#%Pu2%rVZK)GU%DzBdmUDnf_Ni%CSJXQtE>&~ zHpCN)0t99TQ?mD6r^Vx)^U&4+a6;Nne#cN&+!qa@AA-iay{bz!( z0N2!VJ@qBE3{Nr42!N`3BeOEWw zY$J{wl$WKKbUtaD*-mB}ZCAZ1|^AchSxBL40K7|zu@vokJysvdcbSEA>e4RAxcasrPi zz`B6y%@rT)5it*O8#0r_S9o;39+x9+`HqPVmu_ZOD|d4gV>gE%&2K6fv357le@XQ6 zkQpw}FNc|DEQfl{J@VD|-=_wJMz;<1OPEj9MAo7|qC}+jj+Z5D4mp}%D{YG)+L4Z( z(xd`htSGtGunE%Ti*qsuHmx7BQUw%NN2?GKZzQSv#FmSuS+(CV%Ly_C=L1VI&f-N@ zUMXEKXaP7FEkGM%iUc*6S1JKct=B5hkF&t;aeT1v!bzWEvEYxBZ9RpzzqML_tzCO) z;Qhd0n^JiyjN#(;PoE)jw(2HsycjxeaI3g<`_miUM5v}iPllQ(kBho|ZR$*Oe4pZ% zws*Mrd)j~&vj|PM|S<5KOiWje|-9E#-Xt}%}P#f%Qf;h^R!tXoLB|3d)#UWMITLK zf@*u5O_#K(7rWeJapSF(#+B>+dqt%meCA^@5Wbi`S!6}V}uj>Av3vhe{`R|E)T{%>T9TxbaREbLG$f8GFx z4_xF3-nzJv0}cNi7UIuye7F6;u8$^w>uvZUhvhM_vHTIk%uk-XjOu9hsOlXfGl7W? z#U5}Ok3+3x$n3Kewj{S-wKwu7>>D8b|7FQxdT5w7!$+#?-z?hu<0IF9n>6tG-S^c1h!X?t~AqI(BmN5;TRlMKj$#*;JK$B3^W`tO`& z!T&fFLbDGBuPm#ee(1CS+P;gY=i@9-PHJ#9UF|^GN%Xx3x)uF4R;w4Usw^JJodfA4guj&6VR_=N#{BTNs4qw@ZjYoXbI7dWh_ek=kU@kA#v=MxS zo{^O*cLWWw9k}=awgbsy)AMG(V;*5HK0!zh{hV*ThD5*vqDcMJgQNO0_9;BgDGzZs zSjC0uJ%VTyR*_A16Z1pBFeC6)sn}|s_bXRxXvD+%pQZNljjmU)b6A%`eu1?bzFR5SHqe`Jpx&o6rT$r@QXz^7sZ2-S+e-W72maSD3Vw3Gnu?DW)d zU4=4*(9Sx534)lF`_G3Und9-(fnBVjS z=;dmj-pUZNrv1aYO+oj!4B7~1;rN|HTLYg-9wwLjjz6@4pc`3+1HKNgM6sckd354| z)yADkI|E7+Wsj}jj|9HUK;QEz-WE)EZ|c`8+VY4(YyLB`Q4xC>on!4Ty8vFLsM&uU zPhzVVrw~e+>b->}oZiYeCxKrOZ=i;EDJmGOpGh6)E%x`ysNJJQZx;qg;~38d^XLg6^q3^z0}C;c z=;SIiP^MwW@snDk)g!q0`0?U9yU^JQ&oX<5jdDOLqjDT?7t>6uvQBsWNwy%Phco21 z&~K!(E&-}t65dM6z0e|6{mR4#6v+$XYPOTsf8|R01aVO1xsrO2IWnhG_W5E9E({ts zu7miuWGMbYV(?$CnP9F%D5V*MuOUwyAu+30qA#`h=A9Um5Ssi)it?Wz*d%>_8P{4(8z`U|nmhvmV_4bKXU735G$jm?UW;h}_rs=+_ ze?AdAHAbXzT9yX!qhJStL+(3^%2bf$Axq`709WVrQtX*y#fRPc1;>7fhbiu38Y9i! zbi+`%XgN-|(lUJ=*fRrZaQYGh8M}*+Z-2*d(Kum&kk&y-vA^~YSi3HyfLoup z{x+wzS5!8Z+?o;B1H?MoDQs5&-g9B}&i-zJs4<-NeW&gc7%4e$ctsY*-|#F)u^-++ zr}Q3-I)v?qSr!6(6V6S1tQB?+lO%BTLFeF#`0_ReT-%S(@DN5FQiaGmG>rn~U`5W3 zIuP?0RHp+BENTXohEd{ehOIcaF5W^F-Hl#paMmlz50Jd_GhaiHmdQtRLMiIo&%dQglrg!CP6w256Q9pLuDt?Vk^vZU)jRiESHw3WPZ7U68@NWw5-7h<~K}A02Z)S z6pey2&IKNyXv=3*Ez7wwy_cbGoOW#!xvx?h%b-afEj8Zis7qzGZ0?t zbPDBPF@t~+Y*@M@ChLH#C^^A`vnup4|GQ+aBJ;j4`VA?q;JeS==|EhI`LV`I@D}G{ z3}_VTCYi`MLA)~=JK5|{OuVCqwaG<{6t(HHwV@!IJyX;Wx+u% z;0g>FgYOudc(yNzSz$;=hguH*K}4m{RV%`{}Q;TY1U&kE+ z!-Qow(8%zuEToE%En%sLEw&3zr)h#bBNm=i_P->ti6|%upjjjZt1>fGepR^^=^YK-_fEQJi|G)q6Sbv&mfK?UM zJ4iWcP272I22Hvls*69$D}zD93R(Buf)RYS;ddcY`G@z9CaCM5s&^EKV?7ncPjlBH z-r-ulO7?^Rq-FE!7)-YSi^>Kb6Dl0$5w~LKi05e1THf}3TVECNb0gn7MJnNpv$6#uqm7US<*%gzyR4gA`6M1M>D&bI~CVgvRkY(+XgEzufbcL2$(51w_JO`glV z*FSUD&e1K&Y}*L6MVZt0@?r;3^4GoL-L5f?fV3zD^L)jk+UF;mjqmi%k>1X4Z+ zi2s~a`lLF)MExUkXlq^8 z+w^3Cc-swnzx7S43w{i)ZfnX)*LwQS2sF+fA}^1s1l&5dGabrsFBo7;ChXIpO>mwk%XK#Ven}>u-E-HfYL`d;Y-xgWphNsi(HdRswRX4ZdEK8b zAFi35cGMZW&RO(tt5CWvqY&m|d2yAeyI_99Na8q0(WcA1?KaVGsAflG&P}-kvW=Xb z5no;^PA%7r-BlbesF|j@3b53d7IGLVwjVOI(-iui-4B15lj|ktRA_shLFjqCJ4ZF! zl;KIB1dzRp7oa=pm=aMHm{4Y4U|&^K_E^XN%;-hvU#t^$*-`#WW#Xt}pZnr#TmoCl z(%d()Q%l?+B4ch$IlX_*E9T^A0VQ|)a*u8<$xXc9!LmqoPVHAFQ?U5()1WJ}HoeU5 z+^(!{U7I#+ zi*3dl4^MctnRfSES&+@{ZH$}=J^!l&w*2uzX|i+vq|)27mO(9!h_9R}DamR=<*aku z&W6Xgr)6V#+H*AA+Ican?uso@Cj4%8lS%*G@>5&Ry`Sy1 zZphKd)W6xVi~6Qm-eQkco|o5Vx063<7Lj5e2X`r~H!-_^aqx*!?5=+<)(;t&TQue$ zeA;{PT*1c$dhKn^fpArCRR{aZ!TV8bldfMJvpdQzXf0B3bFTH;pwN3gO4M~B*QVaI zfG~G7&MT+yPS5=2^)&4JvVN>i>zD0T78lmEB0^~g=d{u?i=Rl%I7;~)8;?VjN0sTl1?tFdB59sgX;}V zX7da1K(q0R6qQTvosP>%COeyRvOF>-44>a&W^Z(>A2KO&IpiEfTvJ|8|EKo$20c|) z_^tF80q-i(!@Ps$Nai(jUsvnmg^p|k5TZQ8+FVyCc9(v>`EZb|bjiK6ZPyrZwyUX5 z^BBz7xlTRUScp9t|8kh4GN;&?Nww&;loAVk?v-mqxe_*2p>Md9Hb9-wrQ21IW^((2 zimYbpj^8M_9235f_rdd_msb0T1^(rAr?a)?7pEx!r#!MO`&Wz4)|NyqjlNYBzTl|d zT;HUB$v(UKHub?^sMdKW`kgRJP?3v8mzx)I{Nw@Z4gIqCg7W#DgI!s>r~E{kFJx3t z(vD^9e0sbv^9kctz;jZicbQ$P)zP=i>kmVw;nG{WyJk#w&*q+bpy+mCWccx6*z)~r zy^1;FG71s@{t+r7D!cw&edW~+7laet9JTe&MRe9jh(%cF-VZNU=?n^S@o>pPod4PE zH2I>v=V42|k4KjL2k1kpR^&~_g@KWwK@s!HIyXXwIn6^p+w=DA%PyM~s_fc9d?YP< zEmSysK3-F1YRew+?$ipV`YZ2nN)%g4_;XqtTv|Xam>XLts<+xnd>AV#d>8+*_%wne zw`Td;R)4X)`IuuW+NEsWr1*(=xZ)Ng?^k!%&f9I~y^ygP5VW&URQ}V{=%Ys<{9LN9 z@#gH?b0Y5Q4O~TeT3rAIaqrA$Y3##4R_K>K*E+qkNKKVBK^h4m>j)pJUe!&c~{H|Z7KX+ANlRhqIQs`Nb=-(xa2tDQ? z7DW5HYWJFB_ZIJ04WU19ue?m!VOmSFO$yZs^p-8W_GUYRd*J0*z>N2R z+7Hf3*~eQ0b<=1&<)`BWl9L#_(jTQk7uzlpEA34x^9h{jZgo$tDuYd1xb!vlEjE7d z3l`qjmSUpy<&?#V;OT!=p*5qr%qiTv6qWm~E#mg)8QGT@O_olbb9%29XI2gY_;6vp z;>qTwl71+v>K9&__fIpMMl_tWC~=_DQdQO~6iN5aQ8m6C7`f&>z-g*b6VV7;`<HIUP#NnS3K*-iR|ec|y;JlpSEQmK;K$CAsxOkIQ5k~pWX z+TR8S@#>|H$wH)`MVQ2DlPX|PmRrzjmZd~Shvj$wj-TGuEikFGjmGi)dPJWxn)5zP zlq~!zr=O(*9foai#QW)$iY;a8`b19{jY(=P$oWYjqDg&BmUU{`c;5Wdo{p~1LmFCY zO7;cmE%t9ZQl4DO+l-wI*DWeh)w`SAu8;*|-i8tmGk&LwW{;T3@5`ACPbjF!Cjhtn&=R)mYrdV7G`WTVdJ{TuiD@Q~BBGP}O{v7h+KFKjUj72y>u`hWmltKu5j{p_>3&yK zN}_KO97`S?;CxYLJL~>&?VJxM>mp}yIa`o(ZNMe{E4vRky7)vf5epDvLoVDax)wO(isPco31N9{{98Np2q8pNV}f`nYlvz^Tn6`5{- z($|86%3g28=^|Z;4kC|Xxr$GW*A-wdhIhoVr>te^S^!FqF#Y@8oi+l8;YxH)wwX3{ zw7KgxLi-3cVZ7q&+1k;!K|4Qr+9*kOoO-7eSWo(DE^Gf{ka#dDTZKt1-BDo0V| z<;cl_x32n>RdJ;mhf=K_w+e+b$3k~~u^}FWm0Tu-B^i#4Ah}_pGsVs$y-wslY_uI= z89ci^!zrGYjoVc=RMc>0Uzyw-K;rQ+aD=Ugc1{L0Gf&zEH+Y3vITIj(uh>i885bTrD;5vg2&Ur z(nP9){4V%o6eJhtYKa7}iH$UY6Qa^DISc6JRvEKAy97T(;MtDd%k8tA32UW1HEA2$ zlO!9-9GhUuP1Z!Q8#2}oe2KKdUr*@l)mX$O%)KxUcoP`UQMY#ZdIm4{C_MPB(}{rt z;Y|BJwdBC5Vx_C1A!9o4IxEUI86E^T#uK2OJ0g@QdEDVJL^os2A5(RLqlh74C0R+4 z$^u>cR6U$;T7%%c~x23b~y(CGb|_5}O57cw={z91X63Mqs={x@6$_@P5l+d;^U} zDb9c%8K@{U6*wld4s%Hxqyl#sylYjoE`0rHE9a~7D<+SI{+7eBKCTD0+oT6xO@a24 z)A`pfwIaTI(iAvX)b&0xM_<>jzsoj-?)Iuz<{=d@xcc%ysPq4SUE8shZXGgP z6p?JXD#z58goK1p*;2_l$vLiRYfBMA2xXKzSMJ+vxpGeKoRe$h9A+@a9Q%8|-!u06 z{r>*=_-xXcdA(o9^YwgOnEu$B5Djkz9#Lrz2Aw1=O_U*K5C$!{Q-upjc%}08>md?R z%|9Mn$TTw4X8m%0Ej^Cc2J5nm}1in1<|jRek&{{!>29?&@C-BG=M=THgZ2 zz*s@+V33%(Ji$}uWyg7w^@qa_kKTo`-`t;?(keQpQ#9GBeEbtOt7hzv-Wc~NF|j9;t@z7q$Q<;Wv(*9S&&fpp zI>rHpe5Y9ZKV!f9P)`;xBxfxX+`>IxFBODKsJNaRSsLvx?x=YO488H(i!EC5VZ+^7 z%o?)=xyRdyQ>l0*$6*$pJaO|B#J%gp-`pLz=RiMc3NziA?In`u9(9rt5mbJcRQ-38HPyMg?11u|Oa$$(I8`8+}p zBWwtxKKH3yY087benNs4zgQNfzRKX+gbOhlesyMhr@9$i6 z^9fT^x88SksqW)9I>v06p<7W4;pZ}?4`9Ne`h{jev=89TK0|*ZL5h|ices&(p zy3yHqwm7R_FUF_DI>%>RPC0YG)Nt@w5IRQGjgifyFaAkSV7QIOyon1r&!`NP>w^wh zlmJRhp0L~kom8=9xzMRWM8*ngYQGvpp9$^K;a8}T=#+%$>;@?)F_2t%KlD%kM+6h7 zwL?n-$?5|^FYMezNvol#z-0j{Xl8_dYH%k{u z=&5S9`5^67Edv(1?;TpLk0*G8J2N^^hY%k_ej}L=SsoTd%00Zejt|Zv$$`P3Xhnb7 zea0<>pU&|JLRyyS7XWO7Td4tXB!x$Ej(DMfu2q-V1P_7lAQ*J@GfPMMS+`>BG;9hJ ze7(t*?$0>hd!;t$J!?bKH`3w--HBDL_+|GnetZG?((&mp1qd-8>kmOm0bxN17X#s7 zd=R9hhAzuoVPtpO@1G*MH;aSqAtP*R$rqPi6TZLhEaXn_2gt&`C_JajFt#wfB5_EP#9uc4rk^vSpw6>@G~ z^$3qjnTQhdw~rzuy8`lTCCLt8T_xQ0Wrrzu%V9@h^QORsK!~5IZs3Mnu=dK0QD##7 zaNQK1JBB?DGE((Dd-X_SlA6;JCdB;$K#usduOM#&zi2KQ0n5dZ$E1^O zAz9l^y3w{)Rl8>Z4hK@gv06egiPRr-CzG__#v)ILM;n}# zQtt6JdDg@`&~dW>L}+od)QSU@(0^fg0(64_$pXY0&Z1iG{S{vSCAMAy2!ipI=VCR9 zP+e-c3tn`kEhDcMjf_p@NFYnA+91}*x%nmz- znJ5Q>5r2*e$=5|p5$|>v@^~}|ia;cfa?<+?yL(7wmurKtM+UceGzG zG5}^+Ao5HMA^R%`tXlsSI=-J+!5tU!+iHswiB{~O*ps*(5dxb7eY0y-ZX#jMgzfP2 z^^;bM;Q7DmCusS~R=t(tL%o32^;r>U*wGj)kfF|M1x0|Xy$|<3Ut}A*QlRQBQVtb{ zj2XZHk2)sRQ{51V*gI(vTXtqGS8ahe0%rA%G;TLM0rXADa{4KKKsk*X#}k5fg5v}; zZkmw6!XgYz@H-IT!?7z{N7x*weUZlmFRdIxloL1^2;~Tz3wUbSZZ7JmzXIhDLiGDr zpls+d5!+e>5{_NV!W7Zncqfwwky!bUsJJnb?`?5ry@mk1Z%tl#F_xj)NGqscm@st{ zlWLCb#AfrpmlK$~2lEaSA=5jtkH?kS%l_x){&oj!uH=P~37+SXo+bTXG8Q)))P-6# zOa;C!;q%z9Mm;>cKh5)f-w1n1!H%CSpf7al@x_uF&dpr?lR!5~8Utx60yqDUJA)7$ z@dz;iK`IeVFsu>vDh>fe#mw#?)XCt!kRC;6c zX0i|>=Djb7VbJ6hdlMH)NF*_6jei47O`mVY@ew}=b047%2o%FH%NM{$As_fL)#V5l zdM=AYRB6Os(qEOYi%I|l^!hn`4bQWHJG(49KeL)Y-<2!iEfciOFD78Vzz%#Er3aja z#DfFewwoq5Ps1lV>IF!CO~{4M%a-eWTW>y8YA!moaFCGlT5=%juI5PJk}!KbO>k;L zm^IMJJb%-h|9z6|r!fii80lL+4u>0Eiuk zVDa$rktxXMSL`H8JOaNF#Gd%CWtsP@4N4l^4nTI31v3*OiV%=pD%OcgXvKD<@@q77 zOS$@bhxek_8y^A?5&1zEi2R;!4?YL*@d*KG>k&p{2cNJ3gCG?v#DGo%yp$h5o?Z}9 z9KugyG7xwm1H6nB6zG=~`>%bR=QCy3A{S3ho{w*neYS{&=^!1W5!G4PL7TH+^+IAEe8G)~X??&9w*p z1>_1o%PZzK!Tn0xqh-VPSJZP0Ni0IwhZotm>G9l7l9`bDq}WyL0mx|L4_@Lqao8u1 z&Mtmi%{WW_Or?E>EmRffUq=Lk>npn0Fl=;=IJ*e;07OmypUntrU!J={;+uDjoRUaS z&84fZE)uP*EWG*gn(NJcQhV&mYf4>%BK_Wue*D99hmQKs(qu!(1Gs`LU?F~y$;Y@; zq{~1HeiUM}(e(*c#Q}KcKxO?u6oT!I+ag}xG4(nn8go<0m2Ad%cUsf0Wr%57u+*8b z7*o+SGaQ^o8fOE7yu;Xkqsgvmf*ZHjgiL)9%akx+l54y-;`ZwD5)WfMXEUPDw!Q~; zX{F94i>opt!S%m%uQ8fQ7_w$&f1lWuD)K#nK5&=kVo|y6EIwm!+Le8hogu89mzZ?( z(8t<@tTv9_VrZf}*}$)^f+>UPf92If+AKqMCPo?Vdf%izHkZ0Y>asVPTJagjv3!}hd;iypC1`e!8r-z;-1p~$s}Fnkzn6(J)9sEX=N69;o~jY{{&d!|dfY*o3cn&p zD{Ues97`HFKa=s!>SO=zsQy!~xqcLPf=6=2M;)gG&4)Q(r!X`4H{`L)K}AJXR~*wf z-?s^jsgqLz?W-K-%b)<}GQA~_TMwv+fX%3ou3)DxXzQx&+? z|FDhxUK@7x{M>$9(yu3bS#O)(4!*?h?o-(Nv@%mfS)AE|`v*IN;gz`%H#^VU?^aRo z5T~kmbjEm{DQKS;7ODD$c+y>hCpwpDFw>VhcKUK{*rx-OA`|RWmm11<`mA2)#<-m0 zE`|L=`g@OxJ3-qosYP}7lCfXZfPK@DYR2)U<`j!-=R75HzW!Epd4e&@y4oF>ldQXX z@Ba8lRe={jGz^z{AAJ}j`9e04eLlp_pXXt@WGv#RXg#lXtiM(7(Q03s9Zzm1U3h1o z#4B$=XeQ^wlSmw$AU$_uLzSZJ z%0c%#f5JRS-BeBZ0u5{Pwn+uG!(^kZPYq<|YxR}9uraZfxGDQ%2VBc7qIWHv<)pTK zd0P}PqVvwq1F~n|Y&UBW4;kFwQ`0MYyVP&I zLJ96Q>m0j}@Fn`KgVhc7^N|T@*I(EZ9%9Ay7afx{D%12j;#vo~vYSU4dzS7x`nBx( zK(1$KCo(2l0`K5kMjq$L)w&!GvpSq;YHT99|DCjssbyg7tgC!HqA}PIuI$sl3&GnV=Ub=iU;v)UxVOLa9wQcvGf3maF}}7+qoRvPU%@ zNzr)f67`|m{Ca8O8fcS-M~*F;V1`nE4(HJazd)~F^)oMTbZGB22a}?oKT@WWvE zT3Xs3j*~FnI<*yuJ?WNJt{>K~wrq^iD>>}AWO8=!Ysc}!wW>MLYde{7%yi_5S$<&G zGS02_x2XAQwz3wIVZz<*t~mK?W956W~=&;s*P79`ya>` z7prKE@R(yJw=XLGO3qOgl4CN6Ew7Il##C`{U(gNmH6E={-aFDb-R7Gusz7|^o_IgP z7RL+q-m15tbMHf6j(Ms7(X4JLhh^bBA^GAuLX6B8OA~`ldPO@W-b*b6L@RYesd*dJLXgrtz4S3E8?@y zE?w9vj+O1lsuAsrOjlSe`R{SWe7x>3SLW~8L-D5@vM&_-9VQoF^`>#?ms0{H`rtC5wF~xWnf!Hmv_d z!pWUXQA>?esfan2J)`z+wCgU$whUYD5|sXs(tX7=Uq^}Zhm6zJ8_K#_fdj6owJ<{a zo&3lp1w@mRSvMvoHbP_gOA%T7aoqlzvD3NvJ~tZkNJ~vUOFm)k8kSDGh}61=V64>G zs=DuRwC!FSlYMuB7I&$R&NkjC?{GLKN&j&pt~pTZlue%b4;dZDgY_fiU!bP>oV~`1I7xhZF(ip@d&bB)p{X5E1uwZ&L~LuhV0+G>rDn~Hoqbr1+xd=yX#2z z3WGLrXYMZqAFq6Oj5s7KZwMyzB^Q{oTCNchhBb}>S;)*dJy?`K6aGrG+dkoU;`9NC zA8)_kak@NnXUtm=G?V)8VO`T^9d#vB8W!u8I%M3ZlJ;zc<+?X5r~f+MxWhs+<6p?v z_Nb{8UTw2ib~0S#cS!LSj$|kPVBagPFn&yJ!bo>jL*8AWdrXsY>&418MU{04fvQ<_ zC|HaR2OokB!JT7cZ~9n=O5U54FK_gb9B;mcvb3ZBDo2=Ryep1cp&Vu(OfxSajD2Dm z(g|gzUXflMzUo~7jz#XjgJ9D66X5D-^O^%(qk-Ml7Q>rn5y9>Pfc#_tS*7$rD; zy^FmLPwo9#W%jiy6Sm9CrPq$vYyZ|Obu4fZorOr0o3^zLJY(WLOH6+saqKcFC=FVel015LvhSuc%Of9YRh zOi6FFHBU7xQGz{l&3^pVA|r2$cuKQWfERmzc`JAI=? zu@jNoRb(1De=tq<{v9qaxkM`MQR0P7?3D}}S@hl!KBS}bh6;GoGrh8Lit|Lu(PKmT-Yi^JqKE#!_ zZKTZATg^9K=_fM2`nX(BojnlV;r@Z)YSo>UxBF1g^Z}dhaL?M?^z1l_i%Ha=6pvd@ zTij_+-ZJ_kvO6KM5{TmS0J9hdh^?H_K`}eN@lh;gn;eA|d=f_&PH&X>YJ3V&y?_&AZ|Fqd-32R7iA6p8od7_WLziufk*>yYj^gc;Gu}Ri%d^yuAI0NGoCzPaBfAQ#qm(L zJdbrCWZFeMOtj4KCK?4bJYmNV7L%TvW&Cy2pl9)6^-1y2#q(V?KV<^jU)u5Rgcsp% zv}@nh@+wQHi5O=!n*IGGpC18><`8DPbLBQXOo$Zep$noG z**0IO(n_L}o?rts-DfcJcZ$Fa-k$`%%EEx}xh1H#{A#UAmh~dbQ+CIW{&sPCH=$O16hjFW9KE+<~B!?m|GuLxwW6K>-b%$gBiMpYY z27Oa4Q*Xu%e(pb;k@j^JhrIvMY#97HGC}D*GQjT zZ0CN`07y9tii8!78UpnQsNzD&+y1urd4iQL969yYs31cId#VfLxj}ZRy9WWY*y3bqM#j$SI zIciav#9MWi{KvK@ZLK9ludLdKF;)>hCmXHc+{?C*K2HNdRHwDvpV+;r%Cx%-52uzV zfJuxo?-{WM^1Cohrqx^`%Ct8q_aye>dmLe^I}$sdY?G7qqRBK2vA-uDdQ#60Dw#=u zE-56mBk`dYsH9)WWch;pr=UTAA`jBqKTmxfBsTi#1iT43%gkws=W8G0{)5s~QC~Ho z5V*Y?=Vv9}H#SqtYl?PqjtJ!r-zu>h>d&$Nt%pg$Jk|VN{<`=3#0cyc)uz|m$=m8Zks*pgoz$vh zG_FY)gdlz_c8eCKr0=WU&cpNaM89Txf6gx!Ru0hsc>-t$AgGU26ZGW;e`aD&tYHmR z5Zn~MS&_G;#|JnmziysnHrY^6V>F9{a>|V_NE;3BIu}GYy_s5wtu%WKUH}6J)3k%O zvWvuWOHlXM$Cgs$Tgx2+v|*2kS3uIULu(YV8gIwP(g*18m>9^8N82Y5GsP`PxuPaTB@Jl}npRJ7brv zdnh+zU0t~qC%db4HU%8L_oqc110WA9pF2K0QwSq<-10uI-&f5%ef2Mi?=7y?j%W5d zUABzspT|7FdSET86Wa44TJ6$@0vbA6`)|G3pRhI&-*~rMB`s^LQz9*RVIrzCH;1hm zZAGxmK1pSWUty68v>`r0HAGo~!HrA8>+age?rLC9f6rC?L*QL0cWvi+2C=ERxdT)Us`EmcX-5Ak%K}j_s1HR9NgSzT zv-LT@=nasySH&DoOqa^Z4w^jucggUan)IfV*Q>=Thp9p3{du>Wk9OZ5dF=qZ!Frmp ztgHnO-=2Jty720#wG_|J(H^|qDIFzhn<=&a4Pex7BKHAvf=2bn4c)l++>1PSkL5j) zAo_OpH077t3jY4F>!3@T{E#yUJ<7jVN0RcDm9iv*7YD|Q)O_ygx+DAT>0tF za?Y--Z8pKZr23JH90&Pu-u&92H%*eA#PvUvV%FunFjP@(sYx6JAa=t~=wXE}b+}bH z`(f6aPjBKSHL!^Z_n|ri7&ha{h|FdA@2KOjz(zO`Y28cJaDV5*hV^DE8V+fou^W)Q zU_iE9&Y(>{LI3VjP5_Yn&aCIlb)4&@giyuChpW%9MZ0L#0azw!!hNY(WO{%>q%OoF z#*;90(QarSrhDrTdaN->Ie|AK5PjfA@a1j;5Nby1<-6{Ji7L<~5lmj*5`Xg^b_1yD z4(-UfFiY5M-SnyguszIl;(et(BGWubm=HKRx=4mY5`dx{WKr)}u@()U709Ve{WDoV z2&Yx~{|B$7yzd}AmZi4TTrNjb<}&zg$n2V9w3wn>uWqcQm8)I-a^c@A0hG!_${XY={@gJVt=6}-V zyvCo$hhP0f!65!SOwy)F_CC)Xx*QvV{vaRu<$D+EA(-gnA_=}Tc{!B&&h44V_lU#F z90sV+9t{Wov5biG4VU$`P<+*(os9h4M}h2Ch=%+r2>Al0hT?Q3-Qh26wb`h^H2(YL zH%HLix_S^?NBUCG}z6QKy)fa^FOyms|SyrfYs#8Jb$ zk7K`LedEOR*_?|xu>9x4AmPkiChY5w5wCv@zoL8vAXK;*xfi6kvw z(HEVS=F7TVzj}c&JIBco`AnDS?-`}ET%UdSH@o{QzHOAIFOUGwkjS zG8OD()&3-QE_xra2BA0yP3TsiqkGY$yyefT&Pp$f2Kj<~u$4w%c!RhA1cVX>B)(!_ zONlDB-pU{>1nDj@eTU(J;1Tq)dVd8CV9j1aNhA6euEH*<%z@71P;WCt#qRBy)&%l_ zguyusrLYeQ*Az}wK6@4()+JP!cYkG>V20rkoj*(uLw@$Mka9hAQUNKu!rvoXmQ80j ztCE+jA#ZbAX%@HI82B%(99Tu@#M&WdLGh+M6+Dth`?6ZO&Itglxcg)7G6+A+ZE$E1 z?-5oW1ovdaTcmJpAf^_2M37-SZB4TDt)rjlw_-XOt5z||z>va5r}Rx}+h}*dE1;{(bqLhL8BZqZEQ@h&4;68>Ew@l|g~S~F)%+EX zZZJXOOr9|hTjm$?NSv`i$j~JQTKm+?v9a$YgGpQ`ke^>!20m9&m9jNP4rNfh+fC{{ zGY5K>+U6GT0I?*7uvaMc-b?kxDm}d@jnHY$g4sm<)lO&m6tfo~4A%gU87UB*NB%3+ z2KnN@?%1J$Op&XIgT|9(V{^E!Tg$L-+@l5b+G)X476%2#*J%jnR!$;LrPWcQ7Almn5>&?6{+xIr3ccPd+?{ ziMcjRh_J$$fzA0>;t9e-fTfx0W@wd=jDl$3m(h;twj|8gfv$B$mB1;b4eUmE?giYX z*mbaD_=Zu~ZU3L&gpeC0y2*ER0hSfwt|XuqJz5W=pvmzDZYCo9M&fI3&LMhQl{vwJ zgTes>+lYs^kopfUv(EeUw{7B7uXHwPp}1M;m-p0Kb~CsCwP41rVxoDzD4z$AL)79# zY#IF(mTkeDUXOSWVFd)>Ai(|r_us-Bmy^ldfB(Y^f*Vua2DNKOfO?c|2+qF}D5rtc zZOH>;dqqoB(EGtW65^86VGLr|EnLT!n}CSPPq`Ogj(fLwn;%lM>^glH)UrT2$QSBz zgepgGAPTXO+BdD5DbvJpJbyC8;t0>eqbB4G!oorb?k@j=(B=$A-6$i051}5egx*3N5cUwd z+|t^1D<-@IorCkC204?m&X*_-!YO2wTfO{) zXu?fT#I!cYIs#&?*WxDwX3i3i`SZmf8^;=1>hKCtf`N};#V21#MMEziG~#@Ri8EVU zse)dfwHFE)2lB}tj{wgXgi^pvKtq1vdGhgd{)5c|W0jAkaHuYSEaa)I@snB>V+Sr% zAb+mpZzhOqH*%_gLxIu%{$3FB7ja-2&IhcQ_1i+r`R9lUH3mV!LA{Ms5<)KNQGlW&ib0Ke++&(De~v#JnH_ z_*~#nw2G&eKqrolW@>w#kg%YWp+$Bl%G8S@BRbcJWTHIUbtl|nzyLaa@exd-)5w2I zp8x*Qw3iEugwf0~qzXbvE)aiHWsco$$7%Y|DL%(?rmdkAMsS2;Xf3NC;*3NihH`6d&uegfGTr{@( zV?AwZT4LiJMrrl!KDE@sS>3lsZc#5?xHX)N@#M&m(|;>6bvWcwt^Sj&{SWhZx0I~( z+t>lwS+OMR9To}?X+djMxMd$HL8S+dMLXUT;TqMl3IfV47I|UBnNW+?!P&WoHBHUz zN=X_|(Q;_iPcDzVL*uRk??$Otz_S8sK%n{UU6zv}%u4~d$xiIgj#+r@whFX0=J|0c zho3zcM>>g-^&8TY&Gc@$U|z8cDySZo-9f^Q9_6wHM7=||ct?bqWL9-YIevae37V(2 z+%A~8oPb-I;%g}i?1qJt8G3qorT0R**DZhzY^TD%Li={->23GUX>g6QF3n>I_cEDD zH^wP50oS#XY^gQdE{;eaBSoXZ!6QO#h`B?A{(|q`f?5Kj10j z(r>{t!$H8TiE@JOiWZt|r{48G=ha;DELXbv$$k2bi^1e!uys8#&#{f$R1{$IdkW#c zt#yZ+{6yANq`xNT{HWqsM9eXwPOEx_^#*yo`RbRHjN>;AlE$yxj{*mYI}LSM|9jH& z(48n$TX-3{L!x5sQ&`6V^4F2nx%xc<@X>-G&HgzZX>y15dk>Xj(xfrVB6d66Hn(G)O)#Q1Qu}jtO%as8_SRT6Y|7kr<_^Pu?pUJ=y9Jw7Ek%DbF)%_Tp05 zu(Z;CN>vsnUJLQ=mvynNrdn}24UPMTfJ84V`$UKlcGr9dnQ~EIX=l3ZJ zZ%5e0-Gf6iah7KSdh{OSB^nCZi>bwbiyP2C8?)5*rE3o8ob`+|)59G$su4A=_Dw8p zVO_$C8aIQ>=J4%QL;vzaTBX4p~E*xL8v>8{;@Perrv z_EPF2GB0Lt2CKczUwWxeJRw~61TG<8-ljC|i~VAhWO%-!%A>e@?8No(d+sAWFD=D8 zN;@mnRdP<-g?#;|m{70AC1_8o-bllT%Zh*fSg%usSx-ftvV@583rD>81YNKNBLdTGo!@oApoU-4s&E-u^zsUBXnJQA6TTVZWufI0&rAb-Z-O@Dgn|%h(t9sj6?|LLdv?1`}K6V~i)g)_N%;-z5 zA3)lX=73GiWay=pkETZqv?-sS@~kj3E?ZNcx%$QaHjKv(EMazvbv4w&ym4Un)Zeq* zJnYE(o12_|p&AA4YC?RGDO{Pivt^lnX|cSz)=Wp?*a;T3SooGfl9SV*s&S{Q>(b_+ zgE|KZHI*N~UjujZ(2CR5aY;WN>;5#tt7y=NSe<(kEwth_4ih~egv3)O))A@LK$-Yw zV;Ktdp7mNx4UUDJ_Lm0Qg7c(HZHAU&#@xO;JobJ{!d=_bX(rjM(BJg#+8KH36nG!g zsW)PX2QB8j>{Lr0*Y%d`YA!9p7QO92ddD&QHb>`a}?}wEY+7R0{LtfcAFY0rN`i zR#nZ^8(*Vd(?qYF8=3OEw3YVOMDExpct`0(jDfkE>nlCEs1R0vkK?ty0WTkVe_iPK z)v2?7KzrZoywx$I+&j=V+nd>9ab%I1>Zs{1JUuNQ3L4^P<}%S;0GQS-Ox&=w>==Eq z^J-&3cX>UT-D5L4O&U!aGrCt)=r=KCeiqtN_VRI2;tPs@m)qq>tNB|kU6s2PHzCeD ze0+BgyY5I--hBIT`SmBWJC#BT;IfNRF$c@_46VKFQ`^^dw{%YG*S>+(?w9WPi&wpg z9o~{A)nwCVvJ0w#n)*+3EE~03X_N5`=bo|v>`eA_&l2OxUGnzGHaXnj?!whXv+c%R z!eK`pw{w}7+6xn(5f(Ww%z0}JY&!BCu+1WbR`^x>MQ2a3qdlL0U|4D_socv_{8h#u z2sOnwzVBL&8F^#Bp(Lii(axZ8@ma^O*X-u5y-~;%e$)8Uexb_b zSugC8M`w*)e$|Ybm@3DgSm%{z=gf>}YjzPvC~iZDm(xw($(b%regKJA42HB(C0WHM zxbL&?@D^l}88+PDTGU;ahr)HVCNh6?oGS}y();aISv%V zix<Fc|?Rz=~LBXNaY5kaJM4AA%!NsCi9 zx|VNB>xq4E>&AT7Cx+h>??iYo9lf)wPTy|CvKxDISL67~d@eXD2X;I^ZksgVg zSBG%rPFM3pVurZqdq^9`HBXWIuMt`;IIhu1gU}pxG%!F=wy@r$9eS#lrYP{vJB?{=%-<&|0dU zFD`=Ck*s^4-Z;!!WcHZMuZ_XV-{~nOJB0Duxy}ISbI&p5@%g2~(`{Q-dMZZ$aAkv- z?UNPldNt%lF9^@c&BNFlG-+hhp>le_ zYg8_bMRWRg0{hHh^ty11zPTU-c6JNSw>Y5${pT^){1BGm=f%`}Pu|$IsLJC_nnelj zvj)aYzf#Qs;cceoHSy+pq73fv?@%PCnLKgM^1nwBCDNOp_4s zD#|y9eqPFmJ{6+JvfXEid4^5L=XO7>n#{JasLYyVr@ov{Wo~)Sa~7r&youPB*f!Z4 zw*z})@!M%~y6xaw@HX&%i`>93?^8g?z}pB73%HtpJoWW%@Q`ogg`#-yH2B z^(35ys%A8pGhWpj7~$Bnd*LXddb87_Vd*nytJnfpN`Ey1kjdwhJBTA7h)PB@_ZJNg zGO!+-5o*Ubr?B!4_!a-afd|MciovkuS^XB_g}jAvsTK4-4aweK($fji0olCsa4oz`b0aE3K=^bj15JJoVRjFMz)U zqKb0JHOQtgiADbknF1|;^8ZCeU+;9!YK~H?NOF^7)Igw{1%SFSGl!*e$B01&>y* zpgFw~hDYbwirRx!@q<0Ia%gl1cs7BbLMxA?yp>h+c^~$Rv4qtu3e(U*iKnoeezCjrX#5OkYYo9}Cl#9BPRxm4Y&9}NcPK&oweuj%| zEoj{Ei8npo6?Hy{dmhhmys?&Se4Tdg-QwQ?whOj>zd5{e5*Hp2%WGcz_0kj5=An-A zzL76P7gsB{aT%FPseptDi2qiVL1!y(hr1O7>VPFx}-Sk7)9u_HN#%066M^0fLmQ;>z01=T34WW zbbY<36{0TRW2u#=Q#f3nkCk?4<*J*^(A${b6{H0`R|a}ge+;qEgJ-1kBo(x?d5cntwr`p@~oR(^={`A>MHrrask@|zL8*Sr1T zJ3$t0Iv*KBVEKjMm8H$PBAiky-l{8iTe%n7P+4)5u=3*UqM?#<7}atnb*1n@Ihi{4 zsU<4BE5H%>?>h3#PSt8UR9wVqY9d3Vj?N)LN?&-nTXZ0{6OgU;y*#Oy*(&=Fcz zB!~FLxc`L$(SXa=GV+5GWT96Ls>#vkTkuLA)uK`G&jB}*Rm2Itf_iH`VxnwtMFYgv6t@9A&-1=quSKxz zb2_sTZiyB$KKKbg-7%DgprgRED|cvcYE|4XgyMHsAA%c)5-z$s%}^bMvV@c@B@pV_ z;!YGUHG2L5Kz7Fhkm`PeCNqk{fT<`7erW$MSrycwJAbLJ=>imXWy%=uHVe`h7w=^u zhOL6s@7D%rwjz)us_P0nv}1)Re}?ZsH|#vh364CHD+XuQBjuPZ(gWw~%FpcNk9mX_so#Q4C8D$58D0kc!1}A5`M8jvuYG1M ziakl6FJL6lf7r1%X@ z2YfILfwzb4e4CywKxKOvVW=|O!5J1KnN+r82Y-?So6M2_fZyWbOa}{Y6zSH7v|2#A zQV#H?%ln1+BMs~>1nCkoK#jfI&~0SvS*ka)XHkE}LgaSt!;fI@WzZISHV4HY;Tc_` z_rPK%*X%@REPsYmJzItLxSBb7D^1rXoS-VD05 zJREMi1zFx_wh-#WBw8vkz4vg7n~x9>9H;G5d^4A2MEbEib~OM;1nT4v$jrv|a%d5C ze=*rZJK^mk@#j!M5eXw75vKryD}ZsrZYze}5CT?KgnRkipTN=4YmHitRP}CVlfp=4 zXI2csRUna=Am8BZTN|Sm!)jC&Py&fmaWFawf-M1H^AiG0M}(#hn8R6!?gn^JFkJdT z9Dy@kK+VVkWHFGx6Ck{V=T^rF_4$Oog5CS^GhH&r(}yNy4W?P37fgoZ6mP0Efe&0O^v_H?mxTdeAT|5Z6xSje% zy;&p-+eibdfCOF5Vj2*bp=m;ZrZK#Y5Q0C~5ypSf=mxatRf9%hWWukts2r7uCXN|Y zsJ$lo3`Fp{07@Lox-i>FEsc&~7p9C{BQJf)f(eI=qaPZ2os5!^09*bq1Rio651_*bf+2A?8kU}DzeIjOo1`~k0fvOL)9>UOBL|`++f#X7; zeGHjyF}L=r^jo{o<>!CL0#eDxpKOVz59rNrs71KovS17DR=Wha%+`@DEJ7KkWMM)F z_C*MaQ^D{&h;{Gy><&O|$j8e&OX|5ccHQ7v(xutyB6hp9{LXROLemki0LbT6C)?e7 zvJKM9fwC!~@d4(39pMLf3!wVxL~bxp5k{~-PQ`+k6;~O8vxwpG=;o(V@K^a{7e`tT zf-sPT+dlHP3wAh}hMEC5eU7U3!_ks@(~B@m`$Q-sj_I>7B!=58x7TQen7~D3na25= z56Yq7Wb1+?XC4zK?}el}$F8O6v4*jyaS?J%A`hO{e2P?;$h+YwT*kPN>4zw6;YZ5M zQ3JlLTQ}Da!?H<-E60^df%qD>YyA~g8e4c1{eT7{nHGlp1ui5T|A|#@mqyXe&F0aj zz1nhg6hiS*b>(a)Ek1RQO_PJAc4m!0!Y(D)5#R(^&!)@+&|6jo`!f2o;7Cu%;wd2a z5p_}(uY^mQ0onkHmhwLs{}na15S1{>?|D-~{Ius>gK`||<*orKA8>r>uM$Lmzld3f zhGL;S#c$}Y_+cy1kA?d^0sbH36@!E2{S^Z9aV3d}Mlhg=&O)8wQ`&^c$8#1k;cSfZ zR=3_IRZ5amXC=bHAmHS+k`Ve73VA}3IAD=p7X1)-@An^@j1Q7h4o~!>yhmv@27zqR zf)L>(=VNLI`q8WHp@lO+}54oeyxA70Q{X2|* z6Zmr+Qg%rMet!dR1WAsDVrpIzIm3)Amngyc(-H#ZJD)`daQA*^j&4kq@^)!pFLnX4|`(T8HBa4jQ;Dd zM)?ohK2yDGdKRy`pm_Y5t^AV(T*Zqd)_7Vh*)AG@w+EXji$!xZu0E<;=D!K~S>Fnp zmRIJi-HDd3m!6r%u_s~Lt*FA1QBQM(FfE8#=GVl)@)MlZnkZ_$jPs*4gcM3Vu+d%? z6r+`5%6~aFV6gN5DYMaW@sr{oD(3td_J4)OLV^K;e+ttU&~=0g0p+DYYYyWWFCg#O zoQAUlF?UD)3W*0_#=4EcqvjgwIWApx?dbt9IJcaQUA1^8k2l(fqh0m!fwg!9*im8x znq{a}FS`Qk0ed}RsB3nj3MKZ1p!xE1ysOxgJf=4C05ligbO=lDyo2rUi3)ikcy!1G zUM@^IdSuEXWBd}2Sx-+`ViCYC1_a(t_*yU*k(=g?Hl7!dR&i_UKM5Xv&(hGV9$0_8%P%!^|Z z(lr;7!7%2d9y(6Y~!syA95+1sR z=diZvjt-X@L6(FpBoUE1+=O_|g=wZ2Yl&T**IWrfx@n+KY!P@J6-jT97 zybi)rd+oXV4B@vN$`{9$l14L~b8`S4)Gz6t3#ZaYipr6h zc2kL0X6yEOA?ZSE-X;g?q1R`b_DoBtc}7_#a;ax`PG(h|9<5h%>L{kH?$&gBKTg2) zy*Qodp>Op9n)V_lyUhBt>_1NK-bSOp9-3)XF>m?M?vlQgcA#?+Uu`RCylretSMRxF zS1vDB%jlTLjBDf{^iC8mJ<|k83y8=bNGWc1coxu_7V<*YuolxN+VS#%PNbY%s1tTJ zsot-~Z>i%NFd@a-+ZS!F<;ku-RQYFppnlZP3T@?+v&X9SUQCOvW#I1Ud(q@n2xjdw zrM8Lg7qz5%c|q5iFaBzu$9(GhddotE5E(#ub}}Q?JSRj}v8J5wM^TH7|DWX z6*6`%X0F8{V@*-7OK$MScvDwesl}}$3`+_2?UXu8Q%*;a`*cy2?YX@=oZn3Tcza25 z+QDOOg|~@ohQlq3W7z25E-Tqbe#q#pzpySuCoz_GO|P_IZTSqUE54@R)KI%RqNyD?YsTXp!qe z)5)a8%bu?<(o4kjyatPLemvPS-!E$oTJ3qw-gfE+qF%yZN+01h zH~ntS9QD+P)fJL6n<)X#QPj``cYmE%wp(_sXBt@SOBEhSZ;RNwv(znKf>CE(f8A(4m?>c#UT7nTG^5%$R&CaijeH|*l zzBp)4-Ad0;XD*s(4VtUe9&!YbbN*X6@jn!}HU$qaV(Yg*hIvSl7b+ z9zB;^=YOW>?Q`5z`G6Mudb*ZJvsv6oe|qz>?S1cRQ{#6`Ro)G)+wBsqb}!T|An(DV z0t%W-9aFM2g`mS#rIAoDB(q4DrFlgO=GNWukGsI)WcQSjtU}KdY*foOQeTh7suu|= zlinx3(@1w|8@o;2Y3$8-vxf?Q$zMPD(~0TGby_s0RLRQgd6n{|_Ily@=Ie)!k_VOdjeta>i(@cX-mM!qs6LvZr0(=qMy-=*J}nYv|%FoVKV zFXS?mjj+xGG=(-$gB|wHZ$2SmLMYyY>D&2`t^ijyVuI#^t+{79v3skQBvamR#`cfb z3+hidJ!_A{<*PUl63k}it6r4`Y}i64)E_+Uqv#{EE|4fuAdlCDOok>~5&iIKI{lPF z;+9+jz3rIOqT{u?fRnP9os_GW@(LEp>KsnN4rZNS$Th#Rio56VEnY`m=b~TnyLWk| zkJ2KDmC5o^pT9YCovQnPJFWC?g9H!{()-JAml&qh6{!653?B8|<&6FHyG;u>IL`pL z3b?T)FvOxSP2xBjv;VZ0?z0`Zv8`oYZZsh>soJfU%PqAWi-P;_mazi$t!+Myhf*VW zhZx6$S|SGtd%GLo6K>tVk`>fi=5fvF=!tJXpKY?7m3A%GByx01JM8noHbL2z6uUb; zXf^g3=We`tLgnLM=Y1Or|0wL4ikS~k9BDXFyDRJ0O+@ElW~!%-|L2?H`2}U)a2+!5 zH`llBol#>%wSNDHZMM~UCMxpgfxMU+6R_7V=qA#P6>1v(A6H)<2<6`XJuRnB+i@yV zDNdz?Bq1%BIweVxy=+sd>>MG+Hk~@9tRck2lw`@4YzZ@`Y}xmHnZl4|3^N#Gmfq{0 z8PD_l-alF~bI*N$m+O0dHnc`<4%@TcJ3;ZV_R>*tS4%UEZ12m1ySFyzSZKb&%}+{Y zys=anz8jMmF53O_jv{Lk`c0|2$%V;0#~1}}Z{C}yTQF1O)JwC_Saw*5OS3Sau(s_n zKR4$Z%W=GF^n5`>#D420O^K`8ig?HCg!8~)G;ePW*yYfvDe*BP^TEHG6EWjfB7rXk zSj1aF$Nc*VWu~QPu^#bGVX+d2q3D{!4-_#HI=lQl&g|s+B zo{nD@&(>*Jlp*=Gp1rY{Y#aj5ADK(UX1Ff_Cl3xuI0JM`p5c@M-@)pEWZ4s z{pG@mK0PJ%95p|JUdL4^BuZKX<9>uMJUYJ%FuQGTII`+1i1GtQ$TO9ezvd+W7wYN6W z;#0kbHr;exWE5XTf7^j*vs4!f)+_Jl7_z)d^ohOBsey_GWq^m{hAsOehuWtifP#FlxV|{)?o!x8$Bjh80N=@D_B+>=rZs9z>YF}> zu5fZzxzvIqC>LGvG>A%0%KG*ZeOC-UgXR_7S$w?V`)$gbo4Dzl1>bp1p#lC=gVAwA z;)Dh^2LBVasuILLCxVT!=#FsZx#ai(IA@y%;l7!qIB(Ub<;^shmb5BNUuUSWdL-Bh z&W2TRu5O~*895vn?gb;cW_8?t)AMi8 zkgMwSg4u9o%h}_xJgeNrHsV*{OsCH+?u`^t?Un`CC0T*U(_`1j&cO)h!vxB?3N>b) zUaO78)zb7^1Jmc_D$F|Dv?!$yCxhN#B}9r3px2|~?FF8$d847yfwkCUvt43$EYkJ{ zH;@SqjUBC24(Y>L5^iuz!r}R?U)C1u7MG?+7s84X3+8d?rSJh>!7Ix29y;yfSMs1q zpRwOlHzszj!E0Zle$?IM7C7SfxlxS~t2Io?77Oi>cWd-nKQ5y$rmJSQ{zR>je>Jwj zF%~`1q0KJ$ys;R_J5YpijiM0PTy2WQgC6Q0hShVz^mOJa(mc_OWIkR#<=nLOsD(UG zmSgQPkXhCkPl0Rgt?m;mO0}}qSB%Nn=gT5-9Wv5-95!oFum%Lbu4`IAxf0|v=d`wo zR9Xb7L`tcS^pXcEZ3pVG{$7&NsogXzsXbwFV95>6u+@?0jFdjQ9^RA*{gZ(U`|L@> zz54PqO^aqJ-sFY9N@mO+ICHeNB^7OfK9H2;CG8`;R4{SMNB?MLd7)*>3~HuEu02*O z4sIxv;J%Y7^40PCx+5T@Cw-2qqq11~g!%zq7u;1kGg(Ps=WAEa`{m54$nk%Gt>_oa z+mgCHotmxAP8U1$SE2q{bDb-}KeP3I|;4mEPu^yU#XU z2qRC^-WYwX;2AHl&Daxt?{Xq!dWo#+)_m93k)3tiohS_VO?<`J5t_6Sy9S4+qHdSGSpKsiU1u8a*h&S+uH~$Pngvqy@HOK!Lx*EL z+y0NyxNn};_MD9RsV2>W=DUoF$P5Hh$~9@@;r&R{oc8Vipv!_A#;Y&NI`6x?ui#81 zorTl)x%cLF`nXz4_r$R^$}%SuntkS8t=k)V`X@?9(gx@I)~s)99p%jGCR&IIf!_$T zzRlJco8BL-osHOHpp5AQ=yn~L>Jn1Z*i;--$y7ZU$69n{BfZQtQXdaZgNv4l{xz`f z_H|s@rc+}PZ z2xo-hRmh@;Lr!;wd%y%nK1ai$lvKhdsCw+W4Fmau4#l6>Gj5U+;XFhi3wI?taQlOD zv3c5geJ9>2pIHc>rbmI46I7MBG$EhtUEw)QqV68E6ko~CuJshF7=Qz%nYPRPg^>@}0O`6}8 z)uvBUCaK2(8tfc=S!LG(01v(1z^C=y1e`|*acg{1}*@Q38^({WgtK4Q~&6d z4&;vBjgD1D-3$I`y@DW&yuJE-gG`9?*2^Vyrvn4xoP`ZsC?gIe+{?qg{e#AIzcSHm zb-a7B3|#`)a_wP>=c?xx>fD03NFnAm%c zfqh{VN*tVTudn^&;nuM8>$pfv_4Wap3H8v;-?5ib=)p>lcM&H-KDsDOtGK>zSamF5 zWFW^^lpS7P6ul8s{?=2;@{>EzDH;#s5Bk{MZ6^hQ;w%^yr-;(xkhNkn* z3G__~HciyKRqyVIYdq7Me5_;O#dqE=HZO6$+s0&s0YJ9Ph7vV(177+v#euJpdph?K@xeMqYg1*|jk>p*o%ikjKF5c6$ zIp3HM7Dyr*4zvBByWK4S#rQywHvd1Q>@>|{7jV^rbZ!R>0Mv446ZZ=xZt#|T>wxIp z(3 zBtPaO+aTe@AbLT>zC1#pTo=r%Q+2rMcrT$4vilVu-Qr)gFsb~s*pL6cl#3`FytCL! zTr#ej1}8`{+bL!`kNAr|+a6Gmvta*D8x24#AmE1v{NLc9l|%{>d=%7wps732z`9z;^I#1dKVP1=&kFphfieorJQjT#qy8T`giS zm2uUL!;7sf3Al66pezSd?kBgsqNdN8%_8~aI1eL%xE1;T@uw2yyrfIGQe{JA99^cX zfwy}fH#cZi1R0ozV{<4PUBRDa@P26$4ed&{`)ioE3G>o|YhNj*Vx8P^T_^jH** zY+So;E$j*qSmdJ2ot8r{D<;c719+G12Tx1MVCoStfhwW>DFAjsV?2gLgA)$rYH=>% zzNzF3oU{dFA(ao3d2WN90z7P9{c4vk^Z*eI;#aHq10`*uq9;sIRnrxc&UqR?Q8LU~ zFa{Uk|AHG1dCN$&1k|3nOOR!5pi3zMI+Ycag($S}B_MMW#?F=1o-!~f4nyVo?P}6h zb<#&RX?>KRwO2wh5fA3YLN$=sZZPoVL-VLWUuf#@W3{FKFP}s3)Dh_ZKMy=tEHI8q z>4RSP@=q=gg8%bb(*25~xITc(=IO|P7PWRwAUGLf|0>n=m}tb6l!&5I1xX?T^jgx7 z1U>>Sr#F|c#f#%fyF4m&U}MZ{vOc(+a~a6rF)YBU8jO9@)P<7SE2|>4OJeI2(Vs8N z5|O#CzwxO!C$DMv5}=8X5Yb=)g~Zf5X~$NllH{Qy}rBI01 zD54xc$3FiPg=fFu71&I%DfCzVyv0gw4rXfUiZF%!q!iecOR1vyl|yU!+8!99n&dC{ zkxU3Yl#2ewKtI?ZYFNjSgR<^d$l;rW$-e zQ2*(Vl4{WJzgO1wv1eP{5qW!j()TZ zu{<~ZSD={V*FI5#s>Ck)FS#ikYXT=1zkPKi>oVJamcKw7>OI}hFrqBLHEpgFc;_2* zJo(7l#~n$CN}c52#%Ls;MiFPrlLF{=K20E!Tg3Iw3zs%jSgat1n6Qx_`J1*NH1++@ zLgj%70?zsqkF3?iU1gTfD#gi%N~1B?am))qKk zTp7FoHCWSpYTL44(oSiI5U3CZ0A8?lyd^7w+YBbV_hJBe0G*Vg4-1Pcz8Blv(9ytMZ|p9DwTBQ_!Dsh+h6_ICK`WG4g1cBhjro=!|D$# z;52{IBXmO`HLkzU{}>!(_|P1TdF^F>&#Q`XsuFAPUgwf?s@_g55q$nuZg0F}++H4> zcXx)_j5dwVNe#UERWSR5zeZE&W5{@d(SXRP^%>#~3}q=Kb7%6(FIP*`cpN-wJuX%m ztQp(@4baC-#HK4z$F}_jRx;=d~YTvZC&sN2p;2baAme&(aX?= z@)nF45@G2!Y(PZI`f^*D9pRkyar`my{46hm_h$6fuj)m(`3sfV(1r7c}W@| zGk=cQ3K0B(4+b+C2o1S>5{zJ4p|BF@c4AYx>VZWJ*8`TtjbQ&Qa8UpR7svH#S~Ae5 zcQqia5K(xX%&U>V^5#yFcJlcMMgqwB=%s(PWvspzME6W1;R=Z?(daBhL@DqjGWR3l{+oX-cpYs7Q|6~{3sv_5 z7J&e$gW(g{h{N~ZUoR`fcR`NICz7DjA)$u18g$H$!8f7ZE2f4E`w=m1rAO!ieB*;l z>?4)RAloL-_0?C;rAcbzxmbv>HWqL|>`-jNBvHRp0G|%PEO_y}(#`wNh%pw~dpJo8JOK)ds9*sY?sD5M~`7Dxowa{|x1yAhthQ zB7i+SBfL#lR-9*pPZ)y)H@^-iUkfHN%`2yb(F%Y1rE><~u=9oP07#~@AVu1X7s}Mc zwWdPm6n|%He`!%i!DTYXO(rUihhbECt`4sx)M6DmpI}Q{34kt`^axpXw^2KwWL#TA z?F?m*md3%WDpiR80{KP@g&3#`AypwWgUmG5@H3`fH`8ijBQjiZF*Afel|d&P2H zAkd*_o>r^mKXE%)Q6L441%@VpKT)s{hBrlV$?-v3M?j^>opvae2G)b|eL%hi;oEF@ z*kCU&BnzG1Gzg2r3H%jd0|DfA;f}j#1(k(PX5q!RM5EZXq4z?*n312gs zUmNhz^3z)kw17^JfUkTSl{Q$`Bn5S^MA!lU_ZJ_({*ULj(9Z@)8vmo<21{_x4hV2K zDWD7pIPIwAJBuiu|HDaH0X`)J#HS=uaO#ri$W78ps&C{Mj`A-o{Q;WjOi z0t#dzfc-WFLshUFAOd`~MRjtuC<@C)cO7V|R^h8Oq>up&*LD-!I}p1Ahl_$|k$y}M zI8flP9_E;~7vBiox5#((If|T*o1ys$Z)@^SsHI0z!jTmT`6b|;A%FX`zb|o*`1F3p z4VHffes&X4HQmPROAQkv)@mvha-FR3vSh)D-4I@8SsX~r*Vq-Gb^PYM?f&emeOeHa z$R7G=rRkeiaO&Yp!t~H5+V2wwL3nYj$#1myf{QvqlmjIw)aKu&;=rTQ zwf{$i!%me5g^v;-9B>P2YRmTAeSG&xUx{#X`RJc-eSEL@{6t-iW}Z+u_ZBL*Sf$%z z*}ao~8)>V(+{g#Mai5QgJ@wjDN#R8HzvIyg2Md}P6}j8!L%rMIqlwijyFWfvOn+gm z7T?)9sUO}nu$|pJyq)8~41Al&Q#?v1dB8oWDyC|pMumm$ zeEOX}a=eP{)7mDX)%iz)t8=}(MFXY2j#HiGdbYoxFoJBpU8e7L5RYQbWcY4o=o8>kunNk1PMzmeJOocjxQq|kt znE!l1?FR8vA(xVmm*(-a>`O@NCOW(B&GrSqW&f!sKcZ&EC9P*z=vfa!L(EZ;s0?DY z7TkyFQheON$X2Yg+yb-G=Y@HJ2)_QuxrN=DRlb|n8mt+;W;Jy9QQz@}w7E-zi|0@2 zh90R*w+^2RkCHYWaXZwzy}jjpxjhqIw%B>7xk;>I>shP}0X^{fJG!cD%?_NftZ8t`Brw>>^mVc2AWLo{G0#%`oJhxaCiF2&|`{#9H^zmds4{ zhvmwII>n#3_~jw<<*l7V>dMZZHbdb>hhN?L=WMQ7T@}E6^Nt?&&R{V}6N#Q>I=pX= z4o54OTBmd*Bt*Zh?xpTsKSn!3v)C3QQ+-M#)f@uB6SO#`jByL)ab^tjiyiR>cQAVT zd+OE^rK69i{<>}EmJM``UKXLh(m==X?#y179S^^WS)$Xh;Z>gU}l=CL|ID6E795Y z?sHp7wzBE%2l5Bs#7Hx|xZjTVnQ^q&yiS`a5?A{zlhkiqJ!wqU?n#Oq93G3{O(Y)c z8hcTyc;E;7pqab5NTBJ*qTt_8Wd_nNNacKobgJ&6hQr!#4dZe0i_nuqJL1l=x}-FW z>~6F#?Dj&lJ*da4RXeSF(fPQNYi$?kKT+G{783SWdP%oO`>On8UvIv&Sa_C&4_d?NXPK*Nq#F zqIb2A_Nvpk7^H0L*My1L`$3*GRNUGhaz>De}yW=2ypbQi`y!^yk~ZBEZU649Eg zSp011w}f=9(RDlAdIB73;;V)R6_42^_!oBBWcD3M*cD-S&ssmWzi^88pm?(6^j`Y> zuPKtFI9`EUV#>DsMqdM=+nGaXS;eP&ucc!T{#nhDbC2&U)m?pRtoT+S&xdu5t9_nNV2rl@{-g&h2pr>fx=iX3R$ragl36GQj-lfe3JB^v`{g}Jh15wpboxc9GPYfo2`Rq z_IT&4mLgu>woivuAPU6gOz7P&5c%Xp+;U3POKqHkQy(7xzHW+G;`kHg}~Cn*wc zc9;h$F4W~>7h>0ovtPW6FK9>@eDnNc1%^>4)mc#BXosUbpR`Oi9R(*x!@>65f4#KL zJ^W_hy3+K042{o~+6&sc&>md3pqSF%K|2T@>{^(Jk&E7dPS+IdSr!@RjxDywPKq2zFPCcI_N z7jLziQ4y6q=%DvE@XB?v2QjSc+MM;tk0dcH2kq~sxV)`wP;Ud!ebe^rYJ)SQ)EY3$ zW5B_Xpt7YBjW2?5AdpEoJwkU)>zAvZ)=>@3B>w>RyNR`}1xDGkvjM3x8p)l6)M}jQ zpws*Gg`>FnbkFSyflHMP^u-^mWa_WZ@|SS+xa>C&Fb=~3}NZHd7( z$=~>hBVXf1k3P-VL26f)V#bPTWXB(nG{5{cF_3~zFuUM#c$A*%S3kjak)+Z-6@J)=IEmwqPFT~ zjC`;oY0fe9Avg>WlUjAf@eC2?4hLF8Xh4IzNDP_W2&rBl+op=E9VSACHA0Jr?$o7L zOAk6=oH5e~Nf)E=`U1^~P#YsPuNXqjXzc}!j9!PeYvH)(td(X_sUCqnmx-$kD+tD-Au!Ej z7iU&!dRqCAn=_ozyvL@&gPL)h;WeE2B4cNWq*Xs&;xF8%fBk!H;lMVaTBLq?v)n0f z%^Ke8pD5bv_!zt1gQfkpy^(3_^7Zqy(yDzA#cggF#d=Rtfh1!B_6nr+6f4&GWd-8anp@N-!jV}PpvsohXSq`D@#eQZMMqJvaMbiw5d_g?Y<-z) z5B(e-uJsAdS24ORF&_~{_$NE;XINiyZ-civ9ZS<1)ov;fbs$R{YT47V{&3_)?B4Q5 zo^3I~g6*LZ4I^LQ^xXsrh0nV0BHP}kzfzC1zK_p&-v&{A41mY~yu6d+&^7B4 zRU7;dJJoA2Q%5O#tdF@>DpVW63GmE3tcJ#qY$Pl#cO;_GOwdpH<8y9)9rsaumxgF<-al{_LBj9z_IokVYv z3bjc2GhCeWzG?WhNWS-KqFKV?P{E#uQ!}h)O|p}To{Y9*sCI1(xei0KReen%MU7Cw z>hS?;9#RA8d(VfpjY;4;BiqhLj@5m-+M`bF z%ZqIc9HHbo zs8<#SS%Enzq??3IyElB(D~+6HsP5LS$A|bOMt4c!Y({eLhNx*ApbIntx+MIx`?BeB zvmd+P6~U~p2t0j@5hzM+>L+Th5ee@E;WF}@90p`8^1E<8e1YZ_fdrZ$&~mP@=0m8m zc!^qu(xS02)>i>36A*E~^}(+WLO zfxGt#AwCY-gZvH`m?%t( z`LrNKb*~V#l$Mp0$w(SE0*5=ImsmT15NpJNr9M?}Z8QhVk@FwtYjWYI(xJ&igtj_# zdWu1&8t@ZUONMxM{s%^0)DI1MFyXYgXT{CIO9H z6e#Y5W{%qM8evz}n?52goqW*qp5WV4uNd%qjpqcP82rEY^6^@adZ*ypha=2M!A(Ni zXAqk3JO8nrYMLV~&OmNyt^rE`JgA6jUt;s~577~D$K`R5tsr$fn45!Gac+iuR}xnj z+H;2d+(qeI>4!-47H!VTh;Ka{U(&#i^n*oQGRKEE$)({T9U8sQc4-XcEie>7>n3H6 z;d}osH)uEao++UbaqGdIygHSQc`Lwtw=mWA1u9iiK<)_rT(;T_?w$aHr=$JHpob3{ zNC8uVy_TyAYH^231R{k8V=?tUb&1u3rY7RDbVe&8u+PrdP(#{yZGX*TqaNX$Kc>Hq zp#WE?=qh)q*#PqJcqJoT2R>Ee>GkTRY6f~E;)qSvbR{(aFj@-E+L{7Jkij8I%RxiU z@7$>?2D?R#O5wR9LQnpZhInra(YgbCpK8$5oc)Se`o8`~Cv|C};ie!8x^-aCtd1*d z_+>R;jdf-tv@(NuE`X(6PTF(d_(SKH(5S(0B9tP zA3Ip86IjejHH2gl5ALQ$0$<$=?iD$zk6s$Gtc@gnrju?xK3Gc5f4z`u^_Ay^M-IG^ z_IF|w;r#N+GEL1@rx zw0X9USk2z`!Bb30Gdv}y&GXu_XgOv2J-swjt;OEHs4H+Yj{OGa-$CFl7 zVYG`zT`QOAKSY{2)LsnGKc!}_2XKTOfiTnXtRNy}`K=%i1u-5kcrRA^kuVl_ZG^|; z7=P$(km=XUN7jMp1FTY-OasRay7<3V>|ki(=3%EgnQP07I22J-GwL-N^svqYveiq_ z=H06SIp&Y+SHF&Or--hXeU|A-0;-xo{^uyEvjV1wK%8pd^!sk6rnjaJn$$IyBtQew zqn~VMNln4{ft7J-?kpOw7@Y@OrB3*gy$qKLFzD^r269Q_i%OIbp$HS~ALDjWxHJNq zcK9Ibaj%XhcaXfe7B1~|{Iw)WQ;IC;uM2_U5oy<}UwnZ4B?-^`FqVvCz(NdO{sIH) zwgVAEFdncR`x)NkUR?jn3|~KbfBjXby|F~}`iqe}I62>-qyeY_5o@OC?$Z&8kD#Il8V)n* zeuRTucS9CJMaD=CG{9yfu{=lwbQm>X0dply392r>Z8l`XPv>ywbzrD)exg{meEA#t zYXOhVGhVTc32yoaxyqNDxXR(YMIn%g;ZwHu3E)4m<>dqwcH;vSEg-s0dYUMUdm6}D z6NG};DL6YToaV?rucm(o+hxyirkM(AX%8QYGoZI57|f z6Ra3VUHkP%YyJf-un_fxCCi{;ZM0%ENqxQ&u3a%7CFF%%VF@v5t8jQ&`jpGBMD9J6 z)F;Vez9eXox3F}oHSDtQOXaE)WE!L_jSEagu{cV))FZtygecT?dUGqw2$DiNmRyn% zfGp7%sBZIL-%W$=flqEEXAI~Xd+I|D~9MAI~o9v#D!?iJUaR$WS8*kdw6r}xuy?w zyrwaI2)c%-VWegjkk<6oPltcyGNXV{rE`RyO8*p2;}0rY>(fBINOUvwNz;T%@T;kG z-jWOCVezl}$_l<_yhUJ%%p!p$vKC4)PPt*Ev6jbn(>XnAk1#y^%j{mQqeMuvL$?X~q5SC5Lh+u5^JfV~;dk1@yw;!kzuYSiRL%2ISbawz# z1<8e$hj}ZYF&PLmGIS6}h?Nl%?0MFwBS7jWB$gf-P4A(G9 zJC>YaH$eCsRJZ|vX}(&&oC4h*r4O@g4SFY#?1boY>Xj8Mm`R9(2dKu85e8auz+sW|-@&A)hQJ2kn07heT-kzHWF$!q=#9)HsnY@`gb>C> zP|y48m+cTPF6VL!L3)^_2qSb^wrz;zTVBB$3b;48U?69(`|AbFY5q2QUj#X1^8flK z101~r5D$Dl{Yw%#kqL6N|NC$4)7yj*Jn%9(rh>oUmre?j|39Gf8bNjMsmmrf1PR^~ zd0*rgB5dZHlJn8=6#~Z=!3`0-1QK*0JT7=l@ZnX#J^?Nw$fY9o=ScPmKV4f2plHMh zNZLrKmeAJF|}4(4HGWqjB1= ztPrQ|p<^gaA)fESFR%FQbh8u9zkjCajnJBiawnsjV7mm`yY^`j28HfNC4&*$ELQXK z=n|Em7q&tm#TN$EBaaS=+34~kfD1XY{IFXGD@ibLq9y1tAmMK=x5^T!@C zFGp3xTQxmVmpPD#sfjt$qJ5E3)J)JW%4n&HYo^<-v8s1^r!`Hj->NOYPvf zf66-cmE5{*$6e}*eUjT3Yxy}@`Z~@ah-SE zPK(T1C6g&n&n^#A?uMqr`lsr8gO@tPLr2c=o<|IGU$#q$cu-}gHuvjpXGr4%h5+%t zcXsfL{LZD&flk&nFQ5R$uGtO4W>`Z)nV?p7t zWk1f>buN1=h*mKht^C6cbuK01m5@ZO_;2HX;wnyIwk+}#H+kz{`P4dHYmKWr(lY(d z4EK9M)*W9BkBQ9eJ$*Z6_Z&bEHXi5NcbP41OHJwf(rZhQW(Hge`R!OZ&1M(GKzFnT zNcDPesnXbRK5@d=`(u%~(b_+jY(?GQL}r~-Nv%_8=$X4P-|->7^$z1sV0{n8R4fb^Pj$J@bXEBr$Vw-_&8qn8lpe9i z1p^iMm(r!)X>8_CR5PJgGt_8NqTF2UKH1lC-$l{{1)EniQ)#*_fBMFY8^Z?Iq%L~z zvnXqyoAkB+tF;7LEoX^c(`8M)DS63T$ZEd)Af?GW%kWyf62hKH4%wc_5Tdp z@u)s#VYK{=(A}zHS8j)=h?p;JH~FVCgwu883)kYvi0**0V&=U4{L^`jKd*Ow9FI@W zs1u{SB`r8AxlTXIhlt&6h3r-HA`W$P*qZ@vvi*Sng z_554cS&?$D4-fYD`E49qfh)7>b}&)MWn2xhv~szaT~E!=ThKG2ZnGWX(JuaQn)fKR z^|sV(94x4}_foR#)83ztKRX{#V&7|Q7H`?AC5i{Si1;P5+dB@rV7=cR@k#hCYc+vh ziIc19$X4~$aUOnvs}@T!yFxFDQ$uqY)$^I54%q$e!$n(d>evu-&D%div}r?%!OC z$Z@wC#h7Q#%y;Ht(G;!Gi>!q{jGCPL;~(?yJNER~@mr}5RWDb^y3z2ivDD5CC|99{ zBm1nRMln9Ls5e$Rto*KHUmT8-B71-^wV7MoRWHS8I?i=K#5g(`tPkeg_+O#vQt;BC zW?!gb!oop^la-HCy?Bwu7#h~oy!k4q<|=>r@F;5TZ9G`%(o;;#<}0rZR`CWcd15v5 z-id)95(-uQ^bhU4mm$q$$QFiA^O%G>vC#mvc&1srZg}32AT-N9tYG3~ZehEVbf@Zu z?a5g5laLjQX8VU6t@Mq8hIL!^gs@kbvt<`f-G#HL`&U1s-GhH?6l9vZ(DitEbrc(p zFE1K%sUx6G@p+yzWUbOvO71N>f77^JKD2t)av+D}gLNz>%u+4){-6$R2#r}_-@?oX zrLMq9&W|>Nx~20=flk(@Ct21K7pAQb-{Q*47?efI|3n?TrcAXW2YJ}dWo;R}CiA3- zX^>})9ol`8PDfKEuKVTCZWgno?H|3gP`DVHHZJL*1Z-tS_c7eXFKu(^&ExN@u^HDy z+Xh_~Qu;`)#N!#7TRU8+s^L@kp@7C446$&W1J!wcL$}{Px9`OxI#+bdZrvKE2IbhS zdXpfgpm_UJLFoP$nQlK(YjcJ#1+k~-HHVa)>1>})|9|R#ww7wsZuyw0 z+RILvb(r1k?=ouG7zL11CUx9$q1X$nPBcy@_zVOypRp9_?DmtPAqr?b`YZ8c1GZ{T zqoICN3QeD_QLmfVCbqiQ&)#z(?xtDwfYrxJ2Z!IW3cEkZ)XB^)1&EStgBCDyYuA!R zclXb#IM3~tF$}H^I`8T+l%MqnTYM?7`a9x%Qb>(Zz)Y z^To++lrVXF28p)Izc<6)uInyqwulBfTW^bV4u$l6%sxb+CvTYVyA;S|=t#fJB$SU| zdSM0H`7|pRLF6-4!4|i;ps&pDdVlu1&I@LBN_9)|u!`JytKZ(NJ~qjwc)DKdIOtJ# zO_TEo(!^KE51CbBHFtaKd1T3Xx8Ql2HK(p}MNZ`GXw#IjbUZl{ySDzpIy5CC;bV^W zV*0m%RaGbFHcuRG+Em)N=im_~?(n332XWh?yKXCb=+d}4h8(jI*sVX_?BDctA>t@x zS7n9f^xhil*11ORdCHyriSp~9ZCrGrI;odw4RD*p4)r=}EB3VNwc@;-HSH2a_ivwe zpBcZ28NRXD>6iQKZTf)3B(daIk84S>wJiOz+lzVSG)GT%>cHG3oOx9huXA^;k7sCo z3pN|K(Fo4^(EWp;z>}BXdXs(_Ru<|{Pp9g!(@Up`?N_tC!#hW=?qw*vsV;O}W!aV* z(~9o@o=oc!D^khTb7bT}fR(=KCnHg}Lo7ObIBP}I)?*Ud&p3I`l6Z$x#oG(2e$dfn z@wurNXmoYoTf9+R*0qv0gZ!t*G_Ca^qXxRmqXipI*hop+RkKSbVv)m%K3=yth(?VZ z%ncp~mRIUo3Edy^uPU-Y7)hW8M7y+7T>_S-IPNBgd9K9`XGTvl`=CktM@LFbBPOrr zo?p}S6r2y+@5PFX6K3h+0AugCQBWL96+hw{#qN2>IRA|M*~0zYi6PVH)iV8YT{CH= z)o=_;Wu)r*K}9hcl=Iz0b<-I*0Ec_yXT;if&HmA@*O-LEO0G?@pS}7wFLcKi-(fLK zQyFh;(s-JtDk1CFExoGG6Tvh0&+abO<9+%j!+t21zvx`i`ACm$m2^UQDF0b zA}#^^jOkCtQXE8-S~15q*cE^Ldc}o5CSwvp(su0JDBkM zJej)aSmsu*VH1J41ogmvD17=D?MssGkTiXI^VWe|O>-HY!Pf&oD7St>-AwUs{eOnq z{Ls##W;x6h?+<3sADnDS%`Hh2mSfW3Vn)eyS*ADMmTeF6cYnRL)j z_eN-mI+GspkFEwPlKKO<*5p@XBN6TwT8SOvaSNdzsd?zm4*~5(TUfpv;7S)HWBg+j z6>A3)*}2BqQ_iLsrM=Y!5N&G&6dx9_I&uUJ%}m#aUm1SWxRRh8wdoYbCpJ89Y1^1V zGo-)uE_X*b$1h5J(##&n%D5bq399E&rXdSs-$W$y%8Yu%nHIdI)Jrxo7`3|ULVNjF zBMTAha|C^gg57JCZpy^y6KhE`J9CtqAZfGuyP+;Rj=9x!n6EZKrBfPq<<>c)!t1G{ zwqmdC?=j8KS=;rFGm9H1(tj_C*-oKf#oDCs3e*N&=Un4(C9ME7t;Uq$sT%uu^I&F% zBb_s1VM_X50G>vey!3cG(D-Ava~xkaB}&g3Xz8n84)A`YyQ#0T4;`wdb#vG3I+Cv< zzAZ6&R~rlSnNXGd4x!0>S}!dSoSMFUn=F2WmH8*wx)SVpMYYUH(=7GA7o~4&edoca zM159apkyumRqR@JcZMc(_f3uitt+;n__>HvXGqK2QtMlENOQ2Q*+)p+eb|mBi?!-{q{oc9!v!T+5m;^iFH!b_o$89P~I-J z@4A(^BL;gt%%a354*d@}7kQC>4ecx&>Yf{Y61UAcqUtwT$$ZUJy%fq)pX_T>pnk|L zA5~t%Rk9|qkBystaw#5?+Bxw4+4x^mVSkYRf}V&M?5hv)TaPCWsC;Pp_D^oXZOz+b z&oaex*jIVc3$CBp^8OOvFNfPWZSKBCi559KiP`0M^gXoi$_K>@W5;~Cy7kw!2c$ZQ zltQnfHk-nag%6~|q!hGDEtF`3WtJ+^es^vhZ@QUjP>S={H6eVEN#nfJS5T+P*e$ho z=}U*pM4q$YJAL!fT!ChbcaAjhCh}Ya%W;Htb&TC%p@v~hfN^Po4x7vxUoxoKTP%HM zL4=IebR8}fK;&b5;g$d&eiM17X3w62Yd7gk8?u3j;#uE>?8f-wr-{CEzqZJU-_GIBIZf22`*hc?73q1de9GVW0^>4We8ZqQq|E0|G$4;HT@Q z3%kMggBBpFYnyu`E=h*?;A4Uhb(MdW%Fwq01#R;xKR&pu@CTfihAEt!a@GB6zC}x4 zgP|q9R=|>Z$Cs1?&@BJI*~iN$ya4dt+@i}zIt9dl6#^^X<*;ZZk+r;*#+?v4L%dpz zVC*sf!I^%5{p5eTm;VdSpD9|-1`b2SO%Qj^X$Iu$rxr!{LdrhXHZA;u%|e*?)Y{oV4@=QVmnh&n(#c zRr&L}L1(Q-v!#C$q5TIuQ_q5m&sUV-^pF2c`OK?$dPwKPWsA-O_q7$DhnJ8=w@0)9 z%8))QZ3u0&5F#7BG*^#ny8yJSPbl0AqFF43cOu0#k<9fYx|pnQjt>- zo>PY2mWYcU-H7uQtuc^hM?T`nSv}JrOz#2q*vl)hanA93^B_4@W6A-T0@lCH7mOo6 zQ@GUwi?EF%2yap+jx|BJ4A`Vt%^vlNPjJn?5d2;#a0Xu{OvDoA2_;-d8whgeS*gL@ z$u;NznUGpSS1*5p07SB#enBVmd(`wc%;GkCHYh&2&S7RNPx(pRV(a4>ICrT|0XQ*f za|2K1_3%o8;-Vq@8W5ifX3=_ zH`wAD5+2*X@2w|Hw$*dxZLYHqk^9q`Z+@cEfg>1nOCL%VE7BvdP1dhj>g2zWf73i# zX7c=3Kyw}ay$sIftaiuN9ius!R(&SIl52GgLB55mO~87`MBqN*+N2bv5FusWB2W`k zVPWRuta2xCUTQB1Y6VWhtlgu?gG)RQn2#a|jiI5AlfpG3uNrUK*Po7azlVN% z$UxVPR|an8#xx~$LOzfS>SG+G%yN}-|R4YaQmy3Hz>HIc|=M!X}p}; zd;vC(eOIX|j0*`28%TV8d=QOx4+U!j&M+x{l+1S|shIp4!rkK8RZ1^LUuN z>Uu)I5Whv;=8q0z0*I?X!LSRqsf^l57!R za-~X4Tvc+ael_ch^D~e z9wV4~-2lUhYd7+2_4`VxemQcewqW|D_MW+g?(8e#!WB*C%$dHO(oKqZ3xNVLe3 zv6#&?E*(fnLJ8l>|IczA6(n+0UxVyK7_63x)&)ykMJ~etYD|%rPmB6gW%w!Xs2qWa zPY9n`^-&46=l|s`9269NS5_cn3R2Az#<1Gs@v(K7CK-x@FvFj80= zpV|}o7@W4?VzhJ)N!s><#$6k+;68kt! zP9SE)GgEXE@gf8BNw*SZVK!^_Bq`YAxrDWxLj*>3m5M${au<|zeGl0p}N7LN2P!fD|1!IskBN!qeoI#L`Emh9z zLL#Bbz34zByNZvWW?c5Z4+oent|zgO=`GA5Zc>0f4v$3C!~55!A;Q^THrJ~+&3SRh ziY{Jssl%|S&>F~sANcz!C0 zbB}OPN0SmpUX+5x`Th=sW*~g)t;<61w*o7wWUv&$^H{=sWnrE89r{UsYpc!YtR`jz8gB`n5$ZSKS?kDIWo4M zt>@boz*yKEfMB`EJwo*Ak45Ac8Ox65h@(0J#{%Oph3>;{Fr^O~J1uN_GWtPMLgpT|oWKDoD8UfRF_{(JUenfi@g$sb-Vcz!v z^FIy$MhOXFMrlHHd>JMG-#?c>6!QH7E?nDxzKm3*LgRLX*aKT%MtdUiuBh|q7UEb_ z_!mBN?U>>~!w3nAwo#8@-nCVW&8tpw2Z3LXTY3)mJgNET3fXjsOTU5@vNl)jM`jaH z6z;Dl)a%pMKY0CEJ9hC2DEYy@!`}K=-c%oH{lWjo*PF*fwf}$My0_)FY9TE$-H4Ff zvSrECEnAZ8MYvH3A=%e+Zwn#jB7~R<*}3+tbIXSdTxf?yy;>21-_dODS`_5%6ak;C%u22~X72PWBN z$2X$kApz<8f)=bkoYXgi_V#m7PLis#zTby37aobq-@_yJ&Xp^{3dah1oZ)=NpiZqh${Dz0dko4o6va3ESJ^{wnE~`=@flEwrzHPeMn5 z26}wo&$pCndooit{_ZkN9467(abN7!7E@w1NaOoHIBs;;7&s&LvPo9`quBi@?arFHqS5Uy_v1X}#T?dx%QQMPLwB;jdip<|4>e?W zmx&p%m#1qe4Sfk+#GtoSsUwGLTZL+F3hvJeTU-8lJ1uqtTD0- zFE^GJ>63Q1oO_i=U1oJ}!L*ku{veB7W^n(&+Vay**bmduu55XA~`rZ;Ui2ileI!HaTZ@M0T}I ze-8e=uQ245VNC0T5Z{&k>qNgR0^VLT3!|l z0S=ju-&sl2qrPF*`jXesR?AWOrz@AeBV8OkX6}<+*tX4QGg`z6uSWtXVox`}QL0gw zbm{W%-Px5`cDOq6j;TD&M0Lng?bX+l?7`>TC-NV}e%sUN;hr;kxlwQg6AW(W_Ue!s z?d{ta4!d7mmFm3J_piX19IpC$+G9g)ld|#JyX|*wd^Xu$aZmb3v&&nYacQrhB5P{o6&w?trzsj&;mC+G zNth~~Q*S$@sk!#SA=3uzH_n9pfhno_&Ddr%%c*t@bLe_4wnawVkZJAxuRx@q1IE+Z z(A2hk&YKa-_}$_QP9XSzuV9nqy#QTvN4bfSWBdP6p?TOy3Jy)UCyDp9rRkjdU}L#6 z;Uld>DPzczy*O8=YFJZ~GqW#E&*=*!LdSZuAh>w3QC;aidPlxFS7|a4i4L zp&l-@U9d?OrWn`vgnV7^hnRV*lAJ@6edgq6WABGZ;hiOqZqd4g6UW7&7_+NP+9>+Y@@=kbb@f-+ z^%V1-W5^}p8REXX2@895oZZQK%%h(98H=t5lrxKFswxu46AP4Vcl*8B?{@06pcajm zha7gUFi)uRP99^Ri!CddL6=eP$QQfnfQbHijnk5`j@u4OBqK@nXQ|DgcByo+Z0eo9 zIowiwnx;KGUX!#0G3^D9CWjw&nLIT``ON;Ll1Qn=S2U_?n44xw zCjMvuu86(tJzmf@eZ1WG+$e$rfZ@=mo_yDn3lhtsU-o#3YFaXh`PVuF-9*Vlvwdxo zEnlxXs|F;eJfKmxSBfp(I0g_+Hi+zzYSV%lm=_ZUsKLcexd)x6)`=2rz}2D=y7R1!pz7&s`#6E*JV&Xwml&ib5S&7XB6?K2_l zGj!fHU2~P>#eOm5YU{o1LysB)uf zU^dcGgrVkaFY<7fPqq)C&g$)OIa%kBh`Dbj_v<(9YKEeZ+#xuE;0d!yLrGGvD-@LO z4dW#+l0K{sO!W;D8=@7`T>4bj-kk10PX7n-FDVo^T{F;3!%Yos*49x6GDoP(Q zbY$m$d&QG8?qFLiIV@+*gk1u$8k2@~OdY$9c;MNUITfy8?Gp1hi0* zECc9Ont3M5#ueNX6RRMXRmbU&z5Sxx3ACK17~0NTQ)&AvR+YBb-%(+PDC)YR=l*fa zt+Q(iu6IEtpx$|yEWEQV3fnn(zeq>usz@s-D$Yc&IBC(awB~F=U%pK>YqF@(h0&I> zez$x@{obuwhW!v$oY9{)bmehE^40RtYYg?gR0o>7@mk5n{V`1rM_Z(rCC!FSgEmA?p4UTdAZ`U1)~~SUtyJkqt~PB@UypMvS8id8>RF;L6ala zn?^9}gms67-V$_)%^cg(g%kT#eJ=fCs-#=&&k+l|FE9wCyXl>-z4OfaAEEV5$+<=R zUx8oCqtjv;)e5F73W(rn8JY$whGmuCxahL1^Sv0o<8KRy^6cyDY^C3-F_evJYluPNDl*4z43;M)`~quzUb5uMP}0|#XC4i0oq5w$$@mvJg|v#h z*n)fVBJ0T+=nDV@WNZE%?;Uhlv+}L;?Ie_HVqIcds-*InH@J9Y)f=2Of8*A>7qmG3 zdh5}ihl@YO+#4@BpUjAIzuF%nFWslPs_!B1Vp!RZ+_fG>n(>7*MuSf7vw!&&$0g#b zEKn0xU5?7+pr$)a=$=^`8f=>uHb;%L%NCi1KBT|>%^*wB7M%#_FAg(n=5^pr!rFCv zR!+6|Uw*2}SScoXgyn8F%{x60fL$*Wq%}kHa zy?uG;(O`R;NmRLw7{GZxwX<`Be9{8ODdJNCqjadkjSe!K^B$M5*I<%*RT-Ozp25PD z`?fTC3lW2gRJmI1OL+Ok71CF-kE=$s#nOV)@;42QVPggi-h5d7{hF04eyHNyAaTN#@~w>^u0&gp$PqQmv8G zp}pVc#(+ z-*+*#C$I|T{$>km*Bz4aQOZNCg))(Nj7ahiq10Im$r|M)Kl2apfdYi;QzgAdlGE9y z3N9$*pq($DJCg-Pr0;_VJ1g>;|T6o5A(WQCyHP(68iY47wgHUO*R%LTUbF%B41TRgy+jVQ7=sr3)b9f|Defm%}HOTV;!86 zJiGYgL1X!cSG+6*9Ywb7@V#XAARxs#4PUsn%j}PVIGm6H1QWo~YV{JOp+u;-QCUSg zlpWy0exV_&u6KO$!w7p$W?{$(bl;T>O+rGK;-85bi&~qH_y4v>YuW`;61XyQEVYbW zgOgf@v#2GHx1%%+rAl}Jx63`J?nlU7kjz-7E%8@%BOFq~n2r>UGk|;C3>H3l>|nqK zqsxCb(~ga-ec4bfSU%A*(WPX=;rijg3zUHkz(ij>qPSR=+2uv$N=6Li<(af5?AcZ6 zcBR_`9EMf=UbC!kdc1}J9qTReD)V6=^KNEQu60+VIAw)zU;oIBPLvxq#*W!CzG3;K zQts#dwsgCC_LlP-fAKT&v!Z|kfo-Y7E-;i|YCDtYee6B>IUrjSnqbdwope|PTV%x! zM?s?HW5ZS>zhcU~lm6U`*qmbIy$bTwQ}oP}H}6v|={63DYj%ld%oQu!(e~W6ZXWgm zo^3+Q&x|pyGM$HLT3&SVkdHg|0?6)+qN)&E>2aoqjzpNWnST)=n%^Kvm+LMQ&eXX2 zNEh+)i>m3ZeS16;^onuOj_f#<6k1SKZ$ZAe;HnY3EDR$N>i3{U!<#ItOQMC$yS%+I z{XXJz%JzWiQWwQ7#Y@kVR;M~6G8skoaZHaJe~oY|3eRt0ROf=6;ZKGxk#@tjl7<`1 z|H(c8S>0y-w)9^CvuQuu5NCb7p;Y^zW}HUHF1`I4TQY|~2*#cKGladI0J#s=>=bVr zD?C`WK;7_=^S=T)KVdHwl+>eYo1MC1bR;E)znr}K+WyQ+Y}IqS@;7C^Q&E@HYwG&v zjQU>V6HTpaoZRo@}bz7H-ckaN-uhlwF5&Nrk zT-+vhRApiOwLZ2)?e@3m_Cd}Kr}`47QSW5?>roTAO+Q291GS*dJn-xGXA{zx`hyG#^5{W~T!NuVkh#Q+kN& zcI4J8p7^0+U?1SX*xxGP+(XH=;rtpMj<1kcu%S`P{+jU?K>wFz>@{W<`7qlLyluwZ z?z#XTB>Yf!+X~-gX2O9TUjX=vt^)r2W{e=`3l5T)z`v=ux?e7V-&x{OaK1;=TT6yaxy1lQ7iSUSsHw+CSYVq2lvX;9k5M_T3q9h zlOCsD4@D+^USNIOwvEqh{HF_%7q7|#z`v*0M&+4u?~MQtbbUdTdMc{v8Zy!L13)!IgO`pV!-#xi~B*s`-1qG=~5N)36}91KoX!(EFMnVQ>7S zw;-MVm|JCm9&PYxq&2 z1bBxBLeLy74N9A@K<+X$r`T-wb<=)wGg1nD{}x+ZurvU%V&`yxV60`)`IEQ|Qw+w2Qw!9`6YO=Nc?cm372W*ig9z`4^Mr!hJ#@=+JV2bkr*jH6U z$bO7?{Pv&5$Z-_`<6(A>)MW zuG@f-B_5HDjIRKY4qB6?PsLy__5~N|g&im$Bd#%U+lqE@iSE}IsOU8+QpkNk%ownn zC3t|2sJnkaByLL!sW;q*s~35w+aJ1M;wR?HMyicS5SzsY>p43)lId@kIN1zbq2WYQ ze{;-55^SLCS0Q-)Q{ZG!UH{v~okVxQ<+;wgd&=9;bF`(mk+JCYuNCaGuYVwO)?q!x zzd(@;>6nA@2Q&&N2xI5>%s)ndd%x0l5J=UB2vNus>i}PoA)o&LC`Guoeq?#^jF%D1xD#ThMC=^S{T3x4xM6^>)^$?ca$7LsDJ znqw{)HS(&|jg3qQdoS{c#uMKWaNjssC-<@aF`r+Pm5~Cddv_e-F5@Y?93t8(X9lBu zWA+47NX$}w2BDJN9mDTh_4H`?Aspz2?~%%IHi)_8X{05(EP?q%WLLs?%jV?|zs>q7 z@|Ah*01nd0!IsT6k5u`XAQh{Dwb0i^1QDKXeh(hWjJ%oFcK zuWitY8+COsuw96GKV4vB9MXut;14f`>6;m_d@Q)js?sz3pri9e8u zUM2??u8BF4dFl;Q<7N)RDh%n`G+q>Ln;aFi6DQ0K^_RAFhLg43L@JaRgV^P{n$ zF8DlepdigPqZaEQ2C6`V!Y$@CIi_bH`q^s-bm>Ha$9AAOc9NdEnh85on4Z!Z=djEr zHwG*?`TE8iegO|;6^g_S*Whqo++hu~?+!A*vq(Nz4f7@A)eb9s`k zj!c!e&Q#E{e)t}!%U!ehaTJf)nl9haP?GHp3FSw}uTR3_36|s^YPcYUrX#NdE8!E1 zkOESX?-TBqjULd-QAp3y(2~GX2o?FlKSzlRMbq;c?F_YRTWUBPe@c1q!SCucCmgc~ zVw@8ei~+C0uQd7#XmMhh*_@;WNKwa*3U3nNcxs;^UX)csQs>8*My3q%k12*x7~{H+ z5zz+3o%R|67V7hxzSB$4Ya~eOT6UiD!#Tmw2hc!(z%70@;*oF(yqNvp;Y=97uEx@Q zA*7kRN`kV6C^ZKJa%lVP0yj1rQinB`!O_C0BjyE83+`?fWUi3lZ*@KVChay@`yqT= z_SFykxN2CZfve8ha4Y5p^0b)+&);09(LMBR9*#C5v;oC!Q|7JzhEs2mLT)g*1GAQe zT|;mXxAgWI&KHONJX-1*(H49zOMu7sW?Jedoc6=G91FPLCbe$C5PI2QW2p~Y+VU3y z+$L_5=}XXP_s5zEFH7o;C>``kAew=F*%n|l|Jp=cR~ZI$?=pHSG?~aK^%3&0*C1|s2u^K# z$O7O#KCiz@@a0LohHN8{Oo71Rf1V>wA~(>FMt;`5kQBl#wm8CZc=J_>oD6Cz+ER6V zqn89PTPKbGPXw3|IlT5Uzj5Gt^?>6BWi@ALfP-9%tDb1s%9|{BKZ5@!clr>B;=ac+ zSa^xEMm*;)4D(IlC$gB_-zNay%{7AXEX9yY!dEA57vh<|_0DhNXD-Cq{HL1hn;}>6 z2?lRo6#46|K+G5Dak9ffAAJY+QgANEfBpmYml4#Ma9r&`&4=@@ z2}h_>*UcSbM*N}(f&Xf^jNl#@t#`lJV1pWMoAIP(OJdvDJA#!&Ii&LMlKnhW{x^PB zIww|vH=`W4;27E;@DygTi%0=zf=&pev5-V~NU2_=q!Z{_F$ZP|1uY8RYzTCL3zfm2 z&uQpx*!`kbVBNuhxDFI(Trt7h@)JdM@=`6sVCc2UV3^|O*=EA_Ek{q9`34)Vf@X@v zLl?LrPne#M?;eM!NqTJ#yeCc0EO%UY(_U_WiIK?SG}(35?Wh$sBiX@!L}M~&0x zT6>MH)B~5nYntyoXL+;zL%$3bB4~#gWpQIIHPwu;jiP=-$}_744!P&TA70g*ZTFoX zmtNhIskh~qwvyJ7KBBOMrYx@Veb-8Ewn~HtSu?=jF4uSVpU?&ZDkV|OR^hH%fccd( z8kQ-wpYF36_#u|FM|WC?S$1~I{<_nSOQ#YX-FH~+aG(?*V+$+ggsHq_0<-jpY}s1M z-9gOLLu^I5OXZ6VuPl-S(N!UY8#V8ff;!p&6aVM=-Qovv1pUq3Ul~d```qO_FBx(l z&QvfoSEtpANA$fI&pDfjMt@@D$yWbBlZ#R>_mO_H_wr+n$2EF0-z2o3woDq` zmS6%0k0+1DM0}WJBTQ#itopi7sDxL+LZWAZ5!lySe!5P)-o@jM;dq1vR^y&nk_b9d8=09Ptww{KFOHP;tK}@(?nwSJ&v2XiNXvjLz^#S+gRi+<7kTb8Gr) z{)$yneNV*)t%HPvT=1PKay@*WnG*a=(BHP!^s_Q<} zOcIl8bJZ^xN}0Io*qzB5k+V?{^&UBMGsWGpsWZ%iFRV3CpZh z?d$e6jDMD#w_kdXso$UE`eu9L&Jp53;n9SYV*ha_8qcxCGQHwXBF`7QYS08kw29A zN0{=nz3H6=w|ed~-#j>^z&x9O;^(kB-9Ij|r|0|ppQM*+Y>VuSbszrNn@1|m7uj*? zb}u`U}V128RZz`cBbiIO^xSXp!IW^zwWiV=d*wJqW8;Nx3?Td?^>NW)b_4= z46|cEGxMF|GdBg_a#HPEvG;J({d}_&hCWLz>?s^qKkK=$O&`D2Sl_B8u%+0aVG--7 zDYiXJFUR`k89ZwAp{6ZfY0&c93F-qa1>JSGY^B8UT;Fh~B(LW=hYTx^F05HZ*z5{G z-TLE67W1y~ACHZ@r@Fm9{420`CHb~f3m!wVtF@>4FCVAIm$OBen&P(UOq1$@C;dHH z&Yuw^0(okrm#=`tX*{K&1@JdlW~t#|qDmjoy99BGp&SSSSTJ zN1p0Yio9qcvKdXchg{2vSY|G&VX#!}c<16#Jxj?{LJp>yt+cXQ`vg#WLU?_bd$=h< zy|-or&cja6lrLew3CsWlaVX947FOPi>cb+9u!y}?N7O$@)z4%0lh*HG=YWa3kg4zP zrio<`#;jTB%k<|4Cs6iJbI-(Ax;tmG7sPjR`ne|{n@ZB87@f?h&K^CvY^rT2x7jsF z1zF9N|5T|KI(WoGFyFU7tbz8R^q6rsIzT>AM69Q0ZRg&A!gvFs(apkmYA5}^3S%W7 z5p=>0km*H*2@6#h=MO|;qj?Bb z7dJJBergu+8MC;zXPVMLt9`-PH=DY(EIPfiC#75mmeYjiY^6?H-$yPMdL{dYbfhDy zBCC-UyNdx{npk&@n5lEHO&%chYg9`?G!*Ts=FD1;Xxtm=b35d&>me@g_is}~X2WOD zW4mG&$B9PH=hK5`RaOcL&5Pncw~tfBpRz@1dG~4zGKbdu4V0bz;)siF*y=TSO4}M* z#6tF;Ia5~Aqk&ZlZ-pK;;aF;zR*k#j0Ut)Q&j?G#U{8UUKJKKO1EmHh>ob?sysR-8 zpOQPQ{fjOmyBe%t_{r$GFE4JLvZ%~;^ilst#05bNk&69FF64s7e-l(`9YiX7ixy6G zEZ#1D;+k|eu&y@f+q^m3)Yg@ujm$=&^K}n>w<5$|0o@kh&jiMBmz0;OKzJ|=aLuWWxH=la}-v0%Px_!o(1A}OsBZEOP$>gbzewdoPSMN5y&`xY2BvG4T;wz?Q?a@i9l z2of-KhFScLg5yy)2CP6`mv5+Hc)=vva6YQnuX~qafvmidPQ;pr^=YkuEA1-5ahTj= zQtPV&>%F6c4wp~!Ho%2KYzLwNF7h5L@siPJN)|=k-#lm}4fEc8&CH|Wt7O<-H134k zM@akKr!Q($9@xG%Yx3*eSW--xPeYQ7*3|Ied7T92356up@y$AmN%G3myGI9dY!Fh- z0ye~H@y(|0a$MD8RHOr^{!K^2pmN-*G+e=p9if&q%&_k7Vz)!Yc$Wi5*TF&?c189r zPTs?b_Ku7u`?)xie}Ijo=rWLTCOw(OjE(L=Z*V?D{M>rab=N`JtpZn?QOwd+2>JYV z7S1!;j032JyCqDrMw>cbHI{wDKUmSDL3P}$y6jlj{1>&5e+3?5=4AsMZ%0o!P}E(U zWC+o6Wix};Dupz);If_lm6V#oQrcX3R+{p+4#7k*#7}C;K6B!R$Hn8J%)db2SytvK zhM!5*&;xEsfhD_Zaeexmo9tm7T1nDZt3M*MMB+)org|)JH6$v!2o58i`E9qR@zeD8 zGw%i;k(J3c!@w_dhR}LNINj9 zmsvdg!mm6@^(KkT@NCmL3DlnfF;dMQ;y@cPO?g_chjnOE=1f^v-$d)(y@p|VPrPxC z`qOSykom06xy@ssF7IyT9p@vVFP2>i{z4NT=sbnzJ!$$b`k&24Q)^|O5GxryY2Fwr zqSPlpdwqRmsD*YE@a!K79kw60qmSRAy9P58nKCj(dks&5y%ux7n{VMDrb@||g+AP> z)r~y-TEX0m6zh?_A@Kfl1NmGmtYMES-p%=#P; zS>X7Y4uU1x_@hRDP44KqZ#C#zYW6K3!`0=3Xi+1=9)_P_vi@lVFUnTJWPK>8s2}ZvewiEa=>bqyaYlrr2qk$tn?Q4 zt8y?s{N9t`dC`#qI3m=+sgUlFt6VsupGvl(K4F~zW#XtobinjH13-q5;C}_yEH1PW z_QX_6Xqx*(GtF#70s;)rY#x%L%}ygFK`TT|pee!?wx8bm|FX_<2oMfSY60l%HZTv@ z@*}A2Boe4UB_+C=51I_+d!4el5|>YG)1 z8*FFmXIKMln&sVm2c@#YK?`TFtyc@Ti!Lh^!a8HWn&V|>PkmPN=~!HCxv9!eO76;=fAdZ3!z%PZy0l%NsybDCSu0YVM?J%^_ z-v6Qga3g$^!kWoAi%6trh0i8v0JQsQy8{o{7WG0qNf#xR~1WkYwEEUJ8~4i``3m*|JHfT_lo?(! z86e1<>@nz4W&OI{8M8*Q?*0Tg1_nFAF8!3|i|+ZPGX^qxRMzyU{0U~=c;0nlU5b2s zM;7f?QeiPnikO!*glwo&h$PcX-T_-bUj9oyY5u-%$YSTTybST`ALl;aEe;-XW_9QZ z589B*$ljuHqF?BxZPT1wc&sbM7VUjc^< zJ%?L3)+HEK|D1&ge~;Kf?NHXi&B$PGtakaJTcU?#6MIo6GX&jIxe27 zYw9RL)6UZowKnbZ_Z%ga9_ii2IAYd3daDb!TsRr~(+GrmSsWQVr#ebDEXBQ)U>|wk z=${axm=GkRo*(|4;uqwB@n6(QeD><!`Bz`v0}E%{@qoV6#}Caq-Z;hOXGh6 z<>sb3fKG6*7`x1(?`#RchXMfio*mbjxv%MHGR>H7?KgLWik*cXw_b)rY;_?D*vh%! zG7lr{JmRi!GXhCj&%5pIeiP zR&D2xkA-<+w>lkK?bhFC$S%0pgEZ9^EDEmRaKz3^^gtgR=G^M%0{ph_!Pe9&SU(}w zCbZ2GF0b7C_1f^-l`9hudDb&N%V_m-56@a0cSPX3fAM>3+xe}$d~tpwZQOByQ8`%J zciwsv-|zvh4lWn{UwS*5=Z4{M1pNH+6Hq?=zaHiT(VPlv?MBuNY7R2|PHc#^))%1} zj&VdPD?k@x{!?-o*zp?vdS@@}dVpx*hQoI8aYJtZByi1!ocC!u(?-$2e$0a(Fdqzh z2OzX%?fk|~`EP-nY;9}=xNHMNE1=$!D=4gasLsw{VG%rwiCm85Lab>qIv2Pf(x@l= zyZaneHBo$CZ%|bNYQ(v?_Wz~~harA6P0N_jkF)a3;vbjaap?sJTjz=cE}xL%ivq+r zmq|}+)1Lou>6~k7!*^rzNrjC(?;(Msnt>W#=*&C6i3@LW3?v-4%>TlP4g6wOgV{;? z=(Y8`8NkpD)72q25pdECngn+s97N`-(yK3#HAZU2u@MM;l1KwTre(4U$&Os*-dpqo zx_F2VNvvzou@HnJR+z&q09%M`m~jDcDyzFDco8OOyp8EOlg}=dZmmO5Rhv zJy^pXLTp)Y#7iAvCkqdA9kS4as*IHbf|2b)8haG!SGBhPuzLBa;?9xsaedne>;kcq+mhVw}{7I@C^%}YzA8$J8KgB8G^)1Bi-QH0muvAd^{^zkvolmoU*KlL z*zdFqo0ZOMac|^9HrY){(}*=#WHtta46t!Ncn|joXd@oUP@2ro9Trdsv>kP0Oh@1n z7Tv+%D%H+$$R!{u!_!5RfBxjQe*%UgvUCy-i`IW~&E#+5Rbm&VEt9!wnEUX#`J)4d zC@<@xkwJi9{yKT}_F;=uie~vP)`=}!xP-bt~{NNB{jf_`NuW^O;==Mh=$~! zxt(NKDO1S&QE~l*05^6wOIZc@)#Yr?r$D#bmMwrp!BXHc_on?P#o@>4=wqQRBUpdi zaacura|{mS;f~`RTy)j;>J`Y&r%p*j|V{#Evyrk`|O^RFpv0LjB%?Ly|z(|v!CgRyOYd#@z)0r z6!CD@4>HoDRvblR4VRaJ?cweZB`kds^Ry1c@pF4qPa!>Q(kh>q>4Haw6ZnuFjF)Gi zQ?yh_cgg+%y@4?TltfVS-UJ-~@6)V%P#urVKqV7Vh#u6E z#MV#Kk3X|D*uuxb`HRTDX0PrECc8N{N$4h)xmDuv0gMWZ0FUz5R!N%hhP3u_J%%8#2NE2D zjKRYz7KmGSSnm)aNm8&=)VI6^ubIDq*(}rLYi=TtrwR3gsURbdIUslSMbXj)!~kB$&+Rg zL~lnP$*S?*mgNt!rN4_z2)jjY8?rNQaJuRF9TA3d(2Hx>nE+|Tu_X}lzx;zL&WmKc zJTtIjr2|y@%-Oqz%rt^6CsDp<8Dvjdi?0k%QuGz22K}}@LwBkYZx#n`8qa! z&I!Crztoy9Ki;y^WwWWknrRXlndEd{gY(;i@wRkd7&`*5H*#`5xP&3kd&;FFzw@P0 z-J->8ySiVF{k1I1N@K z*mPXy7;>S0f!{6u-B11P&#m9p9FQ5#zvK`-eu6jea!7c73&D#K3^#;cN8^g0gj%?D z3`nma^GA>S@`ipcIm+SdAn8U0oFAOE)o>2ZP}?|WSo_tiuh~B|TJUM$C{V@#|Es%~ zqcw>;a-FY30@fAQ@%iAgBSNNs@Jgg*F7sL0AGtFquQHnd<#!$6|2ORmvEMYdc{@b+ z@$fWQ$^5M7pbNjuaa|9k9)&j`x(u|f44>fTj_Xnm@B9DrA5XHv{RUI63t2~tvsTS; zV{ka14wg$uBJ|`uYw9bx!a(R^ialpYW)R%L+W>vHhHvz;OQhaF2AH$Qpww!%V)^w? z?3r~^>}syC=odJj-h?5w!}Ntf5NzIvXkf>=oRs7q*JEgydy*RJ#gHPhao}Bj$(D4Cb z1!f5MVYf@?C>XxWXy8CEZ5Ixz4Cvd70V6+WWBY%Z?cMCT(eFcuyWMWICo~KgqiD4eECsDytC3p%E-#V zVhS^b_H+fKW%-$C42U`X1PFfQC)q+XW^IM zijrGh{TW7ty$_jElE>Td7qb{H)ob>*$!^K02YupH(=+rukA?8jm(BYVai8(1s)4!d zHsDbJy!YNr*w?WPq0tP|&TmzB$U))r^H@eWj;^Ok|#g!YUe5@s`KABIV?E7qC{ZGj)6y+nMW!W-`@#kYsOVg6k{wX4{5}H=O3fC&!oXLFJA0lLxEcjOo z7$vDCEa@tP+TPFrs_M<*3#?&4*Y-^YvjP*FYH88!t5=7leH7k7fiOo4gV7H%76TQ z{*RZJH^&9Iqxy-+Qo_MTf62&rWdN$#km&vJaa3gx~FVzfUfa@l4HHiM*&C zknEnbQ&X!PM_)7(@gBg?{*g6rJx`6{?gBKo|Vot7hURSE*izDy^7IB zeUu9(Xm?3WS^II?4Z06bhCXT=Z?)Pk`%!Vos&J!|z`Y-@ z!6seP@{O3oq=CO-NS*B0zP?fY*;_h|`#N&iFhH6Xj&#n)Q>i?(f=qw*S`5q*%L!%7s3YJEK$< z!Tzy&J)HVW+2o4ONBajksl{Igdj>x)_H>Rf|CqTnBTgyQ>@CD0=fR$NxrWcp=-%>LoL>E=#IdrZi~YDGl(>qK**A1u9MJ{(GoL4`k9l zmexA{ni zQ)Rm|l@?3>uJ53F8*7WjvZEcQ^%<2&jbC2vL+78EczjsBp49T5=VlAq>n9{Bc7KEP z*&|i`$wkIn<$74fOIVPhI5B=3UA)V~OWooXPu#-T zOCynG=w}Wgb7oO9_sme={0NC`;noqN%MA@oG;xuVQbu{7D82wZF3f>$Rz#;WIxhEH zvuU>IfR(J(wksFd+1E|vj@EuDRSycLo1Z|!!ly7}Q1Rp(A#^ENXq$M`g;fnwNLP-jgg&x@v@pNA~KR@iv`Z?@7jm zs2PSaT{73Ox{=4sOUSOT_l(^Hyi_nu>t(Gsp-X!LB-j|uHVAOl~W1VQN z?yK#YjO6m&m90#JWz(dBe6d*ABgb!K^xt1BcyQxyJI1-T$w;Y` zsf7_tQ;zkpHEiPtj~c@QK)5-xlI1G-zM3_@yEv$Xl}HB89o^hm(|{P| z+{ilA+$^;$k433(-S7N1)L9SRW&c@)pOPB zZJJ1hbx>j~3>9b9d}-HF^!x3>N^zNGuVABkFMqV3#UawZvEGabJw&TB{LODJZ`G=Q zxHZondp*|Z<)SAWHHLnA?{K~*EIbVD!Ja_E%^o>zq{uHpCiO0Rph;Hd@a$VS%X_RU zJ<~oFAfn_vfcP+V0sQ`?7F zer#ksQ@d<9%2dDRZz_IbQN_qR&fD*ADVn=!wrg8UGG4dEmUK7ybj;7}LOEE4&+VOw ze{4XUc5_?&sA8e+HYoS}bl>hh%JGp)Gb0(v_uR8^ z?5=MJQLlVI^?`y0wqZk)w8pan+;jP{qzaX%g)uiy9j}lVXZ6mOX9EjlSf>Z=fEDGL z!};Ff$H{-%rkv(Kt4oF-`Xz+X-Z@zwb&|x$uU`COcjpS&>fNG& zs3i6mAK<)I|W;bj@;B;4T76sh8og=r9| zDA0aINw1+t^ACS*?TxOrG zz`Otp^TWH_D#st<9OC;}hZmA@g9DhkYt~GnM^aI)vgE|QL)-4OK=H^I zXea+@9y+h)@+wv5I%EX(7M&t(=+P+OaU0tmL|o2i;IHqy95e0Nf1ABy1{HbNIxk)4 z*wZt>OUZe!GG)_JrBnlSy9!UGQ8Z%G`tl7G9fj)5zxd6WIBFiPDD4bxT2olRn>#`% zwpI2VAY7TwitXjkb&CxQAWi`{cpe^_Xn2bi&AfB_imo;`9QOQ|1Q-20^&Hz4mHS4TzR8=0)%4+uxBF2);?{j-7tBn%MvXpPw*{-(28F@qF z1rzJxn+Tk$lR~PF(W&yO5#06KIh}Gk{@ZPRkSoI=5$6Md_05AIGVEv z-ua*g-ccEQ)1;u@KxOdM9Y1XQ=+$|5*5FX8q(;ns=gL8pvKN^ErNCUkM(SlDI$0y2 zk5SpE-?vBsp~@qOqU&DPPB3l}1KSO$OKH>1aKRG1#Gjf(g>p5s4OZrnTG)Y@D=5;$ zI$y@0^xRFJSqdc)uQy*5Z_>O@Q$Pw1HvKEGl+6PKg{qLaBg8@59k}82q=`VW`}isB z;*8=3lx=OB{NTKtd)W0E(k=HShryE&&{vHRQ7r#};t~6nNaUZ3LI=pYnQ}w;sb!86 z4bV-1<6o0;81z#M3TUpg4FF%4eI4C#E|H?yrIIim;2^y5j0CPRJk&@>ucq?kom;GV z06X55Cw$fuX3^}bb{RVE9kGDpKPM3{J>=WgyX41XNUagTY6%k;S~U-JLJB?RV*(FX z8T%c`d3jale|6?_jCQL|3>A%Gyueo;8YC@j6Sg^jwP1`_|l# z_M&1`Xb5r0Y-KW4ujX9@DBuE=9v);4Stp+s(}bdgEvnu_*TVz0p3M9CC(iyU5BEz> z@4CNcrP$lfk^|$!vM7&*t3}Qc?`sA=vxMEv^N;=uHVkMD#p7o=L9NO&VaLKrZUG2kkK25sBw|D_=DULh)2Wu_9a4D>pvI+Bzlj%(H(5s!QbP)5)2R z6=Xkbr4i_HJKuHvLk?!JpqwEf_VqVQH{b7XU0Xg4o>gyT8U6{d zt4Am8s=`$R^XqPp+Ne#8RW`R=@bJB{oeaA*AEV1+IDY3M=xewyJi&Q2-&dV-0bH&e z4v)vnf^iEnOijJ)-f-fre?P4Cy*coIiEHQ$1VR>~7&vh;*)poHDF_DjQ~;W=IVp`0 zrp@t*rSkzwfzSZhlZ(^4auA@SWwvn~3aD>u0H1RlaRM@`eC!uuKY1Os1|+U8+L+gv ztpc6S^ow&zJNzU}`A{qG!o>VO8NIp1o7FQiR~363O~J21IdNDx&i}to z+i5E$=WHQzWGhGIm`al5tPn9<%6+UP#x>nS5kiQOTdw5FIon2Zj$7^=L+)dkG3MCc z>oYU$_xt_*u{}1Xna_Ma@8k7;J+C@(u0;L6Q1COv-5envrw0z2Rdzx)Hyb+{{J#~s zEpu%NU?%Z%C5!s$@W~S$AV$4&;q5Er9DHanmb(V_vvWt~>~n#vF?0T-pSPa`dFF1(Pz5sb4zX=7KmLoRffj|CyG z`kycRQwqtIu5Tn)tG&QaVQhzE!-Eh`#eXvi_&MVj7*AlK3Qbq(q=z7z`uQBV`vb_+ zAlviDPOs<5T>TH7j6LqlVoD#~`cv#x!()o*yP!?`QyvL`03h&O!7T^;iE#O!Pv?ql zjo1>TAnB7|IOwx$i~wWFl|Glwx%s>Ze*`HfIMQCO66qc{y;mRZBKtdBzFi2`&m_Wk zB_)to&NDhsn{u9wu@*;b%*CVEBP5cm#^-m5_BO-agl(O`LEFJ;|0fjm1H{&+5!8xI|tAx+(H3RP65JN{{%fpbr08gqtVJ1IpyeCP!)!$&$|IdL=d9fpFF@sm&v zbfbTi8jzwYcd&=xraKpYE-Hl4c4pqsu+|8FYVX6d0nGWeIo!a=GXj<`_8*?ZtXX~@ z*Eo=$zq&+*ZC~Df)ks_AC^m1&$0IRUhi@>!AfCjRX8&s7rylTwD~+_)q9am3!@DW7 z6aka$p#Zq5!p8HjL`|xv9Ze*ggp73BNJ!S9cpj zw};X&3AKz{8IpH`G};$b4P0BqEc{fBm${hbhzQSIEfROVG2e?L7S$2p*#*eX= zKyL`-jbo*fSqn86cD2H#12ZH@V_D1zeG=090Quo0Bmn66lBTnSp9hQD_heXY?-hg4 zknLq#`6F?CAfa`5$rcRLk>TSxd2VQO5B4~ih_9RDH)3NU;6Dq<*rs?;UwYQ9wIaUw zFj1zfaBUd?QN{3wqSrpwG3?f3iU^Da7&NC2GYD021R=diw*2}MM6ZU@d;tDkSHp%1 zOPB8XpUi+S#8!Lqa8uup1?G!X8T$?{^PDBuz;`KFHTp10#g_sQFyF-{Ny(^FQ58(; z8qo#PZ-L7B1GJuW5VrA`OR}J)GIWLK74c|A9wHkzNT&spu>P3QSRPx(`=xS-JjO$m zkiQ`Wt^&T=&MDk^IN&iqj}_E`99qqPLS%W6v26s)68my9k=kAx2o!a|jJ_|ynQ2KMntFqR|H)k5Kzo_7A}-lvEuVCt`FiYr8vEXWuY$ zrWH_PNP>mU&<-&&sJhbU-&EwzWDd231CnZtH3Yx=t~Hlb03%mU>rAkfmF`@pZ&*bS zTmULl6eKPr-Twvd8V8g^Ryi&<$rJMIe}2q?&)_aev)z(AVJd=pox4x4FT?Qp1)?_FnnFnauy%NyG2BW9PqKz}=#GLQtR}TA4`Yj+(og^d8^bH!EPU&8<;k z4~=B|gP-2*6im;7*>dF$*WrRYv9AgsB&*ZA5l0x{ULXv+6!ISW-_aZ+4IX7U4yqB% zL%8dXMyCq(2NG8SGFUdhfJ?>T;X~?-#Z@B^(o>S|@8GbQo*6`<<2rEnIAc&*AC8DO zkU75fQ1u;}4%w%D`~@<&W1y80btlHRTgI=%aHGWti;Npm{MXB#3(zBx6YLF5+XPHQ zIBN>GDq&N9K=TWlXz8(+Ti}i}lM1nKd`uLoeYRnf ziaU(=Ej4O@9th(-*7*CBqvpU@$NHzo2h<)4jz6{3paHw1{all`R%&NyaN{{%s*eVB z`ugUa%|px1ac)$LwlXxIo~$-$z8_tQ!H+fXIdrV`#*ayz78fni* zexiecdHi9B+-jLIh=B|9~2MOfA|j%HC&JMzK80!*6lZJd2h#M{g7s} z`a-{2@9(~68$IS{_v=pw7Rkr)G73pKpt&R@eJ}y(bfb6BUJ=@}|WH`%+-(lxIlrK(T^6JR@s4e1E51msrmVNxGqrIhr z6XlN$dfW>z2IbhBOBK)Ex;^feUD`+4M>uv2P@w5;=N=x$1*4w0Rx8(#^vLw(w=US# z_~w9fv4?Mm-F^dMTy>%4tDZUPDq2MesAH^aXOUIJw=Q0Qm=t4n()S^ zJoH(_w@FHI=ysBkKl$suD2D9Ah#Aw~yfv?0b(8lQrRb;o&D(3_^{0osN}PRs_7{b0 zEXT*si|r!js+#7khB)rn*QBr_ulQw>YAzSwNmidryX&W~r(U$N;YgS?bf@ zx<@GK9t&-4?S>@+0$I{JQX2rMzMJPRSFQ4tH04|6_qbYSDo4_5Ey~x`HpMGy*E=h( z^Mkig-s6HQjp*NGo1{8@58Gzh>g1P}gtEMwwp`6S-BvUIXzm8S_S8Uy^_3GrCoHxJ zvIwazecHpJA8Nld%~i1#!}P#I_xX-5Hr>5bzxHO;SND^)0akGBNaffn#LI4Ued!zg zjm2NZzZjn$P)Ns0RhC+#uS%Z>Q|_nZM2m&NeCK4nn;L)nFX#LAFl1v^>pWE+-c4ED zG#BSC`ni>M$8Q&3+E-CoqPOnnE!zoN$=D`@=K~rDr zEpO6||1~53NngR@yqVZ~7fc<{ckdBcRqI+3t>(I518OoGU%7g8yUB7=M z7Q0`68u4z*bnV=|-kQHW4JvBf%F3LrYfjw@9c~ak*huUCf|7sjn>;-j`{l^Q-b=^R z+U))=?|MQU+K$h;v06|au|xG<%4`Y!Bdf^KsH^QlMJLU#*rt>y=15e}>q!YTpkJLB zakH{5dGa?+WNc8T&EvwYkow8jpINCCX76~BGgfYn)vo9yfJBl&`@AG*L{B1-;v&0y zZqC_6P-y&ui5b+=Q8N>Zd7CTkiPkfJe)F(yQ#+HPleMaS2|Icq57(La1HE;8>$ury zg0iY=SXVNudvVivguRGL;0M$y^?*BxRneN|;0X=tYAv_v;)=CSTlEm7%@n!MLb|Yx z9bGk%OFH~E-14kJEPA-syrmGE&&nZB200m;_fH+uk`V6F@0q+7zbtTQ91XDu=QOU& zlljV8DLLC_RrNax6*Sgm%I@kakZ?2XH7M6{-iy&UP$%=}Vb>Op%Jsy&e`C-EC?(54I8x`WBd za_?#HrJ9)iCVbyqvh&dGzdm5LIew)SsIL~OheQ~ZdJU5&$~2{@6^;IpVI!hgD~wFD zGix-+V}NyFDO?(?#j4AXrc^n(jrN_)W)%mnJ>N0wV@@bo@T*yVa88$1x+fkV>iwM| z|7S~ot;qx|CjbdzH1x{S%(USLKgbRumRrh4|3jFJf380bPQV zhRMIU+$}Rnwsq35FTr1~@zB`0XxP>m65aUJb>%O7Bw;K0tMeREmB`>`Q!QifEYMol zC-fvwZy(Kj?A{~lUwSXX($y@5j3$(#=hz-9D;_1?SI259u*fx1)m<^nt=Zz7;*Mfjsc2ACzhUIzL z(3NI$yuGsv^YKuyTu1N-+ z)ad1n>9aNH`k>@!VS)cxgZ?gbo#FU%5 zcJJ%PRY6X1CE>uiaE!7RS-s3C%P(n)I(v9)63){fm#QFOOfzOwO~meuG(L2`&N+!J zB1?Cw(Q@UD_Y`iXg`-D}o_EFf+Z|k5Tw%J)HVE}S{j22w(HY$rwR@C>ovWR;pp;b{Zz7!z-p`^O zzqY`P^hBr(BQuaq2Dk67&-xt0&GoBi$&a zBVyVEy`@8Ii$-1mF;uczLS(tVbhX5TDHGafTU8d(xP1^q_jBkl%Bf6wh*hQ~3`n=i?>>C%pSe)gISa>DR#Q+@Cm>_w@4WMU4H2B6zBsEST9N`5xEy z58SXOj#_m4quLnrDs@iYbMH_$1{7kbmHB7;$0%3oh3>j*dy@akMwemsYFD#FiXIt8 z7(HNl(Kk)6b?fjwO?hfYx(mlVFIPHY^qTjxj}fzc>IE=x2;Go9e&Y2TyfVNrBL2C>(JGkV0TjA#^Wp7_10Ts| zz6p_eBaU&;QX4O(8PK56>~bDRp~h;j9rIn89!}r$=4;I02wKJ~SRoCv{<3kta}1gJ zFpwYmz?jb~Lz}>n7Z729uVnPn%yrPcyGI?@W}*4hnVHl+C%H7y;ps!eF}=BpkQj=E zLGi@bQK&B=wlS$IJ}s+Zydn(R%AyI-{qa4Tg83Y~te$d790FONX7Yv! zybdJvI390w9rT}PVhHAg*xmICN7~e@W;{(q2S$2n_|k!KToFWAw4OH#8GjQlxSi!n zEZ}1@IzE;_!3sxBggyMO2;8Ld#2z`4%f z+p$QC8}f{SsDc$?z9S09n~22#${wgY5O%3R!e{Gz#Hz~OEIy|_*@+;K56GTD4<9KZ zTf0iFeqd~!Q?s{S_}DYQQ2Xq$0Gd2=Lt$+UCzGVFFzXrYDHXH16}4!aV~5>Wf}2!s z!TsfKs;JdI_H;zuSTHhlz{m9=d4Af1x9z+${5Y`xo6pv%~wp@E^z4N+@&bl3r^Irh`g z*-W)ZPlvmtn`5myCEA)42EZQjV{R`Z_xLuh-ie;h?=u0IlQxHzc~t)xw++p_=iD)L zP+_j?F9gX288hN0n&&O|W8KRy&>6nO;@1=>B8YSV)!i70ku~l;_$(!?TUg@7F9{aEr5U zfDm_PKL5}_!kSSBuKIrE8!RC*wjvVlR|-e0=1_-SC3)%e0-b$fB6nynR8@+u6^0CU zGo3CBT{~(JtD3k2nGsBZQv(jCxSD+Lq3&zbo9QkoEdHTF^Xw`o zi1GZ6Wkq&iywnv@Pcla>wXXzv$WB(2TU-&`F{kg3slR{H;+#(&&B1h`>3XaRxKtQU z9!-xNr{SN<8rwZ5{FThwbXt&Qj;cDY+dj9bb?lm$f&vvwR$nvR@DEP_Gf%3qcKKcG zgVJc0CA6^CXZg9XS}O7)NHeU`G(B;oLc7lZwvNPZOPZ^=EY1_4TB{t*dmsMcNyeT4 z0Y@AmRe9bNb#sW{5qfvNTW#Vu7WA-4Dk?d5MaG6E>udwj_x|CDzpiUr^k|0sfd*y1 zNxGtataARlL4P_A&LhisbofiY^`Eh+1z8d=X~E)0Pf*m{&NL8rDZaj?iy8U^Dw0!e zDs&UK!IXCKSJv)XIlaLVs~|GJ3V=YS=6va({o?U>zpKn^ko_s_@rm!#>Y zDa43p7|h$mzfJmlIBaO4M(A-A>%ij1dv4m|f03PT69-U(@5aI=Zt9l|&zuMvRK9f6 z^=N2%k%xxGfZq6QA5p(3VEC1tVyTj=_^vvYXq3A8bWZ!s?eI3$=wSPZRduw`H?nN= z@o(Q2w2GO={hx&u|KSKVl}^gxrh_TQ-OijJ=D=Ntn|-uJ9|mi|;bo;^M6p5_UUvQhDvSg~Z5-IN$Kp zFHOnxkG<}+?iO5r`qS?p7iWpjHn;DLw`GOA{sHCIg8YV>-=|or9~T%$=GD{p$Y_^Z zYs579In$4>9_jv!*@-JkeUThz!lVe`rE%`)nw8N>OEbHR6SX$R@4U|^L%%K01TpYr zBTlBtb>b6We{dUnl8+kbq2&tzYN~~cL&tKm5AMvM&su@O;IjvV@Ml?4FQ9RX_R+qtQ~K^wUGU;vSM)y{$oU{WOq zdS=61r@0F4pKVUWo0`*KJcTGzK*!Nx2=7~panzw?EcZoGR$KW)4ck1kd zTM;pV)trpQAm)nDEJJ|UGY3FK3wob_b?1>P3_ABLIuwJZ(;I6wyoqPvKM?=VIG^ z*$_4l4kWsJH~myFAsu9&|D_jjo;+JBd|#M~?)Js!z>f$_chF0MizOR>U=IbzPh>w0 z4m9`5(BNlcHy0*{VI}F!`H~z_Vy3h-8++$G*AzB9ia^Bwc^AlUW&4;ve264mTP-8H zC6Qrh-y$tsBx3qssN)r{+Jz~$^eptJYEKeu?zxKa|J(;qIwE@~$2Nc~_T=&tK(_7A zo*jtRlMNj}4rTeHnHHKfow*nk3ewx5RI@lL#%G-S9KLZxT?i%tBnw2S_HP53Z3udE z9cREuaoWnj24a6OM|RB(0Nzgfe_RTVQ~bKRgYsNRkGlqNQP!3VFcOL<0JzzNSAy*ZcPWIP(jhnN90AGaD#iPUZw0*_amaapv z<}^)SP~Z|YzJX{8W(4wHAPG$e3IT!waDE0ff)G2kq0!o5W3CJ{achvpiPX8ztWCz% z5p~qJeXFgYp+r^Ep|Bjp;r5E4KL#|1hC1Z4h8A%CK#Xarf|cv^uNLR0d3)t=J&5wo%a}P2N3X&z3pY#v`F8)ZJ`$^U56!k?i;0CekEb#! z%0^uTzwID%ou#4$Vjj8eo**P?6bpA`XqQFPN0coqJEK9fchz>xo9-KFKG1}UhJ z)b7@7?q3upUKDMrWSsK^)8FaZ0i;%jphnXo_x~^XvGD zigX$vxSXtyU;p7r0khH!SQkKmua(|FT}@n%8DIpGrKr+LQbFjd#-RnFLjYp}NQEqD zd=O%LcAhEY?b{{^3Goz`;=0Rz1gJ`ep2}$Bo1*s|vR?8f9@R&0zv3f5s2~HzXcHh> z?Qw5IOEoM(L&g!%Z6m5J`38;mC$mu7g2(V=f{u?iL!%IRmb`m(*rvq)K{1evJ6}ft zAud3U;1U4_xe>bCwP&?ng9}e_-1rIlA}GBjqwH@kQKhMU@Q5j^96HGtns0^%4zMER z_M6|~K;rRh#-D{77$;IV%c03|7}6HOG93SqnTe)nL9u>0HOzqFqfPQZnf4Q2Vy{p< z+UF2cCC);CEHN$X&aYO0Te-V6z1)F4ZDTkw&oF{QG$3ihet~1lnn}GQPwWo`S+gyG z+Ly_!SDc(zqc-e&7J1xt0KQ3A6as~DX7PRDN*GaLjLE)(vjnKvvmr_z-(M z+uC3~`@@h09n_s%EiD$cL?y5`1o4Cb#tmS164`sPjl(z5qkpT&edED4K>jHoWT4hw^R=_{GT^mQjqC@OBgT&ymu0)mNfHMWQ z?(Q_?dGV<9W35c%S4Y@3yNJ~;+exDZ=nW9yUQ4e);>641nx|)VNqXZnAJgfwaJWEl zU*^8D^$M~Ka@QD1wrOpr+Tyx}FX2rj>}%9u&2hilND_7^-}nj~xt9Qide*0+xb z)cYGyGmuE+c8U~Z1r-TcnV=Q`2LerwyE6C?L<@urU5DHAGRa*OSsC?j2}`72e?x2%meILDt=ww7%Iwj*cdVW-MTeozZxE{E|rb^3Ai)U z=MDixjJRfSXr;;7r|~m^!0LtI9w_1EeCONVl2P<8X`G=97%>C~X#Ip`1M{trZSY@G z7a+DAwhfz)_v$L-maK5(y>7p#WtI!l_viHf`) z&Xbf3Md3H5!^M{{5q{#rV7UP_Db5(-Zkg=y^8fzdfL-C8MEtBb!Rt?N`UxHzZe%}E zG3}mIvJsWszCW4$O_I3CFHZ<&;PY5|(>yR1p!@+l5u+M9fk3{J;Dlxy2N0j;Wm}uKB}#KCl*vMp7Y)^q6bl z3jLp?cmSC>vrodfMq%-HXuhyaD}l(KY*RO0^JgEx-Ed5+sx zlrWYj*Gy|{3i(yg7GD?97N1=1uFW{eZU42ZgY{!?-PAEfN)$+pLZ!NuT7=Up3PM9; z=(edhS8Xo1%*iijwa38mUVgLeXHwXF%NzR~eaC8Zt)g$?O)(Xd2gu5M+I#ct_Ss%v zcw5l%$M!+}PlX^Yu^924Oip@Pk=?I!{E*`*TJJiVhx`+d5x*||I3 zI~oU3i~MDN{aHn-vtJxH=(J0g5k7`YcDkJUqC?auDjj)oXtCll)3l2NXfV4mTx!sD-bCbsdbV7WbMN_d{iQyM(fJM6RQ}v1o0s1tCc&C@uqV^9Swkm&2Yiq_H{uqn0z#CWPL6%WMtkx;inZzLCkY{baoGs>UaeQ00^RRpJRQ z=!;n&`kKj}#prTUx9+G96he;!W>3=i2CjQ#R5d?DE6sK&RMkMj|TYY&NxocMagzd0wAl4W6VKAcZ>ztdefR7V|h6Sa55rt!_nYxLI-ksD4h zvAK`_j-By-LZuX@Izv$v5L3P~O1omE*BMt^bXrH<$3(8|V&9a;)psWh3_J5jI^H;%X}750_9)3!entY~xc zMxUGTbNA`8|=S0ui=L4|(KQ@4Z{oBV~RlDGonm zhpTMyzj}A738>LdE?dTUQl;f#JA(u)VDV?$XO$Vs+N-F`YvL{_{ND7g zX)!G)(Jf9ylLh=U=(b)yWn>^@=RbA*c>u$)6ZEmGrX!dqoAIOa`*zm}?yA;5tCDBX zR^`;n&>d0TVbb5D?F>^veLM98IzhH=dHzMOqXlo}wP9DoCt3Y}kG~QsvgCV4yc7E^ z@XM9L{J$LV1(vm+FFaUS77+Md0cEd1*wCWj+FWS$_wZf46Us3=zCLs&UugRsT-gf=kmIO029R<=n?R2+;BaiaTNp_quh%(5!niyHD6ii!;b151ue+^&hb7o_+np?ylK1q;P_Rk0ss(eV7(G0!Nczy1b|Vr=K4@6a}N-<`XN=Fc*>f9U-@?yxoN_W++G zr+iG&QjeD9JxxPRt2%vi@SmioSI2W!6GSgtM4N?w8C$ne@P;Y z6jpbK6<+B}k@|JcPebUrh{4$hPD;+2HZR`&HAR?y4>BNz!_1J$zU*)BOH-8%7*voFyDdZm5W zI~RBQh*8TR7#G0Vw2*9XP@$*WfvgX+t}HqC@&{(1~2){M`wq+5P0Vl|r9(s4|O zuN@oz)Fc^rJYvognBK$$smdqm;;9Rwm6f%RMk8b5N8=j4fIgyNUw}jx9*wzfdU&D4 z5#!|w^?7QRVi z+%1QDU4-5}wFD?QqS7uoGgP~5i_`bu3r~d=^tkQ2OiyXgQS7?F_$t8!G&+mGkm-7x zu|KKWBw{}l|LKv?KzjSi&dbNi0B)KJ?h=&DKPCc$e<_ce(sg}}#nljU?EO9nJiTyZ zI}77$t%T8OcbQkpdLT-itH2*`>RGc^>?u0=N7RL!FzuKjQJEmcv=`uaMLs?s6one_ z8;Z?h;%6~SizU!vZcpr?4tWlrX?C&~8kmjGj&gfeuQSv@3mDb$JX{oXB1x`KV{w4` zE&xw4_zs?>=@7e2A6-ctl|=ZRR?cS7o;?D6V>kq_yc_!+OQ;R~JCQ}y3T})#hq-C@ zORis#>5V;16VjTJX_iW;4VYZcpqz_z*|k_=k^VW>O_~)VCn#7E#!8ljpwYW(8N%M3 zFr(Rf3YfVzj4w~5@?Mm;OOUK+cOrZk>oDj6yE#)lQAzA=wo>hyQruGzLceq9z0?0` z+~VO2X3OzwVP|lTIMRjf{u8l3ekY~&cR{sE_tm{Y3jth3U)8MgoT3WkEW0pd9G@0+ zxQvw+l=q|Nl8-2}PG-|fJffRQRQ+CTqMzKt@yRl|+V4<;x6%0>O$h3V?eb{TXmv)# z;!4{}E6?jbi$V{Cx8e4A8{Rd%9v3Yy|FVYU;+pH==&i;Kkg*rL+joDj6Kw^=t4m;6=wi6`X}{5cQqKKLu@1 zxg$*;^hX6NnR64h{k~Uy3|6XK2^Vj*RW{Cc^u(Z!aZajt{xWe2EaQ<$pMCa72;*VORW6fJ4C7 zs42{GEXK*^QG;N{H63(}Xi9b)df*HL{o>D9(_n_4Lhy>gFVkgvJC63@vR;3VjG$SQ zl_pKX&xa3N_ee38n{gMDbIk<%74r@d*Zw~AS!FcnqT9)46_X;1*A*SA>iMyMmFH8K zvKgI;bopEr7h2oR!SxkGyIEjD9a>8+xe>*DCb zJ6sQ7HGYnTCnYX(Sg{1?el_DN+j8~Rg(xTqdlt+5lI;>ZKhsi^uRZ8HfqDXcU1sF5 zuK3$>Ww;*Fu=-KMCgBAPpmasXTVmBzL+of{G+isP50#5s>wu1~j^J7# z`QXx;PuE82wkfMC$y=rz@62i2SEPi>GG{tvKOW_a&3w9&8@uPZ%89HSQ^Z@BJ3Nfk zJMcG74o&v5o6g`Ic=n}5!}vD6sv5#tCof(wpw5I?>7hCJZ7liXpWaEb^|hUaJ?%`- zhE<#5`2+F-ar>=ji!&Tri4s<~qJ^2oqifyf>N~Fqy1N5pFjUTTFuN5Fy{zXYRrUuT zHPYux7BWlH8B3`6ySq)6`k9|%b8=CK6cq9*?Q0I&H?hJYKiwu{I!eX320;5=!0o?( zY`DP7*fN4KPsO+nWVy+Yr4ucv*I?nzZNXi<=0th~+0n3`T^?o<2Xi02iJe>fUL%rb zO4C??WpKYMV~BPel)oBtN?A1w1rm;lyS^I7DW;ZB9zN z6e{Zm%v+tYJ^;}zHQH7{LU`yU?E zsOO|bXs)H+u+DNrjQGn24L{LY*{a#N9x}F8qP>^Z-9u+vCde zM`sQ$Rkag_Y|uFxw?mFJ;=X%W8-a-c0lRkPvNk_qsWX7FK12P@)0$14pP1Fc1flTx zdS=fkm`a{%Tf4Hvcu>*^p$8g*q$+y7HN*YP#xlQ-T@4gZzc$?64l%u6T<#_?8cVoz z?@pbY1ggSdlxXgI6x=n-Mm-su?;q~9I71;0_W9z?^LM_b@GV=Yqdn>10<;Es=;kzI zi|U+l?IBn5%9LJVYss3%i=WWFh*ECi)X*j`0mH?e<@Tw++uM`}G(P{Wuh3`BG;WC=Pw7`vth^L%b#J zs2A3&c_Y-oD(TfY`jx5ZdzoW2=ie${P7Gy$cpvovZ~bBM(y8a<>({k>(yyQMp9$o- z#7Zj-{ctY&j)mqPpx#hzl^oiQ_N|?>D{lDBpyDxe{!7BdAGD-i+qqg{+!yenuRB=r zJN7-Td;#H4QE+Rvm)e{ZTs-8|F22MO^=M2PVsAUki;q4P;uRqA9RJ=4+Ji@kt2-TZ z&CnBB%oAC-KO&*2Cx7Ij3uAOzg5a54!MQ_SX7ALnU;{l>?Rq<<-!;M>7!xSX zLY&M=SHAf1d*b_ZT*a*V&UK@1N6RvP^Gy2@^+GP$1voMI4E*3_Il zOj8rE-S}tBc;K8G6wzjTR^38Yx{B5MFO4Y7*uT|#XBQh%&F#KV(Z5dQ8*?Mo%4s9m#~_OVn_9Ne1?g;|bHKFV7X}-e13~2k1aCj% z{OzX}*6_{+wf`2M25kt}^5&{I*$}YIsoL;!VF*z9N(*|VL(n5NVn?ciV3Gs5{zTuP z)vHM&Q1w*5BJ!Ke;k(hSYwK3q0YffPyNKiu1ch@DA9mYOTjEiI#DD8s5zsR0rz>^h zNnof!|5w{YrH1XC!vXakKKlt_b1wSd-ZAGI5`X#m>kv-YlA91G!$Era5b%}V0ZmD| zFT)Y>+(m!{1_D}PQMXv?LrYP#w|B&DB0)glpd(jyg(ucT399#K|ys8{TigW zv7MNMq@$_8kxJxhf?SaX_hpW}`8I&VT#yv8ONJZBEAfbZZ#ZtnTBZ_#vaQCN>?mv* z@T7-!>XiqmFnXv84n0t#!&#)kRdbxen%xUdgq*JdakX5;(Etgwjtcs=b=gQC(SJg0 zdx_e4ZlBo*P~Jdrm!>8KLf4U``c{yOqu5H>Ne(7s)FQIDmD}5fdk@Xq zb;MuOt0$#~o+*r8N@8~$?}I|dA9*M`$S;$EV7jk?oQL!B%?JLKlY_fpJEYQ#v;ZsHitrK;3&x)x32mu=)i*My^ElE)9ey#)5M(K^2Lcb*g&RgO zTR~?qt;G+Ga|AyFAO+miNjNdw)^ER%I6^ptHZWg=Pzl&EoFJIg<^)KB0QSEiX1J-~ z?Fj>y4XeSbhQK&T05@Q4h@N=|2)*^dm39tEQiU&-G0qMmROD6DWFMH=4-#Pb?H7K| zDr3Jeu$^gUKxd~Lq;le%ar$$%ZI*CnJI1mfDROljrFuB>)-5>qCxy^g%;YfnSVTszx_ zQ)kRIxamO&-I`yIJ=mqm6fM%}a&>YnPe@=DPQkWeoQn+_CEN&crA_p8-52mB09SMz zfv6Cb8!b!{o=&1ae_gm2fdpzCms)^X)0YxA)9y_wMDA!N`oZZ#gzy9HPQ|C1^|+4H zstkr>LuU$56){}UB>5|x?;{D2O@4h=Wdr~fm^nCC0O4<4Hu zM8d$hWAi660f~ClH1vqsZ@3ZCPdSdzAwfKB?Pn{3!U8_T`O8lkA~!5vg|iEAI6yp_ ziq2B~5w61q1&QwCxOQ|GV*L#2^GVc{2Dl9{QB3UyEd<<#D@NR%;fTtyadn~6Sw=&^ z;3QY`!r)R}ev9^CrH4{WlY$a=?|&PGIqt~k+<8bIH}L~ReP7{SwR_|ND?`gnAnODm z)^L&0^^-*N1OM<`fHOPU7~*~v;))E^RZLHB;R;UO{iiUpdFpmNN!(SPgQ`Hjt%@~d z2*>)rPKODB)Xev{D(gYEtm$YhE{agjoIn)KO#uC4+V1b5uv{Z2{7 z2iy*1s%P^2v@iw`@u%YgK)C3N!)&1GU)e6$Qv7Rt%()cEzU6H396k{T-Euhf^5>oa z_7WU*tjbI%^xcR|vOcmiOGn`p<_^t&DHwan`=6_6HC|^TBaa}m+9X}}{cS-$rfa}2O5^v-WIMM2ck3PNO2>Ze`;&tU%)=V-kVMS`U&WA zI5KS3(7zBUo8^OSwTSW-wpZBofM&$mQXjoRvU-uci=Tr?noC}Q4F&*eWGCce;2iGH z!)H?zOyBEO09`S)lj_ zUPihB!#k9~DxY9V^E{b|!l^X>#hQ~&|2q%>eBOmL!=L*SM#fdPv6rCN@fsMbVQ|#B z3yA^qtF(zWj!lbQ;)Yv*G_52BCfE^3xi}~QfXl*f&5DKdq#wt&`TVM%HBje)kzUwn@}zg>g9a zjU)Zg^6_Qek`S}8D3Ql3k(G2ubKPly)+uklq|8v3`4;7gW$UZ7_;Jilu2q)jfWWM3O>RO@n&Pg>94IP!ek>$#hi;U}(9(+7{Zm%W>6 zsqqkq;TQHRu;i`IEBN6rvHL)e$Y=zThZ$or9w0gX-~EPf#n$ox|H zqf69#kaP>0rz4fTm;FR0S+|TI4t~bhSJWRS1sjIwmF}Vv&6ji5f7VO*R3x0Xe$rbk zk;N~sfBlPmPqO$IN|08JW|YB7fT*L&<XCiJfxj!V$7FX<&lcGW`NS zdi6<2YL1<;o!Ft`5ZSQ+uNZwz6SZ)g6AtYm6MW2+hxA6f#aWR4@fnsx8-*Ageptsi zF?n6T)K$K)O;?=1r%%)|b%9dG@~cQw>YI-Jw(NNJvbm_s$JobF#S{nSI)-)|sG)6@ zGMLNPGz#nTo~EYsD;3^v=B;PppI8=#U?rDwUDTSrXKQ$$T&7}vU0&a(OV_QF*iV0# zoNDN&E&&Sgm?7rmaZAzJijp!+&v&~a^>5DEsye&-=HKJm9wmsess)60M(1J9ix1|$ zoVgkL#MRg29HG}+ELkkZ&0(uV-_drBmHct#7dXKkff50@=*ijG(#&2JSn^tOexZZA z*BjFflV?)iH4DG;y)by7-9w?kLBQ?vF4h)-4dkY43$;PlR_?Z}T$8gJsNHS7(RkHS zH~4z%v&E>C4;XD$@yX^%fghDq+t3?&KIhh@ACth}4#4%^vj)|{@!E+?6BEHEww6A_ zsutzZ zSkbb_Iy^L8UgB4dX1}Rjb6+ze?n!$%);~I@a6VnmWU>EixC+BxCJzcNp(uHqZQBIp zNvDY@MnUs&3sdjwy8;!QAJ$3`riE?XuTukOoxXqg7~wDVJ1w{tV&%I@A`*lr*|iQM=e$jV_Vsta7_(=G*aJfwk@eVWT|U<24^QP zMZGujo)p&VL2T%lP}Ad$C6xN2=5J=wjyV6mDnDIZA zgsaOOGwh#07sSnC7N=KTpzVEC)ZO6e3E#eqo=~cC_nC}Zjd+DP@rgbc+2aO2v;IqZ z-4`J7*D@Y$9^(@i|4O!AI0odH9$Vxq&QKiRoWL{R0RADgoRR&4?pM6!>K~Yu7tEo> zT-Uz_H+Coych@S%rSAx9BF)?UrLi(FRi+>5nAx)`@OitdR4D7Ls($Ngg5ypT?6nZ( zWB7yKp{<4AwpTg!`)`e56}8w$p&g4Nd>_p$!ZzR*oBpR$p}O{rxsw`>c>W|)_VQ&u zH@VnrTdQmt1?RuWx{ps^eN%iQbdP+wmYdt#(~~IbBil98qr1+?{KYT70na5?#L(6cTt)>%H3AEiLsVi*-ET38F7`T#P*7 ztF2jkH7LKDae84ce;6gOL*&x)3&gonrtE{oLlH;b;tnOtSLfGB`M4MCoyF```si!$ z?geIM*BR;2qo#NOx@cA|=Cb$37C>BRgKxM8Gtx&vH;P7&IuC>%==Bza!pw z-y0RI;$G+mOsZqEK!%`t8wywzmC zXSNF*>5BoJmk8B{;7MYQM!Ky;fNglVCTX_WZC)=wzx0}K-;ZK&qB6E5dic~v$Cfp> z*N7CmV^vuTH_Ba?azZldkAJ(XJT6^9EVS)IF}+*bt8j#+F59SDrl1I(l5Y;)E^~}R zK{JKJl&IY_E7v#varT!`_VoDzjh?LH5)(Ik!S#y5d-Ti;|FwvI=`f^BzA@QX za@?wBOiNfximExmkhL}R@tXA~rZ<%{v#+X_i+rX$9(TQND0BR|=h5XQo8WpoVz&}H z3SEJdiPI@^Q+TyFI#}eaxacDncIGx-p$f$Qyco4haRw_^rX}VOP3gaaDR0kH3>`Gs zaujg>)Y}f-59cs4gZso*As)=Vx_4n9MX0H0Jf+~Vg+ckAyftEN$(SQ*MkplHgA(&F z9Ym&>lA?;XCSUM~J{6?}+tFA!eIf&@i@j0gZnUyZcfvq)LBwm;LP9K$_aMkIL(8 zv1plOAP{Gbj+hZpEPIl@0OaAmKJbD#lX1v1Rzb$3wvyL4bC)9XylrTGC<$|EJj+2j zQz4pO0TI4O9S)tUAS!E;2csV!40~O6Q1mL*#fcYYrQ23Sy=dP6ciSmCo2*?|h#GFt(=jB}ftvT1(-7$+_EFBNDChEImG3WfO%gvC1mWnWla2!>-9u5wD_`dDFP1AVr2E8@a1;*a_*kbqU zgg&yar|>mwg70gP)?UFle>7M&Q(TQ;<`%WqiCZ6`8^CE_yb&x6F4`nJ6;sL&-MR+j zcdau=zgbj@b%y>H;tfl1Pi{L1o$KCLK7P+0O~6lDh;A~oeu#-k!Ncl3-C5u87<+6u zjXA+su3gT7?)q3MLx%Y#Xe>EJN0H@)$oRy22 zk4qN5_s3l*&nN0wELn~DnKXsN2Z>dVZy@RUNU0s!oWwVI9p5t-$J7+WO}}sQ?EAHb zbZTPo{q6Uhdr33H*xO9l#YCkJM)9KzSLwzHI`|29gfxpN_ByQ63x z5rV2y+S}&Lj!1mj4sbS`c$3{)XoU5*@mst+*?qajmb!#1zakQj+ELNUq{y#c@h*IJ zkbX-08F-4umMe}(R}2+Q^m!OZDOJ&zu(+poJ9YREGCMS`ew*Pdgvu_bT6hl5eea zzrT`%>cq(SZS(>ktI{Jz@D3RrDK%G1E$ax=6d^1vB)vG>ZEq3LL_MQ=@e|5X!!S6)~P-?NRJFqlz&6lVu z_AT>95xBpUoL6V$u_}$8H7Uf@iD9PA-&a-`VXN-><)6NDA@plC&d#P9LvG)@cPx-j zh+|@=O+5GpCx?1k|4b-~$@sE>^}mJ8E<-1YcG?gYZJz&KbZf0u=ctC{>totKJ`$&| z90()9F!f?MbkAF(ete+mF~1*YlbN+O7sYDN=EoAXS|oR$#bYDg$`@wM{_JhwU8_u9 z%#?S>O58a~QcZ9-(vvX4H5>w;KtPN3TnKL(qj^!Au7KhS>8Y|0XJuOsH&A}bt+WfK z?xwwGRCUS~Z>)U zd_10y$K#krT2B>pe{l0deMr;rMDCD&tg!Op3`k0S{9aq)qm`p9$dm!#d|m~=?d~2Q zSqscm_lci_E`_V&Uox9BoMalJKhL^NBJQHL_}3|wmd9!tB_}^Y!Xg-&ns8TH=vtv(1?v9vXzAIG|8_um^uQ6?rvuWWN-&jb)EZLvj1>$@w6`ktr z1?QX0b*z~nA-j0uUcV2wVT#ThbAM@3&$gm0Q>Ka)^&d6uXOGOKSz2y_!n;~r6mPJY zU=B}3%b|JIaKi7TzAc7VLHiNdILn-jwf!M7t(+YEtsi2Wvkf%FoAH^e*jj0hE#}e` zbvGH-4EHGgaj<0LhdU%%!q)5$d)yt%v=(wc|GcrICEgpCD~+$T53GMa#9*^c{FC*) zYz~qmZUE9V9DlZrQmpAN_c60nt(aaWEVX09zT+g^f@DE;iNsApZ?$a7bUoEIT1;w~ zU}>s65N9?oCDH6z(;HgU{%b%(dhM~WTk@PUh`T3KLE7ixtL(2#Se`%VW3>OUtDir<8uu%Ic%~!;_5S_e%wEFAIaKe9 zf=hr{ne-x2VTVBypoLn7W`dxu5~@^yEDS)2J9Mxk95YIKWvEs&{V4J?4HhY#6z86x zgl}Pp`{;^_=hYQ$YgD=N3NKL3@xM)5QydH#Eh)@42ow!^#ShjpQnx@frg82caVrjM zeBl%#I6&|$A%URoUsb4pq7>BagOi^?@_<}3z_1`w*w%nD6TjRrlcMqRD$lk)lduif zUmERczAM{0V-5zCe}D$-URoz84O)3dLNsKm5{TebeEEXS7CV$3TD6r0yA>^Fx)Bd# zn^#e|6A6(6NE4(nNTD8zAgu+`!87}_&8_tajh=^ zLE-`t4^&i#xWYcJ_@duEiYBx1TT|lSOx3ghE3^`bIw@2Pv6A10C?637Qsnlf6*d-u zYDw*gEgpkB3H1R-FKZvA@E-xfa8@FYiU}Lg2T$Y+G5*b@6Hw3sm9F4<97b|?RTn80 z`k95+{Er0y7scCAV7TM-T2zdxh5ReXkXHmck&sfyhDMuG?RxP5A3zdu1oL)kEKP|hF@fr5{{fCdg0qlB_HK*K0s!smqf z(%Pe7jLjc&g5oR_XiNld7eb(E2~z?Cm@gnGN zV)0ygnDU?qKXtE{blxtMB-#r^74&_5)pKa|Lb3-4J9*F1A(PHc>cH5tC5I^BZejET z@VrPgX%>Iq1f3}Q$hw&tBq;z5-%Seg(}vQjCMd0fnt`c8)GFD)N-z!xd{bZ;Is8mK zCkzVo3s^^};Q$tnqA9ZKMs_R7DVUpAmz=OBy?>B|+hYZeZIX^fn@EuL$&`#pU2SHrZobQ+u<7>CDJ<#&@Bl7&_ztFA+i==fK%mz5sSPn5nC1 zboE&hzY<{#Ke*;dKs)UgHDY$z9&S%tpE((iJ+YHE`P5jbjxT0xLiiH~JWGEn&Rl4} zb`m@jGmfLDF1HOhzf3_RuS}joXoHIA&FYPuXY;LxR|+72P84zjB2vE@=>Pbih1q%Y zy@c`x5t%7U>kN-|67fD?DOtFR_VycVfC&072n!>&a0UNIF*uhSkvx0(&G970DD$SWoA+(S4Yx4z);}RuSDWO4SMY2puXJm~w-S z`tqFonjzE$Kg3aw0eyIqekq7;$ACy0Ua#giIkX^Xf z{Uelf@M)LBe8b~Tt^eavNR_B@$5qWNPo42 z)X(ti4>E-Wttz13?nS2hoxJ`l@6)i&zOdMwy$2nm=Fn6#sNX`~*`~h>5urvknH7?h(<}JflpZ`&nO5)m= z=pf1PwA7Vm;EmLFEz*lu>!6!1!M&FL9geCn9sY9xvLIMY3IZ!-sL#0sv$B;C+%Q^e zgSbem{?GW<6;bEEf5MI%-uB~*nT>zxlN7)4y}6SU&_Qq?%6V3JVWH4hFaw?wMel=% z+)PTsMmDmbP*PFCTHr9Ry@0qXt>Y~tg)UC0$%X;{seUmKY=JTp`4GZM8{Tt>u@$-| z@ZEIgfxqXr#7M|5 zIq47lh8T$(oZHG)qHuhtaI6x7jw@mG1%Xv{VplgDo)slJ)4!}jrwtfA_4~Mof#y^P zy%*O3+5I8kwvyc|goYoS(%@SXgMwNxkqfR*u%6$Q5@^F8aOWy5#n~kyRve0ZkPu01 z1265M{-OjSRqmAIBn+--PYmCMNRR&n5y`zSCWf^GH>ADrDTg-=bKKp*=!pWt?U-Y4 z>Xjo(wJN*8G=d3bL2>y||BSd@iz_Zs-Vv(-b(4O^ugc(8TUE+K`3KWr3A#DoVj65! zwD)scD{nUUJ%X+}L&KyjV1` zx>9Pl*cNL0FOmBXyQe4CEr!>rxi&KEgP*Yzz56T)_mnXCi}!SIG(1r&GfhZ2vakJ9 z^(yv@{D*7$Uu_A!!8CTCG`HTNOD$Q7*Iu@}`*F;} z`bb##`+nnJE9=~@nfIr9xZ!r=&5AYhAD@csKy?^^Fl?2ogFw5bZHhug5^^5%e8HpZlOMQ#))+eOSOoOTJJ8i7{?YC#$(xP@ZB}*m(ENke(_GTP$Jd^*{uPj$&#D;u>K5cAYUvaiPH&kiHXt-jvZo1kQ6z`KX1 zl$^YXGXXuA+}BN9I+j-7Nc zHUwkQ)bRNaF0D{2<#m(KfCdDFu>8i0V z`sCg7!OJw-ZOnv~x>{qnh@3y!$AGhm&J?EkVQZA>?BFG+U*6{|ZxAng#wIL*trf%B zamsDkH0E*d!gNuh%ZIkIKf>Ty^W3A6e|Hp>gzD`a?E9U#*p(j}mwoM_Io53CjO_S2kayLWc2|yh8M~OY zo%F@@ic6k5%Byg3jqQj`E7-=RpUJpU6hFq`%Bj8^wd|cy!?;M1ie#=mI^+8D!4%$( z?zjQchS5`EXCITxPE`}i%DrxJFFL<&Jyuw6mK@dGdJ(3-t?sd(X6MHHeg!n{a+S>Q zXHC@rqR_B^;;E)MfV=CO6ClzZ7kA2HDYxL{#|N`D86$MAtklIXuT+oVGR7#*!Uw6G z;tf$e4-2CiJzjB{=bx!>YHS?0cK)?Jg~Zqq9}xLt#R7NFV1IvC@H>u9-*`4@3vA83 zotPipjePd}QJYTdqdP1RN?x?gt2oFF@VXsi(s*Oc<<{a@<05(-y;~jIBI>S7{Ot{K zf~5haD^%sB>>r;0mU8{=H-Cxo_mb0pDMhxuuA#jpItLhkbP70=|F#VT@G|{sQT<7)Prl#a*_QkzF+028FN_%e0#HsoP z;;~7tQ$^M#%$LdMa4ssRDAoDz+t~7GczvO?t`*zt8;`4xOa3(sZ~nj$3?w_VZc!;$ zEn;;NiFMv5AecTc`%*gBh>&AHk=}{D7WTLEvg*PNKGwfjEiUL&;Ul}^h}u8i@y6}< zIv34cZn6Xjzk%psX=0TgI5*eim$CmQj^WyXgcfUTc^sy%-R!!#(&iWSHrKt{E**-I zS4U&L9;Owv4TsAf?p*SRE?ohFn`vKaVdK;|7X-StcPx?WiQIk#mKPBll_+nIw`ZQF z<~o`Ao$B~$>91E4v_2-Nrb4p4y{sa*d-@(y*cx)&wb*0#yDxnA%-*+P5|;=YqH;>A z?pB+)&kkPlEO+y>KHcV{>Xr%Bn9>qVj!E1$?kBH+leaHbe-EsEtV>T?HF@<~MO9=k z=^celId$Yqr=_lZ= z&#$xZ6nu~+h&Zj240U#*x6R}IdODdCWE0zdlFfkc-r(`wb_WRM;(>cH67x;92NF%( zJ4ZZRoODTZ&P7Of*`0MZq^cD=JEhOrSm=6O&bHOBbV@mo6(tU60EJ8XF}M8Z^#~SY zMjf}q zr(5JrTByCQ#jxV0crV&>x;jkiR9;#b-P0m28eUPfq#IYT5{XK^GgZTgXvl1tNM)&; zw{hD#l6XZVyaQ0QKW4cd?R+QKO0hq}rcTL-#Gi;A0~ucE;q*=^-Z6Wn{KV(4atKB7 z+oVaBCdcw{CncO4sLkMK)tUpQkU7o7XPF7>~eBZ77JUR?t;zF`9+w&+@)su3hZ?Uz+%k{u)s|`&_ z2iI3MjA>vrKP>%X&% z%vL!GDIP_t#pSloQ=&*EE;`=FuHu;b@a;==+j_4@Hd_a*jVVmZQ7ZO@Zp&-j%XbRn6?NTtY zFm9YXF}SaXx16nD{?5jcDf66A{=&#K0LNi+ospMHopKy3iEDgW%r=(OV3`x6+uT{> zhKz}=7{V?>jc9ZRZ$Vf&vBsQ{I5l-UW?tip*FAZqP?}Tzmbu&Y!!{E0`$DqEyuwD` zynM%-d!S}*?}EPr#HJsnlFwd6)aNe>OL8iIH;0wtO#4+nD?6C zUx{lj%w1cRliqFr(2L-P#pKE!!o9}!aLce^FvEAu=f-;nz9C&F-3aBKk#=C+VkfiP zEUu{EsWkksNH27q-+O5=RK;d^f?ac_&$wVg+pji}TI>wA#J2utKWfumm;9)%k@8*U zDwdXAc*3+Q>}m3qs>s+eW8xPhF`v5Ls$r{anM&{o*ZKBWkA573X>=&$fkFLkRjPc+ z<@Oi9;IA%O%tHE7j@Nx%&9ZH!)w$5`Dw%7uUZ%RPzFYG zea2g!66`l#wEOo*D6=eh%RpeP&9Kp~^|{jGMWw2kAtC!>6@?x4(2xv2!%~XVV8oLREZ@F<%RL87{va|eK^DL!W+z6 zq7QfihQi<_N=Cm?>$j)q5cNZ*Y-f|L*pyZh12*?UD)m^|S{hKp1d&RFP;eRZ^gOGX z-j!!_j!>Bw`w1~Y-ZgtE;!_u|bRl4O_MXc~VfOPPZ*o^IAR>ReaUHv*L*zLHHACl5 zu$0eMKqpAj9+#P3Hm=t!XB$OzN|JeU6-Uz^6Xix^AI|zUP@2muE*)SSd6i?lq}A;h z`hXqE#914}*XSo{**mA#h;Nm0#02lIa&eqVO97}88xC9KrUxc7n!A=Co5NkIy|nXd@XF_ki$W+<>FjB=;cT*35PPOF7vkMfunh z-6t~CnKq#0G&z)`J{)p((XFl(R>fZE<`L@c`lM^JnK))wXncUuv@3@KVX*y<|-!DXqvLa{SoU5#k^*$Lq zx-Zvz+0<_5_A+&^6WLow68ow)Y5p8z_6=Y1DlWf};3c|N`y>xH4LPXeom!s}N^a!` zr7Ogt*gabmg7~mCr1y8F&|xKzYVi>_z-2(|h5VqB7ZH9uQ+w|WO0fW~nG91Upj!6J zBVCJjD6k~}3ja+AMG+q-aGqU(4DYVzTNti@!3L1hww@zUVW4t2aU_>_-wM`^V%p}J zTfKpAN2q}&RLBAb%7J`x2iO3Cai;+6L^I+5^87E-m2YM00C3NLd9fZw$1DQTwgu%O z$`T-Z#?CSt0FvZQDmUZ9k-NlGZS z!WWJv#Q%HgyqX!v+XlFoA401Y#_{D57=v*A5-7%|{q5F2LY8dI<|aO|M&c*vUAZB+ zWVj}PD}_)RV@vY7myg1uL=E~I_|0`;(8rVsk|5|rTUzvIfMM?d3j*>Pw5OtNzmlqUHvin(Q})0d zg-M18s1Yz*=Yf2de`5)&`65}=kQews&?iDJFVfD31ulo-h?)pK6QoqDwG;T|E+>xI z1)UE zgysU47@QpqYNfW7OY(e3bs;cesuD+j6S&X*wOdo!m(PVa!Ql#gGrzBDg_~U^{u)nm zTi}UI5`hKN9|l&LFSK;O3kJMG|0AguZRnqRmI{_EUc;h}E`8zKhBbygF#TZJfKc}v z1PUSk2!>H_L#NW`B>U1Q_<@7ALeRVjBZ!jyC}WK(ACcf&8nXkDrt-x&@FMk*Wm3d^ znrmDgQ#ExNm)<6X0K$g_pCy!uYES&5a8pLlR#~I(<3sN73J7A<>JVAA+3@7;C7cAq zui2uT>cPuV;ITw8&ns~;!@S*Tma5R54E6rx4)5Z7z0y(a(g=GF zfC)IGd^%$i(*#5(b=maFb_hkB_2Qid4{g)`kA3KBMl>51ZNKKLpgACMHCr&;f|~hc zf%D*%HHRD3=iG>kYkx|CrL$f<;G`8)rhXWashw=UE6s<%ozAX6?K(n)=THPP%qy%r zr8k1_HywzDxtq9TyFgsB=PvW*cW-`xnP4lRkOmeoS|pLtat#lkSpl6JRPIM%h@wEa z=J6#08778|(zs!P^nB!xr}sPgTh zh;<^AH>{4!=a5tyFqf}pK+gn{7YPGS3Ibyf0ppCm93*FJe?kW0(Q}wip70bOkVp83 zE35SS(JY#=8Ijr)y?zF->;MEC9BF|LP%P9X%$bE0;}DdbMl_Tq z0;goiffb6{v6hJ3?D>s^Jtv z+_O*~e}aV!#N(J&?egigtEwgX^V=cK^qeqe>_#9=OWv@MbC3(<;c!!+kejlNBhc8% zl3pBG0TGcD*i&22i=*^DI6ou%Oi*13^QVxcpv}LgGGTumuM4DFtC7H=7ksq0^H-o0 zfs3dE)chip|1;Vf42OkbHRvJ9tPpg$PWnDNVe)@xd`~zd4cYN^liV?6_D0Gd(4r%V z>%O6r2h432s)t4DXCMU>h-b)pM3-@-BtY~#+R}nq_A&HkL@u}Do(f;c|BF@NtcjE& z@~<%weQX{;&glyha8dUHRH1Ld&kMm|M$s!SLIx?QZHxGY0wAO2??S*6d66)ZH@Y%a zNKTpj%r8OF0^#PEwm^!BtQ~h&?E@l^Paj%-<%=`bpj^kq>=6RG!CyWGqh4Pn_Kd&(Y~t_H z;#jYj`qV%L{1P)Ww&Kz+q+N-F>H&iIf6A%!BJllQkx&5hXmS{OGiEJ}P1^@9bh+GL}3uzHI!aaFq2-T{%Fh-#bQr|J$dpu&ShY z@sWRetug_bcE~fx^78ZCpG`^|exoDaw?#P6XVzJLSGd>1_G^TWUtz&q?2yU0_?LI= zrQ!i;c3L@cgOiU-!_x(V{rNv=6QAh{mfA0Z^M7h&tCa+1XfWujT_T6Z5z% z#kr-OD#8lMee9e?>ZrN(ZqMK=v*d|9R{k}U=pH!Q9``Pml(AE`pUJ^)sAg&N3WVnx z#KR>HY`-+7s%I;5Ds8s=m^Nm{eyT$)g&ZATd+a&|-{Yk-F<*YW*d#0qAIF;!cgZ#U zvh=I|0m&oDJn<1FnMe?iNgTU(C`#>ogkKm--0pQ{lfT4EH(zlH3j$SjWWMUo9wH+r zfS~ifeJWgfcJPHLi6KOB=Qya5kB`VZ9*c_Yt0*5kAvV|Gb1HthA~5h~Yr-2v4dD~d zvB#1-cG1ZtS6)YHtAB~sw7btUcVW}=L}`99i;?Nx=8d-Gzn(gt>pvFgEv)D2LQFPK z6N}n75!+)~?5sbzNxxrGrM^FeGBasfsOLI7Jj>~epRf5I`|{;vV0TW^FEZWCg-aIe zBuat0G^vhvO^N<|X@PBj@~Q2EKdfifw(f0$cq6yldubxCgVTu<(^hyl0#yyp@wJZ2 zAs;CpwT7(JUuH^EW~q|XN2?9odS~2D-*Fh*qH5SvBT_o6KC=dPo)2-OcgMEZxZcDm zursS=59}3I$0`kT6fDBdFE~h9*ma#G%=YXfYy_XmrO!88y-r;(Xgm2VR9`O16y%(z zv)j$>;uqrP$8jgDKmXXSR_sD`^w)MDw7GzLrmWTR?-`aPPZAFt)v28oR5@ zOGB%vb$W-C^q1g1+iT{t7XDJ%4>?$&$$RuX4t@)ckzA~hCAG@!RQvUegiF2T^yK(* zhsFv0KQL}Phel&ZlpK1kZs8SVSPIrI_-e2x-D$ADtw=@I%r7UXf3fNutJXc`R`CWQ z-eAs^Rs#kDGt_ZonL(Y)xL#NR?rEN;>_|GZT`ltBT9vm44Hll=PH*h`1WU7h_efDW zjdBOe>_{IqcRv95oW&M!Yv>9a=Z@8uYt05wiP6okM3OKDt-nX}EN;9R2wy){S^cC8 zGeu7=zXz3kMHid+M0T^~d_pl=9kn4Ue~wn4N?`SZgvGa`D}P&%Vrz zB1L!?54}=(aq@`K{WkHnw1*zL*q>_fzAh@~NL3}fxz4vm9Fmv{B&U{E*Yvl0WnwSt zPfcYM%SR3m&E)+ur&jdq(=VDbG*GhODMSyMN4Ph~v?Q5SC>G!GQK9G=*1=Ihl65!N zzIx%T+CtdAhSyuXWTuBw-P<3e=Z_7yJe5cnUcd*`A8Z5`24Y(ywX~bu~Fq+E#%^o58%~n%$6dNwK+#W zDmSTGn?Bn?^K^2Fz+2-w0T*QOiX7~2$167{&yqW%r&qrr%MGe*A!vumC3l@v#JgXZ zWcf3L#yJ``vOq{?_gSWfKC#&r*4|T(Ph%dkJ>55HTq{++<#Z?E;j5e+*PG0t?q*MG z1uKgx-0r2O{ITp-*k%#!>G=c`xmi<8P1)H0SM8DDi3Qaf z?%zi>#`LDo52xUh|X~O==7PHWp6ldj~!1Sw<RU2v^B&*v#Tdg=&UlxH)yc z64q9hy6u}Rc~*XG=w8_+B6UMe&JClAzG|~-IJdOSWzI|XaX*y=oUFg#5nUJQ$&M|G zxd|0wE_A}*`P{>+!EIJo`1M2|uib6g#I{$#f{^rTFKG7~kMhzY-u)21r1F&TKK)GQ zaEsxlOY^cZzcdr-~c0cTiK-f&Pyph_JxUR0>fTZbw=1(F}mZ}flzFrbYcpRhh)8~cgT0|{9TGSBbu4# z^ov6d-Pv&0e%=wwI~>`%+<@a|V`ZX^71yP@&EDa^9i_e_5OAq?2GaL#Fo|r8vuuhz zo$^-qZnGB0!iQHR{HX8vo%$aCTZ26Pd8gBBX40&BEf3VXcw`)iQPbDae_i+c)54k! zth?40GdW9xK7YNiD^#=#8Rl(2674DWqrAdu(yjLBzf@`R_Cp32qJY3Wxm~%^VdM#= zD9~M!y|%KA7{|=Ewn}FO4x1b-DGPEu$Yc&L^mNQ;6l;v&5&IbHZcX=(iKk+b5Iw+* zovKpqPfEC2*r#=uc=1Z=W(nfW-Fiow&uG$klj`1qs=CWXtzstmk4ZQ%=grbUf>W|3 zE_=2&G42y$7*CBB{vzaGi&vMr@c>*~-)kN;SB^CkbIWO7YA+@@&hEONLTvMQemPBi zTFBTn`UdNmVfGaaGao0b6bGo<<#m0UT^e-aACO9XWP7=_L4>~*xlT1*77eqkfYaFl zHz$ST_Jvxo%Rw;Jwrcl9-(BwJRFQtq&un8gjmh?fO6V@f2(%pCPx295-wBSQnR!v& zFFa~lMc+R{my2!Y8}4Y)I+LHb5gC4z_R^e-O$Vdv8qbUeefQYF>3QW+qh1sU_uar6 zwa%5;ezsKlvv!Vz^|t^~lc_dLA^8<2{%sEaB^;om&heBY@EA#V62q=S{NzFRyw27M zJ6$_5?xFQvP}WBa@-xxQt!z*$rY@A&CA4Zbq*$AF`%%Cis*o#d%(kU$ zZP8HgfUa2N+pT9?vQi)#uilWb@w-L5Uq!>Gfue#ENynH|C9I?N7oxNehT#cirXfWZ zpWfohwVh{S);^tW#ymKilN6=z|2&uE_>)Aj=P3Oa(G@7T!XcNl4i}1PcJGM`3(Tcf zChCM1S{n7$$k*2oVg#jDeQzksFWPi%e~`8@p$#d$HvfL0*jKW$UWT(tl$i03PL(d7 z&I3;jE9c)&Oij8F$TU`!RnVCS{l=d+$yag=j_RJQG`vK?$^({Sa(r@fGL>m~ir~@- zW5W&BlfxQny#(@|kYpau&3FNpb?oiDe}rz$a2;DLYD8%Eoz?PsZJr_pc)!enZR`=H ziw~~a9hNE?KlZXh7Goe`RB0NwjRox4IlCp+;3W)E(w75&f4?ERq35Rs0F##Ipzn+v z`M}Ol?N4VH9qcW5OfD>`sT?EP*7RUnXGUZ1Ige**U6!1pmkG5z8rwveZ>gqCpOurL z`LYX{JH~_T1CxS3ie;x$ajxO@!ZEI;jSL%bSmuUOS*qe52^(+gkLXI5D-6fSj2n@c zK&FIPxoW%Px4b%`^XH3Nj*44QxmQ|GkAumq+$S#*W6SfdW#;+DZG@C6=|B8>*vrOe z61LS3OHHw=t`(5%mmfeyZ}K3>5|@^i5v>#&MkWHfQ5#ZqN>JAnru9V0(B;m4(-`7= zz~5`9B zbYWFH1a2q13d9Y?7NvLgg3#B!WV8&pMb#Pc*~ZzWKg{qF6aNTtow608&hxU}=AJ|G z`NoP2ZMC^dm#)4^G0~9C<_*>PZr=`lPhjVi!-i;Ma$NVhro#J^7=K{HGUs0^;1Zek zrJaPbOyan45;Y@6 z{TG6yPKn%9;_lftvm`*UEMOzIPT|Lh#L;f&CA;^IkJ$}RZI;?dM`cbi*1m9MHWYY% zc9HC7WlkRdS{y%Uinr9?x=4))NoQ9a>F_e~^znZ=S$O}>9}^BUGC*4~6gGXO zSK-}5jWwLDG3U0F-;;IL5Az~TjfkcuzWc_Ve>FLNuw#mX=T*xF+|)IK1GWS}xb8XY z(ZfuhLF0QlvC-$-Y9!8(IC&6o1!Ci*-7TABzyMd9>Ui%d79G;5pK8D>uij1Hn9VKYPd=Z?=(Pds(i)X_3h9v zZ%e6Y%e``5az^ZH#|3{bQC=||6wBLn^-$$B2b&)?KgqbEcE5qh4>df)%keVysXJqwq1 zHcBd_xV^`u0YdfCqk4OM4RuKD(1;mZjiacpFk>D3_3f9!60x}X4Hxh|IPR?Cvi(fn z7e`l@#OCDu*BZk$!ckdbk|BQFe}sNuEP6@=>-_0IQ-QkN-=n#=kf|6^p38 z?n?3hUqF`<{jmKwHXXeaht+2fCqbmqL(^AvRJ6=+Gy08P-F^iPtjA z&$^ad0zSb`6!2A=lF};b_DX)toEFE2P_%3mzhaB%4rNI|KFT1_qCsnf{xIQ3!k$n> zynqnDe8T7d0FHco6awQ=aSg!6Yy>t==3g|B94HQaS9XanN<-xoE1crJ!>YC9bpK>_ zXXatVXis~V#JI9XwUjWD1=lGO&lel=bqwdWRiWOv25qQ@4yAJc<421=VIi=ALsV%0 za@#0>0Z#(O3Q@W;2jL!2Mi2RuuTlM<|AIc(l%&$X;dS3vlwN!+7wR(rp)^LxMWoRM zk(Ak}WMsRlq&;lLcl618RYDtdh{`(5ruprF8guFdOw*nG7_??`&AJyd|+uKO6eAOET(lT(wke{Jy6puVX z@o?y?2)_hFuG}1mLO#rl%0ms>e}++v8Ug+##qYlU2y(48d>H|kyTS%xR|<*JAdPwy z^lvQOed1LD0EA`v&9$<^Bn?^@1(^Jz>S7T*<&MB~<`}8`8 z7&=-5zadv@1$-P>7|&-3p&y7mMobQRbujGx)Gy0ZXuC3QXhksUOsBK4?p*!a<<{Hp zMhG`Vh+90YK*(5y0b_%geOl@BSW zYM5>yTQDpM6Uan}MIsSFhPxoY>;=l=EFD$aGSuVLfE5IZFpLPl2@W**xU;y0#-o?Q z@#=FbggwwP`Qfwrhodeq+edB@`g@oGW|?ccuk)X((mp;VHz)Le*>{wfPgw6ofiyg= z(b(Go<6M17hP!@$y#C8!;WWo*tK_?lF|EI5F1K16Q{w3UCDU+(=s=ZsY4~jf&$~N! z=O#^o%qS2Ld^NREb`yts;AghlOzf>A$FT3yA^dy88+n<=ZkisSTY>rmlq43q%ml0e zEGz`-!4tTC&Pz%5Gm?GjEV*H5wCt`J&&;31GlTjH^c@H_18)b)0wLY_a~x=r1y7n! zh^`0(30!^ZvdP~9r2fDqDfX8sK8=TPW-zy4{IGpk_E-#LieQ$$(pXTTZGLxWv>2tA zz#a^buubQ1s8HMlsS-kk27LS3CRhkI5WJM1k$&OL?;U_05hZn700gb`Y`9|rJygQR z!)SK{DQAy*{qPF{0n=RixBp%`|K;e^iA+EjIzzEX)Nwv?nC5_ggrEmSMK#cu-n`*% zLUhx@3S@?C{Sdn=y@e?sA&s@AV+Q*=<^*xme{E}Gfv5?FN~#C}X4y1sjwAIa$qG1W z|HAd7@rs}w*#@NkrFbK(K5`mCw3KPXn2}Z(Tt+JkWY#JY67cMR5Q6%U#rK~A7lF7{ zPyryw{^U0@Lc1R1e(Trqs|1RU!qvfX+~AQQmb0MML^1Jplnr( z&r0$q8H8|Ibt9>h#B2>PmNN$AjBgV#hL$1dcgE{S&>AQUalNl(77iI^%uvN>R>3R7}b*wLt~o2Kz)-DAJ#$1=Sy_{JKMQy#X~D zVGJ{x1`=fcha{!^N74q1@M#L3#p8RsBFa0yn|KVeg+VoZ?SJ!^PHp3{DxnprBBIz7 zd_NkcUHZ4+`}p4)3Sr1XNX9QZ7BjxXQYP3KpcgcgQ72ZEz%3GFL4(#G(yKQLNWPVYlEjygXM?`34#eTG6F^D>ph|@=3(o+G z6DbJd{~)QHB3CXURVVO+dfLMGAu9m?W`(o_X_+bLTTohYrO*d#q*2ySLSTO&*cbuI zg>uT03N)^+826|X@l%)Z%S{u&w;c{=JaL1GK#^AfXACe&08UyD6DJ>6AU@ZiE!YVP zc;!O4624m2Vf7_~hjK(mKRPq$vP;R|mMSs|{}Iv-tR+Oya)h??W^o7Au|9{mlk~Ih zTAU&#C0HPYMZ#tIWfK3^FM-c;8M($XwiefC7Yo;hLDUOKE6$}?4(o54l4fI4K@z&BWLvqlqlf$rAbs3%E_ zwBW4@;uHQUkW634Q{{=C(>92i8Rn42)s~GSci&gk+6rsE{|u5Wz2p3_#y197A?hIZ z=?LKsw^1ArxX%G%X*GvEhH@_LyipS5t1V&jIo$V79;QPMtJjwDCP-C;oy{GaIuLV! z)X-l=C@;<@<7^hnT@?uG$-TkFFt0|2C(a% zCM1@!B}}caez=`e?H_q7_Zzw83L|1J3$yKONuUl}_lyj;uCLV~2d2TQl9^X8Zx2ir zIt-FMA_)B|0gS~CNogW>v-=+RvhVqya_A@hm+f-KeLi-WZ>jjaiB7!qgS3r4%1=2#pP6U!DYT1(5ALE6lwKd*%LH zIu_igH{Mb=EJ2--d_BX;{sCFfAD0~+?mdK6v1_D+th;zX@r8d&`4uKfe zlpwktX66@ilmj_ZzpS)z?EA@_-@2QJCvzs-E|99?BO`K>a%i8+-?2Zs#nyytQ4ODF zV!YqAQJOGK*m(7M8>gM8zHCr${sVSHEJAfL*=%j!=gJ%J3R>-z?q_cZ=vs*TAh=9J z-%hjY5m}GDPi_(=$5ob%@7U^|QGb6o=I!%sFF!OF($#G{s5@!Z1p)N|WzXC`+{z%b z;E-6pe8u;u!J(!(;Vp|Mn8izZ^P8f}W(jP_Xa@nKijF^{U%Y)=nZ4LJU_dRp1OY#|{(f2q7>HQGt zQsQ`UBu*U5q*k77t9TkPnE(-LQO(BNm-1YQE;O#wnL)gI^AE4z4Qq_xsKrNY?;Yk1 zP9^P@xaLQamsmAz8(IOg_=cpX}4AXt~ zMz-GfhM>OoN%Kp7DJmB1&R5YvS~!}@+PSMiL*K$CuBY#P;hIp^9}hS!&Q7m}-5cPY zX?Nc!sFC(fn#u|UsH3dYwxl&DpaH+0%9+FXGGFUE z9^3M96gbWp7ooVG08C;A@14`k(p4(i$40dA!`k*3`R9$MJtF&@ZBnDErD2X{Y9M*& zg@thz!Mkn3yH}U9T}RFK;LD;qSouRUxuH$;kA5Ld3-hqw#dkh`qnmHD8c!IP;bg^Q zS~xVt>1_A)zXwi=o%(!m{#DQ0?q;k)3bk0l(mRBT3BSP9hyB)%m_4{|8qZsjJ(l#y zo6b(l(b%Ic7W-YSJAW(4K_+*yxm)Qju&?m~3(jGVUSj=qap#RY@9>VtaAebBPX%EW zKSMN{>_?V+jY#+;ZFr$k@qEvt%pIiiqPv9TBWu7@a|2m8yS`rmh6SX;brE1gueRK# zU36)H&B8&Ii|OvUFHh47PZUk=@jn?bNGK$9lmWM!t*GiLaigKU$K^^zC{w#AbXFxK zthP^VS6WrZyDfQLxpp;d<3?8_a;q7kDfea%uU*ls#32cLl$VX`M!2z! zZ8?+ITm8zNsY>pwVw>eysE5}Sc6Zl!?|D`&oSVj2RT*h#rK?T)T>M68iBvUeFnZR2 z{F-T{-~7sPdGj`1x=u2RvMOqhiOd%(;Jr`nqc<^iIs#lDzTl=TmBnl%KP+ zJaVrXL^c{q$76n)@b)^-ZD^x)TR38Ggm

C1=)-TRVGY^>)?LKR;J5&o&*Xc{1NK zUVFUg#YJh}nnui?T6DK!!M1 z9NVgq|A{Ch$?(Xax7uU8SXekoZ-4NZk^I&o6H_pyN>NAvuM2OiDA~;&%?B9YU(EI^q0lh*^tm$n5Q-_a(s+p>I@#yPd!P5WWTrd znv56C-m&eQ?#sl-MPUZ!bF#GxtX2Cc4~^UII5PaG1*aH_J*iMC7~NxZ+Bf7!*&f|s zS+0VsA)$QX3CN4jVOCQ?Vx=cu;9;QCo6 z!wR!t5@^rMq0DqAdM@CEyZ0}3bfiLR+LsT;>AmmRYpHQ`ywqq}+9S~jMwvVZiu3cV zM6CC`iPn7E#+ltL=_yAc%uppG-3h<9<{?xMHh1~VAhz(&U#EooYj;xTUvgznIBNyr zRL(4bf4h%2b}r_oF%(eb*jsh71D?5m?CH5WWf-dz#n@FPmm;a$3u64M{`ZC&nU&Zy zr^%N|U=Zg1vC=!->8yS9a947+w&e6k&WY^@qU&c17={GLdPEfsC*)hj~4Wls9TdTIVM@h}k-3o?C8*G!pitn-pkEU}TSSlR9}DS{aZ>-XZ_KWtj&Gt9U@*i8T+NYqO3G}gHxs? zXKsAi$ke@Oi;3bLI4ur>BwxCrTug(zTe^mQEvx#&B!=whsm0}%a%JtN){|+<-pe=M zOnn0xUvx2M%6|C^zQ57V2;jMlJSi})gAo^LE^EQAU-f-2F)<0>(4`g4Xip3p9K^iMqMbj#(M=|%AG>r@Ysdf+koB7Kv{ZSs1oH^e&bC*FaYvN{L>O{4y-=cUp!>vQgWubqh@Pj@)*PpDbU9R6`h0^) z|Lb~st@2CYWQR;$rxJE<`#4TKfs>b>V`b!Q$tTtO-Xbzwk*!+qjKa4t>8-?LVfj6D zuenzZ7YVY9ii@qYOU=6UG}G=Z0K8HO1rxOMvj8PXTNf4k+elL zo5b}|Z;=FP+fwG_p~oD?-d*5%Ra~Y$@+5%9+osJqliu&(JT;N7L*)^-v)8j-# z*Gv(0nxh7f98aO)N+9sjNc|Iu*$a_1yFwoh1oO;CrjvQLDlEWNdZmqc*Xu?#52zxH=w2jN**VGs` zZJsbOpL_Sn|0C?bAXYL8h>FN4O{K(!hzJ2iKHuw@K9GZ`EMbX#Psi_1Y%`0jwLL?Xd6OonWm-aSZdwXzLU_>FTL4{L&Ibo z?OUuM)Dd<Ib5*xM1J12{do0anHauuHaiorTFrG{;UiQLEzut6)L)lp>)70_V z%P3&?<67L7+w@Sf?C| zW;JW6TM@rZ1{|L|Tq&0z_WR^mfS=CvR7<7p zoKdYpRcnd2pM>oA+YQ)g^iD4o>repb_UQ&-bA{Y$4%U>{^vI1rvnB69-$&l2h?OX# zI!bb$i;zWw9s~d#9;}pexMY8TPE|A-YtwRtMrLKB{d8qzF@;t(;mEsqA?XC0c#R4j z*HUjAVnd}$J>5S?9dW|+mg=`T93>-wyh9}i9bHP}(6657L@UQ*f8$V1wz_9lb{6Ys z)?)AIh~GN#2Q~C3mapMY$sTLX{UgZxMmdyYgXwj3Y1>US?{Q#a*ouZgQ%>WvLH`o$ zDW73bFSn5 zh$d4#=k;eVQ!X0G{FWZiyPdnvsC4dg@=k*f800?2*Oz1ZuYS zQ=uhME7z=VogLQm-2J`T(_XH5-lZc=kDc!McWwC-0$|M5Ge>mReXtNMTJ{)XAM&h- zLY&Oo*_3zMT)mA~AR=(lUOcG@K9MZMMELcTVRU5btnZ4=NYY5KBr&WmAdut{*-gt3vhT`SU?psn@_n{B>8MX79pW=V~db;z~S&k9W?BxE1yc?gK zQ@jKEnTGR+=FR3@kD9qnhOCs)4lY~AR4wDO?U#KC1C!rNJt5pblUqURHDpquZh0z? z;5$otT8s&*5iObx3d%K9(LUmN`TZ28TH8=@ADGrSx2!oEZcg`_A|#DcLXPsxS_0^d z3!m}61k!&#u*HvdJn&yQrQZ4n?8#9EQE$a24r1p3BjOTUkY=?2F>+9Brw6YC>F?fw z{MbB!(IJM%78+Oy%9sR!ajokg=tJ4s>O1p(@?N)QmXGsWXM}=*8&x(JRKa zE3f(~BBrNBDFCx`d|Cbuw>OFZ%NW%YpUnF9enHy#iq#cX*t z7%?RU2j{`3A0dW(Bd=JINBTYxmV5*l4uT)DO-E|tt@D0`K`*5vpH91gUK z!Qb+YRNpr8Px31H3&`(I{p%OkM9C@kyOzrQD#f=cr2P+8ECg+!!8i}<%^yNc>{I}% zI|tBs2)oM>wd9>-xJIj{7CFlI`RH*Z5%BH*0(*JC=h6j$^zsJObzYn*&zAy$zwB%| z8txeXz$^s`-F((%J>O*f%$8@W*ukfn1-ajA4)9nm9{w)Kx`U#ze>|R=g6u_L8R3K zv1;Qih$!If(PRUPJl(1Vlp?q1l|S0xKk(`1tf~3|o{N@mP&$8*uSc-T;f5N*4c!C7 zrB+mMd$z9qzh)gC`hE@q7Qp~1;1PeU`D*}OgeG1}?Z2zVz2wyP)Id@r%m2nMLLT08S+zozJIpY0*p^ z%QJ4QEN+5`%06i`9% zg(I$um?fkvu-7!)NdL0ULkTYXtIwS)-%dOe20HM@0_TN39ofmk=ALN4RN&}cdRuwS zl>wfAvLlCouNz0*7cN}TKKBy?2lh6|{07uxwV?34F0Y!9Z;&G)Cs7MK5K|mis!bd( zs+Si_%%96H@jD)&L9c@{r;it{!rs3nHv4wu;5{rKmm5|kfvsckp|5Sir*X>6yt zEHG;x`9_5|cOZQ*goBKZgK!?TCp!@?zn{TPtJTT% zEPN>Lgtr~8+YBUH?&Pl*YjgyBpDTtjiaENgOnyMPR(vrU5CGxQ?^( zRymV3k9Y&|p1<)0HhkCpThQ6Wq`y3Z3yxs@l+Jr)`@33q;ar9wzw;F=DAlVH{FBF{ zvt-mEp_zBU+8}8`xPo|t8_8|>dExV_@Z`}m7B2uHnx9acx^Np9og$7(mUQC52_8i_ z!Ot}1i7*1T|{y^d88=MGRTuK;)s+{Xy8bDDMdc}fgm{dhG8w>7RT1`*4zEQz=_mfc zkmNpwv%3bXsJi1>TSB@Ent`>X0 zSD`fydHzM`whAV1_`Ed>r2%`g1ZfxisBU52ZbC3+W}|pJEkUpAxn0Mqv^HY{WN~4} z@EYn~lQ;RL&1>sf{u<+K;#rTZvB%BKh_cy~J*=#%+_;fEeg>{1T%py|Tm2aS15?S{ zcE(8a^9o7@!5{k$!!2Q%%WcdSh*XXM8#~Id47baLEd$)YH>A+81%K&)`TpQXkaM4@cU zE0zMn4+xaye#@9X~N)NzGq2YH7%F33&V@D!%ngAko+ZS1j?GKu|5GrDKu=@&fgB4LC^`lqzjL@cynk32%-n6L8)FJPS$Z
q9UGl6?N+bKN>AQubPJrQ-p~R~ZRY|f zuDkE?e~o!Jy;P%y17GTmG3Q#hr5C))ty`B#aW%kJTMa_(M~ zrwS$+96g>x4wvdX^(ABR8vW#3L_MZzw2L!=t{XshAYQIrTotou$ zjA7sBRw`OM=(k+U>A1eU=-M)n`@1YaD*N9;GX2n!q$BGpn?4`6$v0E6B)O_}khR=D zS&=nScfj)540)Fvc~6QgB3sE(rM9EV-b=Kp<|3QY?`U*w`vH|E zCR$Dff8YlF*rZR7gI->vp*9aHtoYs6IBWt?bza4Seh0M}t=v#?+iRnHg{@Qm}P`umFe!cK3#RZ z`02Hj&J-#cTXu0=RoxWheg@E02cpJ!zt*yIS_kXBiT0Cq8|gQgU5+ZUICn6&ce>9P zo#K!h?*tBhh+IjUK!jOYrbS}kCPDGw37dQ1rD<3Ds*w zPSUe^+TS&6EuV{TE$Wo5k#(H7aRk$5rh~H`BL3A%t@YiR*E={R-4b%f5P_oy29T|% zLIW*jn(Wx$tjbt*#U;_?lWZVs34?j^pZnO|$q5IF0ag`Ns z9kxsU>Ho&MUoOMZ%6ZG(N+v4dDQBEIeQWO%afA1Rama}bWV-b+#e)@Rf3JM40hy(& z$PfLjqMqG&Rbm`->ObCzChgAt^GB&CZEoQFX-LnMtwx%JJG)-<3O{3M zt;y1S4Jr2RCA)R(JZ7hjOr8`pxF3p;=iDMsF1L;BTN{lwjG`il>qH|qM>qMpy6?TM zwkG$@O>D6-OrB#%v6OQ1@YNSX6=D}3UkgDh84*g4ZOk*dQkAA*=?_eDRLiDY z!}C=HsA@YeGYP?!O_&eUuM%%<+XtD#$G)R2M#r-p!}jZI?SF2RBAZ>Os4jn%$!~6-mG-e z43nplV?oe=Y*g|&6sg*|*E7cbxVXyzf-+p*Jld&z(gWXyN!64`&`L2&Q(DF4HW1^N zChS$78}}eH-tr_xY;p|#L8bIjBl5400c+{ewu|_m+TQM&v26Eva~9&nE~KDnWmY*; zd*8jp%~AT*gA|{O#jfYvEcn2oLCDwqfCJ)grx>_l2e6>k>`Ag{iEKA<)N7E7b>E~# zZOq zg6vPG*9BpZCz^RPM{VN#hj-7slibyoT`$doatXv!dBf69r()b-f28Y-!vPtc!*Vh= zOI(Xq;)g~fx4p0}XtZ?LPJpy^TelcGsL4Ot% z=y$-)uC&0v(AyJ@@D*p&p{w61?Nz8T>ZNg*O<;!nHl^&mp=3PTO|~nWvB2&XnZ<P_cTt z;P*&RyF*c!qhfXtF4~Dw-KHgZf5g_wBRkR0LNlfkOEp+P6&{tiG>EJ5F?b!QlzPa_ z{EBo0AaDUOSf$rd`3bUlJ317Tb1uIu^J}QYUb}hVR*7coXh@5qS40iV=5b2U`;w!7 z3*DZK)#2njz%yd*{l2GRVb!C5UOdr?^(29s6tmLRVSpKfxJyzqPj{Chn!m zljC<#RsHHqC{^8f@$u2$D>Z~^Hur<5$o!#OzfY1~<>EFKG7USG=Cp-lCceWfJD;L_ z8txyxB09<$hmaKa&pnA!GCI<_-hMeTrx2cX*Aq0M`|OKO&9@xdpI1+C+9{x4aTk>x zhz77B^2CA<1#SynM})cCx_qBn5s*I(je(v>AE(Fv}e-y z$#cG7a`(@jWv9Vn$lJ3=o9no8s%dSb0raL`xgJ0lraNqQl_91nk1-z{!mW1aWrn*- zVZEv~y*N3Iv}sLJLQBvDZBmovuRZ2Z^jXS#H7gR znZ-PrjWlr{c%Fl)f9H$Y-{C5Ok;C`Tue4v}L~v5alEzNSNnj}L_C4->gZ_mLs^x&@ zywf=Y`KJZT?HBiIu+D(C_g+u>#+mb3and1IGGh{&sfjPxhcyt>)-3#~`LJ2(m4+%@ zt>R_%Pm--)BQd`!Zs^Y~Cf+~589atK3FUJRxK|U@*R5r1&*Tv;zOn2Rco52)&1<#% zGW~0ovkq$l5f;5{WbzGK#1>Fv8sjsB1M>RD?D+9%_F*$j#rw4QOE2YQ?(O%;bb6>g zo@PwLpSn_F?d-MFO~bjYDWO@~)19n$(RJ)%nWquQXJqB(v!wYkrb^8uvOFXlgp7Y; zPl2XeW^mB7ukKAk8ayZXLm5JFCvde5^Ei{fu3RVlJC|phBj{M9*KD2(MHx9QDZ6Y{ zAwiu%FGqD=PYCwQI$%n=tJGl-LshTaP*J6*E9Q&#`?{6nP-*AfPQnM>M`jH4oNqDd zdKXadEBZBd6jPN9y1lN`dC3vfnyJ<{>AkN{P4-O2VJT4oWLx|lcod2C0}F80+L}}( zh>3E_`V|fLj6@+%PRNi$O1vCZbjDKdVygAEtsN!xKZKQPA%hMafoE-}T;-L?-SD*F zW+H#9r2t)IP94c;#+YQ<3PakEtg{}*JZ&@<=zO4^$ct} z($C^rw0|5D-QS-4@^YwOB*m374rC9Y_{X)8F{4AE^2cpS(GYp_nrW47Prru}j@dhg ztB&s}E7s24K0JSP_77%3;Rt4=FWmdpi$mg0Kp}3LG>unKs>n;0G^v_d-{BM&v@eLB zvl)BHQpuI*`yki1S{s$vME+g+1xFWOSwx^`2Tuf74^Ni(J2-|JT`i2Nzs$7lExpW) zh@K2TMqnK_K?SNROFt=Ab36EQ-u0Bd_367Pox$lXQuDuYnxZXi#q=0|lK*^RZSFkR z9_3LL`-!HLY4L+mzXX3Tfe@$ef{weUsw9%n_6Z7Rz(XD1ZCA2RexlzMT2IVTdyyh>*4Up;WWcb)C5#=fHC)RwzqoE(`ay zvVQ#SHFnZm9c%51_P9Gj-8*@m=%9>~{x%-_$2N@nF{IheU+>3?|61NLTHb~nlDhM9 zPZ{xSQ$(={Dh83bJs%&-x$+p!eYPf=Pdfzss>zscyzjFqwuqZ zwZ)G)0jXxSjILKM3M|=92-L?--Efg#;)ax*t(@vX^q=j!5d620`tMqW$R^RPnHfi8 z(hADMR7p#1FQ*qdWFgs+x6B|#*|a^(eC#Jk`E?Y(8?y%nrs=wcToDlhR?f$^nLc^G z#SWlPZb@9$R7<_K>NTZ+63cB2abCObInnq*sYDgDKIF=OG~EZ)sAz|IhG9}>%|Lf? zje(48G(0(CDzQU(HtpLQp<7wnLcaiI3ofty{0Vaq>lj51ajPxhA zb@F|2|Cr;r8I}L?M%>@?YVaUH7&cLH(4Zp3=;K4ss(KtYyEwGARCwy{;84VA+7)T27Vs@9L0D$&lEuJ087Jrfw>3vMLjU{r*G#_ zDSln6e=A^C|Dmn)A?fwp2CdB;qgmUz|4tYEw`&RrF0O?`pMUN(|AD9ZbxlxerNMts zv+vxCA%v{}3INo%ZU4TN;4J>i0f*&kS>3!Obn3$DV@BpUf-QzNvTL8|4>b{WGMkan!S*oD4i%Z$^nNI*v{|(v| zQqnrOk6jo4 zrF86xJbOq0VW|NeKSDfQdaWM=E4_k)NI@PAuv@S;a=8Sc<$%T1B-RQBLms_r9>pMfcCEOitupCR+}lV82&sjyoCiXr z6C?PAfR1Hn*s^!>`>9sXH5h5Cg|?|~_n$1~?8KC@5Tq1?N2toJq%XlWs3;MlH2o~s z#>iz9;}jxzMvxo)D!3@#9$4r+m##|PCpDkRP+Nh=I9XO6i3pU6a4Dav3$Yxm9nHj3 zuXBUL3;~1&I}>*dQH0*|;daFX0w7pOKyg8CFpLNRCjFLZ%23zsSW4RH_n9-B!M~1M zA>Si0b3RjtnN>xsdXTh{?_5y@SZX>NNQ6J#p)wHJ&7<{?tW{=HS16dlVZq$Aou94> zDh2NJa0EGt#EcR~&aI0m)*i#Wqg%cVkuS7V5{d55g=vNV$)k>1#V)_Xe6Y9T=62c% zr95+&v~XnAbPj@G0b+H*m3IKx(HPbq7Pp;|D+b7ho#r&iPk$;JD5gTOozYaURPd09OQb&k}#77aQ3-_5k_WXFvg3n?jUrC zO(HR}QXo<2p}xopawXzr2sZ>&Zv$V?kOEQIT*KH=)o*#~)2A>W+_ia?oX}Rd3h>AF zfwF+xa0%rsXf(nkgm3s2)R|?i>EiJ?`n6yYJ0irrjp07wGUNsep9SX{X!e8#64sB; zt+xR~)&gPOSwBy{hdB$I3T#{lj!(gDPzHR6aD#Cp3zCyTD%8Y~$3dwcOfvIKC$^ms zms^0k&VTr}7w94k0mo6}QbJf2U}XdWxNrIF%eifrCZ$QwmKS)5%-mTc3Hn7_E4i;%V&BCKx9)1v(Ng=>oCwkg7o@wmF(^H<<( zLg9emJCC2^kKQf8T$&SND`8ObTWfiY9RC|P$rbEVaxaNSv4EWm5(uv58-QYf_rl!M z7jpn2K6$IU=DA&5!!K{u=3nz1F3qw^3>HH_BZK<`Qs6;fU5@`he%(Rf+V@ZLrt$)w z*YIBpVrPs6BnQ(lZ*{m0lCV0!seCBO<+=aMZO&i4u`o$0+ad<>0^1;%v7_V$nk4=G zYB>L~nD9dCV-i+Y{b8(RaWRin*1 zZsnzw_ZPwGdua^LgfTy@^?#WTXk}*5A5#J+Fv~_+-U&Kd8JxENVBA^fw#L*g2lgJH zSg?*5L!$j1M@~MF5Pr+tdh_{>-q#U9dBkzKW8O->FA0ml>(7moY$DQ)UJ|cV6_;fw z&V$)S7&2S@_MBrpF_(8-<0#3x`;)<2wcvmQ)2$AXS9bMbWaU(`r-oFkBB&$SrlmZ_ z(iJk5JY(H$@;h#7(A1MuTkW=YOWms|ZI8`4((0xSmBKwqxn%j;dnQc>KAVQR!KTX4 z2}*ROckD&I%^!Ia@Wu4cedk_DVN& zhV2i{88j8D&n1>f=Qc>7K5a*hH^{~<_|>$W?J=?+VxJh_BWD$I({6|~MsD#hivJWy z?EBO`yl^4m0fx9K5rE_ail11nyxb>V|SC>q;r@S z+rq>Ko_WVeBodKIm=`|7eOp_oujT_6q% ziC%GGq;wyf=nV~j3tc!LrzZ2|Gvf7xr+2u^BX8G_@u;8@N4&~K&qqgVJh!pCW0}k3 z#S9I%cgBF0XPf(PtTQi(H&>cJBc(GY-0U9K8LKRth3r>i=&)OG`J#jittN@bcVNHR zPcEPfOuy_y5YfhesH6aS%KL(b^Iti7GT%`X3 zCuR(}$0pXGx~#Ow+T~C7c2rXFJ-Y&c9JZfy3SchRsINMvvL8^UFk0>eV+mHOy z1nXUQ!%sA0zI(*vm?4y#=S?R~%KUT=RL;(OF237hYFIs_EEmVd7h5T*-1{qXb4PLNFD_wD} zR5GFOcIh!>u<|0y&CDe_hcGngsj{>9HJA114Gj?eR;y6)^yz4h!bQyBzJ=1q31Syc z4isfNw2{hUML)rw<$;p@?aHF(YzOdNzL}*3?}{8okY&mTGFfw~0XVipd#9rFVbBM} zJcb^WTJSJ^`%q1r+`+zluXR7P7m*%Tf&RoSv@c-Zl0qAO!E)JXGW_&&>01s&kteVyF{PTdLo zq;vX;YJX}z!s$jk1qP@-erKTWfwEapiJQDSo0FG=LG zjrBxEh0*O3NwW@^KT3ly!S1cTiP57h`82pT+0x{0Y@;?x9^qYo7&0Qo%P`xqg>~Wp z{5w`6(v4B7 za$eQlnfB_d3w-P9?6}PrJwD%e?prT4eDMNR#+Jb4%tWo)S7gfN?|na#&}<-)S!zd9 zR5`HqiSEj01rt9vO^04o75;=*z>BNF(gQE#hHk9{lpilZ8`sy^FPK-E2eTZ`NP0(S zo_2YP-bQHt7?iqj^Kg08QC4oYCcU)1F@?nTXwQ}G<~YCf`eS7{V{A}V>d|QqdD3-R z7!qI$t1bjIBD-9W#1;9FgPZYNs=C_)Ty9uvr zriRD3>x$jDEEP3zhS>&LyNT%C*~US2Okx3n_x$S7Gd!OgrSEc-*dKi2WZq9NWr;B<9+ zhgs;fYcyi_E7yiave`h?X*=J9*JCl&6_|8$iySRLa}N})7}i7#F;Uy54QbaOuJ4J9 z(%4s3FdZ(hQ`zIE_SO;S9HDW=2?ULa$^q|1Q>|w0jl`7Ct|yw}w|?ykq#lP+L|ROZ z(sWh0hTLvGspn7@?;0p8>E=kkD}$AvB^w}1PYf4^%g%p}G^( z#oK8}y7xm64SsDtJp*#bqGT%%p?6s=o~Ha}-D)-`^TVb~IBvVy*q=N=Sw zc~`D#nGwfgu!ez`pSdrg#q8FIE;GAXe*5LHW!ZUq2~1JhN?RuJ5T<@W+?KTo(f-)# zb$9R?(m~i0`HIGQIeWP(zcjvrtS%DGA3q47=E{pzL<>zQ3GUA?n}jps~4m9`j3qmoT6y z?3H2cSSDR#!=8o9k?gsOZ1az06m{e;{NB5_gDjGJJfH~raaQ4|lPA($ox8NZKGPmM zJ3=g>eRQUim4Rm{U>{4&&HO60uuDU{H}6p>dlu95f|PD_Hgs1kMp<=b_?=VPEx$@T zH)?$5@bd8o4C|Mg^>MwI-OM*I&+OqG?$N-p(0yNVPu*97o}I{(MY?&u!&DZQEaJDR zntQn2y`7^qex80Y)Rpa>uip7b=L}^utn>36>|;)pvm7(TY4r#1n+J>-ET~r6p<+ei z=x3RR>Zz`7$2o>$>zKVchH>FXrJmCR?xh2>^ zHnrrta|wTA2Kl4OYaCNjCi)-6{-%CJn4LrG<+}^!?&SEU`4hN_|FEXJYd`6kym@f& zhzm;T?(@dy5k5J=ek5f)OR2NZk7HJIBZJ~Io|+e~^>b13$DL&uo0wT7E@INa&*<%B zPYt911pAj>7tv&rGDVLAGG-ews6<W?13&kl3qVP$?hXm?sFOK{SZ->E4&_ic%19FZ;;kJ$KNUjCf*HX%*MJy>Xao;g)hiTI_H<~dj4Fdrs0YzkFyUhZ2ls0YhD+jV9 zEfres!FC#F2j8%ytf7aO)jP?R?AHd-0aHC>H8ivMYetBj>I)aUwQ@zAsVige;R)eE z4AkGcuS8JB=PM$phP2G}$)k@hAC<&>O1*~+vv%B-?3@;=TXV-hA+-COKLNS@W}reW z$HaOzr?o-h2BY!?$aUU*v%VQb{QBgX&v>>h^Q-5E%%T4;Sxws-N(%jE6=)B`W!7zK6gcuvrYWuN_?h5*iH-A zSgcbg=}wim`k)`j`aIoQ)s^L|e>_+=(XR>;TdZstuA>YUR5w|S>J{7H{jS#Q_D&1Ej&Do`iS4j)BG`nYjHtzN-m|(=?T2zNFx1-|?JGp6 zE?If1`4V{SHUy=A4wjgWp6OdMX}SqAmlBiR&3}^ipSx=OO7r`xnZ>S`yZzqZEq-=_ zkCHAhm}chA^M6m4u`q0YhE#TT({ z*;0Sj>A%(=KRZulvFM}{N{&u++nR%P_g&O*GS8zJWs z?NMq-KeIl5B3JzoOQWoHeAYH7?_I2BdC=X@g>?%BoLRLT7w^}^mP;Ios!oNhfc}v% zeYrdS%8dm|?z5YRCkAw(2ift81eOHu$1brTIZnA~Az_)8ns{(#T->UEu;K2ZOJu~X(-(|M zS6k{M3Qc9udBhH}{Bpnb(mT0Bra5tP%H(*I({P2kdfVZSk+OKj-aV5zkXeY>%FBIL zil1Ofn%-8z0GQB$P#!aUkgG_p!Vz6S=i2+l-eo|6X!#;kO=*r{n1S7cW$RRyv3?5o ze*L5klh(&sktKy8na6qv3!W_8U`~NC!!VL=uVS4p-{eOBatw&rM(;qh7du6gP0dp9 z_O1QtikIfz24%W@2+k{0p|9B9NflOe^vN(>*aPD3*&H*|D=TK#2nbdQ!SWM8Foeza zJDSjkkKcnee2a$l;>bcI)*C->MNm1w2 zV}oTJ_BxNe^Xzt?u}6MuL7LDIft;N|wZyHlpSOR?>bO{n$AzG1bN!PmJrYCIUk0X# zha%|4;mF}H?iT%-lSJ`O*7@|lkLyZe5);Pik1+E%mA(NUU4hTlO;6=Z+_((R6fmC- z_kFJ>u>7z5)`HyJV>G+zdrSIU<;c!}{RJK0e!Sd)_Ubv&e^U*X#h!Pr+rZ1JW1Zgn z10y0!q+4{^^g7xLH(Mi!EoWFy{p*i%@ooBo1*Udq%#6gu zg;kg`ALQ88T0GT$=l?I2Ccy4i?Nb7=^}V6T}L8t6wr{8#K)aq7}lR&#ryb7 z3r{4k1-z5sEx2AWK9=}jyr3wLy#hTvw@nAh#U#^`ep%vc{<&0{eygCJ?LL=prDDuvP*7CSW(Ed1R6x)bl5XE4xFN5A*uN4GmF}H;?dRI9Hh| zZh25p0twoL04ZY;NrM7EmCyr#5>tYKHU%>zkP$*hNFuByFmm02AdIDd41=rAA0YwW z%=gV55fVhF@+ts%7j88{wjVm8fN(K>cTV60;}?O|83|%ed4{uNf`URwB^CHRxcRJn z673+nlZdkPIl}$aUXwgtmdnj<;QAAQ1OS^1(h91H;?-Ye-0(!QmXI-LA0F2ioE<44 zD8JH#W* z;VSK92Sb~l-HKJ5egKmQvYhQR&zZSkVbfnNX6#yX{A(1PS3Tq}2Oq?8Zmn7Fay`iv zXEmQpLk6JQAMW8NLUR44{FqSS{&>_vgusNNIl!;#bbU@(@KQonmOk9<6x)a$rB7>+ z8pINW8|rn7ZT)KC2@+WbacnadM!IKP*LkXdi|qw&C?tV*mQz;~@}Ox>zQ$4sg9ZQB zC??q!qy9c?3i3kLb)orh4Y^X_#xUVh6aJ$!EO(wC$y^9>v3vZmxq|e`?Q@(WI>_F* zmq~9uuPChq+1J7G7GX{$2~2T=8skeN=|U))uO)+ppx48Ld{P4YklZmuLU=I_=%qwV zt~7i$Xm~4L^BY#&wt-VbO8>{;bFDZ^23ymNo!#F?`|6i3|J60)$l7L7#dG|0J-4R} zSHpzj^LEJswCTfZNhUyw_2Bzm_!B5090LUFm?evagD;y!GgR4vX7TaNWL(N$V0`Riyzp+Ck5Z zoNg*vL$7z=YUH3yp~5DgUxc3Q~XaMENp5Xf>o8< zvuKJYrSnHJcrMleTE_rl|Y0>QDqoe%GG>f1<&=G zw3eUh_YC7p)`=ma%C}j5c>~Puv^aRWcvCwb$dagAEZqA9+}?2BVF8OIENIHj+7NQf zDyU^3v)>u+*s*Kzpb&&!@1%{KE)n_^k{702a|N#4CKw#;pvDk{8v;kjn|^t1o;*e} zD}_%*KKzBphS`&lBnH<{qnL}`h zxOC&B_46Ifl-N5YnCw$hnFb_M(ncr%O+lVg@5Aaw+-f+s| zPrW*V5Nn9mj)K*8&-|5tt6xVWd9T8=%E6SuBRcs!;6@R+sXPg_3ai8(--P-@CQ73^ z7;qtoGtT%NAx(ObKct)V=YBMHv14&{{Yi;C7(uy;hYp-N;q$Wm!+pWkZ~PZeski=x zAW74Sv_OTqNpF3O#k-q6(6MXEPem2xUU1!S{A5%TcbP9;E9lUD%a8QrvQa$hGKyOp z47tCsBJg$)|C6|RWbIjOGm83)E%)_wm|XR(I}pRI>pS&%##4R>C{OwUY}EhjuO~?@BdfiaYp6JhO z;M~Te$Lv)jc4b4#*&dqEZ;WjN9XM4q0<2v~T&7o+2EAV8Wkc_d&sM z*Aw=l#hv|3WmsWxY;7v*-#!Sh*E!Y!p{4Vz}Gr66l)pD;!DEqJHQ)kSTXTXFP z-&NdHIVf`D`DR!>ht+U8dZ+vwo^Z)yGT5)8fP{?;=*L{D+Yt?+H7MNzZ8K63go_5wz-6U(xf&JrE8#O*o1z?}#>uS?OtB094tB_6|BU{Ai1kukR zez*J>PgNCrYr34O#UWIlAD3JCs(d?^$VTW2c)U{ z&E%W5Ceua(kQ2~tVNqFqn5JDR<11A$jHx;a>4@)T3J#>z+-QJI#`-5qU*F|#!H>RL zN6GLpdL!rVPRb9wp4n2WB3EKh^D&}r!GE!t_p4@N%#8NX#5zs{;N4F(VsK+*HssXA z_vpT#r`kMz5QW)AiD~vO-0Xm~)BJ3?&>@OHoLr87^updxQzf^*-_%^w{eE+z?JQ-| zKw-C4$OKtlP@0l6CLcim1sO zt=kHkNi5~^#11{C$^cpi_VRP_OW4iNpG1>?a-PTd?#`E-)s2?@jO!DN6*o()bZ0wO z&L)<*yUM+2ccr(;G=IioZ6DlrFYL@cSZ!IXapWSqTQo){Ew<1}dvFy|qB~Pt9G}!#xqWT-Qk*52sg^F~%$7 z?=kA82Q2A!H`PlzCh=Rko)cA*8!>AeUUL-RY`z#c60O=R5T!}P`J9ADn zya|(Qq)u2i49Lxus2d~L;z$9=bPr!+uhZp{29?8dvUlrijy;`tVHI`w)sftj(0wwLerExy?GLc>v$j7t#^h9h4kY+ZFiZL#M0X*Q$&J*J{0X55zXG-uk0A zKSCOQaTKnrktr->UQCGD_K|Ms1@Q&HG*r&zC9f}N$b4snbopE8`az45q5D@>LQnkU zABNjUG#~6Y({z0}LCLZx`DDMc`0F>#U2g%HLlPy~r1t8cAe#0;j<4ez!h?VX^7OKi zTyAge@#~km8P?@G zsll?7-yKi?gL683+_kpE(_+J%Xf8d-Mb4sE4Cx`RUQ*|M@k&Q=j!clViPmAx_I6j{ z*}R^Q-3$HG&nm|LV}9l!(fIH>U#ql*qGR{2L|Y{4$XpNUTWPV^hDceasxJ)_CaEMJ z%xT*bZd~nivz(&So$r=c|MpB-_3ns2>Z5@y7Ms*V>!qT9%{hdv+%*HD1+;HM;(;~` zbKN8LjiL1&qPDlXbH!_CrZ+|*DY&MaC&(WD#}>#%rl}v1aU1l5YUX~{N=$bR-O3hnV%ernrUtR+j)y{_KB z(u1{cwCcz^G4>Hdb>_rKw|*Z@D?WqWyd0#_CiT93Y15=QX2|^Wn&lqb%&Gh{;iQ>H zKLW#X{66yPr(&#jk1GCb$Hl;l(yWqge}qh{swMZDOcpfB=E&Fgd&-*;Ta1)GcV!Md zzM<`l+^psAnbaUXxM{v;gQrwQWqm@~EZzgR&4qHpPvx#unv@@IDYD2~u~BDdo5Qx+ zl4}w4Sg*mn7xqu+G&%cO?W4?55vtW2FE3kH52=!O7Phf0(}MUCH}_0$cb$2~eGjEM zQ=5Bavhe=0tf*U;h2#1$S0OsnqQ%xD(?Mn@(c*q}zhZ%6pSG%-^(N^#+jMlNx@6fS zQd8tBWS&fE0e&gHGT0N8I&Kj_#D?kCAC}%4oS}1egpGMc3Og?sveT?C!=#r{3DLEu zqs1+U!W|IgGZtp&8s}ikbIp1&+v4Z8%np|&^Q$xrB$&RCDRUi(zByy%2Azo+DVyie zh8?yfQT;{cpl^qYPPCQQTit%=0|JItlK&ysS6ux#KnTTxZg`_p*%H2b~; zdGHqVjM;;ux*`T{CV`n*#ts%PaWWM?E_E(VhvQ+2>j%m!CKsyLvg<@n@;AG9Zo42UQc1!WaHE7eL2xMFqGNu z$x<@?(_z~WWf`gQ)Dw#l>~LJJUbpJSp??1pYDgU=x|baNap+T)qsphfh+sdBcMiLH z=;AuB5gtS&ZY9*v<8wEyj|f!$&x^m;TdyaYNu>-}lFsZ-TDmQ!7M}l6%tXc5Osy}r zBATN%P_r31vSP4ekyhIGAoB8EDcCfO0L(bycRb%Khn?NhcOARX5R1rlt+ik8iIccO zqQQEa3)vU7JFfne7wx%{c3oq?ct8uYApTBYV4QG+RPiGu#W`cL2k_b(TB0#|AEoD9 zCszzHy`w=Kb^WmwLd2w5W)#BV&7t|^_c3bp@nC0ZuqW=Y<1C%u9CjQSgb1gu9IdhP z$Bk&3GPFScG8%bd_x2`^#a>IMZN&6-{663>Wlt^}`~_SD#ya3d!hXc7KYV)5NLhl| zmw;+ds$|qaBQdtd^4JCv_4SP_AtXrczy=;!r*3y5Jr+Oz^F)S&_k)DqyYsXx`k-OP zBjfb>D%L`evu1JcV8F^{UUU2@8_hpiwRTI{`sc^k^ucArmhv%;%=-OiMeK1z?JeKk z=*pgb>Qtt|PR*=TP5QyXIs)~*E^%U{0q8Ivgr*5hB)7qq^}Tp(}5jF8V|pM4Y77II~PTE?r^YxT$t zzgkC8m`+A-Lf^;iklT}oF%9>>aO;HRH7)766(j2$Mxx(U`WKih zs0}u=Lv=;;O}|;qN42uOFiSb7Zkl>my-n_?#!r>l^#gB}?%snzuKXzDoX)?tly%{S z^6!ef`%vYFNuP>tYlNFsZGb_Phz5}l7mV`E)&XbTZ#@C~+^tpgBF7+$b&eXJqcC5` zx|O>zG^%Nwogm(yoif=mWjQytfT^zY`OT(QLEqeeW?@n{2VqkN&so@_g~|B0+bX=P zdYwIecM%rBsAt`9sH&~X9gR9n@fl97bZnzb=r0@4{hG!zRTs1AuE>7L(uEph_~`?f zUsADA8ex){4tS8JR%FwRl-Fuwi_VbYDd5J8`u`Nh6d_Y&olcGH+PWuM)5`*rDJDjX4d>P9QdF<4sB`P#jIB{qLaXmMTY zMRmt1Q);wN+~JH_&bI;ZmG`$$dzLwvVw>2u^y#GLT!mm&3nyo+gDtj_ZRNsp({ngk z@Vv1!x{3b9zGIKX^X4mbUz?}zcNUxqpkQEq^2or0dogn?hog;KSytwp&W;b=r#?bq z$>ch%zsI3dSS2=;6;J|1oG)9-%KfdMPC`7jpIk97uE6xCn<=vvkRvz9hup(nw%8w6 zE$gdczO6V}8crpj>_Cu6C7=hk;p)axAY zIejMyrz)8))ZOtG({3#x>|m#UDa(-Y4QKv8Y<+n=lxy6+IxVM8rG=vsIw2ydgvv5? zBq7PZb4pobl67oPr%nkmWeqXPz9;)Sr;v~)d)5gd>oAzXEa&%~#d**Betv(n$(VVb z=f1!9eP7r2x~@RdbgOfp)!(k)!-wT|;K&~-bZ0q(lH&LY>dAeZMCvVEN%ML09bumj zT@`cmVJIJ2i;7|LZ+{~(Xz*cbLKd`L=%pN9=N;V(t#{MnR2K}wh{Jj;v}ER-=22X8 zF-`ST8MH-6#jVzQkL!{k4r?5Uei!Ozr)q;$Tz~@e3|N~~hcd3Wy3mg#y!TQ)8F_u- z5kBcn?<(aV9*41uK%eEg7upaT1>(QVQ3kcoMCzgsxsLkYZQuN17{prW>Aw81?3LGK zo1ML4zsezvna7?Hn`11}J4P1mpDC8;zo`x%8B)UBG=g($j`RnqhMJ(hBjkd)qPZ;G z^o1Q&wUrg+Wx-qOQ9jG9sF|WB@b^vngGti5snx#frPm)wR-O6Us%y9S70FAm@3gyr zTxVx`iu2oFJ~;9YUA;q?Hgp}seDdn{8ZvLB`xW8p_+R;@3;ixrdLM&ZY_kwa6T2+E zhoTVrDRvr^LK_YZ87J~Rw{Qn1*lZgPx*OZ#PW?_8hS0Bncv^{#R5uN=@Oyj}NeAd( zWh8#0 n<%{UoRGXS11&etLq+m5GaMrt zGcevKa*0Ee*DY5-AF4kMtqcF~+!SHf@W{K$t6ck$QhudzP~A1!aVhU8fuX4TR$77@ ze-)Up_zsPZJl6$AGC;~38#H%p`MDYdrE}_={H{bT(Yzz|7OnU-sNP*KjDLv~FU5(T zZ=_>BAlp4_AIBiMz14lsHRS)Z}#Fy-0d;$mB7L?dcuq|#4& z=lRbf=l7=z4Ai^1EfQa#(uqDi@MHjz0kGMt}ssZ!@? z3r%6@;ywZuN5cjA))&V&ex|99{e5!nxzP#v@{$@a%`?4TDU2@zF4Gen{ahcM{3H3Ne@oMqvd7psd#m~ zZ%Dd)x<=Bq75~^;sMdqFH+_|%)swm+t9`q(7>+3x{fQK$w)RD17;eT^nc==-c-}rp z_wh_xx652-rCP0W`rrhu=9+XL>K4*Fp|`-y+Ep|nImcsGHZ5>=)fDmY?Wua4ri1$_ z?q;m}7v!dF@3j8gt`mXs{+HgGT04081P+Rh%Q3#W6TVCTsG9hbN!NQeaOKKaP`aOC zOgF_`{!z8LztUzL-M1lW-#p0>5{v;6dO?@&dv$sDz{=WkhDuPII{&K||K;kqnKWQZ zxx~Z8#*Ej!B-u`n|HBTkfj~X3Yn$c;{vV)xg4&|RR%SgE)&vU4!oB`0M_{4Oe_1&K z+Wa5<9wG@gbo>DF0|E#^e2Dw_>q0hA_doO*OJ>IUwXQ7S1`%@LRZY+!v87{C8^lf4 zUvq!ED1o$a8|Set2?fNHCJDF5mZ&10=W@BQs>KdI3KTt7*>BH zUHI$|X-oSEd;b>#h-J6Iv&C2&Sn@@64NBM>Owoxt#=tn1bq)7<*;kS!^NnV*R@O)Z zk40dixbhkvE>51KY7VNzwxvue+o%l=Y}arr%b+j;#=;O`aY;^`8$<$noJYZ-)1r9v)**XNxEsem^MBn#Naz7UkPXKW@&0kH zERIX?a(!G`BfyA@4PR&LXlAhh6zh&%*vV!hxqKoQ_yksw^UT(tog@c+ROb*0><7$2 zDfL%@CLqAJxOVCo-jf${j>4WJo_JC=&zGXSbGBDrFDiec-7Gpj)%qO_{>gT;MDY3u zi&o4gr=DVmo^xZsI+HZ!{AZ!(TGn+G7vcEsutF|?oj-el`wS{irj%qeS9nNt=jT=`8?1DM}H#RHo4NhJJ;~~YjxR#@rQ(~;Gg@L%_L>4cGq?i*fhNM%3%;VZNqyCB=m$we}P#EB43#S7k8={sJ?LOfErW#u&d}X zFXr-348b}76*_BFv2yNU3SS0%nE)cFb+rqQll1nlYAcQtuxP+yktLS1X-AmT2+KIx+0U4VLq6e-JHl?pEOIjBmQ7Vc6Ou3#cLseNk6t{9UnGch-&ZMM zKYlJ+?#bq`ta$ctjx#X0z>_nR<6GTfC*Y<%CKNPuUl25Dq0N(x<{Uxo=0!UZwJu=+g3q!Yr0p&S*UXweg+=wsmJFe(ebBi6VL*^7)_H$r4OEL({M2Tr* zDLi*kB$eRl^C)W9=3TI=y0e&deU@-Fl-*xpSOnBkMv}xi4x7j#+Mp7#$YJj1PDy@- zu);)9VQgg~$89tjx@n@`w46Z#qVeC(W^zhV#KS+hjF=PjyR0*c+{GyV*lXx#cnTHD z36cTrz-Uzoxpmt;(hPWKzgJ6~&GMznXlYIUZ=hHLfEXy31flYzz zL|*p(AX=PY6+Jkfu&DGElfePizXY;i%A6Qen0uS1bzMC{09dol=fTX!^#pxKwOl^{Z03yi*r=C)gy za5f*gtXN1`Dgv?_c*IVN;TuBO>@CDfxKSs7u*-lcg`Wu^QtMP2z>npi5J9b29YJbUqfEhdOTZ2rj;hrPN)6^j5FKfox}EX-ZUiyktL;* zKIBAP9$1hF!Tjd`xEZ-uPB_#zHMFBPeEgT2cZj4T$WAYQnTDt5b5A>VeB6#2JW*BJ zE&t=apHTT#?Yf(o?#A$y<_~>iqg}bTh?~BUu)lNya8q8R)(QYf>L4j(*%`k}{f3MC zaDQ}fsS71F--~yS)|229bh*6nbU%S(!|NMcGy z(JS4mk%t7lZR{ekl$`Q~TAbVM0Ga5SO%~u<8PKY| z&#%5c$v$zv9!m}X?HR6N+ETUbpyrO}$$jmJK$;uQC`M){f!9frV&j-thNo&a+;NPb z)46@`|!5CT~^I>UbvoYL9D&Nm)gx?{5jV zC*+_q4I%WPSeTHPgDmU;pQ0X$td;!O6)8uQrnE!P(i6PzTFO%uxvZUBXa#1_=50BBk zYX?w`iv_6N{?|P3{}o%A?;8~4`R%u)bAN9H} zI`_uku0UsV7jeKwX{q4+M5D6G+0tI*dROuEFnOOK;&&jMd8j<{VtGV^Rh!7lU=j9x zPRf;*!jxaYqu&i$EMuwDw9gixOVJhImqrWdNnj+|Dgj0>I7Xk8-^T2ajlPq>STq$D z?m3Z%A2G__ImsBg>+JRA5`OYj^H$o~{D+SX`4pvI1h!K16rqAIFr=l>3Yz{o16bcL zRke{!-!4tG3|~O{osYzU{*YbSJJs$9&T6Ts$}0F-(xmOzOvFb>`x>&;#Jmq;(6W^; z?U(Yg20k(~%I_2ic7i$2I_39&TT}&XTn06X(;~eB5o723cd;HVRcek!y8Crk#~V?3 z*LzB)FO^5?)D#BE+AnJ5gk^L<|8K%22l3A8DkU$&W%)2fB|NRbS}ZtD*lH?{mO?A8 z81dUp&9jYOuzMt?b3`}Yg0NR-?Cq#v4XtNqj_Dw8QDf@(IT@t0@#p^18r}NyzCx;3 zhK55m72~ReRB{B3=ifVas=Gl={yd!_`_zw(UpRa{8(x+&W{d{%Et~CR7)kJSZ$mGI8ez3$N z1O3kSD|U;A5!vl3m0cox{pD-l_K0>>UmPhY{J4O27^}3(CJs&#x{FG}r22Sy)op*R zQut2m$Qh2WAiPS^O1w=HwBBE8CA_ARPda2Xo#5q+@5uM8xn}itsM&=MYTKPaN{1=Y z&X4|H7OLL4#9MQefM2l>`96nMRx8lhzei92yOJ+(fZ=#XyY&XT zRDMkpS6MZ`)BUEXG!AwA{No7=M;DiDwWEnySM@~aFs(|ac$#UDXR7?Cho&4`%UL(8!BNB zoPrQkAO~s*dep|fde?eH@_l@N{`B~lYi;BdIcmCT9 zXro_jDiJz{xM!MP6@^u@Fk>ls>E7xG!RYI0U30WjFBH)&J}=YKao!3^k;Z<=8*nP4 ziu~vcdz+q&&cRz{Cl$o%Dxi45nIgA^xE5fkqP7|>`X|36TY2E*O6_c_mSGF_`TS65FEk8m0{IkB?iu-Rh z09RXMmb}WCdH}YuRsFbXuL=29?d-mbW2=8^#Lc!B7o$#?`Qv5xlj7^^dJT?tE=eoZ zn9z(z6lEEmF2#jDAsWSH2Na{x7gynoa>@%QqehH){&oL|Dc~?sZJ>v3wb!((}pYV*RmBE07(Ycao_n) zC+%AvzAwr9#mZzV#Aa|Or0_9i##N({fTuPtWz;DfXpbpj^KSbT>HF=kCD;moh2Elr z6;C_G47*BN5`S+VK7QfpfzrEPBxco&cP+_E?(=!9y-mDh{BkcY_4hY~T1iFrrub^MVAU`@Zu#_Sr|3;C! zBRH8xoNU5rS4yiyOwgPr?tSW4Bq0MKn9#EuhB9urJ5tUZj|3+U)n?s+?Z~$?MHimc zQG!HjmwSQ#oH%`g@O2LF)}Z-oR*Hs5{FFsM|It@BDs4_2R?R7RyGQ+l~;$}Uv zB$C7lrRo^Pg#GCBz6TaqRU3s{gR}9mp5LEqNL?;v+W(Xc=KT^tZ^ll5t2FH zg2(kW+>a_UI-bsEI7vnX;BL+`2rq`1osbYMGA(L`eDg`I#xZP~nE4{I1vb9t#5@l4 z6$f)NCSI{k6-}$kX@=I;tsoCIS!6cR1@VVWfK({i##bV^YK)fh(>;m!LC=k%UedJg zsniJ%10+jG5qV@;Mb3XkuFlJ#^|oqA_qOivQe50o$4tK(Kzv_%UN6zz{e~xo5;s$C zOADdllXC;^Bsf%O9qFChshz2%tPp5tCo=R)WlF;J(uFgpH8WgdTV|za2#fjUzZ-l_ zJ8;N47t82tifI$0HKK>DB!W}UpY6@P@I{l+ej0UtVs23Ow?&zRos7PvPBCva;@7{c z9x_5&svm-#O38AodCRpJCCr|u5y+PF#<$13uTzCHE@ErZtB^v-F>Y%Dvt(hJucb!*N#w1#)875j_0nf&?>4>i0GY!}Zxr)ZRf zHh-QwKgyr-4bR(4bhg%uvZZ~lV3vpv#%bDyxMYoce|FQXSqfGjJok1e1yZCi%frWJ zY!)KlZAlq(Y`dS7+@qM8))}#y=a4fy4j@s^?QfOU#Wc+WXv#`1vU_rI)%uRUVtcqx zlKMe|$3L1D_YoqF(+i1?-bzGXb-FAnljgKU09IV&uUhQYN3fxAi+C={M5b*ct2urWbYfdWK+xNxbum zs)_k*yhBvoxO{~SW1#zKlk(d}f!#Aw9WNJ#<} z{Ix2|^ym5~aPX#=fQU6~l9DIX(<(eQee|uBh8p%-{H=PVVQiV<90|8Ba=XkYYnPA2 zSpaKoW1IgxMKV7*X`+X`o+SvmX?ky2_5W&slxdwC(N(@$6()<0a~Dq{$gb#>*%dB3 zn~lX>uDPlqK`+9Vjape%iFv*KwUX>lARV0vd|i?7r*D#N@0Ikk|L}x`yIYw$)=$)b zKI-xI;vrS7I13`QSWI{M;g=k3gH;hZ{8gjSE8hTqcx{7*?t$fOyqRtEDw*cnlZl}( zh<47}I~NQdMFz%GVm>GawbE{iBHnr0JE#f~tCbe^eHZl{yM-kdjI(6IBl#|S1i?8= zV$6Y|Z9hg2+d$YGw5Au@tXV_~NfLNqv+9CJw01=!j`%Fnw;7B!G0Zq1XmP~W73(0vkMYWQcCBGioLw> z7(h?Vm?C*7&B97&o1uov2TgfP(=AU@YNK4rB3CSJ`Ag)};*TSCF_&S38ssIXqmA*j zun16H`vWx@k&iO(sVXkAz!&!)-Aa|aaNy)ujR1itH3;PGkdYT47mYJvpV8Tf#jjBE z98xr9-zGDg3QI$qFB8icn*Cw8(F^=}GWz3a`EOw=rFMzlfXwMy5d(qtB${)r{3J1e-Fo9Zd zE5SoQT-suP&{~;9cj-j1^1{xejk9tXU9v26w-AfeOI%{EyCPRdiiTq(Q)xS{G~g$a zZ6+;Z9d{-@uigFLooa(NuZJZpH2;@4PgyC!mlF99b7Isb20I!fkvQgXKU3ayw)+#3 zDw06!N|=CaG)_PceQ|8vhcZSvG6t(}i!#5Rdrj0Rzl+>e^UZ3o?#QzqF2~+ome0$$ zNC~daJUW0M>IMF*S+nk*F0Z47f7OdfVf*>fv-x$p?e)(w4haDZM%5~}Z!KsXIxV*v zK+m;b&ab-EF=XanAS%3kok}@-ivPRjH|#d~?pKE_yq^5S6KdyfbHQih=LwB=q}^kl z#B6~Ul&BTAVeuH=GbecSn<=EncwWzf-785V+k9>Zwt2cS^S0mC=$r^yz8m#Q1^cY^ zRES`a_P308RtBS}r@6)ZoZ3yzT5CtX^F~7|B>p@NS9kF8^KYmf?}iy7x~4M2%*`~y zvdXmFFz@@YpN1tKu9`|FhLR9p!x$ve*UAG*_2j(E$@eW}ZI^*^B~Fk36& z7OA9-N_Z(tb(DNvi9O4L+qhCI0$XIYuGC{OXINH;J?A$zohJG(Hp(IsIb;q``2VMc z{^ya0Ei_WqJj=oj+!a~uRR!5A@Lcjw>^I5$0qVB@Wev&cm49tB6hMe!lT|-hwA8ux z>uN~ON6fz~$_|m@Ah^H=aTD8EFfNOF0`}$#TO9|;@+M*pB{Q%aG!1Z4AR5U_#NY<` zHf%lsC&POPkxvi1<|x6S0%gjT`Xu)!{l+pbhnyQ<&(|z)o$GhnktE7Vs1jt$`_s4? zWeyzVn$IXi147%+Q8HFV-r>Qi{a-O3K&~EahpsdZq@BtMAta`;7%Bm12)DnxL4uPgbo@=qFIKxtsG%^-1>^1})0X>h7`0)#wu2VLf1D*jw zDBBH>a%+9C2r)`=__eF9_rUxM#bZvriXxDO*qn2^|@W;^bZh+ITL8?Y_rj z#Z9eF<@mrg0mrZ&T*Wgb$-N2?j;urEyuvmhgO&VO2MT}#+)TqHwuR|`gdHIKDzWA$ zqPX^z5T_W))<1C==_sTHkuIZVUyu@!fuo9(lWs`{eI8U#JaQia_J{bWb?ZUyCrx-o zJ4!l+Cso>TW-|!*0NLah=Q7-$tQ0~vdk%LD_%O;k&dxz$z{Vgh#SQT%dR@<4jX54on(Pefg-g|-5})t zb@u+@o+R5;>pL?B{dr6$sL>BF#)WyvN_T`j zG+>V+9O(t$2%~ymwicto1svL}+4jIQK}<|wiA1abeHl#~spJVqFJa|Au9gNCM>Ons zyP9C?=bGgomSq2orbx5rKZHW9!P_#6xk|lG_9)Wew12RO*^mf9v)9ZWUpBGZ7aUjt zb!xO;|NNKBukVYDWe^up-VC|{yEYprm(h~!VW(daf^cGBj{mgEP*7#Y@sn9OF%Q#KCI3mCE`u%8rSeYhyOfBlqra zW4o9hD};FC5t)^aOepQrpzsTT9VjcIkSnT^{$u;mRb>K-3TN0IdaRL{{L>tgwTd!V zomeA9F24?&$FEI#GxxIwnl5Z#S<#D;?6x(8Bh z6f;ynOfAM9hTzp&HNlY5%pl_*qQWlwkpJPC%wUPLSkLCoFsK1-_(U0P@dbKmm1@ja zFOce9(2cFvq@_ugcy7a6S&7emM4isx1KW;A5Ek#uPII^;5DdnFf9r*ab zv(1Jsa{Z)>)iwa-LG-Tuck%g|BS-8g>Sn&1xlZ`NtX zsw1QAD!U)Zfo5nk8)$gYy*>y!AixpUkUcv@ZYR({@r~+Hh4l$5_VmhnEo6~aoWfxN zmMi~u7)=C&%eblqKyaO%d@$!Y3;dqPRzgMKF1w>)Igo;<8E zraQfEo&LWjwi@TT|Jc~YT>;?I|A$JA;*|LfK)13B0C4v-i0I`0i}mIpJ%rKC8pYrs zz?I>1!!%E`wqffLhd;CF)T3NSbagOW{oLgLRTV6TxQz_2Q{#CRY!m~}O@?geW!;Gs z*3Zo(;9%Tnx&secL5JcXatj&!;G~K76Nv5z zFJbKtif2|)HSvMV|0@VHo36*xW2OfY8;_N9NcunDn>4j1Z*hj zmn8img4eD7J78)N5uD`pw0FO)k}qMi913CJXY=z9qpBi*qLtuAkzK#6q;wb4?vi;| zR-W=MFD!Cjc?Z-7+1}GP#-mL^DUwDApQr5xXN<66lHAEgxYx#s=`>m3mWyOVqxN1Pj$XE%GaOdbi}8wUQQpPhRG5BY@m zUoDSlUOpM&Tc+DysCvrhnxAo`QEAXA9cJnyuu(OCDPH*FoBOu5tR{)Mm)1d^ZZ~$^ z`&OkvJlpDN9-e5Qr6#C;mnIfudlZ(Dp?r7EFjwUkN0SyPu1^-@JFAi`luVhQKnV&R zsq+h`yj3R7rtlgR#yI33w!X#OJ9;30pJf@@j+_XP-P@@o;n@0v>J`^Ry#gs6-PHvR zYeJo^;*FTf`1TxZw{*C^ z8aF`;ZT0+B5&u=-z?)Uw$f6c^T%-KM5p9cBsl`(kmrc6To0&ez??c802Iq0Xro_gZ zZ%)BAVpWhGCZ50M6tpAK;#9dwx^VI_t?;j3(nJvTP=;Gk%#o!ko^pDSOM}wT7932w*gX;pI7@`kmZBHjPBu^DE!g$UZME%_Z|w| zftGuyZ}NwFB6SqwBl3yh?xi~$TAh~oMUHqUC1JQLIjNu4{{-E_ zZ(p}&9Ex!kYMzqRLbeQn`%;_A`BQfXUu08`*~Egf6!)>b?m-7U>;3 z;1=JA1V8Th^rk(}@`@dKIIhP#mK6REPojs3_rjCZQ2h#dP;B7fTu9e-p9dgg>@H&z$do4n-tByyJG)(s^=aFslBD2#DdyE5VMP+p7lzD29Cx4;h z!Kj2AA|9kkQ)4Pc+5ei5V5RQU69xIHZN&*SLpJlioojwNX&EFfSZiv`g!j)+Pj(DA z|GKyS*1kKpiHjQ;Gsd}utuwZi3%Jx#G+pEJ_!cHn$wXvDmmvW|Px6NfG0ZiHKDo^1Pv zhd0k@a!BV<6PQuf@f*-g3}UF?Rc{6w;TReFrfh5ERMp3ypL#Wl6Kh7lNWa@T)RNfT!mO0A>PHDLXJoXdIUU3%+_RKOb3aJC z@;GmwgTaRf6Rk<;<&(4hRSV;pmR7YT8H~!aCy9%#{cBw$0aGWF+9PFMxg)$qj4QGL zuCR+W*3JCuGkb?--=bDGe1?q zW-`o6_(1gWH$}V^j_k%fNPRH`c`d?aQ5c2MLqHoxm|M5(5o-t>4sD$ zN~>PlI71~a7A^V@j}!CT?EW7_d2{9mloL(3Yf(i}&oS)^m3ikhdNf_?vCS;vLbJf& zhFT%L@QCIe<$}T0ibz@ll4sZvZM)aC!WfQ(yjQy>(Jpp>S&HFmmval=Ty(~W6QX9` zB3tOT3%M75xSZ(j>zL);MPbDmmORgKHg9buHvO#5B$?xA*Tk@`aR&rwHBQx878o8$JM3+b!MAtwKB6BO#&2A)Ypkn1P#!;)t2)*?J%*ct z34#?ZY&!74@8BEal}L^Bfu@QM{M8QEC1bmYalxARsBhbPmUCx+M-+&&dsHQu%PuQ+ z7;%tEpGbZWhKe%XrAp__2o3U4_-Sq46$oEk3H|h3fo7M|`P&|yF?GjiCgmR0|VFizmQPw9_5hIdPd518RPh=&;`rZ_nc=F@>0_g~qGKjj5$a z;xC1+6dAg^F1p4mcr(Yr2{ZmFT7X^*(x3DNuY_L{$8IIyZKSdxR~PJJ1iE$@M_6cu z{)Sx{@T5yebeg|6O%2x8821h<3?ceC%8ExQ+1o8Dy|v|!Q@BqKSSf}1+E=sVRI0=Q z!gJSKPA4Y|O&KqpOa%6+|jkNN-a044p1jed-lkeW{4+NAJ2p?xwdn9oQtdqE6=|#p)l+0C^Sss))cDPDp z3-pn|OSX!C*IJADxVsXbc$!%w<0Qjrw%?+6fD!-m3lY*tjPs??mhVFFpV%s+)_pHq z$>TIbkjvs-i(9l!m;~l$Ld$eW2bniv{0{w?i;SsAk}|LPWTixs*xj@!6VoByVdFfB z*k*F6NWSBjNj+`Fgfii;IbSCA`&peWU`gy=KQR$(Gi$)eNz=^ilPMceLha}**mipL z*-?_+s^jetBPjaOW`TJj=U(4v-6u8IJ(wqV#1-;+@AZ|`L`R+8Z+`bJ(jt?HI@Dv4 z88}(n(TTXj&$es~ejIsdRXZyVKM~8aW_i>0ywzGS>4QA6g?bj+k%<^3NPIFZl8mL* zF@L&7{vuMM`~t8K{u$A~!uDv;k*8}eBe_2A_47ZH_I54hY*AQCWTt)p_!)oN0x$<* zqWP`v&utpV4c@uoNxX`##4qDHQ*KZ*hng}iH|=fu{C#w5_x)4l)!}E;$fb?_-Nmop zbjZ;0s+JgJW;v^R8TObldlVUO8kf@mfh5izQs^&>uP{&Rb?bxH*uz zVt6n$^w3`Q9h-`u%?#O`rRE(=!MqMXyG0>jltG%$be-KhBr6A$=2N3X!XMHor(I_L z+9q~WydF1bc{6XaCw-`AUoG){Uchs(r_$t4tZW$5-HWfCx$cp9rJHsn*!C9%F)Wz= z77WLi*xNOdT5&79@NhC4##=l)__KBd^AgX1qtnq{BRYQ?X1EMG&Av8EvObI){6IS& zkguKC8jO1AtyQ%HE3jbqK>K=c>O<;ZKt>QZF&Kpkq!rs>CS_{UBpmwBHFQY zE{)R`US7czLg~c#8g$K!3XN^^U{uAMArtv-Yx=O=e!FU6=-%U^85Tw{RlUeymK;q? zWxSqJ+xdYc%K+PfLks6x4qiRsTH8)OM(H*03!8w=#b%?2q!UCuli%dp8!~<@ANeM4 zQ!nTBD0AY;AxfFdxN@D#cm}oXW=hC5yV#aXDDzw9HLKn9@|hjoNmUDsogPWT?M1Q) zQi1!k&wU&#ey|@^l)LNb^2W-Yq8A<=$Gh)pT(e7UJL9vda6ld(2Ngr)s)^Zahrk;& zZF&l8fi)2eSJu&bu;O>I0?V+$zE6hpWt0JVlqz`9&)Li)*yy7YY74sVcCdI(H@Yv6B9{YFoM~lb?#3 znVna=v~pv(p{qpts*^X|*0{)>B3~%^_7p`%;>~$itO(Zs_ML-sRCxa{^Qc2Wx=;#s z6DTF`pZ!cr^(x4CcS9w%9{XAD>U+~5TG_qSo-#Sy5M+MLs;l$=NV>jmJj(hL*q(K; zE;XCuBno+ZTzSTSzj0k=0C{j+o@ZT*!}Z1lzE6ZqY@TJX=@ypqNY8?!Az)<|fj(u= z>VJZgOtUur>!!+LhXC8*)3f}>$5qZ;%OL5len*|>xwY;C%1Rz&Ve@RmHa+mcWTA~3 z5c-us-%kf`#K@0~e|Qq`K-#5bZCYJa)m;u`uwh3w8Tl}ojkf`!#}V0qof(_;1E}xJ zL@*nSWTgjQy!cjju8gs)&l2%P3z>pWL*17Xr<9U>PK+CbsJ`jZ?o`$}`%vH+y8E#L3rA$5Hjbi(=|FKE!$T=130mci{zI4o<#mI}ym`CO9IkmaN=gHOui<5C<)Qr}Hja0G0mFpW})Udf~&f9VMy=I;_7f2Ip##GUdz^4s)+ZnKF7+DlLWYKdCH07;$dgfv0!@kXuGgC zfWT4%aa6Pz8qj1%0ng@xTLKZ_8-%#IkvBOioprcBlg*a%y#9+pjs5P9rw!Z$fi4>m zPhsB+j@9G+?c5wW_Wg!A!uc}H6plHa{WDZAi^5vcYdQ={4;TP!{To|>q`|)0P0=QP zaC#D5?aBfIOb^YOGHhNEq9m=CQ7>>q{F_+J&Bj}as7`grEO{NA1QkV3Hg?7CKceC7 zymUC@p-D&)^Nb!4(?~xJKLe*29upWFfKs#PLzFh>?y;W}6bTlC2(fton%05r64_4A-IgGtseQ*nG(i?Sr}!g8?N%phE`w3m9kAc24n$;A-`L{M1_w8Itho z9r}!H8wvuqneB)wFXX{htJaZ`0>;E7*G`seeClCOwYHCz?Eubs8azW6{qb@O4RRut zWO{F4EzmIL>+H^Cm+V|XwhTmePnHHAfB-#~8D-`VxGgLfL6~!~tJ`+j|1v64D1YFltQP*R!%{@4RS)(fc!r!f-c*xGJ&=Ko-N}l5g^t4*vNIN&Yb)$=71{gG)v5TVJ~M+fa5C_szT%}!1{qH zDQpvI?lQ)SC}f!?Yv8~I6gne->jSAF3g}u7Jyz)4=auTOs)%b8kHM?35$0aUl6#Ia zqO`xt=o{}#H=ts9Urtu_%L{QkBv?!#`_=4|B)dmM>OAmXrs z_x~x;o9@2j4Dkq7w+n~pGX&!8A-lGwk>Tbh0iNP-FvaG-AF68ZMJKR0YY`ltaSu&* zgHT@9qPU6Wb;{p#m^F3(UGS2ht9IiS{{~ra$(AOh5>M?yu?u(d53$!=d2iJJWWN5N zPX?yqEJBvkJOsoUcWsbNe$8Rx*H;EsASpEF~}cBg?u@+0?I3qO;N~ZCe&|n0 zN7R(tkPWD?E-h@~N$pjEPE^S_WnxKV^9I*-1Y!eMB? zf(^0G7^)cve36A!-e{K|S|{tfAoX5#be1lqB^gS^A!B*xM*horMsvHGu5UBwv(a=G ziMl+tbTpa$o!ogkaefzi#N^Kz>mf|?o1=}e#$Yq`%3RijiP1h0KPstzRT~_AE>`Om z0g6G*Eegy-J?(wBMwHBsKirmrREn%PKU}H$w#gJ#QrvGB(TeIhGT+3MpRm1~)H7_N znh>;@^@r3wogs-Qtxlx}gW`_3+E|WM(#(bzj^Y@hSm6dR_>?)RoEN9m% zELeh%m%|nm37RUVd(aEp8)^2MrCrxIDknz#;mxGNs!=_v>6lyatt~%oqjYd}`2a4; z52r<*9>zeg_Wci+%j#Qe+WRyPQQAvapgml_r}$^OUfO_j?3*X~&^wfZZO$K14|qMb z=ZAS$LF{o{eO`seXh`_nlt+1l4DDuQSsKPCeveXS&t0TS8C@T*^1k6WQ$m~wy~yn; zQ$9MiSCc|qM#O!~>a7kEEZS~$-uUfyV z(Q2V7qdh&qK&PYR?6UD&<9Jl1;Vknp)?W0BpM$d#b91M^#E7QH&TIY6WTsCqW^V&y zfys}5?Oaycy{GZe%~5;*S!l@jYp~ga_SZ`{Z4vq+=eW3fy&C7!$Y}RtIJ?&oT3r8Fi=6@^gxV?AtybPHDENp*d-aTMnS!A~EaUAy*f>a1W++mg1Q zwvQ@7p+btl0CsbEx@EjwJ>zLdgRe@xwa48V_2-2IYB0&fW2cE(JPd#Q4l$eMcY0u9n1)rO=={ORV%bew*ED*=0BA<~k3mK~eAN2F!^lnh0-L#slIS zL}O0I+mHf?3)$WARq}-1u)l3dRy|?#t?sV)zV0_?A_~^NorQ5aY^|~&L8qZLVVzdM)s*{sCY;nt@5@$cVywraVJ z&0HDlrPf-y=IkE)ynwlDjtQhKGNfheO|HwX#rCeX5%y{Azb*9XQbkSsT9ve@y?K=N9rr5Quf5DO z?)8+>zubF;t`42)kU60YjQn|DoyMi?vc6{(S4jCKapI*&%|ZVvMA+TbyT{GR@o3Bp z_B67OIWN)}a?`8@eS_i`t0x42O-`xgL@=M^Wx=>wiLUCSN%k+oq3HE}kdAEk*B;hx z)eihKUA&mM_~jOFb7{3&+L;9nx06>!BUjcSNE5qxYAr_-u}#%_iu-$EW-D_uZqgKY z@qXGhcsM1zTg?W=V!WTFF19iY?Ob~E-Rl)%D@8y2!*iqLl8=9B*`Zk~tf1y4q!O`* zMl_wlelg;JTIF9|Hb#$iPSdVx2jk`jHqr1w0JBtlCdd9&tk_KgMXyus??`?kaXW6* zSiVN%(&IXsQXS>k@xbsG8O~m1CnxoOJjWGT=rAkPh7%UD&bSDVOm)mEY73L*cA?C- zjR&KGLZ_0f5*wPrfi(GR5jSSwbnw%{*MOJSavF@b2DM$W-WwU&L*qOC;km2LaH0ec ze=NZ-{Xq+$Ossvue0|RRsqL?W%vU4YvyOii)fL{s4qj$b+zDe=0jcgS{jC#|mL~v! zD=qPJ^leA$RvlTan$J(~(683?FF&2M+Ip9KWufrs_wxe^djgXDSbgwSFA=RsOC6Mc zyQ+gKVykDaJ&rWCh=KjTArW0zdZ#3RVX{i3J$12r;_$S>$H5~zM45{FTMc40{Af)w z)v67fa38j~tr$~^{89#{PAj~+dpHEOdMl@AA;Xc}H~9rqD&;;M!u$KNCI{mVW$7Gq zs~KY@Xzz=xMa$vnMN-~sxbp^)SU&Q&tn1^&L{XzO$qFnt~|h;u^FKIbr0p)W)5i| zj_AZV!XZa-nSKf&_P*5ahFRHVHkyrj^zeXInx z6lFwHL-H*HAkI0@Lg)P04RUpP7rrLG?#p*)pOoDirJAU*bl*V!5?fnLZZYmw!p%Bn zc@d*;*J;-2Kfw9k3WGV8in+0;;45k98ndxY2$sf-fA;yhL6XML1*v;5BYxy zx6{lLbW1G;&)kD3L}66k23hZW0UFod_?eD&D~h=zS-)2d8s-(}ZTG_$qJ?#-e0Ih1 zLWWLm8vOmz7}V4?VwKsJ)@qf^)zW&G<)#L_RKUKB4d@kSpn%Q2ni+t{w!h#q0L3AwL$uSlB zw31Kc?N7HvXZSmwV z2`$WWSH5R15nhZIG1biaQ7UlvoydZf8UUH@R*dS1+E-%QXMRK=_mm~+QZa?Y$BxgX z3P@&(F?ZjWq)VQDrL}ut^o(Gzr;hc=*Yx^_bFnCdE^Pt$cSqluvRDsI@YH~QJ;6cw z8|AC1Eyy1A{`gnk${D2JP+=xw*qgi4R9+Nh=^hd`YP2C8YrQB}jJrFViY%d5C=((> z<6ui6M@lDtC_P{ERNFT!RsYBHma#XEN6Cv-88ya_9fm`j?Pke1^Rg2~_A-BYDP16s zFDgzHlpd2WKYoNJ5|und`^uEs+$1V!?6K&d%am_%>U&8cPCj|Aj2rwQjW+g1O)BBP z$zv%h2dBuWGjD@Yf0epfMX}iFa7`h_$Ml~@T4uMP_D=UGln!~XC8*zNCI<00PAC2L zKsu+zAy|nyZ9CUvlj&2_@xJZHy;-jyhR)x2wD4HpMIqBEW2(O8*sVoWkn1U{GZ2q> zBPd0Gf09@6Nf|Mhx7N+Xpns1Z)nHklcF?x={(DHN2;9qVVY1OgGjXqiEn>z z)Xw~LTY3Vt&^A>}`To{@{S(22KI==hHhpz~L{Z#B?z~Z#pg$y{hqW^qg{~>3N@8Tt zzoa|K%5*R+lWd0Lr*`>7=kfM>`YGY0@BKf<-aHV>y?-Cq-YKO-D&Z8BLJlPH;c4b!&ehbD&tWlA?5FN2op!S>&&9X${Li_ajbFC zM6Mg=u&UDZB3hMAc=<6!-GHp^x`j8bR0edsLh%J?f~HHM5(8&j<=J8WGr=EmnW}I~ zHh@#I31cf^hTD>WRJ^TlSI7pgjIWBTdl%h;+ou6 z9KqV!cPvd+hZN$fe7AwljH@4s@cE@dIn?2E5tRT=B~U{({xf^MI_u&QD&)Uqeo)bA z{`)r~5ezbNeHOI|g(;dA_taW)nic}c;*fT%)BED1(>YgV2Rg}wvms%63Fg}ZbIjs) zr3-fIN|^b^rp3;{O9wR*c0X$3&cxZ$j*nYx8HgLyh5b7ADUjY3UY93Ngj8Z~UqIJo zzq!9%^_ZU1{Dx#G)~v*{ck6gGvrq$l&?qm%h(+T;1`l+e2Y=w;A!jG=en;?`8^_WY zER+bmpphla{8SlDnWvlGlVOz4oj;j(kI*{JY0(70P_w|j_uZUo>b}BhG4ZSf*}PMv zmkPL=u^VASSJCo857WaHddB?gysk8MPbs(`wRf!NSiQr&3;#kNjaFgNTWoy`&!z6) zcS>R3v$ktbW5t+G@-sZi^zE_IMx!Rq#pI7Ib1%)_j)}#U;i%U$id|(v@NC5`j(Dz6 zS&o@0xOwvs`x{(N%D~y7w5JU zMX1Oud8|@US&t3IkUt*EsJQX!H8)*UfybL9Ea<6^N~^&^K^tQ&f>)jS5_32UH9VgY zQP-)0YWv_l^~em6;QB=3S8#|kw>ujMklF>^K3RS%u0L zJ<%Jpbqd;Q^9$m&TOz`}a2`dY$N}G&a}I@*dNQ#%R#>~s!EjVN)uH-`RlGA(2aTEO zJ}pkgu)bnW;3cC|lddj3!Laa_AWUOuGlIqg0TF zzWCOA?;It;>#i!nqU4v5R5|t0V{$VcjQ3v`{+4;s->=X7vT^Zd>GTL@rZec=F3}Qc zn@`?0;9pez&dBU0ozgz@!IElm;DGj}?;`(T(pmQCmZ`FWi7qT!Ip(Y%E2XF{{lUST zmC-}ye)^n~6&59G84fi`q8C+k9FAaQ9j2~TpU2;>m#)_SxHw_seSywOz6wX;wn(GR zQ>17IC9cCQUvQ<2E5V8W`}(H(N{qMjJjtUbTS1-W2CYAUGGJ?Rg3DSKH`%uNOhwsK znOnA0SGm7bd$RD$lP8*o15C~hwo+JKsNudx-TE6&$8aAEXktp%&E=M)>>nl@)2lz0 zQJ{Pa#LQDXX^vJkZ8LuN_Juz`!yrQsVQEym%h_7f*pxa)3C)}M8r84_|2DnDtEiOX z>?sEqUXxsLN>sl^=zd!06A2qI`He)=R{qx#!$van*xjiyVUdM!8QI zsWGKVw#HdG!l_Ry~YyM;NDNtAlttXcmiT`m}6bfnhMfB;*aOnS# zd%&v+(YQW<^-P7hurK(N$^eljAo~=MR|t`(5`;HCmGfFNNgmNC?W<@)SLa9_vNA<0Da#}m}%6die5o|3tv`LkXO3^*N zZU8oSSPq2S2zV527G|ymMhd2Y+Cv z0)5enqkKGBpo4*sL0D|Wl!(liwFq4z3j-JENkkE#D|)$}*_q5+6JagH^PjH*C=rz} zgusQw&vMh>6acgPgg#>eat<07VOS&(TQGq}g|B-OM33YG<1Y{XFfUXyWH~mLUTEx#d6eIGo$024>AoNib$ug9GkLmn+ zXAEbObUwa)deB~)&4O}?MnwSwa!FqnJ->CFI}#xh%x@D#kz{ehvFs+wr3c5b)CR(A zb}In9V7i7{qzKGr+XYgXlW-U=U;%Y45(~#@$9Y79Doh14`9EaWX>n8CBh3lCKvgai zmd(}tr|1TOSde>#=BJR#A2db3bu z3(r;})q&-Rty{LC{8UN9G~MD!YSVs5NO=|?QB?>*0b(R1We{OEul=de&LY0W)(IVg z1ulHc#;n;$(4$aYa%7R%gJfs^Xg`F-eO{n1ZYuBtl@QmHtdEsSzWsE8B+IAQqiBIw zI6Us5i_VN2^fJfP^5q5_3%CIt;0@9tx-m)(Ik0@#P93q|H~Y3R2fh)Pl)Jnm9+U+)8Mm`xSQkp8q?LIO%|*ie6sv z--%(lNCF&?MFJm5L;wX^LB4jko#b~UKR$9OPnL8`M%PMLniw72k10rc;CmI*Hm1VX^0c5?}7* z&wm<@QeaPRe~uZz6pU~O-SnLqI>R0uH#Ig+sbQp<$2<;dGd>Z44JHH4mWR6;i%4Tb z%rS@(^N)>hbVy7&h<3m(51^uMs<$~X&gg-lRz#);qEFx*blELLde^^_`})^jtn86) zUZ^{CiiQa!Qj=nKHhTpc6RH@%;UaS^LfJ^~r)w7~;ixy0zYYYo<4FhyY@jLl@mvn} z$~aTFgex$^av+5jCUpLsBw{YKoJHn6;6oAm;SIgan>&qgXVCHZZH5qU{9%^P{|=84 z=43e06fME41XR&EnB0V)CgLBDNH{-Ze5K;Lqf60uFfDsn9sKSnqNh3YB3hQfUJ~{J zb@U)B1VovaR~Q~0HYq<*$So%AlZHh3REK;K{iCv1?L9fPc8O%ZRcpK}=W}KL(~$yK zq7#NSgP|sg+!FY)3RCyYrg({=N2zMy?B=p!5G;;UAl&W0DE(yS>jpWa6s}9%hsqos zlh6{`8!;xViM}>I%Lip9$(q~Ap*Jv#Yf<@{{98?P<&eMBj|dnp0c9@~X$#CSMG&Q) z<@2BRM+50OA~(AW;_n3FR{omIpPbX5ZvY$SQgKjB0@7h zb(5AJk2x~4pIn#m{CV0BNt|E^DIyQtWD3HKA9DVG(nvw~-4u>2L}3&(tj%KF`736D zAVf6$5pXP209qnzuYmc>MdZp*oFvfvZLLKmGTYH3W>;^&9Sf=NTf8dZis5#ksz}JD z`btOu4alEf;pb|``-22I-W^ zL|C38sJHkG1}6r~8H=FhmaAsqLG%joh5u3D3z)(GQ#FNjY6!TspZgCDh|qyR^}!q@ zU<`lK{*2)(>2sH}r9XLq2(gHX>fqq=&xl^pXLefh)+D9=^^n7#6JS<>Y6q04AP=}z z=sFzOo-na$Gy+d=j{qPDB6q0*|S+rh-~1U4cYjBU}6Qx+Rl-){#Zw3Vs4iU0bc_ z7yDA>_eEhjLHeViJM(&dT)kBjLx*Z$bw8_P$vJ6m;GkCDe>_*_r#k6Wf7;z44-%eZ ze?@Rr`g1XLKQIJ2so+I2dV z$=*W@nkSyGdHt=Mp_sUbKJDboN+xTJC^q0c zcD2`V4kWoS9Nd7mah|daNENG?Np$9T*J{G?A1Z&(hoWuGtIio#v{S!EoLBL3?-vau z4lQ-Mhq!IBF^L&aS!l%hY2NaLh`rrLC!CA!7s|(dB418SZ%3apWw4ekC27Z6HOyn^7Z;l3X!E4t!TpEF`))k_ z%sl=y!&h4Nc*dD`g$a+uj7xRWv(G5l-@1#&TW4qVud85hv7=*rl0cTmX>upxoM}F! z^-GIE?A;Kq^s}nA*ZYih$<4n$^H?Y-{`b!N?H2f7n!LyAfngqLR)04td2)NNw{a6xxS#YsV?JaMBvV$olvSY9_v>#xoRgM2x$cfbk<}@M-$mXx z-crI2Qg@Phl6vNmhWPmYZ`E*%R2__q#uzNv z9k(%&N;>R4Ed}Mq_P2VY$KQ`%vr&>wc2!B4eRQ!EpTaBG44Irog_B=*4q)H2czY&f z_gEbbxge9^XBigI!VJ?6HSe29N_Lf^HK^1WLSaP{)L%R72&+of>3$lPQ|UW6wZ z{Nz?}s?^4ZHN$~vbJ}eqHDW{GM)2VbQPiZP?ng2$=XI)OA|vEjS69~ITWO46pJ*0~ zcKKA?J4Ori{=1rIcU#U%_a%9&fpqQ=yTx>}_m;m|{nqgAkAPgC9Y5;UAM){#iRgr% zLOhU12)!3`m`YzJ@_!B_Yor!hUZvk{ z)q@u^;VE^@9v&^_FR;ER z`6S&etKMIV=k>-9`PGcsP_Q#>-#mNKwz=f`H?vtUZiv3_$K<>zHv=+nMa%q4uK-Q! zS-bhN@B+6#$4c)yoiyPD!>I^t?_?iI?yTCRa_6H`(0bCsShQAIpj*!e-hw-pwz;Zc za1y_3w{=u6oTtU&blNbxEgkdwFI1otL=<_(q}QSh?&K)t=k0aJ3w|c=4H-KiO?-Ek zDBAh(u!@at5wC}U@Bc1BDLrXO^LWHmHyq*`ufJ(R*ci+v%A(@VjRf`eMVWjkHBW_# zsj&LR4tv?zdr8_}Q-wq08*qV~!Z_lt`WU^dX*AUN(2F3)slwc6?9#wE*j z!#ndcBY(jsgSpl2&=Q#dx8ft!WQncUH#(@JSU@CZ9m?!?{mLQ)`Q#oR(TzVDSg|YbdNuEaql9d~LUvK228mERb~Y0>7}-p`rOVPIWzMYVWTMCA;+vWk1!;i8}H0|k7KmUwf$u=^gA`HwnDxP#Tx79m0HiN zFQ{(GxyZ&$6dYMHCw%x)7IA!jV-(LhQM$UAV9nIfL zDWFhdwv4b@wT2;Clb9PZT}n?!7{qRaSV#Y%h}?{t&|B$MLHGCS@JhX}#0+;eO}E+X z!BAB-v}syZ&msm_wFRSwfT3V^eobQV>?jvo;ugnoA1p?hy&8UFlRFaFx!0^UAK0&r zz5GZr)no*dEQw!eGQ3tveq2@g3}dY|Y=jPUQTut@LtcwZZANmFxi_Ac*J=}BrE*MV ztbs^SBzd1$NL`8>O(A6wl-=ErEu2Tsr}j{mI`CU3mhy3lLB8Q3SFaa;eH^9`rJ{Ew zdXPHZ<+`$v`QCwhr!aTWZbxvLT~7pirIU}e938Yh)KH>X9hmQK zbb(GsYkp*52xcDhzMsoCX9jvnB?R~fb)Lj-J(oVfO$uowl1(d0_m@~klKQvx?R=H_ zxgsOUP*+-CY-!vvg?Jx~AdXHu{)#D4Dey03_F$IgVoJQ47X0I^O=iZPS3WMeR^9SU zkK33X8I)IeWct->45k@k;lPl>WTEOcD^engqAX3!cR8m_w+_k7FT>4v@=E0ZuQ=tv z$T>_caMW?S+Y|{_l@KFmv}ymlu`xM~u{f~;PDkke4hyNZOmEqBP4U~>or6N*!_L%%TN{)H) zx*lidcP3L5!{qwG9buzIH4&fC`2Kbf>kTii-L=j=QuXNC-=<#Y899q~JDhsuz?{L4 zhaXT4$*lEKP5v5~+C=Y2b?mR*t?yCXL?vi(m4c?tT9hBZ^%!M^FrKv)-1`(V5XNPc zy%bCQ<|FInl@@7tg}d)5Z@ighjMhfU6ao1&e3wJV_)Th z$1)P$Ep21Py7j)gqA$mJT^2k}D?GCzT(VtnUGi==mT52?bzyyeLH3+rF4{X(MH}MCsD$lzX`NX5lc6Hd+c$~XRiMAEW!pyEX!_U`MO_QC;C1<2mWhOp zf1fu6T55jEyZJ7~;by$r{QhDn@rU~Vt~!OJeXT|R&{Mu*poE{l`G2DCb;G^dWn?SF zgQD8fT2nW9Ivi@d74%On$T@HKSGeg)B`!Q;0}(a_`=ML7-#zjjUAol*O^&sMl)wCi zBQJ#W>u*cS0?)#$-K#$o2gZA!B>8vAe? zMTij9oBYMr!DuvNBWN~}b)lBFsWj4XbfV&((afQRbjkU9Iu><)yoG5C<;y!V8Qa(M z?t>x|uWhXUq^zw$H3lC8>Pk<$^pPeXt~GM|`)mtTvRu#E zknp<)8;gDST(KLLzcQYNo3qsoIsNWfA4Tj6 ztHdWyFZ2eBA9&16j0MQ+zG1etb`@CeZC9C?#kjm3v&|CA%}i>Xx>6ZC-Bo^(yVS`J zKn**L^=8?y^J{HqdySKB#TcYsY3R8yjX5M*RjM;Gm!iPp-Iycc*>Cb4+qxm!@fD8mLO1`D#H3Vw1-%&3hOar^!#lKfy&&ec z<*$59>(6WwQ5vdky-aAE^KvJVgQHDR_^C53i)KBp$ByYA=;<4eUF`IV*nB%R*4Z5` z>Zhu761M9qr!X5j>11|Rc1bHL2r^7(N%p*E6DpQH25v>B5Bd#}#2#zf8D)&u-B8I* znLstPoak@!8JC-?pD<7{aZEDbv+47>>|?o1u*J8-Jm8?vH*V0gx}i)>)AYrnHTamd zWc9`#V0zgnW-OlJ9FWXlS{o)OQy1rp>c&u4IH zB3}u5P3|u#Jxg=7AzJL?!K>fU{3@!~&yo@&luTHnZbuGpN4?;wHd`KHpW>&1whq#>BH`Dc$p*h4zbAoY^DK{zYE9J>_sD(_@N-l2l3Z$Gc6(K5C zM<>&@E0ney(9X1oLq4Zt@@hLq4C zN6L!kTP`B*K(}UrB=YWxZz(}Ixb7Y z&-w7ZYabuz`a?zl(f}+4g0+1TE9`~h3qIi|Ftmh}9KP=*;9*A)*&AY4n3E;nq>=h_ zls=E4b8t=?vCiX*;7)G^6`Yx_0D1#xP62Q}(EpeV2$Hn;YiYdwsM%ej7iy&Sm|q7L zuK;Va0ReRR;YK1qZe2tG@V1V_h_y`yy|3xlP&Q(a32>&QJ$k@oY+i0asCsj1P=+D| z-kQ@tEpsCRjT+xRi%)6_@Hl}xm=8QG%^(;C@~ls*a^Uh1zO7Y{gJ%R`{Dg!WT$7M* zOA|`a5E>F31p!=qxCVJ_fL~Kf!O2l*WryYa&?!P$`wvo3fJweOt<8rwqcZ}u3+G#~ z%nJfdp$NZRewkbyYJMD8NM|G19>Om`e-oHGO019pPF6fnuAJ`AJDw3Jj-IEDbK4^v z_-0*zq6-uir`P-+7tBv9`HvpaDuU~#2(WPo9|>=ZSsEuVUKhqF_DC^pcuQ&FApPPi z8mbwfP!g!*a=r-!2}0vabD$r9D=P+B(*8L>$Ime`;g~$%fMpuHR@;E`sy#+biEB8* z9ZC|3&SYODPV>N(blLq)U@FH);AH_7r$F$ZBEJv1h{TJj&gf9zrRVMimTiJSgI))^ zvpG}n@XUD*#@``Q2`HVZ+HIk*uXH{N^tAMhP7fB*I|>*MzOF(re0pX(jT+aIaOBn%QSV4#t8*18^^M^kAxD$dOCPU_s?05HM( zP1yk6t6;{lu-r%Jj0DC8M@@AVt*jEz?0`H1-1$m zr6O|Bd~(ch*+TgezrO&tE}Rzx0-np_w<6hq>4bn<2{3-JHm!uIL(mxnNn3J*q;a2x zG{NH)#6XpV!1fdD8-<-_Rga+K!goFm#UDJTK&LHPaO9v2wHwGE%`L2B!Uc9B>~W|E|5w8@$uIckR04j|EH=${s)_mLRdEj1EL z{66o8xah5rIVJ&gMy}lb6B!T0lolC_m%;b44|3L)!FGoMZcEZ1>xqG5XjU+>3mJ=v z_)+o_^pABoRbC>m#B4f&4NG0f<&_@TRw!@CHIU9duk0gnc|PP9t9>!OW(O9XI}ID?`VEOJ>`-|0-96 z8qeY6{tb?wXL8f~Y@)pA>SRYGo4immgc!O3+mL;O^A&&w5ah$(R; z3?bmZmX)jgo+ntEM7C<1au;qwSO(v0_-5dB-ac3rE`n?g@o8LEvPK8|ag|>Op{Ki> zCNzW;y2+kC%)(R>nIv7|vXFsH8z8wlIS2f-K#C+x$^)JyY8Iw@hqga{@I>H4%%7EG z_&rJR`ip$GME=q)oYcbuz|NiB(w5F(Ft4DZMD&4u5tvl!3*=TnxeG*y5X1QwtmrX( z(jRCP!JC6-JWXzpeKi!Jj$!==a*j`g9Z21yD*=uMY5yyJE9UEVK_&;*lmf5keYX%J z%4NEJ6Y>!&|Ib%_%o$z2L+tPvmoEyLyR-;VmI6X=#RwiFs9{$KrxU@#h>)!U zMXAX0D}Z=Kno___BJC-hu~!XHGqN}?lWPA@KNVWf z9q6}kSlsSLS}s7^R6Y!46he*vr`^a-BD)&ctF(5mWgG<@i_v2XJjxc{)Ziau;S$|{ z*E3gXa@I*yBg>AWKFofjFB`s#h}0q)K17wdDdbtWqTXLqTpu#?L#@Qwzja|Oc`trC zeMvi)Qvd<5%XzG#-G4U{Z`Q2h0$ZLb;KD8bmGEKro9dF6jL1FhUmd7WnF@xtMHgv? zW6nCA{uV|k>k_ucoHUjRO#A%HD19cXn^ZhY($7#b#{L^Xu2p?tlHIf~h4c?vm1qCN z5fyb?XLon^`OdV~U`r)sI04Gy&y8Ae;IKTGIT)AW9xv`oVC4{pQF84-6;|8P%NKx;PV}?>4*7Zp4}qj*oOL6HEo&zI2G`R1Giu=+@^fqwTuH8$o&REoYw8i5szt-Lyd zS-i5Kc3|=w#V5+`t#kC!&yDia>#Uc=-u8}0PIwL4Uf8Y)hrN>+16KFBnBI|d^e?`E zDfm@M{dMV!CMYCRaOAF1{Hu|MzCSU~jo?vqlx42Xa zw9U$vx!kml82zu&yi<8{oYH~r+2?bi1yRX_oK1xdUbuJVEnD?+qHMn2UdJr!8pzC0 zb=>z(Gg0(+R;qJh=}C(UJOR%z`xFM-&AsK#E{FAUKe+Ak_}r!QxxPt>mxUn|&r)p{ z(!KYTV=s4=?o=obmYot+poT|jj@$OX-9jzuuP@LX0&!AD>jC;+`oJBqVSKG# zP`=PHQK0ct>Begujcl789`fwQurbbrZ?cED32Tk9^{^Oq_e_C-n0UaGNj$N~Wv3Gm zf+YHIiBE#*nAUX#%m`TA(J#_ZJz+~rU2igv(7H9GHqEXHh486sOQ^(kmmx3pz*Fo{D z^iVX`B54vbE_ZS=v5Q5|IxiL$j6^nGi^Q}WJ+Gjq8`-3y86 zh3y?T<6x&ukdS6yjPqSeHfA768`pAF=8en6n)tfOx~>s3?=i=jEH!Lh8le;K>V++yVW4^^mdr_L z3hlH0b)se>#~ye8#za?fle?B1%H>qBTH;Z)%+HLU`%w>_cjhhKZ}zGkn{dgW zmW`RwU^^S5s?6t}mJpSiqOP%L3ecdHUHDspepp4DxOd1T5qR6B(9)o3+K$SG{tz|O z8nieo)RyPjvfu?4{gax>VV|F>8*=|GPw)9K9#&PP(toVG+96efg`t2Kp7HqU6P63| ztIOpWwRJ$YSC_xayAU%oHcWDSX1IFLzBz=7amx)!tHk3xs;c&V7wNf#vA&xxvpv>% zy;^-d9lBq<)x=_l)ZovhZS$Av{T#Rt419X?tM6)*NJS^g1w3GZ3i*A<7o2IgLZ|Xq zeHE}piVy$j4k@#JB%c~%hVpKML6gK15b;U$Hg_K*P%^~imj4&-iuhdQZRB2Kh$~PfvH@T>si1MJy+x@^$?&-$k~m9#V2F?uI{E zXD04a!Yb`FvBokjZnON4!(nKY6~^kS&AIrlgQMa)EdB6H)#W`8(CQLbM?;DS@jixH zC9L4Qm~67&!RxwzIg}g+A%l)TP{Ajyb1;7)(hiV1`UsCV z4HsMJx;-kOJL%DeOW#H4pFo>p16e@k1R@?doU;#K*K_-CrilLT+8*vvzK*I2)D2K= zk^a$1pRmu0xuZOaj`TtKcrCM2eY{y7#ra*r-;?5o?mw;IP0p}O2^oEjzr_y~s>6}H zg|O7xuy-vt9_&g^+)EqaB*-%{{dqL&Yx<%w8m$)CMX&Dpmv{j%8?C>4)I+?DVT2`F zoWxYPk$Zf4j?u*Yl7303dy&1^@=9&QuNDVcjQMHlF7N4U0Ci9v5XDGJ4Z z90N+TQVmpB>o`?tW?w|{rYW~=4cmNT7b~mYWOV!r6IVv)fS2*T8=F4f-Gx4;6&7*}jx#>GXK(t+_M7@}WvbG~2l$dlFW7nbF;tY5p@;fOE7{)lZF;=pAJvSz#_sRI zm$c?Ly}<(i3^IeRVH^)LozL*|&I4>w2j?=o zR#?oVT`?t;=}d<64cX5*M^wf+uTd!;E!&=j*_6ec%nSOwf75Q*m4wY=qV$9ZAiR&s zW~%{CQgW>*=G}vk&)hZ6c0GC?ITs>*3db?I$Ll7+a>3tk{dbXX-VfklLwI(z!Y@v` zVXRM;#O?RUS0d=L9qf5eY)Bp&U-3tty$tCvjUq=~_kKKCP^R7>5$5?3f=qT^BG#qz zR++l-JVrT>ZK3YPm=|Yh{H%#{{<3%c*k9l}<(a$3aV2Ngkif3CbE}o9G!2Z&(W!eg zg{PewEFXn|mpc_9($=kUEERo7F++*_CC>1J9D#-E3XMCJ6&Z*Yi&Lh5z*G?=f7^br zlK3;?$X)pqu_;wg=-Jj0YK}3nN~7~87^t2J(;62#vwq0BD?XfEjp=0jrl%e_w%=>d z1HE{;15NeSZh^ltP=5JkD8MT^W_uOv7>idj|EY@PSsetL9`xu9@2~G9O@5f*y#O=I zhQ<1t?Xr90kM|CQn^e+{wUMLzZ`UkPt5Y4OtYXhdh_i~G%XYTje`a#*_4wzZ67>Bg zqen4rF%xPe!0Y|o!;~%O$4bgR+vtu}UV>;E_sQ0cij`^2r{2CQcNVW5Wlf%1u*}{* zdCN%j#Dg)P?y`zVz1@DV_Qvw8oU=!4DgBY&+piovz+p@+b(<)+C5}Y*YCP2AT>I#j z(R*p*=Fci!`l#_NPH5b~S+57e{Eqg8~va))rRv&9`&8 z`yXd&MXs97KmC?{sLyD(u|F$?_UH=n^~`zG#!rq-lqqYu&;FXJb{*?<9A-!IT@69Hd0Xeo+4Ng*+Q0d_R7Pt!p()&( z8u3k`Az~=Af*p<8&{b06k0#4Bar?~)Mk)}iK7I!}C_;x$HXkco&;1&J8hF0p_g-zg z2F}5yd;9C)2&P6Pw3YBaJbt?n86)ptdoy{#qCCRrLcqcWP`lh=A@y{(FNzN6G)|?Xu{;`%3){}E9;c_7noP9zr?#Y z@v@Gx@UV6Lb+1XWZFZmM&=awtpAG|?J{leCy*8;gt*<;nb-uRSQwKL#(DIhN=4(&@ z$8Ka}Q^-j}#30gpcOphzZK++gG+9h?>WI~Zx4df^jy97kyDB8a<@l+RM>6U=G>RF; zE4ZCl$<}L&fj%R=M~eK~XzY!E+WOJWl> zpM<&ac!!uhGpeXAa6TfNAHBTeIlb%#vC3ko{Wlv_$-l2CT+-)Pm$ zP;Z&%qYSokzey6m_`cYjs7(Afot8^E`UGH857ws}2NgZ9KG4L@m=Jg?B1*T%DHCh4 zi<>+qPVCE#B)Jq*EQeZqQRTAKcTcQS>y;-jNI&gLx{R6si8x#9tFv)7H62bNdp4?L z26pD%IdHf1)LGRdCS@BbT~)FF+OG{d|EO0z9zFlp)qlYjq&xk=JC@wZNA+i=i&6iE zZcMFnxe>;UwZ-H%2XI)!RhD16r@6zJ@YGWWGmbJf-Ql=l9r<14B_2Epe9+C7R1<_kKh&X5+oO$72yBY3KC6sTY6Zx1(u5CDAsIc= zYorC3v;^O0=|}xBm?jDUNkKvTbI{W80sS8WmJ|O0dRFn_X@H$qlfi-tJ;)Yd5>Go+a^E^FzKF#XA$C_{f*93CTs|!i9ZfTj+uuco-s&3HT(?w>b==*<%rZZxA>orIn381P<%kAAE2V zX>ES}b_Y`aem{z{bp4OjxJK}5diq@?U<|3?Mz+l{U@V5nH#%ZZJ4H6?^-&}C-EC%L z@w14E3F)5vCz8AX<0*9=Zz($vQQZm62!$2>D;xwDB52m+Q68+r2CS}x0Q4HR)1YPC=s9qXBQDKv_tPC~p$GYMX1z*2Gn{{!1y zB-RE{c93iF1#C}JuVVtPXfH=!o#gYiMJrteSy%c@D4q%aE|O`9cun$|3}9q{zNiPH zMC8BJeYTi1Od%AJ1wLQJx2+SpMBa2B{2VHH6Yw0a{`mZAc2)O30s&=IAGkt{r_g96vaKVuWw4N+3&+_@x4H^L(?pn45|jaXRQD_)JNKUv>=BQTtA$J_-fWg&id`X-O=7YHe5}5xtR-!Hc6osu z142P^IZkg`W(n^tFBCrtm2~Mbr`7q4Ox%IT#a!=c&Jn%GY}^6NNQ9QcYTxTX#+cuU z*~w4rvk@`kJO8>c4w%e$u~A>Fa)2_x$1ZiG+ERHHBN*TkR_MS#bPjYgo`WEIM8j^pqd;GkQiD}?B$Gm0zb z%mTx?Lx!h?!CrdVXNsxS)OQ(aAcy(HABBPr1WeEJ;(rFOtt$*2O?Pq_K(i+iN=KL; zCS1<$8VB411?}<*AaE@FhRB*VxzLZ~m6a(jYo%Z@74T$=po_nUfu9aORFl|DXK; z2;rcg=pE(1LoQO%107p<>C23;nNXGVpW+Y!rT;&F=kK+Unw%fIC;l!~{ntLt4|>lgb3j68ouJ7Y`RIYt^|B_~XjWeaaza5t+`6>#(# z5|!I~=y*2c0K@JWc4jO?Xp5QIbC_H~sPeyR&o8BLa$qz7mHpFr_`V}e{$Qd9d&?RL z{vxXnax1N2Wp(Y#ESuMuD$AQTs^YFuuEQITJ4as&E>A!Bh<%r%k<6b-JBm(L4ti4V zzMfByxzI6X9LY4pV_fvv7YSZLp{ofW2KFa6)i$!%+ z_0}pEPLH)~+_#Ce!Os8P(DyCK@^6^)hV~PN^kQp6*FWj+t%vEqvUwY4D!zIf&MD;Z zHVp^*{O4Ml^Zhxly8)f@LIwQoTILH^(z3(kXUPx67p@+1psuzZnVN7o7hv;#DTD!_ zgdzGU=IvfOX2Uq|NbvEn%Wb1HR9JpwHm7z-?JHcHR-x3k`*a=ho{xjF+57ZdQq|Jz z-ir0^>G*lA8uNxq`)r{$EjrG72++WWoW26`kxxDaZ8L*QEmxC1PX6MUz%_`k3ndP` zLP=V?oyfRt&`RxLTD=+INe&HvyC97#zHvOv^4jEYr6t`EcitPk!64+@hmX|y(wmd& zuW)xh6mrA>zP3KGuXZ6KwE0Q^^LVcIIcHgnj}*GVz3J=Yxxp@(=KRA;hT)D8`bS2Y z_THwGpA4NbqiB?^d60CVk5+Kge$<7X`Pgme4!;_vQb6fE%@~U)VPoo1>pGU~G^* z@J4OOX1>7QYR9J%O?h4$YI$zG?H8#cQCGLMw{&uo!v_Pjv!KLjEg@rjZ#0_z~Oce0P^!*Jd2|;AU#OK}d=}rCnY5g0|ZsEI+dqD!zZc zA*(lNhQn|UY9!}bR>v=KdlH8}zeqkAGG#Wi+fd=+Q#yO8bCaGy^l!WvEgl8n$|V=` zdkep1KKv>-rCl~>O5hwK_r8$xrI{^yHma-~VPBYOJb!

{(E*E;loR?|Nyl@Tsr< zciJ%pN(pVrd}lrR^FPd6!>9yNT;P)tUtMmVbZ~sVI>tMD%80Rz{qFhwpI!alZf`ik zD~2+7S$(PL>wc-*QN8i%HVqzI9+2)IkBaB}yTB-2=O*CM^nO-elmS6SqE*2>|_QhJ_d}Z3s74;84?TX;IPa z)xroXW{>`oPRs9cvO5tlQigAA-~?5`8(`N-{K6ll zou!i;y|YeRs4g-q=tb%w4vTs18XI>W#?FX6MG+oZcxDe(9q+m5*}AHM+y3lo!h>fQ zTyzxh9kI>r)S*@Eya%WRNek+wPEQS0%1+ghi>Lf=zUEx{60+qCH`|SV%@z~d^AGCK z`=Ujhw)%IG>-O(oXQzygUKv}WJf9m1yYjiOL;K=mH@d5RkT`7A=L*o5xic?DedGr7 zqwKG>u_@bLW{?@}k~shC+ql65R9|1x=Ef~gG}UGXy_sVzLsvP;MNDVY#m?|-MkpN= z`*DM*9@S$oDgSx>EN)_Gcz0a~E?g#O-~R|Ie81WwYX2HMMaxBQ?pk;?$eLgudEuZo`39z7t6=I z)6%}wG9H*3in;`r$40vt25qDC_)!c-Gm!yeR%W){V!ly*pXB8hIE$(26(@GQ*t66h z`(FDEg(1h(1OwlNs6F@kZB#}!;b>I(ltG7~!6J=Q*4GrGKV`)BlXzJheaHkFOj%bb z;}vfn7zV8E$(XL+k>1m+rJySsduVE2mJh$=Ei6>PGkKRkRDG&O@v@)=)+4Oahyjx<>Y+lRn+kthIFQ8v=Qy z{!tHJ=3gGFi^tR~=cXd_LM*4RoVM7*jP*Adq|M1~uH1(icRlEpM=o4?dKmZBq*hO( zVd6yG=wKc=18y}%4K%QEBXFl5ZZ)Y?#x@)}+n0O%4xS3nv{hfFy5BYGYn;sqGOR9iGqc@=oC-6GtyyXY9Pn_Iy5}HXL!V zoqcxf;}tnJPIJ^wv|VpaVXucM0EL%|xNZ-!H>bD|GCDHEF(H9+afegT*(P#{6ZH=XX4F=RDRny@&1K zZ{>+twqgB*mDC;c{si@FwtXcD$eHZa;yPMBYk0NEm%OI;>~ou1clf`0MDV6q`Vb65 zPD~nlocHB5&yV5I*$NR2qB=H`9t18oPer#jt|LL-orQZxYRjpQIp^#0 z)m1w$NZ0Y(@t1x{yX?sIO7da{xEBXfN*zrrQteDAW$1_86A?Il@sK>84u(eNRjrPh zY*|w>=td%@KCPAO`D9g#R;by@uIb8rP^WU%YHvW%D03>Q+&_y@yW3lz*hIze%!OF3 z>yz$l=D&+1dh9JM-hHKna^mCh%J*UZc!T)ycfZ>;jhV=YNH{KaR&6r`DE^Qs?lyS& zHc)=P($#S;J23oqIgfI4V$caac47wf_9Snk2UmER|MF&*jxM&DR1@4nUY!gp`Qw9A zs)JixmsOz->9`rHIv3lPRT8C5RzioCDj$D>$*s9FRx}=7OZlxoTU7SyphYv5YW~?n z_V8Dp1QboNbSyr?e)~ARvgN=muuuDZ#*2(`D#eBjR`WLY92v-r3OCFG>Mr|<_j{T< zyKiT=OPa&MwOUixj>tDjB{kuD@nhWSYP9hk3%}=)`?4F4bsI~2uUbj@qRz6u*TdZV zn%5+jq7bBW4LljUN)NZKO(3Pp@Oo;UxlQ`o)W;dT!*2?&-y7NV@h^OQwF&6ktE`w_ z=P_SpUtzwOr<)J`Lq~Ns0gmsPEe&!n9D=f$xpqnGa0uXAsnEr<<`mb}xBJ?(zq%1i z9mQ;zT6nxU;O`obAhg)ywVko-@@&A4UfEm563EB%$iJ4=aWoUTTyk4t;4X~c!;eQk zYYkszP%-689s0Y@@t!+!XWSf$f%j=Kf5|Wz#jM5bn%W%j*ltiUq4A^GEuIL(V;W0d*0(h!70aQ2bM>&ad&tO;MTuhW;MIF$8ak~gQ-h5m>7Gv zmY$K~$foi)9KSBxf_4*yoKR?#u&zJWo6a|pl^>nqek={Lsc6sWNN>;YSKb1Gu2rSF zM`^DAu>2YxfU;cn;CA-SIRZ%x$ zV2csX4uys0(7&)N1*p@^F!7RK_tq`jsNT#Rnea--;g(&&79HtjT^U;@CYWjWezTb! z##f$te#Q|kQ}%eP>4Bx13Dd=(i$1Z_41=81rzAg~?Cg}Cauz90`$O=DhTUSfJ_X*6 zd@yQtbacqOXn(wU<=J7Sc%>EMga_>z8 zM$y%6+NB;=;&(I`?|SBgMA-icDEOecRP^VZ1w*Rqd}pf$h6oeG%az!&!-H0zT-3ri zGc;zz^@}shx?R>qDRmkwx!FbJIV)}u$h8Ho2xPPXs8uE~_iLRAWFgiUN%5TVUWa~X z@@KHl&7S#~b3br*d#u887=fWrdYy?=hj;`TYh<6Ryh1PYTbb6c1wle9nnJ#KLwT(D z%|~jw6}@?Ork}kwKE0$ZIZg_?FwVO3t=@GZ;Vp$jXzYz46>a19QM!$4WoiWH6aDF3 zBz}Kc@>1El(_oY9?T@=);bcX^(}WWalL?cB4xz=Ux5{$FZQOQ3gi4@RW01Wwz~Wuv zQ-{lm&B0`!X>IfGvLF$}@-OlO^#Bv$p$p!l)P=|~3@DI)H}u{m1LUxds>td=;S>M} zLA@b@S5AB1UW<1U|F8{DxyLYHBOEc(S&N&vWkDV~1f7dOk9immgh3EMTc99G`N zJVq#qtr}+IpeuF-&=DPsLmXh@cK~(Mlg4q51@YTZk10`C^QGN*3-&XOd^)W&&PbW>Zb=d7kHJ^z9tVdX!*mrWgJL-E zWp6ddpZewiFY){U=ZwJ{{;L!>0A6!6CMx!CXgfrUj_@kqJ#Sa>3Hoh{;D@C2UBy1+ z+q!}uVA%Z+a<-TU=1QkH6VnmW52Vn3_4QW*Qlt+_I}|FT5Fm~Te(w_p3u!T&^}rxR zb)3yipO3pA;TH_dBd*Kww8{ARokYPPsE-S;QwIs;FG44xOf5oCgPY16n>gdcCp4xo zDghr{jj+f38;wNyS~$?0AMo6b-FVG#-haLIF!lrEjsNL zFGozr9VVX6$DOr~F?r>rst56+q_;_edp~=C|LWb!QLD8TMRO^T0oMPcCNfhSOLgAqhH2;5r(C5~`HgLa>pYeS9QvV;Vjq;=O;{ ziQiBWlz7eK?P4E_u-71HI@hP{;2?DL4=Kv!Y;pcc8~`lm1{l>j?eri7rtZ|&rwcF# z%!CIIVUbmA2Ff6I+K{IeFo-h=DKX+|GoWt8{nH3SB_6qpmzHp2I<7t)_A2i1sgaA-Jvl-(A6Z{_6!l~o~x2sRZQef5vH5esd7nI zc(T4s7K@Da62M$>ilBmzoYj7dzUypQ z`xR9W7OAl{MhC6L^yQ^4cAeB_`JW}D?8{yV`|7&Rcq2k@LC>kt!}*mbEHMIOvRiZa znLhhR5ppo$mJPgG`!sk&;;C&IQECVz1olz%=n$aB!)mLDSWw0Eq5)_!6jmHiJw=d^ zJv&D3?13jriJLVPg%GN-?Kr+3kZ|y~*e9VA0TU;Q!Ox&H(mmogDjn<|0~q1taN?M! z@X|a{(3jVwux?H-iR$IV?g5jhILT-BU89U0lblG==QD~Y>tPb<@~Her5KPC4NCbi% zU1Z<^LPnU~6AI88aD)>JfVBpALmY<}lW(ZqQSS6Ir?8FqtxaUvCzc_x4rfD>MW*UE zmOSNX^C8oRfDv|ZFByoqqBakBlqND$5%y*18)4)uN-5#1nb;pZLtYti+EK(6Dsdb* zzT#s{uB>26I#NuMiTH@uMZZVj)F?YWp?8nOo0XofRip|ewTWZ)u#BSoEV_W8dR?9V zsovzxG?Lq`0Ro4SIt*XGS3Ov1uH9D|DPA34c1diG)(b%ufwPc@mKab8Bo-@lHy=b# z#AZ-;mO{Xwm`Dj1vU?uu!Yh>X;iNRiivah~)cCFcALRe+LXlwPfLk=b27;v;jLH?+ zYC>>Dm@-jO_;7VFcbooIq(}g?&oy{!W6Co&rz(#>gtF|!UT_}C+ms2_$V*m z4ESgeHsOI$bLn}aikB3-s-{o2mW^elJ)aUqF8TBSzR;NFHf$rO&QLFht@S^?r57X)S9 zUTbvNG92ZY(CvxQVd|WlUpTI*I=LYHFpxkl;N$2fI|bArS}DZ-C!*8tpA@cL!p3q_ zYu~v&jUmQ{iFoPn|5s#d57hh*+lF&EFlkYZ)WZZ|Y(eIRb;F2=-;bs{ZUM4mJC%-w zYBq@)aGr*rM91d8tMSKq7!8FjRHFLq0rF9_c>w(k=(-s&;37~y^Z&oPxvM=sLk2ZB zewnH`lZ)>U6WO4PheB}@+8{7y`9J^8$GeBnaf`_=!Tl4Lz~MC)-+Md~?cu)fZ zx7N;y+;D;(`aNiL8lS2#$GC-qzhf9yrqEDC8|5?`Z|V!SAvo$I_xgunyRPa+17d-Iakr@O*I?OYWNzj@i9lo%Yqj}ep?!q zHyU?RRsH>D)YiT*=*d}GQP|LnXm3)Cl_s8?h_mmp5dM6gA+_TcTVewk%W-cB)Ed9V zYTWa)>Z=f9!&3GmJ4XD4 z85XQy|F=8*T(evW1}~bkv_Fi$lxDMQot&C~3M(_ZPWH{ariWB0MsMQ9^D6wyPMnT@ zOKI6-d(Q2*j)AG#{d~PrzJxIBL3x2x+;bmw|EOu^1?jO$O6g2XK#$}Se*-&i^A#rC zW7jJYmxd1@E`z@bJ4`nxJu4pib;1(-+3ad}e&bt|mj>mWY-)53>j~feID+gyAQ9|f zyEaBMTD8(Y_iwQul&J3tdKy)wF#pwP$+e-rw(zVkC%o*{@ z-~icA{^>G`NXijq`uOPi^{S`CdCaQZ*JPOPgyrmdZdxWB%L9&PjdkX|G_c?VTR z4diSx{`4hox5*6UOv7c{RknsPfJ%Mit9`3Aok{lWu}I7OCS28^4_gDX?ZWQ!zQhoc zaa5NawG&31_h=nsBCAuom-sS7-!Jyf4tHULcg{MaD;=Djzkl&fjJeaZJ9_$sUD+~| zx;V|P><2bc1olYfR5sbHBroVp)bUsI){OnWvC|;G$)6s6NE6ERtJ7rjmZ1raZ9y4Y zVIM6{MJ)p$>)WMvekMQT<2U+GWH+7uN@d-VT4+PfQ3{!rE#;r0Lc65LA))h;N~;pS zeEh{@JaVVM!?$cn0u7!Ew@Dw=LMGI!Y9|Mj-_IqP|#T+I^Cy+$(>jnevoP_rGylp2nP?ZM14U9}dn z;|c!)q~BwlYUsdLtZ9C4W$5Wl}ds7Xi4tm?7asYN@mLvQ~Qr0*NxPp*43t47HUfs9gN+$SW`Okw01VgH}QLpk~ijgu99Zk1)5 z!_{4C4|eknc1`y21ePiB?k(R6#gX@oPwpq$qd zjZs%?AFmn94xIYcy#?GmLsR+shIxOBnQnyjG(&}bZHk;<>9JjQ;s?pxGX_5H&~kGa z#AlfgEQ}wxD!QyXyMsD3rbQ4(%51A-!f!8mbJg|e`Mkz&e~azY+o6q+0NZ}y;`*&q z(iG_ot(WSpBp2oVvOWt=mbsIG6yqO=P;u;8Pqxv0 z)9lGj-Fe^Ig{jQnYoE|WYiytw?s}By(`7wkSN`fbrDwORHp4!PUxiGTI5^CJ&ZX(u z@t@4&);)3Ux%0Ey#w1vQOCkHn$*t31z>}L)~dnV!=7#ZACV|HMs{9|Z+rSS9Ptc@{_#YgDVop$ZR)7vz2R_kw$LOQ6`xBL(6 zxF9Tcomx@!@h@moTAO4mecNVe^%ixreLF6m=tUGtIf4wgcj^ka>mt&e%x%|0GyPLS z^Q(4On-xVrY6$3@DxT0OalIDBVa1QDEd)m<7QM@oija61zL9X7beV9}v_ER2r`ZoN znd^w%TQVLs#hQ;$5V)k?Xd|MwA}mQHQRpIW!ZZWTYq5<#p^osxL!D* z6?nq-59QQ}TlW#@u2AR%m|L+%mVT1ssv|J=e6E*nRjSudUVJK0g|2qj%>~j?-W2mQ zr8f$ch^@pI0mkpYGXHYyIfT@2&kbo2UJLnD@l*RJ%9IC>KEC-Jaha2EXJt_KK9`+8 zD5*sRY*Ib1F9h&894M9 z9N{vKj91*QwyZ@m_fBO`SNNLT?mpJwujS@DT@oKR>6B-_e1jeGS#xZ5;;cZ{D?K~S z^N;M-%WDv=j{7_^C`pK!Gkd!f<`rbAb32o64U`;D{KpTxeCc{IDDJdzu++aBp} zTlvZ;T{hvsKE=_B7IwcK5PV)|Tf7Pvk997XG1(IC#Hs0Tb&;yARh3(I?;3}_(S%^t zft<4pS3hKyq0(kaPLrf>%JH&Q`~+tY0jn#MJ!)|cQmR$qXlHMAUC(Cnl@u>AT*vQ_ z#Lg4j?$s6tNwub}IZ+hT^7CmTzr*Y};U#}MZJChWP_pHScihFoxFZJw zd9@BsZFb3wP7~tHx=Tipl|jE}yLWAS^@_Z&-_8_bpf*%o>`db?j=nQhS3M{=nxuZ- ztmAZbF0IRgU{yYKdh{|aeQ~RC4Y5>qn!a>Q*QeG|x#q(2fRu%MkUFOiEj_l|CVyG& z#gLgudc0cCzd|zLc@mR%hG(J7u>AN#Kxa}glAubOk#Z_uJ&c@x37b5QNq>xKbaf0_ zk+m^l=s?2=!hA3)*uzx$xX56ac;1X{b(o-vKs&ytgc#dDfS7~R7gA=;l-I6zxn2{A zNa)l#1<0P#Cbp^N#C_^djh8+2%yxNUc$NIP?T3-ngQ>OYiTZs8wmSl6YWG(=ZZF_l zuIbj=U`O8@($J;fUN%X)t>w>%oy1Q9wi9=88M?~8Y;dlJ1$zrRO$)L2MG9yfB{a_n}lc)$H-o0)&v z7r&8IPM=R8vDf6;ogTT&;I(j*_7Y#prU}vpg5OL9Z{(QouzC$iXFy=Y>vFREUJ*R| z*`rJe;>h1(-|Eu733@8K9$asCN%<7P4qDE4RaM5OpJ z;q#$vfolIQoj&6jmTL~1MV9&_YuWqHc#m;<-@eR=C=+C?qb0$Wv5N5G@;Yf%pGfg( z-zM-5>Vme3%C*8wj!#}!9%Zaox;bvS+J0caH*jmOZHW91yPKI8O3BqTB)siZqYL@6;$WCC_pd zXU|4azm_}1a?o)2X4uW|#`o2>zWv2R7DR9+13@$m z&j9(W9naj93%eq7Ka*~InZlm@($K|BZOS@AmMt1uraI{E$G>NP5iA&>v&=M0K6yV+e`UZh>Ga2qne;2D%Tj5~ zSDI|MqRRr0l->})Ztalb7$HA?X|KNelH z?)6P_wkJP*W;#o!Ftop4-!ZfMLBv?!^|9>O?&!Zr!P4v43W7u9$UA4P{R4=$V)g}D zUYKAwW5pdS%3?_fL!LO1+I??@$=KMIHK*$RYf6I)kFFHxQ`i!(D+fp?1#5+mGc^@9 zTko5>ooep13HI{m7u@+txac}*dHE7=$!^b*swDHQ3SQ8~w#c^2k(BA*5b5R;x#2(V zy_;xX0t|{(-xD8dH_71WqSKO8$p;^Hntle+b?`Y0$tMBqpZf@lM$aV(Su9?9#9Wly z?3m`$?+0$wc^>cWgj5wTNLBexfryZ990b6KmPm|65eX8$3lzQ+FE|YFAA*9Yc?4&K zP?NVgH2equ@bCUJvC~FAW!{6(Et~USAOqTG(bjzk_*o4dw62&JPTG7CMYMDd@;NDn zOACO9%Kh?ik%*+JhgR#|2}yIW{to|8MkN8%Gyu*Ri;KVj9mDzh+x6Bz3LBlS-9N{f z;5ro)nT(jNKl~Bx;MEKl<@vzWMcmxR>TxeS(LLH6zC+Z^D(b_%1Vqojxe?4W%Fssl zADsvm7lLymBBBaxsZit*%w6b$>vIGN5Gmg+dhi~6TwcUM0n$qllu>oBmUA81n1Uf~ zJ{S%%ZwFvobKn;|-^M_U@X~)tf?T($bI;%5sTKIg#~$xlPg|`2So9?f1rfFN!qe3i zyx;(AeH~K->?PoKGguoi+?hF{3n~Vp#Zf~+bbwKS;2cDa_!kL;_9Nr6ig`eQp&#_= zd>jQcI)^{5MgP;+Eryeai3}kF^;<*pB2aD4&hG|$i@ytHL=hd=P_D!Bvk0R67dQCJ zDxCD2eF2!K>U`7K7zYgB%hIC(Q+?P>l z^P8lu07euFr(;N+Q7yEsi|yGJwjGhL@0XpveKvMAoRC}sB|vK9rUd$8M~q%G7zPzF z$dKV>VJZkA%$Sh>1dq zqOo;x&Pz|^LKUJ40W8|4y)j~#?9B)>`NAT(M3X5OdxX~juTP_O(XN3%pWpgOW@x_Q zJZvmP?v&5fUq7A>_1bX0NW7-A=;{ebvtpq9q~7V@+iS!l27aD~HJ~Nsqux3=?1E!o zipUdAB(pC+HyWJv9+EZgi=rikv;lTdWMleijZ^c^n`HcUZBb^ zdS3wO*4p~F7#;dJ>}TQ?_g;4ZO90JReo1v*@` z2Hmi1k$LzA&759Fw^6UZ^;r<1K?al3OU1q7sL9$~XTRZmoTk$4faL)g6AZ6K(N;Vq z4BiooY7&(Q09ZSGEDnJivAP&W06)i(SL{iMXag}EhBt-khqi!Vb*5koPHTc6q(fT@l?F4R6o z>P&@o6?Ynhh0F~h@Sv==q~dOcV)vk+0uR{Ob$muMjpT!HmGtD2$XB^z4NRIOJDQK4tsw-8!R8jsg1+mIPXJi~=_)TQuW zT_2P@Frf1OTa5i&Ed1W7js7hIZIAoaV*7aHP{(mFE=!HyO2B^9cn?m8#SXwR!7`Sx zx!FH_{t1|&=Z4;brY9u;v-)_rJwt+o?GEr-59yxJ5rVHj{IdiW5N~Mnhv6bj$g|e; zAVx%iscQg`P6PtbpL74y0saT(=f`=Nva%j`o<;XVxIzsrC+I$eXGJg5S@;jh^DFG1 zl)gOtKm5&WSTkVH6N2wb(y`Pe(H12g55mLk;^Mt;gW8p(ma4To;pR?Eu*?4p-fURd3?(^bbhM4Cmnt?f>}<3q;|E=nF+{1Yjckuipm$UjWY$!tRVfBK=)#hg0a-5`*{b zKnf&w;poS}E*!gVG+j{iGanZ|!h3@L^|Ai>nE(3X+!g@cojnq(N{%7Ess_-jNs_MC zmECB3BE0qs&wIKSO1B)(7V)OmW&b7e$7?vcNnfz_3>fe&0eRS&HJ+PoGv648NUn6Sr!$(&HXqP{SQ2 zC8$5R^SEbJ-C(_W(dr)web2NAOL?)qm4BLPZt>dHZu+CFAb*u0aiZ3L?81%Txw<2o z((hCp_MJL?f5Jdr?LIj{uT*&doPyiY$pntgK)b6G$=vFzn!7b^>k+6I&JM0t#|s-3 zWWu+E%L#k3;tD1|m8=RK7@Bc8H&r|^>#H5>x2uV_0}(&j(xA^PcZh**;U@YT@6$bR zgnRc$K>yqViKBW{=>VmfmgpVb#iK{c4vkdp;$`jlIgstIvA&!s97QUHMn#vjcEng*Fl9__C>B0A{Vwd0!;9euM8iT-etUcRg=wRe?pBLBR(A}l zNgl5xcs=QVRaHP5iKWj(8x#lZ+>m{<(zK+}{^h#5a!cy)GHoTZ<~MsrbS@RU*g8&g zJqE+7HhWZV59o`M9}_w~Y0GQ~sDNrWKg;dz5eBiGe<9qY2#i2wBx=Z)-XI1ScAu1t zn>o;z+ce{6s-&~Iq(fO;Ip2s~Ihjq>Yj(-8i#UB}FUJoV&ZTL!X&- z`&(NgO+s}nZOUz(V~t}uL;pv}6zrrjC1NuyI%wrClWyF1a#C!wm^S%?rWC~2Ryvu~ z)Z=^5QZdY$ZN}NoxqMlQ==w)yt%hlXLCdFt_q)Q0`j@6n-tIRiebU=!OH?efY*Ff2 zt8N8DWal+9`>!Vda4k@BNF_x$>Dh-nnwuZnMVU#q7VkZE@9p)~HQazd^@JVGD&tYJ zhc@-Q=F-O%7n1+Xkh#b`BTVpqAHi==`w}*MRD5IB%7n481};fe@?}6Fs1TY8iyhO= zgvmBnSsf;o!y!HGmgRafh507R21v@CG{%7{-tqr%YIt4UQ=Kz@<66nnAudOVOGra6yNKYS~*?V zA|<<$vUttSXxmSo%O0p66f|Ca)8cy7^m1WIzT48MYy5^=4_LPiQk~4p5Gr@ixmEOm zXD&#j@rNVY_KWCc*5@Ag^7nAnEsjdG2nqqA{mnjo*!OtiW+RruTZPM1A5BKtO!IYW zfC8-xc4)g{WA&{h$xD@P^I%mkGNz>Sn3DUOI5+Z-(odDE$PCzinY%EZR*`E|2;&5# zz|L81oe@h&T^>&Myt7wvRL(EmKd0a&e?TSrCDqpMWWYVOHB~8{F7YcS$?oPaX{9_i zp~2OplO!~+-KWW5hTQBmqcJJmTSsAcni5(JjTg3-q^=LzpQ2s&D$Y_hg^Y=e5-k_i z_3yvJRziG(O}NFqnSuC2wA!e@#r9KO?2Cn0t4@*I_DjI>e!I{?LQln2y>8D!uQeNC zvmbT0-F?Qx+|(<^ZXl?Rx9VGsb+X(Dya1B}FyR;%ySy zqZ%n_x|4i{FV(zh^{YJRHft&{r|p-r#D%J`1B9U|r_cCyXdz!OVkWmC?HJcpHrfXS z%IsA!cg9BFTASbZ>n}mFDM!PP=c}z6vbH(={7t{EK#KlA=NTiZ)oI;yTjLLh7kAyS zdH@!{2Qwe+nmF;wPwu>c&9=$&KdQwv4@f#>k9_W(O6MAnY!262$vW4NvyZK~-Lyz8 z&!<75)YV9FI#;;2i5utY=EI$u2)JEF&nRZ}S^n@iho$(qAQ5cYAg{;D;q-*mV=rNW zgOBDQl%UjY&8gwX^u;%fc_Et3Zl-5mOe_v^Qy6?D`G(9Kuuj$w|!%T41Mb$;r2Ok?nzLHBmfZ^I&&{V^Z8 zR%?u3KvKz$s{uEgU?N?*{Kf9Rz1vj7WnTNlD9@C7DR5iks`A?FHxCODYVxHwJ$GYB z`@d)gb~@L73FEwaqOM`FG5XpEo@UYRxjFnGltancYlZuh+%Fr-AX;UYx}v2A4^0Y7ngf!! zg(aq}V9GSDgM9L0OP60jOnW4Pkdt-l*V+2JDLiiIWw}w6h1t|(kLGZ*w#vR5w^zRF zDy$~Tz8NgPuBYlx@0rTBHEK!xyr!<)ERCh8bqD$C1b5O zQ03j_kmFXJN67yS!D2;~0&#e;u!KaeD=bkL{;nS%0N%_*VG10wpquYZ@JD2_zte`o zN9@VTJ6a0s$u+0yueps3Zi<5OlxHEN%Ul@CewzbT zvBE4rx3DQ8R4%cohTWn<)g%&PcAbvv9yPWsebr>Qbo)116{;IMx5cQ}TGI;vPw_E$(Occ|TSF#XMk6-qyz&zlvv$s_kw?`uBDvN)p#zHS4C;{rN4^uk0k! z@0OR+NahU`si-_Hx6mi=`oAOr_g3ux(!U_dpq)YDfxscM$oelcDYXL}j?zZ{@vHA&R}WGfg?#(qrOv6a`d* zv%w`fnwE&PlBb5(RA!lYeEN7tCyJz8Q(5Tfk0Dumn0#C$&bBF&m*25@Vt+sLX>?o0U)FLT>`sCZ!_~>^HP`JG zA7pIj2qWzjjZxHVGUMZyhaPXAaTmczIS1!fIEXhsth_=WLL-~$+=KLi2r!;t6e9^+ zsG-h@;fEQkf6tJ=TD2~DZ)Lw%uyP2Me?cKoU+z#Fam=M+uerK@{}lH^jDLF{)%8&} zc*qRQnx6dq{we)-h2zjJGpfPePZah?P^uqGl)Wt{y-)lA9mmNQjjrJXLc4ZC_v!rn zU;%yCF0!I9yP)C5v$S6f;(2Roi0qY@Kp{fTvb_A_jtQ7xR`bZ3&6$NJSG$f!7fmcl zf;AI7L7NLxCkTASH-Ot}uPr!5W}GRt;FQ~p+Q+1suA9N(FIsor(@DHuUX_AL5%uS8b;ljmSMwwSLz z)rVBnKcrDP9~Hml9GUSOKkheA_Ac$HYi`8Y7_xU%uxh-d@|Wn^k_S6WrsOCS*$Gyv z17_YMDGw5U)1JH)*VIxYQ=9YPKG%Lxd9*OuvG49{3jb~?bR;qJOD!(Z(muDcEmY$r z`OB@(8Xqg?J&(B`aai7c*)M9-;>>S}phh2c+UR9M>E{_qS4!k0c-xF!N(^n-#%J&x z2QA81&e&#oN0AC1zOVqy#Ow~?>h55FGJC}a96zgfWgWBdf^dQm(k<#qt{k_31)juNr8hK_f9>Jb7rvQa#pW`DSyFV zn@&Z0kQp0%Nc>@mA?k8ysiX%>ug<11HprJ)M-N_MseQ{yE|(O@%{Y#mGpom~gEml+ zK)ZP{Wxg8&PF$uuuCOocjF%pd=1N2u-XW|OSj3Q3w5NW*YU?FfMrbGxDWshGc#Sn; zpmzFBHDku6$S{-e`ACE@jY^JilzUI6S9&@4d5^EX4#C7i$c(Yea=51A5I5c!`#DuStM$& zltuX;)Mp9vLs`RE{ZG3l>KAki1t#Y4vAFQZbBGZO^?$kbG<>t3B4AmHs9nB`waEwf z7L^^J!bvLHB&QtAd}J~1lZE@WWa~k*EgodONTmA_!yy0+)A%<50pI{i><}>vxpa|R z7*1%3h?<4ajJ#c^@9;rlKyj+^bwb=1UMiSydg9-1jI_S-56uJSH7I~;3;-&|Yyk&_ z!k=DfE(j_|dir#Z^TCXE9)Xu#F3wSG{^E(<9;15b36`I;RpVjw!D`)OsC^x3sq!DA zl|`nwfJy|HPO8B~2DIwV?tjTkc4PhZC&hNBu<@9Uxez4$E#S65(mFNNiO91Z2#mDO z6k(+qpdBjaoD0eMkBX#V!vGYx6mpit)>~2z3?#2T8ApRK*hEhU^_jsIgwRYwk>Qvz z-kZK(L~uB%<3cVJZ9{wb=jj0d1`z`&CW@jw3q*!};79fq#vmpWbH<{B(42-`(Xjj{ zoI=#b4g(6MT8hyOo^)8tf3p@juiom^74v}lI$wN_?8kr|6z4>L zK4Bvs8xQ=CsE1v|y(GRzwP$nQIt;WQ8iZ)rQ~;?4_PIrWW79$m%}c}JUy-5>r?v_#SC@gbNSn7MUUZa5gNE4N@#}k$}wlp1^@-(UWJtpeXs=pqXn=M%{Trpx+Z% z1`xDhmUpqa0*YEK>X@Xb(`faPX)kO<+>C^6Vkn8G)0e|0_#ip$S@O8u;d3{)U3z&) zUQsf{k;Fpe+0u(l9{n}t-7^tT3x2l?u4C4Xx?Cu7@dF4iJuUMWwB7qKS5?Sj)0-fO zxMeE+>(eELnEF89Po@uk$I%u!bO%?J+x4GfNPGqxSdtf2f!Ot zgBZ=8FxiWDe~Ybw%%c#{JfLF58%w>fX*-MsjR|L&X!6#6J*kZsDK(78EG@aGi|glT z;6Wy!qoQ*1+sH#kzaaK!YDX<$)Dg~Ei-l8$s2P-8Y5dmnsG%N|MWZG$+}rJBnC`R$ zoE45^?9wq(DQt06P*MQs&*N7_V%bG$hhYK@c27rOb8K$6tCDF(#)dSf;WF$Ht*=9$C1u2F{uY0038Cv%BDXt30SFtR5d@P3iLB>Ez~lf)kihlnLe94e4}R5+}x ziwr>stSnaT&qiG$H-e{k^Z_v{R$ft!1lXsE86C$R4A;&hw60Hm5<4BS z^T0AqNw?Y~eb_3aSMJ7xXrV>Ez`J3TN#nzs1Efhdbw&a&mw-XjaDj-}d3d}iY4nT_ z%nic>;9e#TD<9-Ps9kI4&Fh8pNeVStk4u@oAT8t^zNJlKpc5Gfvk9Zl{k(W$Hx(&R z7G;cK-$3^zAb)~SwHhsJD1DRoo`n=oBO4M$O3a@a0O0>X*u(^Vju9`6%-97l{0dZU z07liKJh|EAi3OrpONj!BMDC5IdtI>sZ6CyiO8rSC>pTF~Y#{>2Cj^})!7A2EtVM6F+=S;~3ba z$R|YV;4Afic0P--i7t`{&-D{vD?Gjc;k^p@zG6C&nvzo7kcR7cn=Wx>g0quaNn-$9oTNyTQ(i9e1w zZz>4PWo*2fI6%BdpEpL^n67816}--aTv_%f?@`MG?Z>X4x0v*Qkf{{oKdU<%s4_iT zK?$Pi!O`n4mCUD6orFgAp^E~)mf!uIg>m~1+zD^%uekh;N9fCRyh(`jAJdq(l)jX3 zraM&eDszooz`*ZL;>kni{^4792F*sUpehUGD5ZJMj{a%))XG1Yl7zpddF!5Y*0Lam z>zmPg>$(?r%;vb&rIa?L6w)|bWV|75T4~m_*5-;^F8_!1(x=bzm>+uzw6+sp2IS_* z04V(HSei}awe?JYy11s|`poF_NMxDIJ!c>8!S1`SzfSb$kzQL%aHfwY-R(h<3sy6Y zC%b~-w<6@3>j$CoU;DkMc9ax7h-8kvTh9(W^j!VG{7Q??%>0}4rfm(66E=BP6a~|^ zWLk6#lAJ@8r^BQVWf4XXvM+GlY|aF@TXG#PIvw{@Ha3pYD3h&u`O<9hv?R4fqCbWf z6>}!Qi_=WMhpJV@*PH|2I#rJI$!AoDIRCcY~kKJ%pc?vs_< z8UlHhy!-V&XNaY&4bQnPzDf0D2g~YF>FG#rlUrf+OWxHR<$Xf{WY<`nUgJUbyDmqG z(rr2Z;E=+~*wr1EoJ~}d^d0E~UwYSEP z4GT@fgae&Vkej6}v0?p}2I{k;`?f{oie&ujZ_J!BmN}^u*MDU13Lmap##8Cs!QhAJt*LCUiI!=JbI- z=DFpvyS@&UH@RL&>ev~V*|J8uRO_Om%7P9KV6lK-H|Nrj%L?_d#*dDmUV7&K4B`fjF#RV zF=!c|(Ys<_gZxli{!dkKbptj8!xfe-{zg5lRp$rWuIFau$Cx}uQd*h&SnSOlo~cH5 zzSk<7pL5{?w6+=&j;sK{SX0w(5qlj?qeuEDvzcT~16wbrNgX7`#N|13PU&G>7a0-A`-lK35H{H4B#Ye&MK_o&V zGY5IUm25tewa=ma<{_zdHt>=^5caUOS$EkH<`Hn)C~ruJ=se$IVcVlPD!sq`XyZ-| zWO)6%wt<|DMBC8vEks-9Xl?uJ{y}<~+gN@n`C0nDirTow>mI*?gZtQ^W%8Y@z1HMp zjlJZTGbzpl3I9VTo>HB>=@**ZgryuzKWqLPmNmlzFkdrw0fRN9a)+(UgMznN3KtKb(3b{12^#qjYZRG zuKiKF^Ad<9MBcy^RfN`}xZ{4n{guRb*1L*7x)KUiVuPedeP*PibMJ$9aQT9iARGa{=WBD_c&^(9ZbI#cZ74Up-{8kR`7MUWKVaG@5(sO<+BHT zL!WuPRO7$9P~>V~>+Y(;;-2KCAt`_*HX7$Y-ll#aCZO6UjNZ9sOS|(UTC7E05MQ6z zm=xm1KA~)OwO%ts>56Zpzn-U^n{?}FK1ZI~Vc#xX;h`Ry#{FAtP|avNnJK?!wx=Y3 zcx>W_ELUz-KWvf$Y(VX{?Cdzq>;< z#^2UTqpEO+iL?U3@|k;oc5R~=OxSMId&b%Jn8fz8g=UsBL((O&)kVCZ=@OGlFU^w^ z4hD@$QR2n2ZYKje6G|t&WE&Do!=+`bJKE`Gg_-8+!ibTEqFUIW>3b|N{^(XfPc}U12`^Rj{YZsF@iI3%2?La_ zSY+ls;%Ux)d)2~%r*~zG@On=^cS+~3+dg%+s4R{lDGdnZs8QS@i<}u=Ww%~hS8(~4 zrpMeuHPZHonHFMGeY@_msuC52P0oxN9(pi(uICVP$UiBdYbv(y2y5q)`we7|gK66$ zD_X`nVa>nML&-=aUs!Y6(Ir*NG9s74f1G-X>m@y8;c1%7+(qNp%ahq!23KaU5hleo z=v}-^vn8^v##{Oc(+@7s{AyA8T-T*N_5-5uslvms6R=>*pT?#;ExI-oMOj@wvX zwZ-|+al?-8w{gec)7Y}k0bXaBOY&h+@F+zVglNC7Qj61A$Q>p&o>}$>t^JZ0|8fo? zT$l04bK`Bs*l1=7G(r{k0qt8Cpr~I8;+%ybPET}oOjUQgbe{}TMouib$ay5B2#cKPvGtJEc>x)*C$r8zqDVZL+ z3Lj$E5xU!-GzWz_I}0r6%g?R}M8lI+DEW^EOe;J_sy(#c=J~w(3m6EA1FFG_R|rcZ zYKR(DZQA+#gDif5Z1kwp@<}E?A$Ezjv)i_{`4`FA!>f|^N-rIu8AY4&ZhKM+b=b- z&|I(?Q4x=#c1k24n17yD`-)p}j=!ifw(mk}Gt6|AmKE7RNl%e=0U z>qq@7kfG%*4vF59CmIM11qUaPYK|4iXoGu9-ZjNnZhJPf)sh%|>OPCpm_j(!+{p?W z8ND!>aHUo_`Ug@_n&D($6rqZ-;To{t8KJ{mgY98&?E3doDEwm0JhgtvDO~3tElGk|dm*Cd?OzkLLuLj>g=6QA|TdtH-3HoEx6qc;cm7W$)9Z z!llG(uHtKY(|hVt-8Q$mM8#N2`xK6{1i7T60`-{%mNwE8t54%eP8^WB|>q zKC-hspE*L$8~tkuy>zL|bMGkP4DSzM_{wHCocy_iw}}_$Gh)@q{t~CuAJ>(6m9Y02 z_wENN=fe@g&PD@CnBEFXXulkt<23=2{=*eZ^QaRG810IWH`%tUY*{O6r#$?02xTDt z^+yH=zH!z9KT%?b%Q-MrltQuhB#N`5r$_Jvxgm!i&LN8fl_~tsAKYmJ8r4C30Z#WA z+yT9wxLbz^7&*D(E>4Ps&N-GX`HG_onBfQNLnZf5V)7G_E6D@YAMl<_zdVHV{o)-r0jK*khmD9R4Wi~z z9RI}st`0&12@`ENUJKUva}W}myn=HJV5@`X5^VB#I|sYsLL1zrUI#@{Ky)%c7;JnX ziUAVhWkfbNNojqe>=Ycoz&|Ow5t)ObYyhQ+Lb5UL;t_`Ah&*`a+;-BP=FD5fROeU{G@(Y6p>snVee&3_NQ6rQ z=vZXC^*^Q_YzUD1!W$SajK#X4QI;U>s|Xl{pF>e63{x>&h>2ndS(qpW|0SsH7Yrrv z4|2ku0vd0CtD0ol%K@|0IsO;aBn>?j&8L^vE$$tA)Zwo>OG~tKAtNEvQYA-$%HR=o%`;vmYhE zjs*6T_OH~61p&yt=_@=2#}CB>0n8TDvk4(}L!q3Xf?G&AB9H)cj)*?t8$YIU$hV{) z04$1u+W3`aUy&~JXDJdMC>-V<=lC-;#w%gVOvSLWkMpp98qqZQXOk>`N*r!|_8bsZ zpcc2P$~Y@JtYr~EEee)a2aXTzd+#I-bP_DvZ`R__UcpXm6r6qZ-7753zY>{9PXi1O zWe&vF*q@*!@dCa=PbX2}i$7??d3Y1Kh)LayVbZuF0Agv*`@pJGHiu8*d;yN)iG*;X zF^S^pSkjf)yBS(KLJoXngm$Ix(M01y0-m!V--Pk**=?!wG}eJob(@>|76AUnld{fr zA>`P;&15eXXMYx_gb*6Zl|^iLf+%03uNEO+ ze)$(}S~}j4Un1s-S~H{4DfUTVv5(`C3lY5`b0KYZuE2>@r_0lI=_7hLhO$!Y^SeuZ(Y06KHQBr(B2$d2JTs5g2C6_ zQVwi#v){Gn^T-YKPf6&z%t@Ls4HbF^p^RzNkTvSs0w1x_E4?8j^7+Cug%WUG(#Lso zio`iw9PV+S7cwP+mGMS=L!SqIKJldP7--=Mc&@1A!6hjoi(5g(@(8KJfb{@d%vK%l6HcS1m66t2xSHk`R@ESai3So8(R~*2 zyr6nnX!|$`BzoMb42cbMOt-YtJz-9DAr^54y9O8m*gk-%`HtgHSDdI7Gr(x&!Psrs zTms=DUYK@2UHoQ6rqyo($48x%FFxgtjA{4NDAA2gW}q;zWgbNax>%h+GeLRyC)5cL zdW?}rzmD;}38K(9u&;;ZeIw3H9O_w%h08pBivK*SGeNfs7`q~RIzNbXCg0cSTiSYy z!3xxhgFtnd`z~x2;Fm$QSOma4{1KUgG2^$vh1$S3{$T(_uYm3jyi{HT<)cVX1H&1V zrjvw#0XKn?(VP$ieY(&iAh9`*M|D?4U@bw071|vB)qH|(GC=1cqAlQG z5gq!kufS!&+XC?fW#vDFhKSt2N28I5ivZdJCo=Hgun;RMB9&&rP6)#TzOw?F7%&Dd z^u|(AvjW~8g4)mm9f}U%xpIcS1m8np!wEJ80UZ$ zB=)j&^0l3uGn#((iJzDUQq8XgE~wGg4E$hDGYOwb?Jj!k;R0!lFKjuo{4CaF&Wcz) z*W>LY!wZfZ3$LwFZ6UrX3rKg&>NJz^b9`?%Qn{C9)Kx4U3_iq~3KJ)P)I8eF-|=!n zdLqeT8hZQWP2)(4pyTMe!j@?I^X!+m`kmT48;M-kme!QTkkK@Imh5y(noK=qn&i#w zHpxmDTJ2~p(-0#G$JVG*+8#5|?)Te-Y|uV`6)$d0>6Hoc%cjYLPb5bIE28=zDLGG{ zd8HlZ=9%h~TeDYF`kL|>90ks+Kgj$Evbqj>zW-D7-|0VnJ!i3}Hx^q*p_coIAFVvQ*wXIf)( ztdIT>wzD>B0Kk^pKX~YO4@cG0w50MRnrf4-dab$;)EyIh>SZxAF&=x5pY;d#xRf zK2Fi$R_*$s7xBr3e2Pun(lkPtRz)(cXx-mpL;Z(~2u5|z5}Ks$#kWRVeE%P3R~`su z{{MAv*Xq!z%x*-EvT~Jcx+F=;S%fXgO-RV`wA-doS%hMlB3G{5x7iZPk=$2I?%QB6 zGv?^`ex7HvyWc;5f7muLGtc$;yx*_a>-~DEwzLeh*~AY->R*ImosJJ<0j})<4#(4* zCJ);Nm%BC9eCj2gZ`lzh7xHnBW0t26?NN(^Xph%(Zktcy`?$#%_h&xD;-K+GT~8UQ ztScs0euAuLnrl$wiyk#@<>T6$HmxwP^ZAP}t0h91mMk|SxA360b?_nU^n2uNYSI1l zXECR9ErhE^N=TmqON}elSqj0guSg_IOD|Z!doZJa$n*86kp|;?} zb=a-YVNys|G+wGkjGa~a#aE?8dZM_h3CR&_Y~yq!v{S-2+L?ATI)`SJqx>2_Yp~a> zAwG6VT72*BVw>NY&gRVn~^nw1l*8{;VAB8_7c>?N# z2dc`(Om(%i3gU8FgouZ9nD$_1zLT5#cFb40ph@j^CJ=FR5sTL{Y<%~%d21wfRqZV~ z-Rv`Ty;4%z^4!p3MsQE3$35*VSS7Ng75w1RzyHRk!z~3rpkKOzayn1Hrp$dH&AjgX zl(te48J#^~P8A+r)K`k0OnYQz7~!*(m3<@CJLH$#ag7;44bfybk)RD~SEQD_@fF@tw?{^T#|_$1!6KJdABn&>v<=6sk>s@el9PfIT%)qHAlSJ&|zy_kw5H{~el z`SXoH#yRUecz;K(jbe3z>%^wj{n}|)6duN^smmbHKUCN=ls2x>ZzJY7P{ZmoB4BNU z_Jk>NXH{WQQ;8~}Gxlc+EAdFGs@}ySleO}8*JpKgn6{%de#XYzkqYs#sm^{5pA%=^ zd_vaIr~$j{TBcs^eG_b9L;P*WI^w#cBA)ClDaN(YsrcEU&|3PasSbO8UYp_OQD^nM zmE66B7L2wS2EPGIwUt%TwTqQ*lI9kvJ-5BVGffEce+rToOVl5}Q>#dGQ9k;IyX2wu z^ToBOh~#hRKE086vC*`4>&d}9qgARiF?PQ-JE@FV zMd1^gy(}*oSxkLtOR$L#wnXl+gF?g(Zoi_wmRB=f}bs^>sIk) z?~DWaxvsmh0iI{oE6_HQbR3DW!K6gs_~jIq*Mg z*-#X#BON9@OsN6l^k}kxuXfww;UpM%GE(pDdYC%N{@BDn8aa@5Fr!Rc9?Xkk!y(!(GwNOEf zb2r#Xah;x{HkkY{12!DLe`8IkwrKp0j$RTS`BFi!3UA#d%yNWv(JjxyY5al@X~KN$ zkPPhlBh6N`-2SQLf#46i#kaHyRYn&)V>&8a%YJT{n7^cwVy%!*&yH-1&@PDJh%O8S zE!;*YR0RtkunUh-!uEvhKd*RnqpL=gdPMaf&sFJtS)$^Kit-D(b*2K7CUQd;>bO?IR5c|%hr0WzJy)DR7&Sbx(-l+?|MYcuBeDAkg!x=yha8nT;=fJ) z2V#zsZ_}2m#NyV}lkuK1o{uPgoureevLOyFWr#$b<49C=cYY}!lu}lu-*`2XdBo4E zZez|HX3uHWY<;!2U<`-MbT-c_c3xWzz3)rbs<*5cgv!i;;9-{MF3NRiu|6(dODlH- zs3zPqrOC#B$ILyb;u7neHkiNX@6`}_r24A1lnUrq(C|frG#myqD9I;yUrPb|$V3&* zvC^tV^Tls`Z1(^jBKLDJ+6H>C?M!5&I!^^;rw=}a_&$UpAOA}LvG5S}T~f*JiedI0 zn@<#KRbJ@^DNEO9FXaSb(NF`Qx7DOX`3g#PaRjS9?cn*=jfdzg;fu{bdDS!fGO9T9 zV2O|-tLfxq#eKZ<^!z8g)|Y;18$a*id^y70CL>0N#OEHTumz{qwC)cNhR@K%-0SBO zJJ&k8ZIFxF2di&;{ixZ4%!j-5jEgD;-x8ivAL&i1DBH+UtP2ZdG&pRl#ejxp$9=ye zhdL=MSY59lbcPMGl#f~iJ=T~LsMmh``linKZc2Mv+$;b*W#o4+n|}=;7Icx<^)%i} zcBcP`lm_H@%T3ZJvKg(xiAN_MabEpMWbK+No&;)dDAm}*;ar)XNizMQ=_#cZ8q!Y# zi@zMS8C6>rGiI)M)!e=fa-0rloE6OG*PMRuAm!3)QZxA^x5>I^-J1~FM5pO*$2oL| zmImvd6}e3&^fz*WW4SG(N_*3uaX#=ZPNu&W5Z+iXx%F|7t7JLx$N0iRll5L(nrgW5 z>#yoGf8*PD>&=MJn(6xq924CgisrOQ0i;cgsV7=8vnoy@Dn9H1fMBLaj*5+PI`5sG zp15P|lXguO+6C49!*dxL?WblfAG&dcP36TYI;>j-IndWT&2n>BEfmY=FaA=vh7;?m zt6|}j#JpVt18SK^D%^xNF?PK62;eH8$*C%c(fPWollavjRs55euu`tgXGZrEX^or~ zxAhNe)M=U>U3|Xsr7zh^&2FVyETzUyt!&Ontk_~(qxnYU8vkZnUsGy;uAqYgk$K~& zD`UW@jKTl4zMSBvz?kMnZ6A3=1h~ zP9+$|cB|}|R{q;pV3ysb-D6H?wb{5IYMz}WScYT) zJv>P!50FL$`!C+}6vY6|l8`~3eh=i|*o%YcmCRFC?z~T!qXWj%z=Lj49E+#zM&;@} zK`|eC1p(GuYV0xVaunj`r4p-fp!3m#c;b*04i>R+Y&*~YYgfmPDB{OS5Nq?K_qXxr z1Tgd_W6EdmhE&8wn{r<|m_uci|a&xEl9A;~CCK5M-bLvz81w)vSfi z$?#BP9GY9pV;*4vc!)i^d`mMhQ3Iw-uHlcw(qlhH+ams@we}2%Y=Lc2yUgQlTh*V#@#(U{00h z&@{6M%)!M5oXb@+I>Q|}PTp-U3`r!Y$>)cXBrZFOybj?FrlC(t+aMCEeVX%-jkql*8ofr#w%UR^}g{Bhn%^@iSZUH?@ji9NsCew-PEHasf-gu9( zDG7WX*o)V4`YVzMBNW$RkiTyhY2YyOedSeV_+rIkh@}eDcR@_M1q70Wst>>z+J7bt zG*piLGd6`=USU#lFi1*7s4YuG>A62VFECAY=t(FGN{NSX6<*r?5+se%&+W?}&mf*; zYNYs8PfPB8!t`#wsHLftb1V(cFQHs!(nu9YE|kkcDao3_Tf=D(spBM;QF?gSZs7%z zvodY)xwEsIl$=vV+`=zyab1%hw3%y6^mq<9`K-+T*~kn^eS z+9KVz8pJ^88oxB|H^zq_Xzb;Rr)7@yid{#(@$Kd<{PlWHTH@OSul8LI;%PansGJb3$ zjlimm#(ttQb(rV?_QJYJ zPUpYW44ATkXXBKB#fFIU1|^RJGLNl~FVfKrd0vVNSWv;l0tV%;_QoltFf1;sv7?Z6mWQo7bqYl(??{kB@;lTnIN*;CKy@$$P_lSAi~A zNkg5ZAdL*C#(2y1ag>t4_7(iGMxDZ7F@Q$^BLcope!`F7bUCkL!7TgSu-`d`aYDNI zHi4B}Frf(Li854AXa@254+gX)zzRASvgNa6Qe!iC` z(soi#IeShu#jkALUkj?<#Yz>H(saGTX?}n9ikr)|Zan%af}>q)H9*v3%UkIDer8gw z%K7!VM%G_OEu<~P8)tGg9Nc?rTSrC6<2l3j?|knO4|8&|4j2L zJ>Qb1R=0158rx`dpU>Wxfs@Plf>`b8l&VG1c$;L;@U1I#PJ$t^>z&78Eg-$_Xmj7^ z;d7>oD_Nm2On!mmHFP{>E?Y+9jCel>CK>fr)`6;B&YV&Q#_BM>vSZPddA1v_a zpMISud!f6_c2muQzf{QxaP!54#>DX8IsRP+j#h|e`@vTczuZ4bn7039r?mISg(D?- z4n2(YfSzH^k{B&7bt&lbq&jZ*kyzv$Pjee>60I?N5sZ3$tqv;O+tbK&ko0lZC!AIa zdTu+W6T#N1w9D2O>a!!w-RhnyUqMTEzw)LfXFQph-As_B>`ZsERIBpYU~f@;1leS2 zce8GPfyD-K`mtT0(KSEc8tc#fD76k5jR=E=-c&R@ZA4x8O`C=Fp-`$s`ZOhIXKKpB zmMv$*vM==qF^q=9X3}?^xbS7+X3NiCNad}D(>CPJ-92$P%UQ*Wh_UwRMOB@PESGNe z-SA)@X(FbpU-NVs&EMsqd;1>O!=W3niTT%@yf7;#2zu2zjnH2KBrq}HmeX+i;Q5~Q zUH#BVht4usV*|?ufq__1eF5U+Oqx5q6ovYPc7Wcqn6o zaYWdHoA~$z*N(_Yoj?mf_zfK)6FR_XhW=34>1VfPo51zKT^pDoLiKCgJlRXeCOv;^ zYGur;or|I^u1j-Lkv%yyw%z&=Qj|5t4f|xi|7VAq=jWanZt(e>LMBuv7Pq9n^G<(T z)$M)j^#XnSMb5`g0zK57p?`QR+y&svx7?}f4WGu#)x;iLob8Rc`Gm-{`)hbQEoJWs zSumS?EQu7|Zi{B@qv_iv3txH%8Ei57WO23b@y}m$ob47qSU%q5^R;}4aQ|l8ABmZD z56felJ71i66)t)7B~97BQd0a@@c#EMh(oG4Kihl#6H1y|-xRV5i8CE@$zYuY52B_< zdxu{GCn99joduEf!f+KTTefN#a4VG_k3GW=-IRS!T(NFkD-6%tZyz|4n=Q5*-wJWC zI*hDWJUi*eQRr#q&-L(h*hU<_CrZ46W*4ZH@9(B|xNb?zcupEM+3TSf8QK1~T2t;x z9k!q^mC-DHF#SbAdk1&+kpX$<*q`lYr-MyobCB7Z1Z4MzoHj1SeZks-br^iN9jVjl z_ZlDVSADF!S3@V@^1nf$rblwww zkd^(d*QCx?b&K{|^>tS6s$$Psa>Mp^ziJ>Hvh%oxO!dp( z>!ohCH8lRiLHOD(;-J4H>9JFHk9x|enb)NwMB2=d)m+s)V3&`Qn*B!d`TgcT*uN@? z`LJt%IyA?T6sjrD@RD*bN>7yZB+b6xU@=oammYM=R7sCZaybiz#VKRE?Pq(c7Muq~ z5L(+{on{UA(l>Y*?>#)u+3rFf9=}rm(EIQ-NlpLV7o_HW+!S{&s&X7^aMK(5@?~7X zd4d&J|7k3vub$t|_h=4FHL~*RJqCRebn~kPH`U8U_DTBt)*pQw$E7|ulH02~*Y2Ef zxf;q!Reh_u6eDDPYU=*F{#Z@G{@5VQ4Twt~;fjAo2G)9vJ8_$aiuiR8>$Q)`kv7@- z?|3d>tLZcfPD5F~KieK~5N_U69pMNhIp4qck?d@rad>sK9tDxsCs z-zB}-MuR1*_eMXDOOJ&2DFgV{ctwYv==4I)ki(IB8}UG{@gd2E*yihbhc9<0R0c8f z26g()j1uc?7riq%S*M`+Npa}5OZ@Enb5z}SZ7~DO5OLyQukSRtYZ<3&ZZ4YJAl7iqFoMCx*dsWE}i#l zBf`lCF8kgSfC2{O^7~3cE|rk?KPWYgP;cJj^J*+_gz|+ZyV#;5tqbD$%Qs1MSVGNS zO~(YOxc0(Lf_`>LwSBi>8n|)$`aM$2Wf|0OEAk4d%k94klNSRcu zxe}of;a?t?7e3`_R4;RlK=u2`etpD>7C-x(ygOyOPx-M~r?P_!^Vc%ps-feH0Rc9# zIn9Um<&`|Sx~AKq3ud}h3>B;b#Xc4mDn89*>h-pe9>_l0)STvoaN_9>Dr9;WxbR=+%as^5Qz}&SEySkErXFC3W)gktv8R( ztkCCGoSLkzXC?7W)x|c|hM{X#Tk8J8)E{DBumRuPJqlSBoF+!g!(Uk%i^JUvL*Gvt zk01VZCV}7T>Yk~YQz1yxm0*T`r0eMB@}ZJXRjh&v<%`p|?c#6ob3CfS2C6x=b z+s6o!OV%ygCxh`)-~N)du98C(R-iBN%5AjW3_QK{Hm#pj&6JkeoQo;_8nV zY#e5L?RTuR2QRNT#A%5{xlM4&l77wo99ppRO?x#(1)y)3>8R7%_*-BevaZ=r>lWwB z2O;8=)wY#Q{=JKoFcRv*53&08ZPQjFc4LI7rpzRu>gT)O<0b>`GHz zb86PUtlgm`lhYZ4EJ_<9@)I2-Leyviksi`oOe=By)iPNQmq z0e!QFli&N~P1ZT6hDXJY_Bh75+$R^s7XQ+`&!uMb0mlnDjTX7EsCkh$82Dd|BmDG* z(w#!EYpP4^H26sT(qWh*xJzCW$vJ0Y!5xpByV<=g}`Q_RP7nm3Dl1ZM~TAn|PSa>LF6~>q|$1M7a9`nD;r`P;`7E}Z(>!1&m znDiE#-c}+ba69sL%=1u&u<|qH?@Y7kUg-|8KI)T07LAiRA8skHB@X?ZR3=K8{@Ju< zHA`r_tW3OCUvTd%!}WXvyI{VsU^GS!L+q;6ml zjt2B;O7}{}tO5feLDQ`@t!n$byPj|5n(jL{%w1KyXjCvFw0&CVk2H(?&pClwCkgBv z>lK+?0}GaRxQfPdDwh@AqE}<|jn6!S?Q9mdVcQW$@4<65TTS|c?JhJAJ-+$y@kHE; z+1J^pvlauaNTibIY}>PAFkZOR0G_;ukg7~1i&EdkE(xqsYp*xtWO|z2P@U01aV^Zr zn1@RZ^F*NE+ieuK$9poGIF@tLHV#S&WI>o*B*kgr3frD`yV8KIq&bN3&6b>xRN+&De(BaQ!)CldBgr! z(AGN%A~o@!gAa}q0MMnP;|WVc1H8jp0ecyE%8bO~h8H<-u`#v(FdPClHIRN2!vXLY zOLW2vef;ijt$y%vb|DWbEsDomU!^6}jwRF>U|}2*LeT}xo*GZ*{rvjs@ii7*zmi0!nhsBl@_A8}_FL%3^5X7#gqV z$A0SRKf*o05Ws+6kcg3uqPRN;9v*UGne@~UEp%uhPH<9|2*5k;I0WIUy|Y)fxy%Ya z?Ew>*Bj&pCqwjDYZ;tgT=;ymD^7il6_g`qLuZx13*m<^DPve zLZ!rz0)k74Uj^_Vb~;LN%7YXPR1N$ue58C9$>0oQ{04r|D0+g0%;qf_QK8ygm}|kc z$(W-9AVf}4+&duJ#968(e=AMQjU1m!!3{`o5QUEyAi_I790EgT2V($yMX#II!ewug z*x=&OT>^TM)NxDo2QEXfXllkf|6ZkNL)aWAX{3OG`9P^KHD##b+$21dZR0l-H$}0=yc*^xc>EJwMrrC# zG(C_40c(Mew&2vk#ZncLay8*xvkBoNt)ywLLK!o>;+b-3kXTePtNut{;`irOQD<6* z2;!CHoT=O$9fVJ_-n98Hy<4mj-pX)SvWwk5%SGsA#)=i3A*iu!(y7vmBmUp`ZuKR8 z$0ZHF%q*AKefHBO4PG~x!m5p;%tPn}gy zo^8+xmkW{35s89WPjS+07hL`o9V+1Ep$vqOz6~p5UIi(wL)Qs3M@-)I(x_fP04}xF6D9%_?XQykn`(MDjU&X^oKaJd+;7tW(55 zGpTwhGxOR+grzDD0mnV==C+s$J``qN(7+CLsZ4eDIpnLnH1JjExwHI|MJlLv=GO5k zFQo(Uf(DfC3YOw(_-3jaD5GW*KCc9$PN*(OGO(?pfWn#e&&L-24UzPn#KA=R2!1he6MXEra4M7))02Zv41xMRjKXTkS2>14tiTzc zqKGX9^}sv_gd1TNSvYojY_l(o%FaO-pYNkQurWd&1l;DplEM9g4!-%~) zkD9$<$SN34fr(NwyPJ7sz5*X`BUyFO5QNyso9V-rf$w8ncXv$cf-xCWUl+^V#z$mO z+GBWG59}hhn!9>wB~-=f3Z5gG!U@uNEbMOo*^hymf-Mf?=7E3b$6wrPaR7eq-f9&x ziDQ+2Hi*iLMrm5gH*dy&Ll-U$Khx!()Eiy&?SUs)ibMh;mK@c^$~}ftJlN=5EN(}I z@0u|Ek#Xk$M8!2@`zMOMUe4h$>5#terFU@s-!pWCzeTi}(3E1=iIys;&0E2$$h>`$)Ls7mk zlVq{)3j)LN&q%;`(gw?{*x)IG4=VAvO*DfG7GQML2BmF*p4ixf@a1G`;@I7l&{UEX$*$wii1!!%Ypzr9Gm&dQz=M@sAmtM~;yOx4W`Y~w%pXNoB za?(c&4jz5a+Z>lRN$dfjHeyRTRzv+Sx5E3o2<|1fgzx{Sc3i5aaMm7zw3dGAe_4PP zAJB<8sG-n1Ck{LKrK%DAb1nY#gz%04HfZobgq;}WNANi%zWAO#b9q6bX1vZiw%y*% z)k0`XyCZkn9%`>gC2PLw1cF9U>`~6cP95fA;5teKbY0o6&p0Ofnahh1&wk3?sbi5I z=GoBtSE@H{#%RG}=0UmtK(mT!@r6dp6jOIfl3^4x8r-?RHhi?K=ho23p1}v{wTLn6 z!MV0;CPC5{-UL8(J^V~iV?INY@sYLX;|#IS8`;F&%UICfd!H2S93Oa5q8ceJzSBRN z!?CMRi>UMxp2+Gs^i(eGCCxo>PG?e;sMs2-i4^_ioBMt|W_A8kB>7h#yH=IUy&2ot z`zoLbCz@TwULsst-`FSv5BNQft~+F1u(4TPe1zOX;UI>VmHb4E$n+ycWTf@8 zOaUxHX;q_Uo_$UahAv+ZOZz}mb{sX+PkJXva{H~$!2I!FtXJ-V3l&{^C=+R0zKjLi zkgsw-&|m5jBvx z{b5n1WcjWgsrQHwV}c9LA-Vbu9!11S{G{_cMyxVSn&fZdw^86t=>*eD7;hr z$C!ZY2=m65{+aag^R3d4bk$w{)ERQGlwptg#l?1-RXo%T&|!|Y&GlubJ{&PBU{Fc5F`g`ui znGL$kiyA-EQSEB|gv5f{J?UB-BgQMo(a!6;(-Ie=A5R0mL%K<7BR0lGl;1wAay!j) zcef^c?*sO_Of|tVBT`H5MOL9vWY)b*7U2T&VpXRg9_PB z^U<0~s(ZQN4Q}Ep>Jt7_A`fc1WIhNn(yLB~_b{j6 zyxb1a)?tYx@oD9;{R*<<_53RYp7q_%JWS?ek9?8D@qGv><39z&H7i5$!QFGp^N z?@nE<^-MHBw1!nf-X^_Jmk=eka64K2zN&L%P4SF~LSE6s?EHq1%zE zNn-9_NfC?)r?W%EoS~e1b*FUO#~&H?#EER_x`M?0szX0bE#%7omef%vU}mU4(7TG*PvGQU`oknW{Xb%03B=?af7LoQBp>{|;_+A`O+Ez5% zxL_G_1ZhsFb>P@tWgWc|WSCcYb?@YpxI*E%pp0>2we2&C{$kD)v`L1-D4^ ziOd{#{<7b$*n8Dp_au=!)^xpSqZ=o{aeBAVJg0Z6vqcVGPN*u9E~h|W|qFPrKR#Or8kP;R$eW>@vz zxKM^{V@|@1%I;DU`|Rhll@|NG2x=bs)hYX3!-uj` zN#zP7rTmA7N=7ntt?PUIqiX8=>-L1m?^-`n*+h>FIrZ76w+&$!B$PCzdN!u26eFIB z$k~tipXYxN4KX#HY}RB;Zg*a65C|fCm@Fbr8EJ^z&2)AX(4Z?_KF(uX;|`52?^novYZsj;@SrX`LEBv66!raOHQ0ZW)U6a!r0rKYRnM zG(jGk>XFtLX)ZrZ)Xq&4u#*yUdhQ2Sw>$EE)SM)IM0LqMX4YacqPLV(W;iu2l*cCb zw|jbjsPswEITswkK{dmS)JgT-6t3#(&JEl*71*qjg>e)=Cd z9|ad1KJGi(XJ>2|T(zU4|LH5_OIKv}HDltTp`!}aiHt`^GExz8;d1+i~`Ri;lr7sGPZPuq4% z?Zzu3VmC!UYqoA^&{C+_S)--z>ijLcaarThJMQh*J>s00BUw`%nM;v!;@hM4mTZ>& zgH>JbICf93Su7{8cKf(=wuZ6cqI=+Alkkkn;<*Ppp_@N}_v)kCpwPvZ=o6EPQPtTG+zV_;mgffEYhH@MM2cyh#?sM)R)gFHiJfV? zrRUQkV+4Z^@gF=maD%&0T|btR*?;Z&A1o=2sVU-P?L=WsdS#~^qlZ{l{2EaSDsoHEf0aWwfy+ zcek7Q1P2?{=u8Fp?TC?PI*{I_)(olx#OBH{@Pj&Bv2rNH11umQ>1zz0-^h4khV zBiMq!==5H!_@(GUwcqM|I*Zb4n)AzyUvK@EzZ7K3rK1LhJHshwTG%=nU0svM?T9@F z_RziVZY<9C`y{P!nsCm*rbM%Ey(LS*e-Cs7i|-a=+4l6A|%^B?E1)EPPofQDnxxbIiiE$h@3T=PTkSK4DQOKOeqiL z?(<4UDtmhIdQfeem&RM9{a#g!n2=d;FY|LfJ$LUhPiL_{ z*T}_;Uc!XKSN{9Px7^EqDT{iq^a!DKsGVf)os_?h#B5-f^7WwK zSl3UI9=8u9j8>4%ZHDI}dq#4NJ0D9E0 zxwC*b%7G0*5QZe;eLPQv4dJ%v0Y*dXee$|AW_|bs(hMJy#>_!djVB!_E&u0?#&`tt zCe-oa0+{O$Nom0@cCOi7Xua`oW1nUVc9Q=iyu+H^CB-X%$Q%FwRW$}LcnT7m%j(PJ zCWGMbI)mroLN19`QH+64ri}WF_cy+H0juj}_5=uu<Y(q9Rzz0ytbx*|E^<`1W?vS`}e%UEC z%DY!1YhTciAi{kx3WD?vVpavWZTS22$<7D7VF(_$hkGCZCUap$UrB&jE%B@TA)L55 zED|owv`*wM({+kq)qwEz^Kuj@I`GfHhbHF0jYf+C*aZ>2%{-f~V?V|Q9FPn-j;peD z0_4J_WoxVQ#GkGaxrgh=wO9q-Pnj@Fa#Tdkx>1^GLwfgI!6Nd^pN_Mv#U|0m(5!_& zV~bC7r!Mbzib9;uuEoYtA?c$XB5XJwJ3+h+dBFUAQJIyjKB(feJ!evu4ezkK!u zvn3R<>ADrq;zGH1ZqLP96$D62eyT4!ut$9lLaQgIq1PP%Z@59hWr>a=&P23?Fvnn3 zl)K^%*Wu41oNjVi6d6@3&ru2&Kd)fta7PmDBUmW_F6*12DpuKgbLN5*3vRB~7i-g` z+PKDbO&k)p6ujHN@r7jvQExo*0m^}ipsC)@gHFS&c94{!M}g%?00$UY@~otJxHI7A zT4zrkn%CzQD*4?D(z0}p@B(}|>*hV6m$_q4m2cz!P&4zOc9E z?fa}5qhz@Kgwj^PN%kp3-*ff54K9+c(8!!YimGgJ6g!HlGAb(XA|mI>j595~K~lA`dD);zzC zUwuGK8%|M57r+5#8P3S*$i(OOpoz^^7{O>c-~=lv@$6BN6+@`^&bb*aHb`#ez|d%k zI!G>AVn)hN5h(>_?~s>bs#4*bI>TVtCd1jH-CeFLY z-=j%Qbn+N%XI=q(zU|gZto*&D7 z#&!Cx|d8N5eU<7%e|lI{ElTz5oe3vx91MG}Ec&~_N-I`*V~K1f=?ypIWx4Zf_Uf6fN5ApC z*vq0m;4aqBW&GM6H|5M|sS~?gl9v;5)H#zSR;|&0CHKvkmr%_5!|%VUt-Zn^U2dKy z@{f_)%>_y}dcH74>gyVqM3r^P$qYrE7Kef2#)pycp&xF7jOZ!;tqiB$w)b_?+|5)W z596APkojwxFw7nwChzrq8zVmbw9%HpS8{oUiRL%HL%ZV}j(}SMVLC&p>{4PD@z_U3 zvMyI8jZTss@8M&~G5_o(7c^KbwvkLncXV!#A5?u^8$)(y+l|z39qo+qC$fbp+e?m$ zh9)7tE+>4w?ABjqc)qYza4{3H3|E`pIDu>km>KlD&`pTH&el9lu8<~ZE8EZPY$j3y2`3Q-k5nv^A{o1W(TJDmM*yI)5B!5XRQWEc%Xkh3Z zZ7KV4jMN!i8>}-3tF_dDwi|Q)0!qB;*V9g#r@{Vx_t~VXhF6N9ztEsbH&CZKTEx6R$A{Dn) z+O9OLoJOqM-^ktp#9+L|1W~o5$!qEk6K|c7i5xTQtX6n*u&2WJQ?#V?#m`b^K1ZDV z0W1vMU>D>A zVpP8-r>H?s7a!Mi6W};Bo-V#m+_12e<7=V@ z9ogG)%+UA70F<14#hp(O*i8w$%I9Pdc5 zcEeiR!O2O=rCeG1d6%k$mtBolF$tXUYomL`o{;r2MX|YIl#bX!D+0x7I*A^~^gZbWDiZ^(=+kSBkmyom`1K?1({pO| z0xriDDAIXKZ93V(5xEasYK=lFI&|2p6Cx+C=P*|{jYcMJ@Ry#s76q=TL(IQGgt}T~ zMlQB6&s;{VDb{0vyTO9+_@qtp$04S9dmhOQsf_isGI+eD`m$vd=bT}`FSRnBTW7LM zr7P*O39EQqNOO&f)}nWxR(u!O^z`qISWpsS88QSLYT>-6nn#}2WbIg zr0a@@B^RH$kYlU|g6r$IXmxpYunVTX8inY5)l}%8OE|f!V049IvqAL9a|KT>y;GKh zo44lNJX<1Y!91Ej%FbCugH?6|d6zxxUthoZkW~Ngh`2*o8>@Q$ykHlM^m0PihO*e- z_#WG|j|Q7`B5}p1DLt-5S@UZLGsBOBmDGU@ZVslD5~Ut4E{%^TlZKd$7OZ0U)ZR1Fh&2>Wsw6HWZD+2Y zGp#Q1L?ePYoGj8Y0F+#OAw&!wV4p>}@Aj}~4xV)N+D6$Vo?OCoDC?(kB$al}+xdI9 zJG7LR)N^yqn;PPCSm)w4a;lGtSYDejI(dO;^XzcYlvAB?F_|Sq8Z=R-Azp5loCKL2 zPXo#+4wq=O9ZyF|-i?F76%ud!CcN|S{L$cP-#?y}n7uZ&HzF0p_U_3sU#KCrW`F}qx%sx!UWaRmrBRXw0zVSduH@t)Vzups+@Z0yu`-Q7{|e^)?)&q#WyX}J^jR9m?sU_+kS^+%v%tbi z86>3{kQ;9}kS6-&Z`jyxkya2aAa&iAgh+r78D@IzX^9`Qv9XXQyv7d(hIpcUO)V`pj3E^x}k%g$^TaJ zjs>B))$FBwDl$X8DL5h-pv_+yI>zeHK7Efhb-T5a^kU4Po5vxHIUCLE9_`b1cfaZT zRct0fefT4PBu#>KBcnW_`xN4}j>On|rMTHNb~Y{N1AkfP_@&(%1i5n^XO8I4P^SMl$h*lqpXjNX9(MNm?d!&8-?}M=9ZaD3ba2Cb`x>XlE)9g( ztgbKFHdFNC{#QSn{pMf6|3tBK3c7%x>xX^AnkGi~BqtcV&{9KfXcAYT6gc?&2w@rs zx*J+|QlulSQp&~3Kt1q$sL;l{^h9@Cw@0F}^}?81W1fs8;<_t-=UK0P24~JVyxP)J zG1#N#Gh-;T>S>wNpK)C%JDcM$x7V=O`-A;1J=XiS51h8et8%rF|V?;v01PX^VATY z&Uop&lKbu(-=-3~)eF;KloX=B@f}}3sz~IpenLhJ=7Zi+I!t=`-(Ge6Y-5))c8_Xm zD|~?XBNBb**5T@g9qIdolpp>PA;wMS7du{A$MPm zq#w9?pxm`Ddqe+@$6Y5lmBlI-v{p*)7Ur}YIKx2Q-x`co%gQ)Um8dNidkD0TM~L(` zI884jH6j?9nx@`Qbq?7YlpS3!7X(XTNPxJ@nd9mL`gg&o;4?3U?IPYXKqcyn)})%p zKN;b}s}f58DpHSp$pHF4kz4-n(oC0|(fuW4Xjxy}QW0Zaxo z`#q+N!kQUqkG@N#V7N(n4pT7UsN+9@g{bcXhDaNs-2hL$1d?nNhvHd#;GQX;&p>Ak zDQCDZ=|xN_%kzQ$kEzNsY*HC(B*9k{bu~qCZ=PE~+mTf(p5xxJvABZLaqi&_Gf=+?_M0XWCS?>S&-%^#gD1n$6K z<-#+WR>w8(8lAD=_dUzgt*-(5fQAbbIOe1WTM4N2f^ZP>`_wUJOoLxkv|C)t_vK;x z@1LaV`(81pwUDU+Djy+JbOK(*-3=1>MBsOz>Ehj40s?qD+QbGoiPt!u{}_jv;S@i( zsLs2CR^I{JA$P&bF!DP|k1i;nZVGt``JCisJT_bG=9f9Qx zMI(0MZ!U)W`|y2lDU%O$@I%XO&Bs$1TH-=9oC(0qhV`o*LHZbzIQwD^4zNHsJUSbz zN#DPop(o0j=+V)+!^eA>_zr2h5qI&$$|Uc6?r3r{&QRQyz?@W)f4&FRA6EctG5Mdv zf@`}5RKXJna}1Gi14;%3moXCvc$SdijL~DOiKDL&dbhwS?|SkcJK#xqT6`2p0F)c< zPVo3erH>eRf};anF+VLmNxbyD)}NH%?#J3=_>*Tn?Z%Ta9^gxKf)hx}`^HB`ICmUG z&R%D+jJPX~t;gi#s1M1pRdA~fqh8#vV6a{6BTb`CfDf#+6NfqC?lm01Rm*I6jrsSB z(SrB*Xrm8;a9o?Lg5Oja*vNzX;JM~A@WEXraMUG%2Mj}u{hE{e-okLKO#e*6MSyKE zvl9W=_dnn_Cmx<4arQkAbl7IbQ`cc|7Us3QWJwc-eHaXT;QN<9fabB4;2MmdQrvyy z#prB>+*y}+{`(0P z`!p4w&a5Kz)G60vVt&&*8+|@1aJLcBOZ9zGQd}f1lEO*YW0V{U zu?$hXbskJ@#(+-)Ya0|3irQQ=C{{;B0jRi`pS-|F!@P?04kOUmUR)h&U(sEjuJrW@ z>JE06*i3Q>b`!{a<2wh*jH1vEZQ#o9MP9YKZ@k+V8ESD?-32W3WuxJOU~O%X12$M~^TT7xwM%pO~gg1$K_dAH?-!OAVe zJ^BCT)Wuurz-Wt)cgb;haET&xeGhFnx^!X?U%=76<sUb1XOL&*+1M7xi^Ia&*Iqwd?dQBIiw3Gsylm%QKF5w%iw8EsGSCa=b$qK z1{&0cYmV{by5V|^{^$-G_EfV0Pd2F7_4%;ZzA!3B9*?bOB?=%(5L^q)nEy~HwI+Xh zN%gvNr=Mp2H@?)#Cr>dp6SiDDW@0VI8NUASTn#*X;5$X!gU;0ev5gy+j-MR=JSw%>=e{LTOn65$OM+3O{a8AYJYcmrlQ#F>mQQvf}K={X8b zCqswzMB<1BZpXjm=K&*2TvJ7OsZ3qCDhaeAmeH{dv)yo#KccJ>_p$}o7}L!OfP2si zoXna0#{=xqcK#TwuBYr=`)z-JiWY;%7@H zSbYHsCg2Da%D55GzM8i`@u+!p5GIxYY{#)xyzQlFX_wmt)fU9o;}Z@62T=GJ+n@>f zGV-(=-{j9PjYNe69Dq73MqkY+BDbtK*&5Qlp|(Ob1ZaIFaN!KPJAD80#NuK8m7e<_ zkQN1OaRmO~ZZLSI)d22p7z>Bu%*pVfaGNdO_yDhp(f$+Dv-3P)tZ6_Zkr4JBuxrPZ zf0(5TRc;8SzIjWA$y1rjoI|ZkXRzLIj znAvb)MkSEB&b35!Bqqt?>B#Ni)wAaQy$w09B)Zgw`L~UJ-!0rB7<6U(ALYyC{3=f4 z>sH4H!@Qz9hl$sqd?aDsX^sr?uB}-o9)+{-UbtH^s)~|Vn*~D09wJxiXK#L$^(4m` z)~|IHmwH7>p50E;?i^m}%xkg`RvY&(GZ(bU`usjw`ER13+;dl|)thbZ<-@J2IdeOH z@8cG}H28wH=>6|`b+!f9H@$1Q1aUpZafc^7eaF2z?KgApdQNTKxenT~b@5Yo!I2{2 z1i7Dn#4anY0+Zhcf$?MSzW44!%EWKbs(38_YpZdI{#%viwCZUVMn~K`A`Ms8Y@H$W zPV*N1<|Kd5jMDky9=wS*>}w63GTHNRiw zeG9)}=cb$in-ff~%(>*T&qU<)dXsH(P-^`2SnzGHdg8j#q^iiZ?Q9gc`F?2noPDmk zA#-!Yo^SrV-v7XnJobS7!kL*~$9~x}FzQ!5@wI4g^Eaz2I!`U!zH0kI$~Hqw=3>0; zvjg30)lsL4zZS~6{ux8}UhhybY6DeWTVc~V_%a>zY5N~JR%s`nWPLI2#6F`dRUKg%bzyGXUa>MJsWWkkj>#dBh zq%8Eh6%ew)@s72#`S>-EGsMeN(%TbbZBCWi11#k>c#{Hf$^3hT)3{lA7hU{qs+~Yj zcwwMo>>Ae@W9|gq3vQqn?mSOrb<%(SOtB+c9PD2AGx3 zK4?BwGfaB;g5dt01N3nosO;=_8rAqbJ;WBwZ$Mjh{Y}`+H(-%P+V!_51cUHh@AA3h zXK?UDBBk0}?r&vfek>^KOqP(t6?3x7CGk!HYl?sOH=4dJ%35jn%Pn5rZ>O|oi*Ex} zV85WNqc89RJ7k8tKV3YSm$9O$j?*wBhTEyLtD^JV%Hk(8()wOd@@MCJkE!?D_e;xW zN~i;y8%a{Qb2m}F8vxI`v?qLQaoSW*zVU(BAA`K_Mfoq|Y+7pVCVg9-Jy75rrrKo;OT!p;U;5xPe{d=*dT09m@D$qUJn1;8Yx|Fz;J) zu4u>o)agp=_zhHE;0nHY$m^oeSs)yoqza<;@dDH9?H>;f5TA1F9J12%>_?!(Y#X8~ zcMef+Ygw`K;nZQ0}CTCvf_(46z-h=s}P%!FlL8^uktJfjp>3iJx2 z|B;uwHmyoYr5uZR&OfJ2+wc@7nuHw*`KylE4)qu`=XXv6pL1}>lOn?JoFI%DzelQht%ImFCE+1SP4BxZc z^C)Lut8m5688Dwaks3WYe5ysYOI${N(}efQyMy7IN6vL57^iRQEt#QPJomC}gbBZ) z^d-E5*JqqgtrJLTQ@~9%AiztWzGAH0jk&V&L=NXq@FERRBYuWtaLrLd(01Ca-Np+ zYBr>sZoCnt9c*Ue_YA7-{0*PV__;diDbc@ATYCLo`skp9PxqF%YEx~6QS_DD?(KZeMV3R>GHAOV88Yd=E4+li9FUuQ8-~7m_IB&9HZ}x@X z9zSdPHfitUW>JGDd2@m$ZnoYr?TB>r z+!sZhmdny=d1(8#p@`6R_n;kXK0c&^5<#p;CKNJpFGEU4(n^JNh zfZkAkcXsEe(i?Z4i}K~;N0|Dyq~`SV_WbNUu=8xSk??(|YqIJypg*r15%bGwTax~2Oc2VVZQ_89sWOnyY;9VYV^4mlz6nufj?dz-sicr zoJ}9)KM$(|b?mN^psmMGFvMG6Rw7{q%nJDoi8E9fkPhl3n-hiQv)@4?7#Ep|X*p2{ zDpK$pa%|rCLgTYb%x{nAO@tlJ`i~UU1XH8YOJ&yBbi9a1x}{U(0>GoJL?;Tu>Np>O zx&!bZP@YP>F+^+%K3&=UCM=_l)q2=@nG0w#YQN^tI5N&{h14kcZwNjEhL9iw5r}{* z(A$7|7rwyxF6LJ-fGsFKklL%e0z-1ip!PcsO>p9w$8KXLCWG#VXOS?4C72u{t__k+ zz=(>O8urp>1R$?4;uPQUF}e{nLV3A~YLWR5fr-ew*0tuK4IVB1!3QrS`e4Wy6-LzQ zshO(2EEqCnz98;w*v<-m){JGiAqv=!dx8CEw_YOj7Am=swA2rwdn8AL`y8jD|* z@fGC(VEY?DNZRUhHwW2AM)Zfg8Z71tA@x_|1c8w7|F3EcCWrBuFBB9s~BjNbM5_hffdf-&g2f_a+8xfK9?kIp)F48a3Wg?#z3_CuI{s zXHEmj2=5hKY4yiOh>@yK6$h0$uQ4xc^-6^>Ljulx&$JyM=FxqDGENvqSb$=vDwx|$ z5cv|O-gSUiVjvOp&=|0yc3Rk6tMMKMn#fAN{&UXeSG{9Yb?@c7$@?NaVkoq_$v5VI6J$2vgStS?VT6S?S!Ctg5;1 z;wZLH$C}D!&f&{FW5V{((y0u*&4BBAFt3d&e!`Du5m+G*Uwd=~=u7jT_>6!@@|&oD z&p-w{jSQ`-Fs|MKktRZuktnfbnZnmXd%68xsLIR*8x5}1`Z^jF2p768>Wg{0U4@uJ zYnMCU-*|2@_wMdqSh&9jTsoOtC|ovtlROF_DbKtwyTxZ#iwOz2WVNtH_2@$<)6D6e zF1S|bVO1Y`DuKujoOp_m16X{93aOU3NwgjT6g7fBT087yY@s7r!P7QaeN?b(%Hu zeSS@zagg|YiI3t}kDUF2Am;uKv(c|O&8=QE{uwzX_>GjN%y8Z|%-UY=?rY7d<8wE* zvyxz|m{-79JZQ#+y9IpGO8nw{i4K#|jxjh)BQk8aPo&R4tu9lMytNZ%@w257y_?In zb1!xsjwC|$XWjV)0kRW{Y(wj9Z5Gu{N|0RGX=|x}CHsh+O0dD*xP#`a7|FqgWEcU61utFtOGI-rZ7;=hO;7Mnp->5$#Pbv zw;iL7`1FS%b_a#9h~QOBpN0&ZOkpRPh8D&`oSXTc|teA7Hyy4TG6k;FV{ zjS<_9#A@@CPe&svv{S@MXdd3ED_8ZaiG;l>Zgl+5KY$CXd5F5q()@{a3r8R!cGAe#e8{k z+{rj4N`3-cQUfaLJr6*v#0}@1)%!4*M3RE_GiH(ENCz)Qw3?PNy}NG+Cdp~QEb<%Y z6#Y+hT@n9b&Z3_n!HW>hwa4^q8EN)(DY5Kx5`mxUKMV-EoslK^E=^$wwLBs&F-RM0 zoRH=iabEy(HW_!aR7Y=kWzwxBv9aWk#`bR*CpwYUXNop#Wt8M$#CISJlA%)@HP+#0 z$RCpG!3!dZ7Vt4Hg|q290xq+R=zHHUWltDb{de@+z-<@9#rR-FkNr2I0?Fi9%n5bn z+e@46(B}U+U2tLAEJKj{wE_PDIJ(nRNfdu(;wZkNfY*{RGHXeS>`!5B?pKyY(cnD! z=tnWCQe~TJ@4}$jmaC)QZH}RIJx6pdXf+#>qu=5b{U@ALn%o? zzhe(c6Wm=4uOOTu1kWCayW*=}&z>p`U*{xy?b8kElj-%~cEj%m`8!@eaK5Y*d<}5vB+~qXfls-Ph^;AXCs2t(3x<~Z!xBM?g?Pld0GlqJFkLs1D zDz&6k{YO7&)y#!*%>4A|x|0)Nz@%aCbE=lu1M7q8pbjTerwQVsd@Xzah)M|VdDa|iwE`SL}t>W)M+ zxqWb0CEHs`A>hdFGkfb7s=CQE3O~gxQm8 zwVU64sK`Yn@#M7W{5;c{+(6Op3svqC-Z{({$06nL;u8+?JkB=wu$x0T+(YB2%oO$;UVMHsyZmQ*N9ddV(`Rh=H%Jv3y%3Ko z^R2FZm6~ATICslw)$Auupkv~QFd0$v7C->lFVfmqJ^A??Zbf)e| zaeet5I%dX`)Y*AMK6z-r8wb`j^T)K@6gk}iRFN+$IO@@|E5m7IcvX?^1;Oi7musTa zdtcgUeh5qYXW>q-16s6jdEK1a)Ju;9r_#oJWy$-bJfqt)_uOiEwj;GIYq3`i z%D>jSd1tVnXKvyRdS9;k*aB-}s1h8n74Vv_!tW9;o3AqHN-j{PS=5Do8Q<#?D0&W- zSgBg>(P_Vk{vEcN%FA3K5I^QMI4*MM{=4p<+?uUh9&T&*gc0sfoqIke{AN6hI&*)i z_jAZ}d~bCHW{wp!%-*6E!l~T+i@z#rU2sh_-M&L-8fS3h3{Gjoy8!Cn)13|_rj$lg z(sndo$Ky*-A1;$3*iA(tB?UU2FndpXG0di@f+o%1HpU0?ar-7YJo z$L9C!C-Kz7>F!$MLk*|LC(roOHKX`rd#UHbc(zo?c@wOtT;pG+3SBw>BS)0(SZ~)c z|A=l^>F_!gXVLKrM~iubS5rrSznhp}860eEq<5?_X;TTUldtiWWmOz*Nw>G|x>MV7 zpf_?iAt9Vjs2KB2GWfmDaxs+dafwu zgybpfjmK#fk0vqJdaI@Ofjv;1xS*nu1N)->jjNI=6Hz;Hm zyM;rixik(Hc7(i78nnSiH%xE8SF(er(yJt|+AepWTi+z{NUhGkLw9ZPyggLB>Dc@u z^NiNT+E+TANrGZ{x2AF+f1s7|VK7_SzfpUrWv$0Fuaw{J7ROrF9v<>JPkY(0@TYAx zTLVp<$|CQdQ`t{d?p5804(3m~u@l&OYhE&CVh;cvg zWfE|C6L*w#js7B?yI}0)^Q*=*uQ^s9Y;68ZIp^KBF_PQ)yb(5W)~ahUA7+LrVtt$1 zIb|~sxq|+QCQrF|0hr?qX{FaEj2lzQZ|u1K>Yl^O&z2B@-lkaG_zY1`PQ9NtHtA}ye#`cRM$S!kgXXIPaXz`HkAgoq%&S;OqoZp?M4rYH5fWv;wkD5dnJ7YbkX_YxXBiU z(9F|mFw5g?ZXBps^aD<^R48rB{B za**=W((~T1x5Icy{ni03PN$`Ae|s9;cpH0j<8jJtN=b9gx##aSa}2dUP!O>0uQj8d zd(55Lc~dm@Tv3DUVB+Lm7UeCRX=`*xpHH7&JF<|iC_KORsFA(@0AZ*9I5Y%2We4Y| zIXx%P_NQ(jfYG7*Zt$HMXigi8iu<$Sqo|rh5a@$`Isg}#1BA*LY%xciU4YdDH8G5B z`RIXaTnC{wAqb9P3HC=Q1eD{JY;Ud^n-BByD07?!LzuoSoWt8IQ*AmV`<1ipg~KxR zW<$}JEyx5HJH3;na7Wx8R5_^M*f?$5Lpb{-U2p_Gh|r6ol#f6vgha`d9bmbKa$*C9 zbAgx_d(BvCJvKsEP!kv6^NI1u8fWnYb?%PegYBWAz}^J1B{LxjdXi9&5D5eFD`ATN znZz@yqkx5}cNmh?dBxCBp_B;dDml0q5I_(+^!Ei!f+!BG226sn>4PdkhpgS&qu@dG zh4l90>%f5w!=N_VREEB))pjkydyx~AuMsE{z}rt+2V>$$;1=f0Fmu7LTXYU&1n_sl z-TxW~r~z^xTcZYDK`QXVCF-z1e3Q9*pDqE^`jrKjs44sM7=S>gY_b*{Ya(3A$^AV2 zjSA5eC%9oNID=n}^`Atlm7cK6;t7O6MCvkI4J1p8Y6dj+rt|ElNMA-bGvZQ7x8NVy zA&Y)Eb5py9Y)sgu`R_$rQtx2R#+>~X+$$O0 zVY@X@bwS`m^&N%Rq#d|oC_fO{>;gm_kd7p1l*U#duA*ww+O(PT#&uC&2eflR3NGu41O*{vJkW#aaFx9In-@F=Ia&=NXwjeumcf*fU30q zAYFT3OKc0!EOLP|2$3E1c-D3dgjO-R6NEF?{iLp?!2Kx`+@H|RMP_p7G2y@Q<&@J} z;x{mcic2+HcL`|J#)5jmP(6v)=OgqSJ0qd~hcIsCA|181w4a*6OH2i!9zz>aHbJrr z1n!Ha`pOP=&}=Au69IHz!3Y$eac*;x+}#nL5pM|6x*P@~aj}6>R!b;rP^p&;h|bpb zEmX1b6999X803O^^w8tSuvmu{aKe-$LC>3^L8{R%rXU2|Sqs(7X_{qY3fj!Yxop+Z zaNhxH?qjzU@N$Wldf};WW@%huu-1!z~km5J$+er0t zXpnbGZ0??x;6o_DQ#$^C&Ol6@*J%19(h~mbV&&A+ z!TzQ6QA#&PjnN8VsH)2$0!9d$%@eLdFLe&?Sy*abk%_pKON-@>4*>lIO?A-Z@uv

x%qcG#41oZ#<>6`i4nTliu0b5oEN93aRV$5v zZs%%);S--J`~rHg_uNrmS=R8hPl0x~3JIIytp~T)&~b){J_@VqO?9je5898-Ee%$rtAk`Oe}cfKCf6ueTuN-~BMgj;jr)JM16#5;Ev^e=LOA+pTNdmgw4Bae(AqE}-dTr#H)gDxhA7?bn zUqZp)fFEG3CR}~*g{^k4dbjSY@eu5VJ{GJ_jBXN zEW!R=-%c#ubX+I=T+2M8{qMZI7K=)r)@gK+VClaoq-;kmf+5Hi zd}W!ijb6MLtx6S6lF{pEPPT9GhFyNf(?@L*xv+*>P7^$gqWfCwis26CTSMBRoJM_I zL(U@>69G{yw64vq^Eej9t|(tLNG)4WulLA?sYxbqT6GVTZ~i!pG$cy+SfYMjgru_V zn(Wy2XnX?Q1Sx@@T*pls>PG<*9(1T6F>z$$h`52;tnT4~NQSZq7mY3)x{O$fo`K51 z8WeUC1`-i1p7|N|&wls$JIefLlqFp?U_jLW)tl|6KK8}uEhy4J@`be)m?<5^G=2<+ zZqDoNJ_8NeX#!;nNA{YK)gyqm3g9v(E@Iwq(B$sXBnI_lc<|Pw!vm*VH`1uxeG((a zDsz6~YbV#kh%cb99rxW8{@Gk;9u=9>I6aK#^K7N%Gja(5^8)WME2EOPIsankUzdgQz={ zN7OnxD8RK+OXEwchg;5QAH}Vqzg9pB#D5s38>XvF6IQN0tc+b{_S!(Hvq88|loE2X zWA@~iVbv>dSq5~D)|kQ>>-$k@ubAkbxGyQSe*C0jE>&g>aN31qy5XSlt=SNIW$wO{ zUu=|`y}KO7dFo7UJW?!lZTB89yC&hFbXV)HOUCc!8V~$686uN`tW>9p@=4kBxOVd% zW8Ku7mR{K^fj3`ob$W>E*(SFJ->@bNcDS+8>caia*|mR<7Aj`_)%dDzeX+L z1rsDSuckGRQ9OgNReEEiyJl_*1dWHu1_qi`}Hx^taqSRj%+Gvgwug$j>9X zYO+hurY%1Z;n()^-re5LS2>wiuX_FA9n|3@YqY-c_VKkrRikD?b~wLkn0MyMoqO5; zgs0caFYW6(RPg}5pEA0ZevzOQT>3KO=a4pU=V#P#dT@Qk#8#Il*CXot75{dTiYni< z`O}ilkIoXEr;eWeTIU8Om8F&#P<7Y0oDAF!l$kqMqFCQeCT88F_(qaJVmB; zQZlE~_izWy!dEH!b}ip4RL-1JPYd$C@TG11dK-A96RN|!Hrn}7I@N1>$1%aYO&{hs zg2OC7k&2ng*`=kK>gaqr_r)FR@RgHZ!X1h+_4|Nb$6+txeE6HE3cZ^ElwQYPyo$Z! zr@ZbziSAwQ+0?h~86+)t|4|uHja}FL zoS#T6Rws9IZ@mM2zQ{(|=Q>PP+$<0rPwT8|+m!oEE5$6m)8Wk#n1-Tzy`$nd}F^MCB*OI=#Zzp|psp6me1ocF#nr8&L#9^owH3)!-Fki=|MG}x*9 zA|zS&%x(6W=Jco2p1OxgH+1qNsXYFPelf7!e?72Q z?iTMvpI&?FQ_)tcg4K^h9Dko=%&#b?A?|we!aa zGIMewOT34=|MHd9yxx@VJqwC}I$-6_n6}OisYQ^W|-M&}Xk1kT$XqI>y zPwS$>=C7la$J*i!wcOvoA-TxBo>O;~$1ClZ9k8x*#GAZNZ|q8ocX6b9^Fk`O#t+^U zo4$$P#oap_G>7ZG?4)MY8#B&JS9|vchvGx^6N@dz3*LPaOHWpV%uC`#MBLG{Ozsa?>B) zFOC;gZ!dR0ZR@Xf|L*7=W`ZhLU*4^Cf`?9<&fMu@k6X$MJCrnV@3fqiXUB2sl1HkH zHje8L8`XFyY}(DJM7~t{-$*&f4(#N_PiG zsS~$G$uri?o5ztKHd0=Z*T*QQw;qY*W_=#Mm@SK46)Qv{?d*zJV_9^_Hf2U)c`m!Z#SiWl#`=pp563Kl)HvtmXh%hT zit^UYQ|qtaw^%0_^YR$`>ek^cnokxkQT6`l@W6cpM=sAwqyMHhwmS~`%6b5qn+rAW zLmpMd*Df!62xnQvCSHn*@=q$rOA`7a_>{>WGq0;tRd1&sUwfoguR#0K!u$Dx=CD(u zbh*R!e~ZiqD(x_G| zhZY2X*A-haKFPH-)#J&HDYwq@xk@lz#g-+dAfiVTxe-ecE1l_13!1 zPaA@}HcXV1%C@##gU0$)g*enn`-yS0ppM@F8h+oJ(u`K7n)pSqBTNz`ZDN2s)QkgE z$tFU7_d4eo9m|*9``()vBW0v-d;FJg`jowe-*labDK;n$@~IlIxX(bu1i>3$;ya=y zfobm#qdmM+iGfV9c{;Qj$-0LEK0Tbj8;20qpi9H6DgQlh3Q)#604KQ zrFa>xVV?jye{}%t!_f$C6p# zb@89c_&AdZ0J{Ot3LI}3%_zj;1kRP68unB-snuUI;Hsrgtod)C=jF{Xi!@MSqix4n zIjLDp3=Eoz>uvG@*DaVXi4$frB%eyEB^dHLolnW-A#_(PmNV7EB`(Ok;mObj3?c#? zgGgG&&6N`ST14q0yMs)MGvZtzE=UNYJ_D2O_Jh^}$IubMzQC**nk(@Lqq)W%gb3D~ zH=7Ri4#b1X3BK421e`m_WdQmykU@nEA|gTGa)C_AZv(~eXomMNQnH)?lF>KYLBrJ$ z88&Qbqk|mrI4(2G*gIU0!di=PH(^*3M9uydWQ9EghzIZn044zRV|2CP_f1kydLtUb ziDBf{qS`~R$VOM032t^!v$|oX+X9RwbB=n9=LXIO=XwKV&W;JU$d?;eSX6N=y%60N zhOc2bFSgC1%SE}@218mPeaCl|(%LLo{d{)>7*G-(UuRyQ_R$i#cvef{(Zyb~}Qz!V3arcZb3le1C4Gwv61+w-#0wlT^ zNM_j4iXg|QD9^2K*D`X#cH?*EHR$uO zVIP+0zxRS>Shh6YB40~Jv@-#u+i4?j1`huTdV>{PX3Xlz!<%P119 zAcP`YK0HeHLdAPqf@@k4Jz;Gx;7E%=9iSg3+poOi*qo&5u-59b>f(hm%^AQFFlIqc zoj5oWXi*F0&0x{Y1CE3n2%z^PO?0p!DEAR$0K4VDLYPK}D1BwvO15Y%42A(`H{^MP zaO2xv%pQdJ1M@v=1iSx{WPK=7nS)r9CF_g|?jWk_&|23L zCk5>|hy40ba)%;%HzreQ@{)tlX6dQO(95j*7)1j_09nerz{V`EIZJ3_9D4-`+xG6H z-gQK@0RU0Vc>@@gNLLm{%OiuGgXeojALe=i7>8S1S{&!DKx8cWQtnwsQwI?J-9|dV zi&}9!pvgfXnxkPHqO0uSDykr!p^&#)b<=TDf&8HAJ-B-jn4k4+pTYiP^cT^5ZGDC) zFVT>LTQ7T;JGbqpYkx2U56d=>*`72A4p4XWl`^)hVDTWKuJW$1@;8R0y;6<%NA!cu zgQ@y)Na>Xzb!atw2KtXuCVdoE!^41vA0RIvPI$H&LvD{ixV;PV8NA8iP+cHZP<6&x zA~?ahz+yXvJ(C;(Jqg%#QH0Ct^OItedCulnV%bYh3wSsLP~{`m5vvr^Fh1l@_EOMVfM?6-cJ^`;M6aptNIxcKCM6-L_XD z?4^+B^+M2i50U-hVco_SGpH~FT59*S9J|9_@Ju2Y%ft}_R!rM2iDz~`U+fYc=`#R& zwsyIiVM=$UA!;I8DUVWVUEq4dB!0{bs>c{^ZPQW<)?=KWWq@v_Tm7;lmbF63vBQx* zeN>GWznUx3wQal4CML=-97Kkutp198{!gT2&B70V&xDw?B@Oft?ucMblnbE)1a~XR zu8HtBQ%j>l$M!#+1O{Kk-6f=%1TSOw*nvq^Se}K^jt_^-#$haSk5cq8uX?quT6&2r z59&>OeQH!5Tr`Y8o7TAD8mKXDJx-k=>O$aOL87wHz@D*N?Jf#A8#8=AFhpkYT8HV= z*!D&TH}o7)ACq|Vu`pE=k9Kz>poqMw)_6tZw<%M1~QA|2dNt z*CbUm7wAh1o`d2sJSWkFbYqbD^>J<^`w_-xm%>gFY4|BSny`>#1A=psz z&y-mbmXxT&Gh3Q><5t!ds9NE@6_I<2In8|!KC*a4pOf2rDtPkGaDKRGYnoeomzl;X znf>DD+dF)+plMI3YSUBB#~n(8)AnT&YYpi6W@w+Jw(gD5;pv@~z8;cNHAYmKG3oP@ zv&y9hzOt&tjqoF-D|d2hhwSj+sj96NN%Fd`*9t@DRilG`&$_UZ-?##X1sy)&{tNVMLIwe{PWAbb(Q1%xW@F4nUfvG zzx+jAkD_(N4~1RBt3jwppt`_0D!14w3{#Zd*zPEAqzo5cTvbVM=??{-vn+;hP<@ z@$BLqT=9}!x?Q1JhtDlICrwJPe7pzSTaMu<-9=!jP*X7aPgQ=?ZHL<~|3 z9RY2HgecO-?gb$U9v+m=cIz7>-VK&9-;LUogqAa@M*$4VkunVlYwvngMbuqckZ!J9-~HkS?n2MofXLA=Il1>t#-QYF<}bj)!4iA zW{bTR)sQd+jp8>JdnAT|$h9xrbE_P%)#^feT*ME7_%$!q3r7SRd>J^h;SVhQwbeSq zY)GyFWkYRwCa&?CsdzbOquTLyz;Gh3h1zuF7Z5PSXFxbKmO-W6AnuX8&BCU`8%he$ zzpItfO=BwIlA(RRAGqp(7ok=s#u!L{j%<7NYzOR+F;#RMC7Po?YZ*$Fj~@E*o51#Y zrLo#8$}?wwy`X$v+-DeV>B*i#L#uE`X>5-qDn>bIQ!GiwDSn)W2v>CsyXsuQ>E~^x zro}F-soi`5FrrPW7FOEjqe(^BjN+_oe81hj=5O6if65dLq=IwYf@)n=jrL%DP_a^3 z(Jq8y$>;?#aTsuy&~wPdFQY^kzkPWkr}Qb*ZmfuYYW)`$7LSK=ThvzC9_gdY7`ih0 zHZ}~Gh?)9zG;|Kjzp^$@mcHHJM=OJl1)fniLvG+La%DG5@t_Fv+S*HilSJs^l7YO?-R7d|yh`m1ogT>W1o~qI5+*4 zYd`_4O(iSY?<0{732S3ZkUExmb_+%nJVF;s$c6X|B=9< zdi0hU`>w<6%v5s_J;W74w_N0!Kt`0#+OB{Tq#UK(-3$A@GYpc)$Vi(BVAg9HiV->$ zqEMQqp--ZmLc9Rx1IeC+CJ97L7Pw}3Y=P8xo#AwXj98UVka)f@p)!3zrzcavpS%91 zly54ky$$3ule)88;WRJ>NXdZi0CwP0&uF=g?AGufPyTd{USHQr3QBc$9*3!4;EaPu zm-J!AI_W5$kXpTJ;!5Qrdk}Uz{j7}dSsJn3npC$;T<2pC4??$N;UHnX_RC`yq{lrU zH*lZ;j^@;H!yG$`PSBqpTi5E#POf`Py9Lzp6AgNCar8qUqSc6)AG@9EU{BY^lfs0b zz#@_F?WIo&Nsw~tYdr>w(_kyu4^+cRErx#Nv4Um)XkJO4z5|Fn{%jDG+;d7j^FS_74`>?R}@J32WT%c#RO zayu_i7_o=L$Lc*ma=>jOvXnXckWBzXt!1pO#t)N13aBuq{@!bbQuux&sZ?l$CI*=qn~tLRZLPixHw-4$LBZ zG>#n$^)6OoyNQlx-JnvxRH|46&+!sc?cv>=3c(qMG2`!YlpSFHh$tTzi6A~9QIQA} zSfq&osDI{%T0@_LyD^Duo#py@S0vPJ8L~NMU?dFdGVsZ8GMDd%V2c2TRLh#k&sWrE zwi2+K`H0Kph~7L;hN1hN7fG#Jgwl;_mVfk{+Tfs7a;!FETyV>Q83z3;oF z&QP7szQa+dYK5gh#;aR21RXk%L?CVwRLgi! zlax8?j0iY*<`t(EKwJCdkz*^A9sCu3oaSLRz^aMVWnvj3623Ne~jWAt@8;rU880<=hBv|1nWg`e1r#vX_ z!GC(rFX9_bUJ6m0afGrfT+G)y?n{IMB9Mmech#od9*K&ES7M5021y#Ao^vlb#8UHI zEj6(v3+fMbX1(?1?qzVCD%GQ{YdFT)>%XbXF(*$i0=ihKJ*r6Y%m}^hU6Hhokz0UR z!u%7I)JX8b8;e_z43Cth-<8@p`6hf*lJPR>I8(27uO$v7nnZ|!smn0~ypSnj)urtJ z{H!K)b-!BASO3Nj{>Z#2BMZ^C|HMf2ah9jO=g@-=`E_}%*e z%N^!x3gubkx8elYN2})LVU~W+X?<$ohOs+tsKzq4W3g$MkWy;W#ndLnG{~ zH*-C{rjJ!XC&}5}3qTyTU@1!-5U@_f9-@PA1@Y0g0B@hn&@MJ_)xsjBO!FoZq+xOK zWQC#V5`V(+do{MTo2q}?cDn;So!BjJf*hsT`~)Oq0=thHo<}$`df*cOfWDbZ7$9hU z4CoLBt2HK*?g9g~$q{0WAVUT?Va1c!$t(#Zv~^$zkKW3np#^Y!8(I^GH><_3v>DSt zBvcGn0ma1e&A?Bs*D$wMk}xd##K5Jrh8e(elTvQude0A3E(8vFadOfYzG!59Y$I zJ*Z1c2=jJKD+T~q)CRU>%^*kRWNj#(y30Ikm=VXBN)Lr|+~Bs=Am7ID7Z!oQXCFg1 zRo_uRGW4H{OE!{RYX65kIt-3jaj|(>R+90bsv||D&clNfpp$|_v^rx@A67W~Q(t}X z$Tv2EDWZqhbUq~?u?OXan{7nvSR2w6GFZX>IpYcUkID>69eopl>PD#34)MX#P9mLg+IPKvG(z_6pizi8#XA+0YR1JL` z{jn`2f@E0aWQr!S`J22r&KzCob94${<`H?0YTuA`SuQ)U8>I>WeUq^pJx=zyg8IA_ z5Yp5eJ>1=gIjqQ!oWWZN45tw5J_^npbymT&2U)Y&6$N#Nq>RZVM!?lG_oy*|MS1a? zfmBvSiU+k$`4M8|h<^N=eiPFKP)$pNWL_yt;Z z$3kHV$kdeewN{URJn0V*BxM%RjLp-TVq#gdSaPfb4s!hLuOe??CE7~%zo!BokO72Q zR%pDpq9U=#17zzL4MNL(uk8plco>}mN+K#~QoUowccK}w^_6o-$BHNqvCM2LC)!gz?@F=+plD!`-HQ?D zXS%RVoKDsfd%i3OU4PPjfV^n-r#|-vDC2*I<4@}m8A5sSV2NOMs z1y>;Hs>0Ayv`DMWLRhO5iSpV}>Lv0(b-|iHXenDimXRqc9E_J8rq_l-=L){S=oYV!H;v`jcr1a((|lqlzp}-a@$-?brNDQDtn z;Q@&}Ws>RL#x<`V)KKH8fvf;Rdb)5C%70^!D8fYmBxd@Bk4NRfx|J79J0V4!;~Z9FsptW|39A* zqD5rDEHMwHJNG8DnLQJ61|k1n4=NF&2F^vtR7g8Zhve> z$~SdO6h5N|7iudwsnE<61KJQ61Ri*p&xQNcV_DFw8qm%V(Ew&W@gGRG5xMF6abrAv zqmJ9Kzma}f99$JX`Ux_G&>`jzBfo?X4%#643c(8;gM%LO1-y14&I{#LyZ_{Qo#5qN;6k8NSrlvWpJZ15Z)a5 zD`F^;VzK9}(f7VOc#X)ECpq?c1*ehDZ0i>m;x?d@z}_1Uo3{7RH=QHqfPFPsHB4qX zsOQ{JtpY$SUTGwC21&h7cbbHx7Y)?qydUAGsMB%OR3K~fs?Q&=9}u+$b8LL{GEc~I z4$!KZc4&UrpS0NV+lJl>!&_P6Cn8-&zNCoHV{TFs#yTAW+s4YMG=M!ZcYaX(1w@>f>H+_^9{ z51c5$J!1ciI$aDA2S|-LoLa43LdIv!KzEU0xoX=Yk5w1paNMI>gT6qv-D>FaZjbxT zfFRf?9il@WOC7_ZFg5(Edb2P!Juz7UUAOPhQf7ye((JqR?Ay&|z4K=- z%P(wr7Uexrbb+;@ZMfoL1e2d1X6S){{GnWwJn5M0#SG`+bN7`4+!A~=@Q83?@V<1w zqn+u#%Schuho=tw>yVp3_xQYqa2P|qIlLY2!caLLn`b(fhk$@;$r{#1r^u)T_1viy z^t4Jn0UPVyS6^9CAiO8v4e=eF_uAO19HuLYTQG% zr8>0ITjG0U>MYWhio}FEbZP+Yt^_gWpdSR3QB;y|{g$r7X)^n+@C_d~LO*MGG8HK( z0x+N#BMX4a1Hc|pSONkQR*H;zDxza#^hk!Y%P1E@7lt^I_c&e{))|B(dvkUE$dF!8 ztwIG0_I;@J4t(w$43lNG-XkM43$>a#uUk8g-+V< zHEwiAcie_R1kDTZX+e9<>yudN3P`ga2d|l{3%j0%;m;0=4%OmoNc50s9B25T#Xwn$NpJ(IbGD{9Gjq;HdW)R-tc5%c!V-am&HN*0yl1$8gdwJll&;e>a#E>2qZGh z8iEama}s|aBVvM%!5pg~Q>BusCUoyq7!}5&_hx{F0pCaKDoDPjY45*QDf#_0GG6~~ zj&u<|^z}OsAs0&lXj(X{W~+9?ylEc}q=sVnf*iixtk-hVgSF7LrfM#-FU-b6=ZQXg zzSL~-4bhY96%hNM(ub$T`FlP4h1OnLuP-`dT)k*;m@L_2%e)3=w*UFL+4^0|*oH+T z;PE4_Ev}Ov{VBKkLF@^myKL`Yy|oKuf)hY4T2-Dea@$Ya&LqY{i#POU)M|Eu1S zC-N2I%oY@`GElf`vgJUZs>=>%Bd>aS9S1UK$qIJsV-SJ?T^Ek$R625sP3QPMfdCdM zniMo8uj4h?`C$YLfVI{}Iq-u*c{ojH2EhTR3xHXe@FKQkVqP#NjByQ-1oX6=c%fH!$1I2D$S?dWEOi3 zmq$ow1z69+3gGCw8bTvsTr!E+ThHe35?q>Spk7&|xK?qM5e)$#sh!s$n#+uMFy|vC zAfQbriWxHnR?#tm^&^1;m=lyStM@lB7vv}e16d5X8&7IlpCR=iICr}4SKRX;L{&K1 z7+uvvU)1U=sQpKX6eI({h;|@(qdjwXjAf`jbG~J@kO4s22Z0xqM!L(4@9c#LH3wKZ zBmq8z^L7H>;^N1O)C=8|b*7;?iHJ9JKR?8JO5Le8CwR6$#4gNd#~&kLzylM#K-^r- z^y!(jApJl6 zAs7)<#N_Bh#9;8P z4r+D|Sxn|5&W>12#)`LudIrHvt$d_bWQUG8PUD!Uh|)N~SJky05@a<)IsgCo1AP~? zTNn(&FdL499&kO=C17?O{2S@?grf?NWwnezr`nk{G>|hzb6A0RjY)h&3&%)Q}* z1_}Wz#*LQv_f4M*xu?fy6)N-)yQhY zFenio*S1&3XlpQr2I&6b|Mso4orwp_#`InCX$vLnEiWnIb3=+dpj~tx2kjLG5kmi^ z-Uy%S&V4~Hh>VaBwP1YL0-*%_{4FNj#{iGveCIZcDj4eOCi+0wApi0$Qj>6vJCfge z9O`Kn5*jl8C6yEN5k~^Ds^!r0Bo(xQNGsrzT;ON)BYD1=MD-ie)V=l8Wl9jK`)hN)Y)*2?`Xoi{Q;RFbT=e0-EB`MNaQ1kX8}HS)bm!%8ODS* zMq<22q$AZD37gE^Cyq2d@DQeISCa#^x;R(rJae4u$-~Vkt`Ka0(Xh1?R)aF6W^%F& zhXm+%UB2r*Umc}*ICZg>#!&){L#Qbs@chHGk8>t?kRS#jItWyQC|Dpd69A{scOi>l zK;PZWIo0#%0At_B7>e3E&$*2{!MtCaRL~RZA)(|rjb1d~MA&MCTltyUcc?Cqwv(*@8 zsV`dR*r}tvqSab9DunKBBPlJAD21?7g!p}-bBN$?&vxvtZCkx6S8*@4P(3e5#EvH$gJH%-lN&bv@(Xt3G@@ z>zTct+(me8hckpQt>&1vqisdeU|Q8|A$pjs zaWOiX0{?d%#yoV)M7n_zt--)>=^{1#xgn{`sk4wAN@PR?+Kaf442I2AM=KQQOPO-+ zsldZc>MVCqbS#?HY?5*MrM1bi9dqP3uQb`bqcPk@IcUyCnC5bPnDMU^H}v__&g#G4 zmYSw@NjIogm8vqKIvQT}+RI}A3UIby;cSZ(08DFHd%UyAOK_3}2^Y#tc~TBf5!l_@ zJ|yQW+OZZTK_AY3Y%>?|$2+I5U|~)eLD66vLJ*cY?k-aZ#I-Q24y1>8&%~?_JjZ7^ zpSTe)&-0(l;h9Bd1F*2Ta5MP9y05h~ULXJ-Q?h>9b}-Iv15epj5Bqq^8#t3DTsf2h zdGT#qq8%mwvO9HHi;LKk@5v(13EKG?7!-hRX~Y1AmKKMm9s|0H)OOo<^?~0SRqxAk z>LFZ(ARSE=82-K{DFzCb3)-U}=P4zQ3NeWHE&}m_Y37LlBSz8JIIUkyR$-$Ic&MQ) z!7}+m81=GR?RE1*jpRlEC0GFm_Rg7Et{yH^#GdwKwS2}UV;1wx(Q3Gpbz{{SYnfho zr~T)MKB@jj{G1p3ma#=7R>Mda+cl8-cMth38h82t=<$OvZw})ivPk{>z5=?xr&#~F z@*u1s%1cQajG>`ub*iE5-_@?xyiD7sThZHk) zn#BO33?rQ=HfmVRfR__ov&401Nf>@o){cGU#aA6eJ{L?P5N{BnDL+>$4|l8#b;jKU z%u=~YCM|F-0H-Ann5o$?^3BYgt^y`}$AB7oeuQSh?o0VKx68AoMa2^@REHFJm>%2C z|6kVjB$7aCjqWqNwZfb_VV7+$x&4mDOb1m{$yr##6k4Voz6OA%w4AVbN7W>nV?oKh zH?kOkGbV2^Spwd~4waz{sax~$L@U|bxl{6B(PJfXG(@#FNDM@C=d!fbUFxxNKG@I4 z(ydU6zdH@X*0`uga}Q_3cMEC~seuMpri6eCZ7XEVjH8RsD5f_d{Qvm+_JF9%?EL{X z(=5_p&GkYh-q4nxmn0P#TqQ&^KrziWDJw5zvF0rl7&gCU;wlMVQqmz%LCsuKGYgrv zyyOM(LMocfrUKqa8Fd(D=Hu@<=e(Z*-0$!A2UyBw=KZ|ydC&8l=R7AkDfGxW%Nu3W zWwd;(FL#G4S>Zs*YqF-Yu3K$^sCul&Q$D7MNa+MIjuMxs9PXfUxOlD#Kx_;%b*Q8|k=|Q~8CLodnnN{-TEa=xDoLEA z!(CjxEH|{-0nv?hO}e6q4}j!~l==GZS0xF(FG12O@Q(1FQ);^|>Pysh96w`7RjT8B zEp+3QBsWxkEcfZ+uV2Fgfz>mNc;caSxj}!q@BfZ;2>sB#Rga5Me{GPH{J-I$cfNx{ zo_LXRe@H&khv_8MAyep61`s3>La#XhUMXoCGoR6Tdg@K73qZGP^;!VSm7$VUb!~15 zn5f&H68rh}*m)A*a=8=|9K#ZNNvL!HEPOzdtRW=1&dYfGWv+mvnEh7}MzytqDJ+G42MpcVx{LzHiM zQWuh3WvXEfH{73mc~OXaSCiWtDalcFM`TBl0lUK@fKfmN*L9+^*Z>;G zM}-(93i#C#oQ|r>D7=Y%)i5ze3T?Bg%qaKz=`J4*&r)q^KeX`5NXsqNsJ3=0y#6J$ za2!-s6|>vAXR)bcy{M2as3QZOk<+ds11dFT6UR*)h-mT9ux-%Ttxw>~MZy;sg)m2S zePhB%lD#he@EyVWJ)UtFyHi1Yxd8LHTTHt7>jiVUMfUtVU9$BWTB|giQiwu-csQ!wR8^3<=}k%zL@7g1szy+m#xp7At-29GBswVUHuo;$7v>88LvPxj zwJ*Exnv|orn&m!n0&7n2rCvUzxjV%h2WXBFUMmYdy)UB)HnbPXpT*yw(JBkrQdu4X zrhwLbn{M~v9~<3mg`ZS1nE651Td74G%qUIu?@2<0r?lrHp6T3D$So1U(R2{H*3#*0 z;kDt?KOaSL3$B9`D{0d@iT^)Hnf8-lno#$Oyv5P0-EP$=$~4&6fLw$ysC zANS1j)@iC6dCMh`wZ*pBRe~Og<8ZsgkKPq1Z(|n#1Rj<0Qu03ciha0vF*a#2w z$Zw!@S<=J2Rfls*YVIRb!iyk(P*JN0^*q{pJMt~rS_-AO-V9fU#f_{?-s3nwW{my4 zDyQXLWVn{Pvcrbyg?BuMMyB~&nSJ7U?_QE9)9U=O7PDn_mi&sUmlDpxWiQc4rG&VI z;*#I^h#Dy&n}l;PPR;y)uC*r2nv9I%`8$}Ze9r*Gs`}c!QdmZfaW3-w5gY3xytklo zpehP4$9<#>Bp5C47+KOPlN%obUhE^AFX&&t+2=p+wuT;Z_r+yJu~a9SRSk*eNn&Ub zS8o#;$fUO`9tV3}^pxKQRkFx!?c4e44{OEBSOF_xK&i7uVobxUUBBQd-RR1Fv2-q( z3Ef1=h?0pF%6bj66uHdLlN^d1aN!26KR^PiQRngiM`}npOo>tyeZoV9q(LbpPN<9V z<6%m0I=4Jfn{Rei<$zYjFQuuHs!sf?=Ye5`CDS%3f1$b7>9JmqXfsrL()wPUT2-#? z_-?`a=UQLd~H&X+N!l}Mg>|I&%HE0y?hzf=hkcfS4|&;pGb*GsgepW)3Ula_!H%xAnYXU24;I4TR}xF5vL+l`-M7e6^K0gTwQ^WuL~S~A-y^W$ zscneI+Ze|ONIF4>0tqBVz?%f?@Dh_Y_^?erWZCZn@8G`l+4@~PZzP*o7CHF}h4 zcN~n)rku-KyFAdjdM>S~%YnK!4$|diRnDQt3nhj=q|k!6UM^v-&XR)*rC~#=2K{pR z-<=SCFQz|#ghm{UotM?K@NJjv2QQD}we(k5(Gc@|Y7mNsCeOSC3TRvQy_(>h_r+L6 z=yIOOLgr~&wnK}d31^C_PY-(PV)sks+P30(r&4xnR|^W)dd+fVm0^i-h zFLozeXkRA;_tn%hr` z+Z)yl%d+3>(djp>%CyxWMc9sMj()f_p1c^HQs7LzqFj2WM~AF2hfvxId1_*y^54Tn zM7~eiG_4W4H80)gAa*Vc%tMc5C;y+49oBXm%0nFEv^wBqslkLE`@vBVuVDmZY*X>4 z(Gxzg`cH53^j9kr`o8D){%;{d#6ZOgn;nV#O-)R>afsHp8K1(Pf&=*);v80`>-=qH zv%B%Kw_ci3S~sQcM8qZPj4*E?BtslBb;@v=NARAYhF$_n7^6fHM&TXUpj_w$kCLF) z119f_s0T<$U8MA(E;3=kaOH0~-jv#r;{EkipnOw4RD#YmUH;^YW;QNK(VLNbfZHW- z9E6Tp3z^U>=7D!?nh=#qFh)$^_Y_XVnr9cRMDo;EJGzt zeOYnS`gt9ls`LFmQ&q7Ip(UW$XDvqfupJe@BusLFPkJ@e2T5f&I^wq8epNaBLhQA+`VyQ(C%zxpMO zv{CQ!>xgZ0mIXSeS{ZZr5<%#D6ap)fRrV4s^4&9FLo%p2H?R9oQn>gSs9>Z!&x0?{Z}dsUz}L zQV7LGcA#=@&o`+M+ZiMvN#J#{HU2?JKb?EhUmOhpQV&T2Up%zFvKsIQT`Er_ZZt0v z{l>8rU|;9taV+vZbPTp3smIc(>}oNZZ2(j{BiaXDrOW73)mz1()TbL`v4#~ad0x4X zc)l`H_kz{rNlf~kvH31836;{;OjmQr@itrP4Vg+rjM=Hrjh&~Pkhbii?zU{Hrsvmu z1wWxo_Z;AMOpM*(eEnY-;h^w$>&l^(vyByb3))CBMx2o<*{WTH10I>|b)sUah}65i z#2Uz9Mc{HRr!4~4Nk3mgn z)g=kq4^xM%HtuIbj}^T>e@awHO(8_Uqsj;eNCjz@IPts#3M{up7xU4D$A+E1xn?2c zS`f3xSAa@biv!i8KiiV1j3BxiGOE!zRMm_1g=u8**p0GW=0H3ab{BXklsou#^OO2qT+QVFzkbtRd!GRRwZ8D+5ISZMq;s>~m;gxD`WTfQ}7A zIkm?3un*~FnlU%6-yu`7BYXn8$km)Oe$r=kzoZcNPFSJ9;^kqe;b3cJk*~nH2BcuHUZ7O>eaDfNM_Bh-%Sm`}3{xcFZ}g=>ll_2; zk*s6ioWQcQar*X9iYy@w`JLvIrVY#^Zc3ex*lbS&_?z+q$K+`LX~w!eKJMT>LBkI( zZU4^V+Aq)}w0YF3PDv{+YXel0{&{D41G?JwrC%3Tyc&2r802X6@6M?OB6Z{ZJ-u^2 z`AEXm6U>QftSeD2RC|Kd+%0HwJ=^B4uEUt`b|YIkwtHNkB)O*G0IgT~=%yy+ljowS z(Yp3z$>TdMHNT*;UOmAzM!I{pMbHSXv8FxR#@EJZ=cbT>X+)cL_R5W=>x};?ogg^^ zBX_0jwDtNRtF9V)t-J>@(33e2=_@=o~N?8>wOo4PE$I3T5L z&xM=2F|2~WAI*)b$V=^xWySAU?1xOYA2P*&DEpm+3rVhPOFv5AKC)NBz5=ZY%Qa}K z8~2NYK^~Y|0rM{34K?5z)}g?Z5+BDbY&SEWTF)tg6(!0yM^3AN8v`b78Q0i$n&u+3 z7c&TygMo>^R{BzXR(FeqMx;FwJjo1YA@T)H~rAaxWM3JME2S#mIpTk;WX?-tV(r zt2HGde>yC)BBJ6Qv-N;+#Ip-ZRvor3op%woS(zdViR$MvmUUZQi@&0W8*_bB1#7-% zSrsL%!{JBhjwW((xsUp*!iQM!O>ZSjT8EyjvWl35d4{yR_9Aol)R}R_^?_!fWbZ2% zLP6_a5Zd z9n)<6xO8y0Va^N<6|KJ2D_<=(BO6K(;Q{pWF9i-`UJq*D`!loy4^<(|K@-h@c z8M={akb;GJX9ILL^-W`jxY46{02UdOs?bWE_~da4{3>_57{@3-L9c;`8FZglx>CjiBUP1(zvQCXjQIU z`ERKpVH`c*AxgFttrr9z0;LvU6XOG+BQ$jX(V155&j8=a^qKJkXGi$?37tT}U6G1n!!J)Nc6t;-k~j+ERyj*mC3 z=^mon3$I1z)_henx7r#&N>z{3>#GS^ub;3z`jfSAAz@10#@l0pPGxpnX8W*?d~EOOfq+_s7_e zOtWn&O@&IUjWno6M?SJIjrM>XWn!nIxOf4T1mA$WI{Hf}h&!PgU(-nt9B_O%K5@r> zFuiu{WA)g5M|cr4$0Z4RvJWZr?fI$Rv>BO7eZT36(kHzZs#ITRKI##1T$xSBU#F>b zN%QaB+@3&JRXXN?aI{$}s}3_RMjEL7GScv4IMNuCq*K<5gh?sKkw~NSf9??vQiEc0 zs>5?Z;D&;?FzuMxUTd;tYI{e$#O0(R1!0la5XUObxt?>ksjPRA{UUbLz~%K9 zNmtbctwe|X^kY&jd$3M;H{@+nP;C93-U>A}bHtP?T0r2ruQU(TA6^>(5h7vUmk7%C zzxwI;cU(7sReVZb+xT*Jw4=*dxI0fye@dx?IL^bTM)EnYU7Mr^`aT#Kobc@tt+YBW z*5~C(j9DRShX^$|w zIG0Fm&=aL3r~S03x_t*P@jv3Aru6=LNAmYSX*ZwuemfXF{V7kR7Gb3qtoB*=c5-pT zs@f`ZjJ+);@LGQ92R|WYFfr<^g3DJ~)+t#fmlyKV+da8wffoM345?9_x?+6_jf0a- zwNH2j5ZR($jn%i+$>l3n*M=S@_x7F7V81ih!o|i@~r1tUFcSH zPCsddthRXW9~t96s7P?Wm024gweeYf&+m65mHp|xC)=wDtL8b2W;(jGsGYo8DQb1? zl0nvHY$$V!CrF>WC&*X9t+A$zfH#0Ht=V*=b?07NJ{kXj{Z)$jH+w%Xsza89_|EL+2 zZ+LPhM>*^m^K1Ec=7I0v0y+QY=%@Qux`7NmKa{=gX@4uF)-94NQD?30c_i@ncav%# zH%bh=$Rjv?wAuNQ-$0n zjm3h+U5^OOA#iiNvP#7R9fnY${HhcQ-GmZ_&5T;sex*@2_VqTu#ncwLPi;47@l4;e9g_Ly1_kzSPEWJemOUGbF?#-z58;)Ta zINpZ1rs+D)#zcb~+f)h)O3BOeTxmP1Pu%!c#=l4pkTRHTP)ilfnzu{K05_Q>u;Kac zfnoY2%CwOW>E$LJ=P6B)e&6f29Jkg=HBT-Ws$Gl*XQ1kw@=Hlx)aV@%MM$bTWOj=2 zqrlY^=l`O8ZHRmbOL1>H|H3A6%KE7LWK)@B!S%IOyi}BUbh+h< z4zbDR)caZhs;;UOwNsUI<5zSzB{SAJjYOfRfy_|vdQ|w^pcfJuAJWuqIky?Qt3-V{ z9SUfE^?VEf6ZOho+>pOzfi&Q*QFPJaU=bGJ34n1!{^fncy4IE6EWAA_T`_JY=^spN zs6N*7gOC2f&j|RP3(n*JF|l9`yuD|{j%|#%3&|t8{`|g{x5oJWX6x-oiFTYYh8UAAshF0IKFdTxD9 z|50$a8oYgtNV@&FkLbpE@J_EH^ETGqJ@@kOn8V8deeZZvQ5vo0prq$ZwJ-&&+hPKC z1?=GR9h8syljZ%Vd=izDh1#Z+YA)^GSHpziU-}8RH&VB#I-#F2plhA>>Y{7|x`^&# zDuBHVsDnZGm}D@nJRC8iDT|798MU-%jhC~8xo~O+$ew|LYeSxf{aYC}oCe|}m3+Xz z*toJKr^Bj8yu4`Lx_b<>+sbl^@$x@Du^uHG6q z@e>-shJlcG<|qDfuhk6yCc2qdrZK$(IVpYoSH;80Eh6d~c_i_(Q8&eN{=B75su{Aj zlu2N0&=Es%Mn4mA5q@Xa&0SlSxv|IRAno)Ja2(fAgNd6h?WixU&xpS4I_)z3C&820 zm5WRP9kzv;=e&=Wr~qjfmqci)boNgb6k7w`+&WaqQ{v?mbCpUY2>?WRvnZTCiv$4czs*pi|^r!d}89hN6oh?%PYo_i9Z48IZTc(P`X@ z>a-VgeZ`G}4S94XLD+Nb0~Ps~|F<>YPX+9;P_RM>?sYb@s{Mnd#;4tQ|wMEuCtWK(o{uy0&G~{bOFgJAqd6uS%}yRg<^ScZ(G+!1V@;P`bsS zd&sA85{14`nS>`KxSWT21q=M8bS#Fx%X`l>x?`j7|B6cIp ztF`bkN8B@Gbmxy#T^C?I$2-wKH8A^TEQ}|Gp%i|?*VH@39EwmeXEjsV;ls4KIp4qS z+Ow%h61k-^N=DHtD~>^A>g6vaQu=gM#&4>2)u185-=KHhFOzBtr2@nBi2xl#-?r4P z9;rQD5GUM&;i<_l7L_^o7Up$0*cO`8(7m!Ax=*1*H2K`}^0F>WlCml>cDSw*3BrN` zw(1HI@S)w5c26&LamgZf%Aw~T{UCQ$?VTvI%S^Y>QvZ)boNt>CEXsmhb*bskErQng z^$+Qp_Tu%7rN|~R;U&Uv|F*T&sfDTiPfbpqq;)ag{&VK9i(NaN79KD->Y#gmnjJGh zTB48=J~tuJ+fNKJEVG+eeXppPoN!Yx>d*I91nJGL4^*z&-G7SApxoRI0a^t$Qi$diCwxM@uPL zzf+4+htDe4FIp4vtdl%-WYq}!A^#iS1ZmHIvHw++rn$WnxpTR;p}+1zp$_8TjpzV0 zL|#t2RAPmB6W`GIl(Xc^d6V-|-2-h8Wu0#C=uK+^D~?po!tA}Ft}RDrS3!0nuFBH( zrjH!@7{fL&dI_W0V7rgb6eKJZl7*uF&UfTT;b$}mS8QKp$R=xJa^A?XTo{C*Pi#h1 z&-up5)1W2?+;GZaNUw}iR%I|!6>6)L6Rsr@!H+|^E*EZQ7|b`&+?Y1^5@ZPify$4ecE;Eo^rGN z{`yab0^ple*nWE33~=etbfMhGo~(MYsK5!E8j7qe0RZ2k z@*l!@Li;5S*C$_b*{?M=SN%K|vx<*uh6k4(1P+ojh#{X%uJTq_4K!`b#BnvGqFn?t zT2630<>)CpgBehL78myq08%K&G2+8rX0kmqEM)*1c*miYLT)cHOo zQoqtkjj9V*$8bdGXc$C=(J(N0TL!TnXUSD|`-upki%_ZX{JrG>g(bupTt9*6d0-V9 ziV58L*WmRwtPbKeO~N2@-N(BjbL+=&pi$o2s546rofIy`+_r2TmmlbpDk9SIKQ*Ls zWMxw6;2H9Q2xS6|@Q%?dqR6dUX-T)=RerRe+D>tDy$?Uy5l2puOE+=J#Tp|1sJ&8>SFTQeUrC{=1{S-I?B* zA6)&#I1{p2O(g3~1Z#vyG6-NALMp)gRq?6}9Ct#-Bd8p*kx7gq<#c=f6Or|}Rfv6^ z;BWlro}V~BQ+c3Dj9X${*u(p3eJ=g2nCP_MC)4cKW0M+BlL( z4jH0LGOoq!DsbNpbo|!*MHLNTC85)b38}SKkK>^iFC5@q42d z@bt%*nPyvToAqUr)(oV&)Mo}|nZA*-nWWb}ZtJIn&@RG<(wu@b=u;^aT_Y-56;>2gE^J)q;} zC*npqO>BmKgWO!@5i8kXAg?;XQ;6wwUQX|iMO@)$r=)U|O*1b$KTMq?(O~R`mEA3i zGTo1SUDoDEFU!7stx3dFLR<3fz@H4NmOXYe-#EmkptG1CnT)~t_^)`3fNlJHKtYKE za4et?ez_9N#z?zIK~jwKvT>l{T|G>V3;GyKNAo89Y;V=fn}0mI-GaHRs>2@``ksGU zCr6{(+T6tz3D;jbf+|RjsROygUR(fiZRVC;teOcn(u3`;&zhuA+0^uIj%!7i_EmxN{+8xt3t8Zf zJ&s;Uy*q>Mu*T$@7b>vr4SMlhhcMfj7td>Ht}+Gij?$e;qeI%>s;Wt-a-c9u<~jyl zRG^D=dhB@qchk(b)0W%l_D^r8riH)stK|o)<2PKOe1eCG3P6Z*Ld!C(vawiu)Mu47 z*;Njl)ZdeTbZWPBY&p<;V-W;P2R(MQS<)idUs`5*^o8cX=Nwvi*+K1F>@BJn@bxHT z<2}GPIHPJcPi@Ja8FZ)kP5BR3sX=Q<)w96##|EQ{O0c72(c8 z%-01vbo_@@mE1r2!5hUtCeB;hJ?z)8g46=n*{$hGU$Bodr!;c=qdyHca&*J5&ntXX zRo`BP{j$tBK$p7yv^J$332_`>h?y4T+$r)BP(qBVg~hR1jt+GBX0 zt>^6$Iz?nm-U4LgsXioS$;xfg$lT%dDX9u2*L6rBsf0+XDl6Lg_B>)SQop z)A&JT>eZVrf&vCM*R^{5y9y^^gx!r-H|AC|0`I=Vj3=%%CoIiJ*S_#(^-6}|7&ggi zS2JAUll-^Aqp=(LSveYoj?%fv>A%z3Z6VS+U+KkWX7TCWd|SJp(6eB0qGape7*D)g zT9IwMRtHxK{(`)$9+OrEcAM#!x^pLdH8by}fR&4^@|d5mGR=LTjp}jmd1lKlq^)Ke zPY9(Mw=7SSo}Kk2KOU(Z)z?^jbhHgnZp@RY`}ycpY!(u8WKy=Go0R4c(Uv&EQ_=#C zL%L9B&Ayddk>;q4ld%%H|JJrtyUg=5_i6z_*i%3pEvlm{lh>iRJ7H7h?iH0z^U~l8 zv7LT?XTG@z<1ft2Lr3-MH05`=3L&O+wVNyNf0N$s{%fXk_qC+w5opuBE5*!KYX~vzpPEih{6CuC=R7-s!?>FF&u;C*H7 zcDn?%JfOqe=BashZ9eSyuopK^d^>sKlvTBpx0X$uD6<6PD&=Q~tlpQGh1^Qr9IQj&xI>-a%#$YR_uiC=x51`wySW{K zj*=m$;l!qBRE8tGBixN4b>*eQ_7uy{`Swh=Bgu7fy=&~j6_v^6C4EIXoZbdEJ%}_^ zAdzZER@q8B_Lgx~s*yef1Cb>!H2&qWsYL~QthU|yMR2I8rRE^kT`_$D8!nGCru0c` zw?X((j_*;wOQ%Ul`MVDcQ|yIX;Cj}L%lU_IM>!?~5J_zK?2~_*KKfv5x+EBRQyco# z7^1X;w@n|)GI8x&PQEDUM4!anFQ}lj7px5-7d6tZi!aqSUNX6x``7oNA?Xn~x>Gu@ z6V)I!r02p8EKaC727oMJ=pxJ800*ssTx%as63Pe{H_Z)AbSeKyDoynZ+ zdLmiq)W=9pCij*4J6*W;P~k`_K01u%=cI(Sz9Xb6q&RF2gSjobdTtaay)sWBeB=wd z&dUoYQPh42j>_W?TY@T=**otxivgU@PNM?GjWC2G=tKT~xQXV5^DkkCh^Fe{#zj1$ zGA10mU*wU z`6#hAvcVtf)G|OiKt_R-)p@JkDtDD`Eg)nvpXx>wNZfB3`u=U8Y}EG*b4jQWE4(bj zbolv->#E{Epl+?lO_E5!U^Um5+c21lfx-}PGDv|gymY)cw4r)&HbWnt%r|5R(GCCm z2TH)edS6GVqPj}ayj6Bp!+2Ev_=YkDe4D_;jPA#NEh^VyKG<2GLn#a-G^ha2P*ea1 zn??X4DT7->cbcCI@h#Q)ZW<4It^u3s09q}j^oHtCN)3Fe2iI@5gn81$yJ{TTkYjyt zWKYPs5Y!Wk9#C{PU9Z?E9T2lyA%V@^r|`903F=~qY`)52o%@GUNuYVWeH8*VW|?E@ z`b_(?aZe}P0W%V0a*vm4BmJo-!)OU#$K`UKoZR3mWV)uzC}AF{egl^F#&hc+mVUG_=wo6` z;K3h1T(VDr*|9*4*czWJ`U&x({uHQgyL_m)cp$oZNZ6Z@E(0Q?!I{Zca(k1Lk-v3- z%*Fu}XCh9Qm!}|@;VfFroBdcln92f2WMAG*s)-V& z>JSs5FowfPx*Efe4-C;Lb&>B6hYA^!pae`(e90RhX89i8@Upsai08v6sTqgkIPwIC z0E5-^aAjgg#JH%0y(I<^J>^C4tY?#0deT16W<0Ic?J)hceI(Ba0<@4{9AfKMlU*5D z0g%l@NhDGp$K^n~Q-+J6Y%%kQT^og_*Y3X7l`Bs=w<3p|K4~>JySu`Ap|I<+O08GC zmjW2Y%OokLN;C&5TAMT)AhV>X3M1pI)yd(pbyfIDF<^-Hi0nwo3Lh$M``Sdxd6E$! z7dLc;j9O)?*NYTiN=sh7lGtX*Um7xb`gPpp>6jJnGPXMD{LtAcY0Vz(*5MeVsqQkE zi9J7I9t3ncuKGTJ+syKL@ex|9*N#C$CQ^^SzQ)&Q)BcLDkdchTdQ3?0WC|t6lX0Bt@2@oRXO3ra9?<>hbR4T9loCnx^Xv zLstJK{K5+R;5nz<%22<|#QJ&41);L1v%u*A<#t5Nftpx#_-g3{Oq*Q&J>cYkw45H< zF_=iyV>d=o4f>TBE<`}$ME#^*I_8gIi?s<0YR8a0%5*zWxieEtQ-&2Af!?ZLQ+a)Xk`{N-F>JHO^lj2F9Vsj= zGTd?DN2_o3F(24ewr2LXQI2wu?h$QdqxOY6<%bn-$bCyNRq;=%1AQi< zg?IdxGB!xWh&CB;vuJyV;?XG;cLAMD*1Qf;A!}%UPlr3=rTf*2ikLmE6Ao2TnDSM&Xx@eR1qBN znR!?k6Vq5g4J^T{c^1|poH2X}2UR~h< z!m$1*e@m<6jNH&Et@K_GsB)se{lje;*ioAT{C9YvmS((jF8sF{F5K`5*CQUQ8j0h& zN^`0FcCe~&$f{>j@++SRE&;5FAW2$le5ip-+9Pz+khQBmS^&DA1jE4H-Vw{aQZSEl zr77fASK9&n5af5FNMD5}haOOZ%WXg`zX9Va&8Et{22liJ+|Yt$qhV66^@HVvjXKxjpT5?rsp9`I_jSBGWmDu9TzdE z)G6?^h(n1Y1U5R8Jf~if*PW`0(ocH%2z2)PBGrVXiV{8Pn+&RFsG+fCysUS(DwA1# zrZI9%ft|utAn|{wkj&dND&kN2(lWqmA%{x!RZ;;+R%Vyzo@4bqMJmpsoA%1bA6`cY z%C~TFS}0ZF_eeJWD;wF%apDKNXbMlect}l17Y3&`I+`c1c9i!T0sT}e7(`XZCZj2Q zBV56)y^$zP+liqzYCmj?_)~pBwILs%_EwVy8%@b@V2#6ISXB}g+UK9AD;>SBtw9U# zRB_0q7U9N7S$@T);K1iWxF#kVbT6fN8Ax09u$(wbhDOVaRqQPk!}VosDUR`H7R}SI z)3YAc7fEUi09sjV`ANte)H{+KSMQS6`92JG5E5YXJq7?6GC|BhW3jH{T-1aK-eV-s zXLeP$vDVvV`7f8l0F&i)iZqjc%t(K7XU)rCKU3ZR(m}D)m95M!dg*wdrU@J~hB-%jmv5+4Ny_d#yRSf?lss z6a(n2@VPOGdf!q+Ya+I4-x;sD=Ue{#u79U6OWiamYa;bRXv7neMIp(hs}?K8QQk{4 zes8JXVzP3eS#II+Jn7HD9jbgi4-bmRy^P*hWo|A{LWWn7gBAT6VO#RoVbUvaNZzn; zgbb_-K}E#0blUPb$nCupPW^yGdW`hZ!^QfX)znYRBwCO2ylOfA+7#<3%aUlRqC!U@ zSDRFF(RVtO&#;0ed{ZuXQa}DQq;nCYD}~~Qe5U%zg?QPCjVR{NXEra!aB zFSPbf?HDz(a`U{*8{190oX34VD--UU2Ig>PT-NEG1p6V2?T0SrBd$e1-tj7X_m5Q! zg5io~u0u;qOVCoIJomWX7C&EDB!lW4Kl=vBI)`cfJTL=FHrdw95sgspBbe>|fy-w8 z`Lj1m-687S#lAxTx4)hbU(dX4*bULNBS$y$1W`I7l|EYv0SCU$9+vhR4-at%dm5SiLwcey0p|)oiy?%1}Mm z9vDi5#^LJsZw*d?E>ELLHTzf@H4Y0!(|5qE(Z!5zZ>}GsSy2qL0y>VThZ}Pp0l3kz zZG1&f>o2ZYeaK6P2$p=ALr4Rl%O)mG9EcFAt`mSEG#-qp8V0rGKWbx}AfvuJ)9!ft;wI+JoTpp50w?Rn@~lud23r*b++fh*Z(YQ?_Ln&*h2{DaCg=FxpPS z%?79~bhjf>72UVv#&(3mj~wNvoqw~`-StC<@ctnnur^nyE!Z;r^@6%Z{nOfQl1<~~ zpx3+lCWnvvvRu2@=j+&J^K{A??LTt5BrV3`feiG{thv%aOv^V+6KUj8-V=JcAB{Na zc|s%}pSiF>^@kn?{XERCrBuWGLLMXm&m(5}Y?SO4R33?I)i{yDYo>pVFkgFx{BkU; z+g0moST-ccJn#3AOVQiKjp<}j&5)|E&&N1s=;{c=)pg0H%I}PNF2H;YHH;@+7+E>H zdaIu+=l9Z|ab7WM6K*>&i<&nnKj!y;R}T5bl)5L^H*4BknYa3eqGY)9Uq||xQde01 z^I~pOtPMgF#ugMEoT!inITA7_g8CIraO7)-UZ-(uhdVwyI`79jjCOZd;bh0iR>f0} z{xZciIB}HZzENQj?;yWbb`o>lyK8r5(yG*b=WENzK&N2cGsdG) zt4h`khM=8WjfRXmtIf@>GJk{9@D%Te)Roo}DqQPY%-o>z8I{S!G5TPYyq;SYJy)}C zRPkLRW{9Nx=I@>J%!?D8bts1S^;rI+dGLf@Gxbv(#DYn~ zWk!G`u0;ImUBv#9R*)=26<$?B1)7OUQRQ1Zg*ZQmaX+$A92qdqhZ%2!ZX_H$!u!6w z#wpxVsf0%l32z%fh~V*)*2ENVw%qP@+*iaSa<9il1YA)8O15)6k5Np3}l+=e8)MYz7|7-#CpERO`1W=*wn^;u#8OA98yj50<#lI%A!A zWqL~v&MGxkwNLyV6(FMf{p(=pm;Wrex!d2cQ}1vF>yNP0Lyb9uT@B^5L-#gzGK_ev z+P3QOKjyls10d8xIO+RsLVv&P^IMIRlEY)xgnys=XkJvd?eMtU1sF)k^buTuB#>-= zFB;bPmD6o!Ug*Tl7dYyxO0JiVK;^DHWW~T6j$?@(ZCrML<(M<2Tr+*3R~I5$DFMd- zoI8C=34sJE*4R$2-)X*SOs#KyIPzI)`FPV$Y9Q_ir0)v2a|S2mk7_g10k1SVu`Ku0 zZf&rn@(&D3YZ?->cfUcY{P+|XYDYpHN@J#zNvJfBFia;)`6Hu^EoVPTtf@g)aLzKK zj^?}N7jc7Ad*@k%n-c4~d6UwE3cWsgV_)bdL*&LY*g%V3da&4W?Rwc$x+`3-sw+uF z;T0cIB7Ln>3yFgz_(fM49ZL+Oya^;Ej5>TG7^K>adI*e2?^;j!Cysy*Am`@00JPGo zX%+`nO8G15zig`V8`A5pUZ^*zvBL-pf%O<8d9gwM(D?>Ky-iJ{(z|?>!iVyQe9pqc zf)xLald93=<6x;2Qt-?!IqE{=Aru93f{`!M1l_((mi%GuIAGkYX7!+ZNM_K-K+|U# zp3xnFR%+tt_yYhCs%*Voz4|kB6$E_Ntnu5^Nf;qbX!RJUiC8)>y_@SfQ|Sz}iQ$HMePSw(#mB zx`7Q0s0@MScc8X>$$iC|gropEL|N+m8{cW;R#a*bH36|w3q4hV>#fp=EMX7Y?s?=7 z%RJ3;xrK&lg@BPGR9Yb~KO2lOsOD`yt62pE7teQLSC}scM8D4dOy4Xyr&uLF=h5Rgpk?`Sa%V zs>r*Y@)0;8DcXromjoE0`INg>{x=P2s#RjfnchZcCF?abD5@S=tj*i)xf8eFwWny6 zbMLidPE(P^QGz}|7e#a2H@gemHGykK%0)*k_4oEUGd;hf(y$Ebqfk@eh5h{Y&<7%9 z+f2|z=|O8u;YnSGl@=qK%&bmw_m0DDcaz4^94fD^O7ENli`2oXSEOOTxyqH9pw_9g zMVTwG;FpZIF>vn>?y`dusnLGqUcSH`%B*E8aJ*6Zk;2`Ew0riHBJG7{A?5k@l+s;V zd-xbQ->lIt$?$hY@4{;d{(WhLf}yxHbuzWg%3ovZpVV#gU^{t=*}pMWe`(qfsSC}a zFy_)cTkHijmiBmkS2&$gsGE0eG$c;@5`M&Ly?}{TKCT>^+T%L+(J42(t((wIVXl~YHFU-(Xzzw)Zik^mEt)sqKvp?C2IEv z3F|dyrbGW`<~CoSx$jXoRE2{c7&^o+mzWAHD{(7uHb@(pXgpBQBL<7DFeIv_-f4tQ z+>1;cwZE#~;0e{`{y9O`jIwEp=vVY->M@t>E12ztc$=S=BTByFLW z*U%@Hd%LpBa3yAP*^kLU?ski4?U9a^ND-=@;OK}IBl(84ZT96`N#C4fa*HHKk`2Xs z-DFWn4-yd5{tuBgJq?B~qScyE?Af<(7nE@%i$(wxTJh`;_9FAHz;DjYZUN!OmE`*$ zbPvibF&}d^Kf3`n|H1uXyaU}$HU2yB+gsZG#1Snoe=dx}v^xW%9Ty*~Jrz7?&wUT< z&oK5V;V%i!YZk};aO~TjwS8+((BNhKeVE65JsbU+DjCu%mDHGI-Ov;#i+!|qVhN>a zG}5~1)}9cg|@J2Qba%CrG6=3(g)WmwVoZOqPc zcMnSFpEPjz+z{KKn^&XbJR6OJj-9iP+SOG;tsyGl;A$=7>TYe#l)5o)S3GdYi!r~v z(Cl2yWoO(_yJ>zu%bSbO+|6xav&;;E!TQ~aiN*4*(KC#=r8*q~Mcv_4sPY7Ca{ekc zX&iZ3^Z#{X3+5T#T5>DxZjQ0lncOB-ABnHn2M5v0(9hF5Mi~UIa{S%d!wv!JFeBhk z_PsnNhKM|gX#V@cLpf;npb>F2VU3(sxMg~L-pYhO5W##4rpGw18G{Ya2MuvtItcrD zvg+jx$bAEusGwYwjuNT8qo^Gm zxd7ddqcKS>sdA2XPcA0RY4Yp2NxK($-_;FPQaz3CCIQErh%=K+FqcxCk-mEc#0FEA zwRKE`Fz@m;^Y7mD4c!IlUT5t1>4D#K=co8Q@W%sY1f>sO0ks zb}TVpH$&@_v<#gSVSd#SkBbPY0Sp5Cd?&_v%V=9upJg?fV|3p$t{ACsKx`A$Ew9q9 zsKnhnvVGT@6m!ozkbF^Ms^C4zR}29iPI>Y@BR+6b2^ziq_aSkRBw-V# zkg!~Z;4rMm>{#eIE89(88aSZ*_46O}&~4qMDx;c}=q(C&hJ0IwXi?^^Xe_uT3KCqa zk1|E3uQaF_3|3kUhC7p$;hQi}!L?}j9T`HZ67VX#V966^!JnBk8vZ2p?_Gb9{3BGq z3rAA+$05`Z^=*=~5?C$sa1Lu2yhH9drIvn>BjG>Lf4UrJCH8ny(D^WzrP6bv7)Yl~e22~WV zA2mq+zZ95w?d224DSq*BC8!GPDr4XM<9N;sffV8WfkK2|MjphjLxQT6>YX7ZfOf#4 z=j4^@FHhQ(sp`}`LI>gv|5Tuy@}yu$x+49lb^^wL6UrSB->D5U_F9Bqb0te(>Yz0! znfc2)_1kF9mU=}L^-%1l#1SkVA1DE0Ivit2KT8#JfG5>GrDGrAA>nSUMs4zIpPz?N)4wS@)b^J7I#6v0ye zU*y#&Nx>b(G}|s+vEU5C`s+1sObJ!- z1Kc`Jk7&|_Uf%AzR|n1BE$@#$BqmPqj6PHO5>#ZlHJa;|n_OoPW(^fax!$9%y-->b zj&^mL6IoSSGk-QBh2VzRnOS2YMban6={gX_&8;e3=vrVeEaI1>X6nQ?v2gWCSSm~o zQgj99+|AU^)M>UKyi8Je)oY~4Y7$pd^Dylsi4VF16>7S~y(_v$mF3q1YIv(Cd#9^Y zThB0H#7Ik>A0iDwH^JNE|Dpba=5MxC7tGfY|78|@O-)29R2WT{u@c&ESmn@7eEyksIq&#EDdjzUPh4$=9 zki;hSnK#%{NDx-yRIAxf%IG@kexc=@yRbJ-mrR$}6ShZ!=3mt8CTl-=0lF`hndccD z_gRwV9OGW+nTiVz_2!r69;qrNz#$b3gYm3pyhRkI)&6si)#X$~1Edc7P=vr~5$-OD zb6>9&45z|$$m9-;D>El9blvr7H~fD_NAI%CJwJCs_ptpzCi^M0py0f5or#pMtIl<& zdlu)HA|y08k{Bw&4sXoclY;P*N*x~&PuaM!U-k7zP&+E7N|=`>4wR=QJ8ei+Xby=` z5)WK0vW6T^_@-|9$zG$K>n5h|K)r%^3}S}Vu%8H6pslF9mGI5E%%gLW@pq!Z1$Cw` z7HN5dIS+p*c%$W&nJ|SjQRUzAB#m!Rgz+9S-B z>2xHxkmaA=;EFd|bDK{$Ij@<5Kh*=~!`?Nrh&{9Zsw?;7BAYJctxJS=RC)im&3o5m zVe2e8tHrFXGH>irtj4k+OQ)B(B->OPa(Y2{^doHO9vMFMv~f>Y+s~_qJWH^A?xDI9 zUI=}jhaj$XsHR_W6FEMuwDpSirBx-DkFIu%$pD1bZ7o|x({qJZMe+$e8B7s)zqF0W z|8m)6H>&^Ji%Vj1&V3!*;ZDyIm+i-4J(}&?zQ5_~MaP_1hb3Gwnj9FA`M{8Wq{F`+ zFHT6B&}3Z@M``u)l^4n*DOo|A@2Ckj#MvEjmF}rI-a*LE6%tPNvwspD&;E~^P8!Qn z%BJtGDcaWUPRWznbuDbqJt8)b&&*2Q7;|eI*q{}anYa4vc0ZDDZ-1Em%C+~avU(-2 zOGsM{$Fg(R|jl=)Cii7M_+c zqWgCqtJ`GPZ2d-kanU}e>VB@@-#>X~u{OT?4y3)KE*BiJMfN?KR9jc$)T-V)?&Ffv zI{|92gr8S*dd*b!t=X~4e*x8|SG%Xm>{U7&5}<`Dc(UlE)4caUDH_+7Mf#6P9yH!{ zs~|P&lB>FA{X)l%n-$o=Rycinw!J{>n_IZsnPr+XE)7ue(j2K zBr0txBJ?wMJn(Nu(Kt3tfR5>Qtfu*?*?nb4D;ymw`A^xpza`jzv)D44wzG$u?xkJO z3sBo*N%ab_-8}hXRYH$5Q;KJfyu8^|*wTEvL)qW{cBHD{(9P3d56eE}`09M^>7Nu2 zHH(8m{hg-|dQjdL;?x6;y!23xrZh#rlhB8E4}Dgs>{G(h<96Ob$AN287+lh)`01QF zMui??lN+g)P_#Bkv~-^XEG|SK-n5)xOvotDmB`RygZ+Mr zeIwS|@};)C^xM#!s$}b@AoLy>zCHH95OlrRcxx7BF?ySg>zQf2`k|LZ0D3^Kz>zU7 zW=)Itae}|G9@vt)6~*I!;C&YQCWec*SsrTf*vp0(#+ zI;R)AHHPx!qxI5LiH@JFPZfLXWbQ>-p;zn68 z6uorp>BHkb^0R6s@usx!s9PtM3Ry3M{n%+&!}A(x19H4FpPa2>iI% zu%e5OP89(_*454CJp}aiIXsLAfSweJHDo1S7*#zRp5$J%-9os~rEBl+22tBwtJ}ym zO}?pN4HYcM4>_EwUvX$5VKP(V4Jv!0lJl?cH72#YIC!bpAb{_o>L|M+EcfYBZwu8+ z7ymO{+$j`@GEdMo^vm_H(f?KOCGi5#X;BiaUM}gn#jRLWz#%02cFib%#Bs<(4Vv%oZsW>q(lpWd)l(szWeP*L<5dKRgmtLp;@ zEHgu(Ec{5OCm@vuw$kdNP8=lkK*R8?5-U|L zpdO<0R~hO-k&kXoD2t>5BtO98$^my4ozS?5C#b14XN*UUdfwd9TalNSFVGZtvEFHz zq4HC@h!fbZeREEVc+X*1OR@-N$qU3;fz_9vr8>nVGSvYQ{Ymn3rBd-7@z2uTM=QRl z@zkk;{9M!3@X6L}bb+9}(?UHZwa#BhELH!doW#(TGU!-M=Ba+y(v(o~5if3K{HpA> z4P9AAvo2F6Yj4NVkZmG#s{{r@vL%L%tYWr?8qtUwj+vVr>uWsk*4 z&g*q?h?Qqj(7h=fDndH$TILmS?6)=K1VpRjJ3^P`7?K+EEpdn)4I3PEI!b5J(2s;p z@I8dr$$L-fPL-e`F9Wtr5G|O4($O19hj=^W8|)j3ImdDcC4{V)xW3+aKmxAdg(ugy zHXZ4p^#jZym`NW|_fzjAk12+zVNxb6I57YGg)ZX&>W3(ChGL_Tj7p6UJk+9s8p{Hs zcw*vZN`QwO!hj$p^WdtyN#=lTM_FkbLh$hEIfrUewjJyjCu^!&wdpSC={*Cg4_)05 z`iA9SAlX=VIqHo-6{@lg)K>fz(z!Dg9?BAqB~-v6t2MG9GUo%!&fWKMWmr+da{+(D zgb;}*WKphT^1Sp%y(xii6zogNU0$cC%AuE?#;NW(a*i62G$?eF`NVM)W=KLYPSq-o ztGc67kVdrm4M3F=KB4V$A4S9>oh@nT9PM(nCK=gcfjlngQ{3y~P4+~zk8ao}9%e(H z)oFHBtc8bG=&cPkKiE%$31SZslze-HS;*|if9UXQlyWl)9;ztqR(}P# z1c{b1)qh!4z|<9qx}wm_dMs@d5H!YVN)1hWfkyJ`N2PgYh_)sT38+3LiY$BD^8n2k zx`1mSU3tCpp|S=LA<%aEH&6xPuMmiZ4n$9Ll%El?d2^5iQtwBgXWsjYEs=CVvNrf(S{!vMSSs`+|klgz%dVmK^_kNl>=Ks8e%L05F?085613%^tq95wxjA@;YDUeMn1 zockQGoyc*l3O}|d_FKqo-S)t+byT?@23034otonb^q7uBc?Lt^y3-(5Md}!;8ov;| zB;M4&N^LgPn1rbY4^NsUQ*LAD8z-HGF6Y&M#xIX_nQNVwPG&U88bbTjc2a65!WYL* z@ALFIqXp31%u{(W2cx;N4-wKtCXp`-%D9*`@X(r=$#3lXTUP(JG8zjb^bn8ahl;Z& zWTcm!gT`I|Aqxbv&QAtq{@VXe^6T!>lu7H$LothYDk~gOxe0Y#K^cWwO5$g$?ne~m zrDqLe2GuYZ*g%NQ_Df7AL|_lWf}yMF M@VGcHqc=LhbC6_te*b?sFmYP!-vSXAN z`2OWPf!~+AgWp($uZyZ2_7dLe`#);UsCmEIuQ#mC;PL+QLdS=PdZwIr%~K_cC(^Bh zN~=o$a>w)gqB!9|wd+T1mb--)zqgm3`)ZW+Mip)YigNvYf>3TKtja>E z{L6gpyy@O%WFFvd<@;M-SpS>%H~3w%7?Pe)mW2VCZN=R-9S43zcPwZaSqZWtOe! zQ_gLGmsm}~FLq~+me5xpW`@#V&Y@xFZEFl!Qqa5>*eAhx+2Xjsc^UA0XBxvZY23Qu zDU~@%-vzJlgP2wtxFklwf~6n*qf>2>ZP3ir{$SP6#%ca^T23XmgY{cJbDz2ur>^u1 zxfO2vIUk+!xok$nncaKZ;}(!`)wq&W8~*VI2HXG^z-84FtrYe|7w@y>IK-1q!!OKb{tugrQD+#8%7&Eots4H{#?e)>E zEtd`tw%9t1-j9(cr;;uW{Wo+f-yr$;p*MQ`PFb}<(J$no_xkyt;3$2PZnJLpHXhCN zpO~~J2EK2PuemnWl8GCf3QyiOL&lzTA+qY;r48dZQv8zm+^6JhG4J;9^iU$ulw4P#VGII3Qb@ z7&mHlMS}C4s7nz~fa8RvK=0bPVIq<8hr!~ud0h2C^z=@Ap$tb|R3>gYBh*ddRe~US ztih;KaSC_4C3yW+F7iRKB8Ma{?EB{rZGBFIycK#CjDQjM@n@U<$GAKc$yAYh~|i%mD!H zGdgLZHV^QP(u&c-^dN7MRhG;{+P79Uwk6O?0W1d55KgQ7`M+Ro zOZl7ipY~%-7+FN{NHjGd7Tc8kcpGK7XvQOZIxp)0Sv^%E@u7x{vNu4(l;V94Q_(Xq?1wBD1|LE&ed1 zVqw;P)voUgVttB3Y@r^z!n{hA5-Ev>q9it{vfpOqtp+Eh1OQy?2C@NH^-&)NP>PHA z?Ze?-NX~{_tigaf8jWAdjCt+r!WQ&yVM=y{gQ_Dyv58%y*))?Z<)zTv*gi zlyMe&8#FuKmsj^Kq**=P#1nmqA5<~vQ&>i_fk*aY^@f@v7Q9vW5scXKhgiDKl*kCa)N`J zPwddnNF;jQsr@Z;*GR3ja@41AOnR;Qj+o~NE~f^d$6D=x0nzKgx04SdBox#X#8EG` zOcVX*InxYFF!i>#4vX{LpNyHFB0A&m!iq&u`S_6sxK7-u^*1|8!N`P!3JuM${^=n5UH6O(x^X2ElVZn-l^f z|Bo6G?swM$*I3MRmwBC$l9?DfK@#Vzww{&{38PHD4kC?yFhd>+jL0A^w}}*{8Tg&h z0kSF(>-4v<@=BRCLU*V1=~SCy?pETS5!wD(tWpB{;7p})&e$e69)|k)dUtp7ec!Os z3{s??TlHNWbJuLCN!uBB>ed~|hm3X5LTWpWp59JVg>&t=A#bIL|7?Oiix&0bYovj(v$qI{5<(-YmStxCFLE3p0wHdYtHod#pv+Wpc z$wmIEy3UZkudup)dvsapYI`8wsShOd6 z)n{tlYT)j83M!$(;+#@O1*>Vx0%?DP1sZ`($R-TU!~V*ypm@Oe3EhEo>_z5P*E5z? zu6|%xG|uzee!Wd%Jx1xtoN+w?oaCbmC;6(Kxr|qSSF`m?W;!mR143qthwTge5RJP& z&-=%Igm%EI1y_e0A4iETcy3$KT5{o2PUU=&DEC{f`dE4G@Tok@_6Yiw zTqKuh`+sz^?gjzkoBNe6*z_lp6E#>mR@jPRJS_2%eMxvS|FIh@2AK|Ie!E{gQysZ4 z@IAPO)M_=L1n_~?R^*gW{%u=C&xq-w1x%Q;qmJW6CIRIGj6%DL$t*h8+&`W}4 z6TLzo13ewKk3tW;WyN_{*}6%lk7AoX71aAk<#Rjc--0gc=}P+>tM`6awvIpn3b$?A z0V7Q@A6U&9=kk#-GYy>k2ig@(;NPey})x3vcH>8@BZeV?_y#onzZ;5arIjmLC-&3iwx>WIne;c?JPf z)4_26vT<=m8Q~G>ZQU>vk|oulZa%EF(q8Cby!uq~@sx_+f{rKeNFH>@mVrBiSGJRG zH$QMYZn$gdAHSn(+_6>phvv1%Xztr&OGhzi0B0m}DdqpFfr0wJjJK6r`r zSfpWgQK>}5OOatxh76{Js5Bs$rX?peMbkK@hA1#ppENP0KqMs{0us>7W8M>)PG+Qt zsNfZ4@&b4Rgb{~fW*)!qwbyouF~HJ zp4dBLrw%E4w_MKzdPaH>)+H$uxNSE83q8_|Jr^GqpF(>~SKs-wJ)A^*$&Sb3^cHh$9W+K@M`O<3?j}z(YJH;9e?@DjmvP;RwKplrq_o+l&k-pv&?% zlOnx~OACo0UumfJMju-0_6`MuS4wm9GN`+XPWy>`_B6PC@v6-RE=kA8JcgA2& zJP;V&Fi2qE@w{vrL-qL(CcBkqq5S2IIR_Uq4^XIn@7a`H{z!&N#M(8=NJ zf$~TGck`5^36CjHOKJm-{8zC&+}ZwGtTqY6an}C1)ugtw-VS_6;PRr&f_7JTcx*=& z8Gcg})J{|(aNp{|??Sy}tg z);ZyM^r2){Mw8cq?*+MwV70VN3qa0{L>noX&_V`dS#lhP3+7=M>Jy025fCWi@9w_S zWqR*CQBYD!mLVy{@_hY99ngc%NG0efTCjW;J9%>t);Z3ZK^h2dRXxNVbI!K9-qvYL z%x?mLnu!aR4CzG(CsMMU_jjsjY!Wi*F%%k}&6=Y0hzpwJD>~ggi6)M9~ zARn~DBWouy*Il8&-4XG~;708$hsUi-u6H}WQT9xn#?I6z;Xr`Hgdj3?w|9rQnk;st z7J$4OQCL!3dSt(ILK=-!FKQP-p!tOJ(UzvrV;+MTg!)ZAOa>ktBX>bBu=;ZJdDcAY z?Zn)m0+nRjiJJ~=d@X=EZvCh!x!ls?u|nRvFMsL@j6xz~q}I*Im7=C>^?3s$fD_im z(t}>$7>PA2Df?ddyGlLzoYbD=ZjsF;RCgB}9%!6rZ+g~UWKk$U`P$|QFb9YYuTIcp z7-tl*L{(o5miHIPJM=d>5^kYs1D3PHN00;Ry^WnjfzvU_&gM$Z$ zjk6Ed{uQwcHw6tse2G&7Qk^kwDF)wzs=l2}FUUMl1kBoCaIUm1r~CTc{`tOUC)B^h zA*Q-{2TEr~O>f52Ip2^V0ms9xa`u^iZ!HsHc0XC|1^9B)#)DpdvHH7ZcQY5bFF;M< zk9h+6Xe!fqEtNNf2>RTc9A_%fA2^lN&wrj0J<@Rj1Tc{VjeXedtYHKLU^*y)yjcee=aVIY7K&E}i~ zt%7B>oJhv6i>?a|?s>7~T17&=X&btPg3~7|U>PK6wl28lHpHtj3wff$>mJ7&%c{uh&@uLg8M%Y&mQNB{$aK0k`v zfn;vctZb3MOiXfC*skm9l|t}Cmk&Umq=sd*XOH}!|AfHL9k)-8_8yvI)xBtnqgzIO zP40zo)kqH;=_&X*6h*Rfq!NrA`$E@+#)>KKzxgVdekhO%=%q!g!HRRgYo=SAf+EQDQd+L~X3L2}Zd_ z3xUOL0Qt|kb^d5vY0K;uLRbXc^H|)SF{O>g{p~yM+R>WdGwNFK6VD8}uX&|6#(LSG$maM!f(sXJ}fZr4CGM9T|;>G@T3`q9s!*GZ#0- zFd{$`Ow>W8GmEDJCrE}UIzY&Gb+AJl5CxtIpf&VrLu235S+;0jG#xc3lr@9-aH~9B z?|Tsh2GRN-zJoz2K~oNRJkSQ;H!FVOnFB$4Y^mu0&#r9d+0q`LMIZFN5zwX8Dfq?% zf;2&M{Ob$X+vf&TB8}x^z1ng@!wZ*ZOTtN}G4mnB)n|}G&fj#7n>iS=^sd|MXoBLB zI>5Xp`mVkSAx7%?XZ1WXXN7gHju68{Hz$Ia?G((w0plhxN;#g)hT7$Np_2BP=W@55 z;2e_`8H1>j%7*aBEfH5~pC)}Jka+R9pD@Nk z$$u$T>X<(5Sl?s%`lnyJ(!LN!=}#+$?JzUHEJu9$d3hXc8WN%;Y(QvLwL z2Wi^!T3yjlv`iS+88}w-V${P)#K4o1`lC6@wij96mmK#s8&u-0N`bN*b^z!ojN+O( zC-3s-C5m4HAU{q8)zC_}144}cy2zy(krnA~1zkio2Jlce2ng#WafpsIqYC2)i4sYH z7Y%d%VZnimLKs?C+jqY$DIa)JNxx=2D1d4X$GsFzfTjSK-Ow*K3m*b<0PKGDrkwuo zuDwvEEOE2l%*>is)MJCecQ+Je@45MYuhe=ZME8q^QI|)fsif8c{)A{#*&}bagwK*) zXH`-3DJga-hM+a<#6nf^e)C@C$-JA9FO#2L`mI6+41vuZP7hwv zl|^Q)Hv8mEIb;D<>EX9>t!KdUJSsV^n{g;gY*|$_>qWg#`eGQHwgnDZQ9`)B#u*&9 ztAj9t^tTe+lkj_dar2A&r01Ms!p@o$6$t$|#0imReEv_Q`8XVrrpjM%PQr@Bnm3@@ zvtLWU8ZHMjDN8{KODvFOr<2ymCYN783d#;%K|fkOx#_qzZ0ZAg(b`+@8pZGiUL!$gxm0Y56 zrrX-mmwoNNI=_(5Q0`*Q5Kj>a4W^YIY-B)>O1`%&@^8JhZo0hjq-f+$h|U0KX05&`*B1GMx;4_K{YX!HJ2iQ53plKhzbvYVL*Nnd z@RE28AoTo~=r`PA!sOuu-)N4PVl&e}1iC|LOH^9uT009~u|KK4rgbm@JG1*V2}}aJ z7ugVyFRp^bpXVOFOyk?cXd3h#W%d#fdO+LnLkMKZ-&!L|E7T!ajuX_f-bFp~S_}L( zx*XECHKL8er&GyM4lqcnQD}!*Za&40kumADUqQYLp#50rD3f0zvqcs?Ajsj;+t<2j z^bT#hQt1=;)cNYzuA?{2wmo&&ohV7(h4(e5QWqOGR^6VbU*>LXu%KLd6dK5d^D1@e z`xiQDcWKrNFjv%+z~hCFyUas47G7-Nvq_b&*gP+K(6OS+z(e#n@(8%<`^yIHK3krI z0-#ZbTq|@s#ZpOc!E8_Df6$wW>AAZiJwwowB+GQmpwZ2=bT@5^CLi`3@ zy|<$&jJ^4s$JB5~oz$>I9t+{I(T1ZL zN4^+RSM*BHL$%%DN(+M>A|Jl_$++tOV3r^C0@e)2CvmCiJmILOJ*+l0pma!&m zm}h#4F8u;YJp4_srWH)}uEgzMC*ZTRFS(vp4=Mreg}RJ4qw~9I98&&MoR-C}gHzCO z&GVlZWAye82O4tRcz~v(Yoss}4CN_}H&ZqkNpb1*pj)*BA;rUq%6uqxb+t7i5=vNq zG!~S4-zf>H9%POAI;ik9?W)aQ<{t1lG8PSW>EIiF*dD8rTQeV4V2G5Vv{3V)vpU#> z!$V|-17FK++-%-6yW*9;5B=v2J@S|$!x06ddALDN8sFDM0HiEM1}m2?;eYJ8SGr)o z(TKk7cB;MmH|71_yzaGO`mME_NMS)(4+-I(H<0oC;5p<+Lqdr03*0AanPwL3eBaWq z$nJ{Y20L`}KX=(ecD|&&STRnFVMOfk++o-d`2LNbRjP=r6XpishtVnJsWDK}C2=)~;zJDyB-BUJMff?A?YprdQ6EF%s^xg5BpE&n~dp=8(`d+FV4 zA={gB9lw+vHUG*ps8_rX?=Beh&;3ZS5fkmU0#Gl+NJrFm9p7UFuzGP`ZxcSIO6BV|8tye*`%$k zrWQQcS9c%%7ZN9c$^_JgSr_O_G1yr+5H&7=5b4rH`FW1_+2fBqg4#IDbhM<|34n*rI9|o;brlyjlI{bB))rQ4>SOX_( zkIKWE!DT}HC;7!?(cT~YabM%(_-=ak*2k1v#h1+U91$9bp~KYv!EyVqoUi{5 zxiitX8;gqn&`qlSnR<)3XXi}9C+6xZg&loKxevT;S zad==)!&~+|(cZqxT(B|wRw(+~Vm5%YZhRU+;Kxgw(aXWgl7!VEKPdq39S((#_1x|WOm|OJt8+Y*>N?5R|c&- zzR8*+IZJY(O`BX-ZP0CiAqrrm%{&nBZVC*WLY*OO?p)^mjvs{!29ZwXazH4XG`(nc zR*pmSGTEB)=5vJbg%dq+s`FvG25`xJtuox6Z7Ziz1ePZL6XJ4hk_Fs z*28vYgSSa%`6tO0m$>irZb2gbJ2A=gb6yIPJJ5-+l}tQkFygSl+$FU(?A^bd4but` zX1Nn$|b+ z0;IL0tM4hb6jz+wgUoh7Xv(BGU9^D(Y`7DH89$w{q2X&zy!H)s&m>BilGLe_ahV%l zcB{~d!Qm-}#4+gPQ|E8`jUo3O^`{ddRr~Rdjd*lZJYp`Oa(jyhwr%Nk{*vQ{L7& zhTqnYm%_R~2E@}17hh`R9-bue6!7;ceyGbCGgQ;=@`i8{3+OkyT0_y>EK*(d>>k=H>TM=A1|IG*{B$3y${9@@NO6<)(>v4Gd0!4Ed00wg)7*Cidf$P% z1FP*kEP1DGnZ_%lYjIWol6r>4 zCX7NV^GZd`9&q!JV+L2fkeE(uSnn2mGr6Hq*gYfCMt?sg?c&4iNXZ3p-gPZ zu!P56iqGSufX-q(_;YcF*U@)ge$#0HxxX<6jSQN~Vvso8zRPBNv$X!yNu}tH{$uq( zlRo5$eK`3Dt~!ibdaBZlmlv3R2d5BfvS3)9;uzW9p?COTP)e*uRos@qVQ4dN&|(my zDM(spszQ#zbClHJID@yAezS8aAOyx1hLL&jv0(kan|bu+XPSqo#Z*49sZW?)3XnJ zUh5O+v2NRvYF;tR403l}1CgMVh5{?iF@O@LlHy3tjj%Pm26dIzw{v z3s|4O!1_cVew=5+p=;X;>*Gz1xte4iKu8{>Y}7a0lr?cD?syS-n3sLN-=E=*WiRW0 zU5KQ278{7=Y_W0mqpkfKa^=t5NYZ=|Jhb<`X>Z#XobB#3H(ij>=+Re+TpDxULs@39 zRc9zhWqnNpk(Er=sjn*cz*mSlU+qJQB}3`G^gFkxEc zRh_9@U{%1eO}2&Okf!FRzBjIiMRz%%(+daf>x_NoBLO#>OU*|dis6UJCWu*1i@BPF zhaSK6yS8~qz9Oe&-+!sd^5@)-xR3`i-F2UcO1#n~b?}O##$^5Q6vS>R`) zt>UT1uL;r)n;o2lDjpOU7#WXhr*9pH=4nfhhFb3p2JML99DPI1)hnAP5FewwKQg1< zk>0k8Krlr|_I3}C8L5_4-`ZNrHnYcUgG<-o9$%XZ>Z=z4v!O7mru{XFO?U>l(bmn8 z^4;a)b%J};T{au4@1fD(ogxgZTyQA6rIuB0WniBQ>&iTh=M$!ndto)pEF;6=mRqma zHOTM|W+snog$-3E-I?70ayS}56Jrrn1D+5|7qqt$h+ZU=HrI(rMtl{51;iW@jzwZm z2_Xq#gL4GN@_=yJ$Wv|4_80wWC{I+PA$zW~!P|Z(+E%x@lBTN`C0IZR_|rcl|KEQ! z##>hF|I$0=7NxxDm(mR~0oQU%>P3wtMY$OD`gqK_HAKPT|Kn-xK{GInPr1?PaH}DH zm@-t{$DWem|MMS>?_Kt+Xb!*p`^VPp=4BBcAI2DtfA)w{u|o&UM7G}z_l$p4G*@S% zSQ-WWq^!ijTwaD1V~qLn=&KV*@nL%7FAL7Bf8jf@FRI2UbvNS>d2`lz{h&PIac7_= zdcC*3G169N2(fkO9i3hc?tZy!MTsMx(-hJFH#i4w{^wbYg(PUh=&YMAGDQz&@9LZ| ze=w@-m=u8|`-F=Si6*L#vGJ_O7w%O3V$F({&#u-AL#>#gCqczTSGkwzSd;@n)L7uK zMbV;7$=tzIXxkMgj~{_@uGMgJLVTVwDdR1CvXL56frNOd(>K?04aSok_uet?=Wt9+ z1WnLrQvqrc<2oqxn3Vk%V1lI#p%-dcM6Ld^n`&mEK246v6LX6T_O|4to*3k0RTZN58hmB;Y_@;21jTls}vi&WIv_Gm@oh(LF$04H7IG@C3$U29H{-d#BmhI&R=ACw-AWnIq=rCrk*5 z!s`|POZ(T*kH=JO>Vs>Kb3Cv7V2{;QI^iKwuh=1kODeXlC!NM=7 zT;MgJ&a!euqC+W2jMKE1xy~h^;XuOxn<%NUt%aA7^SQ{^l=ZR*aKV(0%60w#RYLX$ zc_@IY+IAGj5$G%zMD^r^dA9F=lii{LMc8950KPAy#- zmNWX=kdPU5Yr_XBRwFP8TtOQY=<(9WRyg`NVoETQj=R@>({mPWx!tf$F1gT9E-63% zj-!FBnZ=?}&qc^$wOK^}ngB&W?b_aYIbQjF*9wR=kkznBGPXF{Bbx@P8_cZek&~Aa5y9dA_lQXdFbED}B?=J}sVP9yUV$*6uQt$B*HFIFp5k)wIXlUXfijQfa+ocF z&CAazHyTBfe)wXHP>04E;_oTl`w0e@gp7TP)BiNboi4R}5;2wg3Nz|-)g_H?bxnEy zIO=CyuP0iiCgb)>lsxb!3XQ0cROC?s4+*6N=J4$HKp95PlF$eI!yISyZa}l`d3|0o z`OqAO4HLR=K9cFdmAhDFYDb5Q5m`xV=G-f(3fS~d<=%pTf82Ymz~D0o!j*=w zk5(Kv`)w|=#GK+9Ax(NV5nm#kzFIoQhu)39VDfr8%M z2^^?og!NAEQpLCom7cDJHeb?WZ|wMP|`v=13$^qkTks2gQAM zlHR9&v+0i<)YAgZYO~y(rW4G}Yn6=mE%D88tt+xDJ*m2Mk~#`4K0q6=|L}4wB*he3 zBjc(<4m>j{+OqeQ_#ipUy>iM5a^DVj)ZW$nOrQNU?~}vsCl6N--@5FhEhZ}Pu&rEX zJ-Io>JQ%*C1IGr>ND3|}e>-x1`Hk!igLJ!UCzvi?a?A}F%h)3mS9#oCMP(3v!Es?m zORxGGYhhTCWns8jlDjh4Mg4(DDh7?ydLkNTwi8(qtypOdlGh;n>fF$eJ@1|23Ju=G z9!E2x&+qw3Z#>WvetGwDebbx`tB$THF%R2Yws_poogU5i_BIm3*Qv55k?BJ59M7djG?5z5ptnn~7^nit@boOlAjA}C3{z(qcml12i zG&l0)Ir^0!P?H4&kb5E5e#;nP-k^cKqLB3QR+YbljmvtZm95z&#`;3rOYT?JF_XT3 zvEmpP@JL#vZhzOvKTx3a#DW@zG{IcK%}6V^AKq0sAx?1&Trx9QzskLMjB7^qQPXl- zH(ma0&=*h8cYY4+$0^@5hi`V=Rwj*f+pGh~HN7oTzFnclO7HUoiSh zxl_yymE&3{q#}LmXEiQe29T3ZDbA8LzB8q3RRcq^KtuT~efM7ia z2(EnYdSSZl^*w)rclH&lw{}RyZ)e*gOeQJ;Q(MXtq>!{K`XZ-1CX|q&^k8#ynW&@D zv=ybPOb)w)XxH|97-TZrdK1Q!+_10gl!5DGR z2F=Qvl=rs3cn+ZHW*?G1MW+i9Y~brg>%)#seQevM%FPzxuPS!r0ON;x>t9H~z#4vR z_A%>y&1u`>e6OCd2299+$7YK_Lv140qMyfYSAIytE_yW=KZm@>rME44&{4Co=fwRV zyyIV00GL9vC2cyPHC6R`yKDEp;z7=_J+3TGKAQNfS~LPty0008?lvGL`4P3&_p4%* zCgzMXnP*MzCOA!_MIP3hctY|gIgW_k6jk$OS7dwOkg3VMocqAv^9m7pRS-?(QBBrn zTBYj(Wjh4g!R6jG)w6Kssd@MQm>VKf(N!gQ+GnTM0~v*yTo21srDaRIS9#JnIB+B9 z-Qpb1mVqIxl^a9tS2{qEBXrlD6q18aNs` zMI_zDLS48q%DNO?6YA^j<>Q20e21_9BfJP#YnLgx2Ft#Db~<|H8O7p=(L(p3BWUNP z#zs29kTmh`f@(l<)uNhh8m$0U%S7OF_J;^nB+9c8Uf~jpdnhi@A2iq7Q5s7guBelG zX@EWoTyhLLbO9aJ{Ew38_UbIOmmF(QL2{G|gtNC6*`mM!46lBFpx1;{*~x?#%BK&M zK9qgQV>4BFNoBDHVViPZx?KdK4LJ6a_pneln2ZNlEGPj=!~HTxiIwQ}IXkfXSh6 zX?nIV5C4iF3H>|DBG4df4M>vL`eOj+p|Vt`-K)x6@lw~K(JwgwYDuYnf0zu8qx*1vE5Wtn&|GWd4v*d2wX z0Kb?j{vmW5>FE9n+MxPVS++of-4boP3KCU_RbO*JZ*q5uRwCyf#K{>^`N=WctZa!1 ziOv%H5=6D86np*N_*(~Hlg7(u$jfXs$%jFw1>clo1FPDo6kd_HI%2(xX&k==~8%0y`ycuW5m-5xIAZ@iXy%} z9!>ZhHatrgn}qNt=cgCR4)pRwVTuw_rl4vytu2o9EjDX=;&Hth-yCln1~k7^i{`k@)PMxE zS8ayOMJr08ydt@&+|SeYo(N5}|!ovCt@M z`d|xrp^U3qjAKoG7}7o$IY&KB#q?!&2r#I*Zvbh~7zU%b`LmZ95K`XA+r8L1!LD@{H0=ozP&OV<)&Y0%bXVHpiyQAE;!T^P4sU^ zuXHb}4INYDST{bL5|DRdKiJ?s@RRVmtx2nRPq&-wSGo^=j49}f)N(f*E(3%Z^_mR( zqzLo*EwRh`U8=%V_SvR94!OV8dYE3%b5ZE}dy79PITtt*{ci;#yUp3~>Id^`|EWFj%X$W2 z(!T6ojmww}$u7%jDG!yE$HeV5t?$^i|fmSIDiU%^y4Ww zel*_dd^(bjkPr!hV2~JMT&7-})s7oAvHqbg)$7~n0*a@T8w#|SmlD@9 zfxNkgaKh)b*+4CNLz&`8jb7hE`d;&(sep$4Sa{L;y0)hjk!wx3T}r!+-OiCyg|#|+ zu|2?x)|D1N<0S%!k4C;Y^S)-<7Q>yvrzqh;Smzuou~VV0$)mtAu;7Sa zWMh$bb?{JR_T~N4o2yga?_1QeYMbfK)0)Y-jY`W$_NWc*J*Q3ZH^qfrn_X9kH8%S* zZ|i)+Z;1m(J#y^5oz3MxUn($s+zuc^Oq%b;YmV%O3U+nH!QDq&RXl=Rzc%UTD_S{+ zjbPzjY9;T|?8+Fn_?^4EP3AEvbKy6r57Jch%_!v6ojEtDHGo_7J!pzHlGhT;EqLfk zZ~IO-C3leuM2+iauk*eV4rXr7fg`I*-hztMx7JYWL7ippyN7CG_dluB1}@R-iO(}E zVu^C?2V-YVhOEr|#=$kL)QHfiZsx`@l}K&Hh*B+vlW?0bKF~ zL8{G}W`A2PddkgMRPRiX((t2Sns;9_0S~(i7zqz&HWnSLNVe<-M<+yz%C&xC$)BoMa=0(?kRN^BE)fzIOy2+T)`YY+p-|Byz%etxou8!D* zp?}Vt<8NQ4f7*cEYXIypX6Dpn>ud~1AXUYjzvxCtL)FCrzH)jtcitjk3j;u_Huag^ zk`CpwWUapC8Xd`=Tl7)7LC4O)V*2B3+o+sr@BW!zzZW!I4?$O9@R>pCe2m|qq7iD( zI`jLD1tleSD~MVDiQd(Yme?&3ZrZTUQh1C3j6s))H;Y6DA&YuAH1LjFxuZkj#J#6geZkXY^$lH2Fe?{Ljn*q^vY{-Iqy|kVPu4)MuMfXfkRG1;>u7XE9L)o=HqIbE(#!$bvOPe z`42Fkg>5s};9v_LsDbx^EQpC$EE@qr9(-r)h!NM@9p+T_f3&q&Q$Rwp(vwyw35Ir9 zEV4fIia6Q9)k?4thy}&85MWQ{awVc(!Z`v;7tI>kjCoAC7da?*L=(VZ1(t1AxoNFSkMxCWQ6HofGA`~PvTT4kO5oC!jqXhhKSS0ysRwL*y~N?Wxx zG%qxa&kLUM#WR)@2=4ziDyy0$$?(Ao=hToGC3FV5>5eQ~J1^K6|{ zH~8&8Je2@Q{VUeI>e%T@U+tZz^#VgGv7Im2wCS!!XDjvI3mF}q*NS4tb%$y1ln+Ee zkwg7zFPV#4n(mtHBtX)vs_J8fvmyF!-c22aJ0aQrfclYhIbm!~;*d`Xa*QlI<#Q6) zl~V*wyrkN!pE0jB&x#%^3KI1^0Tdz^+Gd-AbaW`Vz-&*YJ3k}v-(G0P4eQ* z6C=(m1ZQ#fy2|MF)d^4Lo*cPSbb91$G7YAx+xX@0%^tQtHj4C`EH9-;mpG3;fAIf4 zTJlW^BIu5=5@Gy69}(ute(RfMAzk+CXXX|*TlWUw8Y>3RgBDv`zFeD)oU1vA8m!_QJ1$#86y_I zo1y_%{>O+gmZV&P0YES9ZGxlpGc7@PrUM$|MB2;0na z$cck3g^;X)D*D!lO=OeSp}YU##)5)`S32@%f`*f@F5OJaV1TdCloX?k0bMu68KHP{ z>UO#f%(eJ%z9Xv~JsiJiLhmH|Sd6zu{DKKln%JX8xCa^mu9oXCb0K{)8vh<5)BY97 zL$p;n`v2TgWB1ikj2|_g(p|kP#q^$=c(d|%e9Y`dkbtC4c?W%C6}C}&$&J#n!`_Wg z?v3~jb&i|SuW#SoHK9iFUvgKJPQyABMi>rSK+6#!Odt@1Z;rC%$5j^Gr3JxUES&m* zjxzwg^8L=mZTEqT$0=??$DJy1pEj+^-xlTaxFyKa(>r@12S7Y-;ayMZ8x^Qd<*V{V zn%*Cz=&Uy?fDUE0<6{EPPG1#$)pjngN{Ft2fY^2B+)< zf7yGIzMdj16+Jk`{%yT90!8?t=aG(6Gl$)UOg}sZ?gPWNr9%qp2Nhqno&Q_?&HFsF z6II<4xp-XcOTjSs@f<_Kv&PjLZvz^e9ZmJ$O^Dxg)wYVBsn5@m!`xCUwuRJG0D%MY zSgP#SjN(+1E{qR~=)@54mu%s2W;iY>W8~5i8h&gb$YJ3a?9y??)n>G|9P~EH{ z{YEqh7182-7pkLQnR&kUtFme$X=6cSSUM~a?2%5yF7aq+>XBhta^nm^dQh)bk@2iG zVJtu(Yg0b;^R>8*@)p?CsIb|n7&U*W2{m)S3r*}+nt@?Pmmm5 z=><(rYh}#Zg!Q>oF%GNWdeu>l8-o-iE5> zV+u&rkwe3LwGFj4@he#JuAX03{qmVt!6oZGC>9r`84dN*SI+FKx6 zNZwZNqT%rKrEwk?FPOgQe*yA1A2z}!&~s65KO@0BLmG`$YjYJBRadBEr>fsjlbkjH z5qKwMjvfd3(_U%SNl{nK3#(&_#%d(@Tb`xdt3L?Qj3Fov^r@z8KE)`S!urAV=f39U zOLaT^u;jW(#b*rI?~wGxZ1e4pjGtrt5BW7Qt2{q*^N!Qr8lf8te`DLpdY?|FmIx7F z=6p6RIPRQh1=g3O@=sSsGFGg1Ygy^5FBn$t^JZhoHS4#Q!@33U-`iJkq|Vs!UHudY zG%ovq17P{T_)ZECX?wu$py?ks7i8Zp36T~~h38%E4bevv`-&SVZM4}LjTp)+j^C&q z{n{Gu0ZKHHGfBM2ZGE>$>s;5%<9DIDsv1{jD?hO&uC&r{$+kH6JyhY1S+^WFHT@hX zQfhu`u*74c5*S%@>KdSS?ZHLCj~vuBo-Jv#mlnB)JdH^3f=_dV|I`_^(`MUU-8FX| zcc;zlY4fGz3%J!NdqZqY!Lscv1(qphU=Tz7XBytS`wmXGoBsM!Um9b5zazI^ag2!8 z&FQwK?V_9N(Z-kr`e!Kn==mTYXX2yLF|OV|M4WTvahPzV;M;JR6{{zF22L4R+>1Sl z@Kt2Fda1eW#rqmkOm0!@k%;X0!i2Lm`d=3aif~$QfR0BS^$-#?&P{hf6<`I5@b57b zQKstP^fTYn=ZAnAD5akNJg@8r9K&Ta#)dkqt@k(CF#BHC`{rch+4&xIU!D2Y$LS9$ z4mFwrn?OSI&!Fk*4;hZ`n9XxeurLjuUTWvKp<%Jh*4k=wAZyaB-7j*uJt!R_?`y`J zn@=LsWXmq&IqJ)v}E5E&qg%+jA+S9#P{?rb{oM;%MJvW^I@78~h51Lf44kZ|C4E z{g3+k-gTE7!>cW-Ks6IyoPA$ZCpgUEDYUPK`vEe8`J1B11Ezph!Tm|E;4U+!3m3^}UhO5!s=L;UN4@jV~| zt8CkF!<`K^w6G<{EOsIcxP*D6Y|c7$Q$-omHLXAa#LOb!PkAOIO7E zjZPr6!KuP*{{0-pnP1H{`?t)82xDp-2|eM%5{v-85ctc|Cz^&lINuQJ(!%V9N^Le_ zLuaIoZGg&qmC6Ddtw>imZG%j)B%rn})A$6`u>UWkP>?-d;8_@MFTBg0_j$qI8w}o< z!6#fptYpG@1v@^06KaUDA~Xw!S?Nm#BcW(38_xi->{rNX7#P*OFS$d%tRJ%b z$Zfo1PrLLowdgM5z0f#CV;b%4|6$M&+Fp*T@&u4t`_OlizT1ki8VE_s9g!?vN3LHn zmrDz&o%mJs-voDFojgtKEL&S-(1PJ%$RSK41$2roD9A|6RQ$>VH2ZUMg=Rc|y}se3EQN0P=)e#focSdm?$>8{G_qxWXLsU7$MS~|P|p%C?B00wyg zJHUxrz;Y}hlO-^Dcw0G{<1n)j1%#7;(%L5r24O^{FSXGahfR}eO$IL#vMLps_yplPDq7OANN}bF9D(BTez9FFfY1 zI&LnE-s}ZYYYrVe8l@2c-L4WeKv=4x4S2!1$D202FVWrvmx}f_eqEUqT@S2~n76bR z1g}RT?I%ZmDZ6UA^WGb4=0|S6sS{&9GSU^RpccF zac*KG2m}Ydo|VGalX%zESrFNy%KU%Ra1c0tsqJF8Uy(VXv`R7C*5?}8KGglbW=OZl zT_L@XXpP^UXwS!WkbOV2`lOM@fvU#1+TyzvA!@HA;2*>Kq#xB~R3<&g<747(VXA+Q z%vS@ebD8AN7~|eSs&^L>IZ!iWZ40lf=Z>o&ZZeDZJZXR@x02s;H}KTPeiW7J`%jA zoHN%NWw><>O{MjK@t!1VeppYXX^5_2>qaEX-rJC90U7e|FND0scyr zeePJ0O~Chu>dE=(NKDOnRE5i;2ar@-sRY+_t)beHom^L&HHYV1c+?Q}4Rlac^(G*q zD4d>wJd5uUyN|%vpoLRA@=F-$6#VqvVw{jJce1En8EGGo)}#L5O%hC0lYwvkR?|a_ zquYS!=U0I_{Rp~V#I#vIaGbY&7uL9jJI7IvE?M2m4Jr^Qob@X6wPbAr+P(=(cFngM#~O-9dlWA3>!0B> zKIu%9z2wG$V-7FKl5q1q4;1ZFaR-gm*fu;;9O#{PZOCSIF$VX|@t9+)C-umYRe{y@ zrM*yyC1WP?JYxk9$n+D+CZWZV0gKdw zuib!uNez;icX9W6Mi_4EU7?Un$2+DLE*oy6prw&dK}#*4rQ$k$HPRDY`ggqLXVu>= zZ9+#_Uy6ewKD2`z13=hUW%vz^XK!l{VSYs0Qa)p5VFZNFjZ?HM9NVWzOC40f*l=Tk z^3FnCs<$sq^b~XFd3Hy7>_&LgX1!xC-G^~UztxS_oqO(U=1ouZyk*&+7gwY-kIWiWc$|u`7bwKMkil>_YjtW) z?|CoX)?PXurOA@Ld7QI2Yt;l)WU8BQ>#aBS17o<&Bzm&sl7$W851xid&DV0CMxnrP z28e}4nIh7$Q0ZGHH}fuOlNa!hpTl~rP0BVcM*9eY2`aez$vClO^IW{z4jv&F1hS*V zdK^#eB`drVAs7z{Gdl0Z_3|++4l?9hGShDIu8ZF?V25Yn+_=JWdCCSnnzfoMEvke? z`omL`^fE8_M;l8-cnx)N?>*ZDVR6qLIUct>+uzCWyKg5Blww3U!qO&ZzmL82GEhyg zc6!lE-_kT4;(o8r?l!J!cr99I0o{vw&vp!jJPYjraaQEWqHEx5vm2EwAicEH3679c z91Ols!ew)<{IGAesnSH@F7shEFLowDqA0?Hli=WW4i!z}~!N#*f~LEMhm zTx9PdW?#HWL|42JzbcX4(1a4Ue#E7Nq^_GNp`eg&#Doz4E(_B?D|3cLs_L6sukeY0 z6(W!eSz>UtDjB0?r-o)YMz4qW&-C>|LK^Xg_Npo+Bhuy{Ox~7s#Y$}iQNlv>{fN1s z?C#;=0t?ygA+6kS$35AxL*CHn3npcWhwUa%0DqQT4(y^tRK9i0RxPKZa9oDeM3?UI z8q$j)`20etUBnkS{HI{n;0dCP0D602u?NgWVd(& z;vvEqcsN{1^FMO~|LtFcfew%xVm4E0MA`X>uyIK8QYQ&%hyyo{P159P`FDjY-bbQv z!1!fXD~|<;#m`&`9@`hYDowOok%Xd;PeSpcY$McA-N4kh6|0(A9IbLEL_y*Pu<8~K zkr0af`~@aNwq)Y*-^hFKbgv>+30VVA6gteZlgj&F<=><1rH`c6IYPYXk@dY{>#Do_ zmT2SAE|$e6{iqBdP)AvjVS=~;Zn3@6bcx>mV(fv}l)5MoDa567%YA*QHQ8}>z8B&v>U75WQjAXt!PGl zBn~7J5Dov83LjHiFE?9TwCEg7*Q50;R)XZu2oPX;hea&r#V%kY0i!RMq9HF zS>&8K2RxLehNFqS)UqX8vh*G+trnu_nw0hGZ^;Tk?9(VrleLZs0ldS|3An-xfpNDC zycs?g7HQ_~MfTnQPKo;>+eI~3fMaJ+T2)Vo-VCjmj9Q+Jw*5PHXWR+mv(qGm3povsUf|lQpjJ zZfJ*v$ohKgJFm%+BMMdM&yqTzCWtk+s19T1uoFXEZI1Opqa$uQ%5k9q!w|S;JD~@L zGr7x8_C@S&1`n`|SJ2Ft{UQ%<(8Ptter3eK7>V~D#z9h(TI+H5Hyvb>$hLNqtJ0Di8;a!inqgzVyZDPy=FErV_UfA& zqwPa<*b$T`wqH3TRf)aVly&ki?aiu7%@d%5TR5${r8cu8?x;X$g!ou8v(q@m;#wtv z&RKbkgRcK?GqUPWxWT@`ZJ^#aA;_{o97DjvcrD8)&rsN+Fr4%PuhlSM%em1wb-(20 zwVmSOwTx)QX{nPPOoT4B4C1jcG+w>3czP&r$Wz#s=$Ck6L18Xa^v=w0qTBxHfVqMd zllWYv6bsOQYcSXLcbLah3cwC^>xV|8b}zcD!;pk==Y5SEK5k37UW^rNeXP&&F(2%* z>CWHL@Bbwyul8BTeNCG2tGkDTnF$Dt;JHTAk;YF9+lJQ#SJYM1D7c?zw)#9g{q`dh ze@AK6Xs$-v&=KXCzT$ulZ~LlJGgfw-Q*YPX3SL&srXMDm?~SW^>i`CrAN_Pwd_AmG zZ`}pP)^ayG{di-H_u2XVB&_C~6Z3lQ_X8k2r~IGYXS~U^UK?-mulHD*vBO^9zeM|P z_@0q9L2juQ|Ax{^usnqI&6m^W^ljYqUSV@q_J#$|+9vcf-CGDgyl#oWg^+FerWm|2 zFu=W*K_`ZIit}}Y+n{E1vn;cRD$#dlG+?|*4=<*Hug-YK-=3lGyo!_F(Dr$pSP6io zO)`Hiws20VWx-pS?NJ>Ih#x|5LN!|ECzz-Z;*Xt^YdhJFZB3}J*v1%se!>Z5d}{Z| z%1_XzU&Ez3j9{!|veRI(wRf9Z;B9+(dZWRxRE|$%J~ZLKBX^*?3QD+aMA2(kE~&PJ zE>G(a#PjiN3Kj&V`P%LJtGoPqN4M0J+OoU%N4iZi3@ARiksMy4Hnw{wlr{~3^!4DA zyG~hypF_{g1DOvg;K>F|R699hl50iD%3ipTiqZLv68SZEP(a=bTZsxnutwJWh#>X) zU0J_BnrKGW@K>X`enZkk5EruD{=h^-$hF+!`asuL+a8U_zNQ$CIRV357=Itd7D;KF zies}Z8kX#B-`huNA0F|qQsIP}0%PtFb6{0Sg1Ypc)KCMJNeCvvWho0b$_1Cc zy}CKdi$={{$@S&vFvM>^BT}^Nn`2j3#cl$$6R$lMIj9)R zAJW9kfzEa$UsbB8d6Fpmgiu|r9E($?6%rABq*jSWQT6a14XkRN7{UT2I;|S((ZPGP z(PZ145$@Qgtx3dDzr~GlW0BEG^}NOYlg235MjD^itf*a44&LW4aX0|;{mc4WCV|0; zPEOkYynOaMiq5c>DFF%EiIz--}K7RN3P{BZNar*=N{`gKY4zw!7$O0ysRC6e9 z$B;V*&3aPH6wIle`ck7Zg8x$VWLgE(WEYkYU^wgD*YrY;Mbx}!(@)azy}Lg?F!_k^Y?wQkpGeeN;5ILvtU|dmfi+xS7I~=x6+!b$ zVjp9=9ZY%07<;(;B-}q+6ljL4|elBrfyCMV+Xx}E=ww-;%;0k{rRhK z%OND^dRnSrED7cM6VlT~R*S(ZtRVY!!;y4{Y(d!zCv?@4A?DWl;r>#zQ6bjtf+(E78$)C)L$To1dC> zl5pen8x>F=RynEPwk^NgwIxI6ZhBUfZz?&lXRx1PX)ojIo(-gBE}QlfC9wj7`z-h; zXwMvCWFb!6aZc>|V!1txqfiQ#LG*bzh{q1YtyD~{u0dK(BORfp zcFS-Ll9KrRIfJ=8Awr!zWyNGht;9?@lg419rh*z_fi&8jl~jgppn{!Fv1r=4Dm19t zCACFI2yK!fkt=Jjc4MV+>+W8#;5Ho7)pwq}mn3#6L`EskO*Je4*0 zDME-C0^-oG5$j*Y?F8cQ!Z9f#70nQtbi1sT@q4$^3yp8h8jN{C`3BCo5Ut0m+@Wbb zm1PrKBstFQqW-gh8dN6$>{gl!=D7|r0Au6D>1?@6X%GUgL06p?y|H;ZiV{IW59lIy zV!nTIkJOqt)A;fQLYc$Gdp)|$vE%NV!XgYGAXPe^jU`8c3>^VkifNcKJCqt8P+D^0 zE1DN;bN1W5$|uvtAb|;nS%;a-r*vlkY_BW~wx2gH(UK!Fsx8fnN8;*VpX=~dBbf83 zngn=6#~bVboO<$rwva$eK3Sx{x`V+BwKnx}y3H+zG(^+UsN;w5PnZ(r(k zScM8GP#c$?FQJZmy`erSg$^8|W>cZ0v1U(Ju4NcIe=xGk$s$YjrRE3%9o&447A}Y= zMA3%lJ|L_BWB-l%kxs)?t#u)xa8k2D4qE%}wM41yG^8YjjVOBauFHT+lV`%PBk#H? z`ko(zhL}vR$#xj8Dh>>R$=I*z8cL7PC;{A~hKkE>Dj6=7SI~Q$t4X3D#&xh9k!C)8 z;E>s<-g6jK$?$ClWwt2hNSNnJ6l;+iW@D*G=|{!9KR6 zV)#(gb>yDNVY5xOUqy(OgX2%&tkI44{7pU;7-nasKjtsVh={S9k`)Rtl7^dKEv(j; zY0o6NiY5pkgH}y2l}+@O`{FAzfNFcCdX7>5FPHab3zhGaP;Simk5 z#YVmtg%ytL62}1caeaFu+4(i)*$g)mbY}#AolfoB>$T;F$<>ET_z$XkN7n1lEL)VX z;-Kn;QQmc#BWo7(t;kArh1XD>_EI%QTzpZ3`yT9M+ur1-KUl&)fb~NhRjEZQTPgjT zY#tZ(b^$wu44Jx!q;!rdjOWW2AX{prR+Lx$9ZXMG9;g_K2DgV}RI~#C?F$%s$Y zN1>fnd(*0{hBX-G>TkDrSV}Z6Uf0)D>ufiZjc%!K5L=h7T+j{^MBONthH>s&E#Lb7 zbfNDz&+q)>Mm;wO_q}0Z%mE1E!PtVcEZsVpmN@iCB0bRw->bwYM%enbJLVZej5W$ zL`>u#E8jZ5P+QvHZobiSLVxmoTTo@j*&n(o7RxH--+B+^DX8X_7wdgX8ueXdtx#xE z62N;e%**_p(^E#Uz0MU8p61 znpGB5O;m}JA?aPs8qPa5&GVnMet%&G>QAy3Z{KeDREz0A+Rmd$u*DEB6R*bs>M z*y7T{D3?^#hgMLU(k%8$TpTa;S#?V|hVqWh`#4LMHyVD77H8ZuD-tC?FB_a+0}yWUP~jjsAcY0^dAny;i3 z>1`uO&)sjE9gu8y*L9VX(yJ3*#UR_EeFy(}3TkaLddA;`JS@dG zIvq$mn<;mU-7I$-=shT5$*rVMkQAZWmdl{#_0C=^`={2A+S@?g5^grL_6wyQM3GYA zNzWB9G#C+o)9JwE|GT-MerWuV@tmol*)2vvz%fjS)O$C9#D6KUYVlQw@r%>i)uu0U zv_Po4cXYVpf@&xMYz-#~kPs@>aL9UaVh)^mP_s|-X%~-5!_gHr9+l`?d{X?9eZ^Uo z&HX_;-M*W~)t)$%b8I6Fr238MwT;LCGz2ON&~xt*y=JrmOF>HcWRZfype+Fr6&lNp zla>cQDPT3BP(s(wojN-FNSzymZwX0`LX>ZcE5CCo$JqF*TXXl9qkN#I0WR26d6z}$ zI&W`d!S?$3I(-67!H|)dr@U zD4me0seY^)kz@x;(@l5Dju-_-Q{{JyvcNH&y{RCS^m=e-)QlI+2Kkcf0Dus&++G?-=P*na1;(bR5IJ;5|HlzvP;?x4eMU&~HTz zEK!ji^CQGFM~cm6!mV=BPO^48$}aFFN!zkPh9-w#RM5`AIugJ*C!Xpht2~lCwZKwE zxrLrLNXW>`piPaAiO`)48)@+TGT{vH=U)+%BI*e$5;f%u5B+E2y$}JZ?U-mO5ml{o z1EYq5X{p#9Uv|Y!QFVsML}?m88R60=o84qsLiL0?&lNbP8~tJ%f{rZy_xcfwmx#Ia zA-*p;Yi}{wZf+KJE52$Ch~(!!wrJ}gpMZGRnG=D zZXUEx_E9z@z%zN_@4r(v%!_0rY&iJY`330DKdvlGwpFuOO&QUSHlIPep4sVYIiB6q zY5*6C`kO+_0Ty)+MRSh}I;|eC8jg#sjJ8!XHbWM(ByZU@fzQ+|B) zGVmSKFo>4PP{OF8uJ=~fEf;&FjTO15hQuEv+2hq(U^$h6|f ziW$7o^baiOv?UtFR&W->4)z`ILz<2`g!HCIZ+xZ zTR(R!Ry^&Pfo{9^W>i_VYxFIrM?CZ#Mh)(3cC?fOI2WZR1Co3jQCf74oND|1)#!2N({+!+~-06#Q#Yogsi>s!oXB00IGj2 z%i#aoy6ykuN%_OVrBQ4=v47I~&Vyu$gjOZ+x~SC`&OHL!@SJdBXVrZ91TFVb)mqEy zcLXFbal)D~;q~h-r&TXyP0-J#qm2cIBDCF*+5y2^L&-blJCNUV9gCQ^ zbJ|e-$GuEYag2CE__Vyb>hVM5s&GOFc4tp;5fG1S@FAV--Ep ze36-NJ$45f^9u`_RUysY9bO)4NEl&U4U#)WWsx~*S)0>2(IwymuS_ny=$sKu_W)PH z_Ac~>@sU{qtS?&Q>id7}_~b;p=)K@a<{9g^s6NHV_L>U}I|7>y2M-+Oad8!Bs?S{^8 z;!}a{Go#$it>DvOC@DqBKkcTXu0L^Q*Z}r>I(WnhcDb^4g}~&Na7AYGPrQo z>4_%g>{#ePi4plc=m)c;VGdg@$+;J8*<9x0irj$1K%ys~W$y8}29Giu-dd{}XZtfQ z$EvgBQ4;ULv&F)w3$WUDb)eY&?Pn#peF)_>R84POK4CTd;&H%oYy9S4U-V20j!Wx3 zdp@QQ4;g&K(cmBLbHr-Lp5fC2VRVbfZlnE<)^hsw+R*!&1j9A0{f@^*jkkUOMBNL@ z$!k`F_vV2nQSo3FOU!y}#<2o(7=F3FJmYzIc5Z9B6n5#tq4}1_^>Ft(^ljNu{d)aD zxVj}Pex8?_ryHPrWSFgZixPi@<+0cuck~B@c+G!y)+p6;J)Xa6JA|rWuU&7-^b9&? zoodn7!o~U{?bQlA)Ja?YBhDCZc8BNhR%>|dv{|QHK0+_hE;23nbEtfn_@u! zHlixl!k1TxE=jJVaSHE0&*Nd24o2~+>1Z_cNc%@@LUTzl8QkUhpTo{P;;6Csg8U23 zh2|eN=?>0*TIt@)RAPB8zweq@|JpxH&2GRtK6gD|Xsa&*39iptYpQiyV{+ZeT;-M{ zT05mJbY5UHyr6joZO583-be;6gL-_5x5=pW{cIoFw<5=)Ub^Nz(zh-}$D)D2z}+8B zKBYa_ul~2pb#5=az2Y%Pse}2OenE8%X4`kf9Gf;_A!x->wNlKw^g zfU~BpoE2@-TF2n8$ zhQ<%Pr6OKu5sP(t@49~Ve{>vi`V z+v@sc93TDmy-Ok65FCvDiIRqh=RD{{3Z9)V` zWT;mqZ)_o&61xArwVl_1+Bi{?qqnOK5IS8X$)LPU^({6W-+gbh$Sm`&tb5AeQTYGr zy7s6h?=2hfftRT=y~V0aYo@+Hv}P(QdPP228WEKXXsnM`S+%wx-Wsi?Uexfh>a^a0 z5elf(qJ#n>Xw|A9MS)DK7NtS}MXObEuL!{hpnwT^I{Wwd0_|G!hl=PT$@hDlbN1e6 zA4Bi((BQh?rEV_QIu2X0x#9^4mfeLT#L8{URSAnQv3PX!d4J-Sx(K^V0YOpdqH7tB zW$y}LzNH>Qo_P07K%kyANNHFF1w(YeGB>wg8JK2C-B$YB$AkVH8Di)TOeLCHssF`} z?GHhJ)p4maRKu#Jh@~Ef%yRX=p!XwsMY5@=-16k->sxV*h~$dp){$PF8@hLHgA*@f z$8*;Uw-UO0T-cwmGrlQ&e^6Ngy`aGyQT)M=n+6e|-XtDjku9S9Cyt#kw*0Zh=ek&- z95hJ&{E_oE^+F?(N>|u_12QC{cP0W177vKAhsFq96T^II73e{JCzEz*(%Lw&XY7uq z-PWfg3mGqdnRjCoIqM16KxAHVlGyqCVupo@M4FP}DkJrQBw%OXB-lARbQm+`15YP5 z#8DN;^wc0@B*Zmmu;zG_N%YgKI)M+uo+o2ZiuTJ~v=24sfxof7Y9?5g1Y0facoV%! zl+N2N%U1-JiAE@L4Bl3m-<)IJ;vV_FP_{Q2{pRk|fW#mVced05j8e3E9u3`~e! z!tC7GJSvu7(xA%Q%Vd&Xd^&~KB-+glGfqT}IbrxjR=Uj}*2IKV2J%pjv4-<&bfh5C zFODnAKt!{5*Uvy|a>k>&5@+T4S($(=M+IPZ0P<~@p zzZAd!f5+FZ?TAv{!~T8OMI;=&$cKP2u9K&K1U>>__l3kH(KhSgh5O-3VSt-zzbX*) zV_%_COsnz0=W{O6Y*A{u&IETQlAlbA<`GxZ7BTG&Bc+N+70b%}NY=O> zsT>@b4n~#xDf|i1g21HQK!)EYchVkYR(FRQPx-ptP1T!&)-peMmaDoPI>0#4f?)ou zDa@&t9hD^F2qy`A4lp@EBh8&%0+GkEHy@}c8O_-aQWB#0$2mi8O>ZHI)Z`Vrs~+XO zx^z8!x$uSGrSMVfN72%+fvKa3l?meeyh<6ggy74n#>>c2xn>J!l!>%naZLBr);F7+ zJC3x-3pUj=>nDunR*uA(dt@_WQ%451sRXBK#oC zJUwafA|xrWdnTqpCxir?-;7*NfWASfO(0mW+X?QI_a4gb*$0^wF|2!EeNzulh~`ee!WZizwMd3s?(YA8V-FMz)t!#VzPTr0P)?CM#_yd397acLUwGaF?sew`MZA1KuULdkFiB(a3l$5jBBP} zVayxpf5SvciCH6Wfx>3>6lSO!$m+Wp+;2C+!D18jB%tCec0%e{cYE9{Mi~@b*&L{i zpH(+dWX*WX9S4y=ttb)3G|+q8Er{i_$AJW5;%zvKm@pqBtv=L8WeRb(U-4X;G>`?e z&x*fO(=%*co)yp}DUz#s!ZM$xEp1JBW+R@d8PWEBRxdo@+4FQCjSlrzD9a_)=-_>*@&d?6y!q&^_mZC0O#DLbcjkdIQ`=|A2IKDbi7-V?_ZvdzWw|G+CT`tt;z^j_?fr`B7b0j*Q3b?u77KhQf`-lNrZ zMMJvFwlTk@TyVTJB~Z0J*ch)1nx$z9dLjF>zQu9*es8_0P}D7CiGa5Tu6Yq|nD0>e z&h_rwGly}mZRSUt7aoiX&=1JYiuIKF)?Trxk^NWVr{je-(0^4lfWJ(Ai=MH`AB{I!T3JLmYQI^L#^A(~0XqB@6JkvG@3Qk`9n`qT-0dX{1xLK*cG?RMHBi6Z%D*3Oa6LtxWX6)D+N+Nhozvg4_ zX{Y3B|KOFeI4#V$MPD*@BeuBQKgS+B>g3oci#oM;XsGFq*0DQ6=QvySB4^0ASO(71 zrjHEL!??;hd{i?axdQD`S&;9o)qw{8p^kein7Co`!L(9sD*y}gws0&;Lx}zDm$DI1 zExf%zZg@xC`?{ZS=2A<1y!upZRE2QBt5&FhLNYon&e3B(uCRQ7)CF%_XKnmo&PGCk`8nUTE#CsJY78fp~$yrm=pW_f74WfT7y<%pf6t#Q7RFhk#S zLVIzNzIWIrR@y=CK*fe0LD3F>G!!1ANV^OPH2HSk#lywxSDx)K*Mzs6&YF-AZLREm z5)n2SXWxsAco_6efcixPtVJU){k_t%XlwZ}g+?0cIobZjHd%g4Lyo(pL1=gU(05i15lx5&^ zq*ycA{MRD)rCV0mvmG%<0dOw4;CLyUA+s&|AHC_zyYB-tYD^9rcXgd3d2_sY|NhE0 zneCkAd;1<+MXXTlbEKvHZ{n`u6GKgVZC-~;wm(UdrO2DKB|Xg@HzX_c$p^L1AN8t6 zqzmRk_n#R~I1%6$%iB^JZh*fp*A#?fXjnIQk#OUC2vpmSYt}k68gwdk)>YPy7ktL(+jwwxhCvO~L8B1*B|{~0y6w<5LzUQpyTMm8df5J0w!37B&JRBlV{NUi=B z+ZUA`{Tau--2|)->v7G8Y;NMc(Fx7X&x%+DmD`mp_MCokukKTid(^3+m`M9HdVjqo zP**i0@?+-#qs3n5#w^m1lCp+LWojn~>Djz8J{^M4{@B;^qn^Vsw&M)hfIy0V{R za*Pv_ycC_4oj>J)Y}}JWgxSbQNLC_!Sc!08a$Sg~^wfq97xM^xvOdQ>S=Xa2gQkhSGUzSjI|^`i zr1HcglSvR)pQcip_M$X*IE|xRbA}K)DT9dz=hsJLlK4SpMbh$s(S7v&AnjqEQ><=q zEnvTTon#?k^y&LZ^c+Xj_k~!}EoR6^yzSfnTEsRXm1RW6&-eOfVWx9UGu)?OM%Lf3 z33)ai2Ftbabc5@#o^l9Nyw}vjBY=+E3x!Rs!le64UuDcRv+&`jAmym(Lbc}-BxvmX)juPAUipOk(5IWA6D&uMH5sF4oGTe6^%l0v| zhq~lo_F~lL!#sURL32XeUr?38y$EKDDr8h7r*;VfGw)(V07+S)PGglWv3%pZD7Y3y z7M<5clP+4ck~*axjT#iA+$2??LJM>rQ!!;P4%5i8RYB@QC6!r4F(e;9wbgTX{deLg zVTLcJ=bTB)w;5-QdD!Fnxl~e7yXP?0fLU|6Ovz!q1v)RmNMIIxruPcnD?<541Owa! zl_}RWtYYKJ5_-T0`D(T=!{?{3%)k?lE4Zox_S6fdA_1vB-&wj3VGvoHdxFVLi|YC* zFxdK0jS1RTe{~8SEc?TSHO;WuDXg7xcnB-mLWk}v*6waEoUG1|asKOyj6fI#3&tZP zPkr8d*#2E?ak0!Yd6Q?zQb_r*O_y?Ac+7*bp*lY?1`d5 zmn5j9ErM_pbPx=!cE~zYUX3N_rQ6%55JKHaT?G!N49l60 z4ZIb$!K>GT2za`hZpTv$FCyl4rFV!;mUrjAsG+P@$PQ#&;y!PPuca9yXa=`9JN%Xi zt#6(yg`SkQgyS4Z+s`!{a8>r=-7Q(_^8g{WoVsiDPswt@_sA)Es*As7@!E+;k2fiGgL zDsNovx|(hgFIHuC-Z~Vol)7s^1tA{hmZF$BddGQ~>ugwdP$$e>pXezyGF%}xJ46eW zd6#7PIAn;G@j?%Z>~X}0n_O_Rut!GA`iLs?IUKb2Z8kszN9c1khI%&0Q>n`Y1RW%^o{Gr(uJKXB@M#$hxVak55qw01?cnxQw-&}>WW1{%6HTq z9fjrGkXL1z6>;rk{Fc?J%P>z9X+meR4fV?I`eini?x|B{C2r@$8b4Xyq;YPehkWog zoiURk?fW&y)tzWHSK1KyYAxmZh7(oJ43jXvm9TZ2$)i~()QjirsRt7$NPt>|a(w73 zY2~}thCk3R!96fdm5G?KbwlERbQvQ}tKa`s)E%ym)SPW@Ut@sLE}NM+ATa~9A{eK|Y)~Nq5e1P}788Vk#6y7mIl0f*`~CTz6V&a| z-~F|ZV?xe3-|y$o`_JqBdcWVF@8Ii$zlwhQrZ4~Lm!q7VD9Yjg(cq!z#wa&O|I>eS z^}mk|(|_|m_OW3fyD0CXi|m#EiH~2D|B3vIF8cV0kAGsg{>N{>HDdU0=^wg?e>FF6 z*s#3e`4{EeNB%$kXYhsSl8=A<&4L$(<$N~Ey(DMYB{_q?h>96!o(+$|`u}orhkY#X zqI`xif)8x@ZH7N=7=zAZ#C*D%_oI(plJ}|K`=bdL{Z7R<^FRBY!asiCA0GSo@88h$ zFGXK(d+!fEKl|>#{>1Q+qki|leEM_$?GH=-d+AkIf8m;IuN!;gpOk-j;{W{9oBsTk z%E?ovPMbdGFaGkc=FYoq{(`E7_uTv4MT;N&-a~)$@ZbJj%@2OKe8rDe{^O5-Qn&VJ z>mFbK#6Q*l^QO&Po_^-%&u)G3rC;oLx#eH~ZRg&7ukL^C!0T@u{Po*M|GoX4j?QE6 zp6-4>b*AU+x%0i6S57p{r^SCia$c7(uUzdxzUGyadoQnHmwYVm_x|XjPfe)E|K@jo z=d*wOz{d-3_=m@u{^b+D|M{=KS2X+Xw&5fH;HqDL?zE<6GyBaZ_SgSUGs{dYJFkP$ zZw<>~^M+j#C8B}5`sOwto7n&S{+h;~J;}XGKiQn>C@((Key*)&Pip3))$^j~`aW6Q zc}w-CBdN9N>r0}wCzc#}viMBjxr*_5W&PWekA0nA)*ou_=o|QK&A2yu5~&mXq4-;a z(aN%?T9X~UDZSsncuC=mH`)fHvdxon%0H7lw!0=OS)0D~iNR?7sl7{wm#pnto%(fe zYfgFLq-ZeeJ~eS*(UPkw#$R-#cL5)(-#)c(Zts+@E!lkOulYn(UVG8Z+SI{|?`kOi z*4crQzPYcT>svTw&4}XeuAbt(4f|^n?LA9JGpF*Xuwrteedp4Gr|(Ozn7QE&?l-n^ zFuL-VoY@UbF}Zicq?|>%+10bs;*~q@0QU{na%!B~e{XqV)-eRC`lN z9pmIx+T35gzvkk*cxT_8S3EIx7}HprUNY{`|Cl8z{Ot!4ELI3Www(`QLf>*NTaA8IIkqNt})~9&4@UoopkyR`c zqhKpoX)Rk+S$qEOx`x)A$>ZKQ)myo^B)6siS-rDmCQ~S%(!jO!xr*^c>6WhE{WT?P za%)PS8;t(!`1rg+_F&X<{!?ThE}qjfFi&s$tM*)9UD%E_nvDJtzt*N_#h+}PY^!m6 z*Xj7HgV7EAI*!}5`8^6N7e`IynqpKaiK(mY9q8v<-CD?H&+ROU>d(w(4w{+`R{LYG z{PEM;CNiP%7oF<8zs}ZOcPnA#S00R4KJ#;J*G~r{?&06Hp9O4cDtX}4uPfQbYgtxS zQ46Tol!l_+3mz)0m=P5wcXdsvd-jE<%Fep27ytE+qFL!#^LIp3beGZ{Q#RgZJKR)x zqBEZ3nhyT-;F`>WY?IWoOta=49*k~Yu4`6IL=RfdUd3h>?arC_`tvP=(YtjfAthsN zm6wezvFn^lj~k3W9hGjMiX0)9HI3u*rt01Z>eRZLMqR8tD(>zpHxbE=eS*~Zio`{v0A zlotlw(p8I3pDi!aK25pS?lPrr+0ICd<|W`T%+x}p26`*? z#?mnrNEYL(?9AO!+}%F$(Dj4SO^MbkJ0q9I&om!nwOIiS8Wt#OEH6G+P<}*~4_l)} zXJ@cWvQ5iweyY_JAbDWrQ7pjBic0Qza>I|$MsZB*Jebo!jS8JK; zx|a^uh8KRDTkVVz$6C(R)neV2OVG71O^mwpmWuKDuMb8YsmjG{TkejcnM0IXS8d_l zw#BDY^(nq*24Au?7!_AB8W~6pVQzWh!old7>1Gd*+P>mvj#v6xqt1JJzf-p_eSKnh zl)xy|)t&}#ar-CKJinHr3GD<!+{`5xqp7A4f zwSi~4R;Ra)=}dH=p3^%iC;dRr){?~gM;FgC8!4~Y-8b>|f#&H^qI=d=Gn2cTUr$bF zIK{^{Cm*X^_LZIHW}=GQdXA5qck}u0)}LLTGiM-oFZMFM;yY+Ll2y_3Wj1ksqP3oB ztQ|;nRmnSG@+am0yt0N1lx>;4JsRDn{YdU=?EmiR@(5KjVd`Ew;^wD#r);yn)7>{d zf8g5m=4tlXA)-)Np%pZ>9fQ|k@NAJ8Qj=8at*pzidk4l<+?#&*wHln5v~|J5^1!Q) zHNW3C`A(+w3GO8Mxbl`M*XH*>-&Wp^ehx-6XMMl0y7tf+iRh+=!W^@wQ*g^D$FV)! zx3Ro^)bgleQF$S={{jdBvCvg6!^1w`mVV$~W|WWpeEikHsPh*5uH3J=kqf=lFhc=B z6XeY0uUkwBM?acdV`|X7H3k(cOI;?}Zh5faFYvF3Bv!zdO3q_uMY(K_JhzO49PqBL zsF1ZEpWnZ@5|Nfm?il#u?4@I7x+>vz`K4T1im!cl2QFnrgnE_a?$EePcLa2UuHK#X zj;@(}>(F5IwS`zH*(Pji#;?dYNQ+!RbMX*E6*lw9-3_gIH6W11l{JyAWjU}%P6N+w zcQ2Z`bT}HnW`zG?$n`H?%EhmHQNF0tCBRqnqkl&_ieC#u*d~MFKFfv}n}fOsAH~Yr zRop}JuliKP>XplYPhB%S#mJ_b70%<2CNan>me(zEe(xPx_QK|GHJ$xg#@h``kLxt& z>Lxm%B$;0vJAxD?_thRZ)dRqEH|$^5oZ7u5Ogp#cchWQEo=P!O-xrvTU5dNa(;XNZ z#7tgA0+@XpTiY}~4;(cqZ?VrQ_`15K1#%1}QO3;jUAaQ@2n!yVWnjciXwb%}2qRR# zePoqDX1Oq>kPWu5J?m6srKwvwS~yUau|ep}y%0C6xgv3N-+Qi{!raKJM#5F+f*gPReGT9(ctz8mQ-sSiSa~C=c^m=Vu(lG z27y~?b7?T5z*BC}`<)D;7T~hAU8jzGrPN{aW01NS?;~8c&{At zYpIIc7NwWrh+mZ?4@SaDcKhOU=YL+;Fc>|)zcz)b1<97I8GiPjg>$$~)6+7!OzXzu z}~-zoecGKF-zOoRs(dBdPP1oidJ9%(wqJJg$Kd z!qOxC`2>dY6jQ-IqX`q9N6x2gLzoXD5@nmev*>n&Mj>2RGk)jH{$tr?c! zpA+iO%o#|Wz+XI?oF1Je*kO3{cT8ySZ|mdPN){IK3q}w>G`H z`6g{Jz-u=H7Xm_?Ge{-*c-!3asO00S-wl9$Z_P&9OkFPV!ojljKCU-++vZ-&$<|)_hmIIn$hO_44zurDo>(p0{@k8D(U;WUi zX zFdBEwGUvNJ~q9&=O&?GoK;gCjxsQSrWv3qAf;{* z+q2sM3>1+R`;8BVAjRn!of;2%qMar`1wgJd;Lp5eV*w1ixqNcXjMl0=SE|t# zM(cwD92r!km>>qOIUbpnY}BXlPd;&lB$%kMq2`~qh555Xit*(|*&j)SgTYvpgocG) zVtKN-)bNXk>#@}NPZ=Z>P0Gg0#1Po+8?tdJ5S)#PEWj(lRlz19U3dk8P4n5K+qB)> z5);RN4|w<;55;0kWZn>m-9q3~1jO^c5!_G0n%I~AH)t<7ur{$@80wb8QSlvp#e>nt z8U~QNV~T}p1iPJY?*{Wn>pIfOCvS^)H`bYv&-Rt|UN&W$jkE-K56oSYyKH9&?lQx| zm6$WAiEdVGcMtyStERoF zg&0>>B8GCAsq&)Ppyc)*3wP~Lt@SU%5b~wQMR-PA^SotydaJRRa9s0uC@zw-J-I>t zu>Q<92c!14SoH0_nZW@VgI%Ym%d0S)aC+2YqoheEs|ks2a#a?*tRKszuWhR zhdOg#hVWvz$F?vxg^WkOIUx6#I5)6%*%$E@gHZxYI&jIC`CLQMEP=MDSTMGDi$Q)| zpv-so`}{QI(pTI&W#*FQIkR`yG%j5I&p7Eyyhw9z-eDlj&;bGXwdp64sKc-z_(Pb6m zc&6~{tn7i_qR$Cz{oDN@+~(dQ#rhZv{JYGv-(@(4Qs`uI$#pN`*tbIu3n)#-XEhui zcsQMW;<}dxzJR|e87n>G&Z~SC*;v^%aimbS4R4_2$Yw$6AbRw1H$j3quM{|4hmFbcVG#UkHOwa znG~7JTIUAQ&Oekm_R<*-VA=r!DLyAfZWBV=UXLKcuU zPF9RKDF`Q_Grt$=$}5(LPD=XyepB;#b3dfDZ1c zLZ-`OYzw5ZObVnGaF0ae+#Wad0iudrWhaNT2pSLTV_U0OrQv{vQZ)P77xX59PEF$?mP!!^GPJ+8 zx4VD7ye;vosWkQjE*^@qu;SKV*?2W}M|9Y6c|_5t#dn_T;f69z=TiFR_}TY@_A4X? zleV90UGO6$)nEs5xNK+BqVYv%;qgLLcyId0uhld;z%%j}3P2i`p~K4CW$1hFK>5$c z?j`;yH-l|9*CODwv_P0UjKF>x;U(c%>%N@#GFuX^f{&Fv`eGXc6a3T0y9!vBdF}*s zZ8&|qw)36HQ|wA(F{d*#XJ@8G(@(_($JXO1SNAWn|}WonuNNrL;W5oJ_j978N?7vvpkWx0kD z1ZFsjI;9SqoGW9*CI+P~B+nMx5X(xTS(g@IkoRHF%SR=Tt(qrO5P^jsz>L3k!_rR@ z<`B#E`A~%VMZ8tQ2U`?M{Jf|f@|K)bytQ)!fNz4&4T*&iq6DN;NgRn?9Y`!$ZiKZW zLIZ^|OF{n#oY85MjMzE_4B_|z&m^Hz`UFIPtgIz4$6S{3u+6nw zulP<+V`{8MB}NwOxAI_A0^`|Gd~6E{XG|xX#b3Wh#8!JS(SAJrr9~OrIX>^mz<1vy zhBBoWn~0ehwPGeI8jW8g;;qVLs+22WfE_U9x~PjvUs}-rk8<;3nUUROXRlc=v`6DG zm%^V}8PVY)yf^f=B7MYdW-E%%U1H%J0nyB>XP(?MFcROq=k?@GLd9!xcZ-ENb$Ts~ z)Y#!?lV^^~{T+W<4ydP$5%xZD-HQjB`}^7!9!gigD^C6Bk!$vg&wOlB)hEiuaNpig zcxk&b2rR$kC$|~sEc5*Ar2N*x%B06g%w&567;Fdp4_v!M=|0&aK2-tCUhdj*Lvq&+ zo93wHmN>vipFVr8t^DvR(E~;A?g%8*+D`F@d(x}tnfM84dyo%qcMN9E{-xfEXVQ;$ z%7`h@g%<|=<2Kjum&R1-PZ_YREA$${vD6Y*gFPRK8JmE*H=jjXZ(Ag9gOth~ht5p%B@(*u=LSTWb;+%QmkB6+%9vKHK$ z{OWoD9rKONkKo^$VMd$9cJe>`>7c-wu*?+ehPc-3?@9vVrNZD9h0c}dirtMIgs?4t zvjmJ-D6F@oh01KcHg9KOkt0I!lb(F!zT%Y4oPID)&D;$%8?T#rdPi26ZCeNauK1R9psVEKoL zKrNYr1Q|d5%VHv#ArLq?8cS=G?yCH25i7+@&aiMhx#ELLm`F-62NPlMEO-beFnBgn zktmRM8HN)nWHj&&7K0(FS=>Wt8ULd!{GzZM0MVL>?HD93^8x`NJZDu{fQriFc$i=& zJ=`k`#xx^+lI!HYvy3JnXVw}B3vs*(_XZU)RV(hi@X`@u3BMRMs56tm$!uH3Ln;i8 z34o7soEb?*fEuCGEau&K7m;scd6nf&ho^8EKppI(scM`Vv3Y{iFw;)Gx?>HX7u@P} zjdZxON10skRE>5_(95_+i~8Dk8p92>6IQjBcyUW6G_(lJK%I%3@3t~6+I zPxQS}ac|G3eQ6V|SLiz0BALs1jzfkm+ch1T9qs1+Zv4%1pmRV9{S0-hE0VRED@LG@>Th63o(pBS9nZa%?J(` zBoES^_Q|S?v*?>LsU7i9!o8*2N8P*J(hET}p4-HRA56A19;)8~Cq{;qDj1^oHPf{U zm9t4Nm;C#T>a9Crb|D{4m+}ubAAMP>N77ccBRKQlPj7#M$m!3HBPI(LEu7fDxFY?~ zgx32WLitk{-@V;dQ2x#bjfrNP;2OY3V5_-Zt2?h!o*CEXv{!k4dfc~JDzW_Q7Iqw) zsBGq>oIP(OI}Qp{$oG#JYv%2}&j#bg_T7X(vecBxur#`h=sHBbA9=jn8MoQ zY~MJr;kZtDNNIx7o=19%r#D$m6+(|fziF$=isXNJDRhx0VAg$LR`?U%3{E=@@}gBP*fU@%lo zUjVF|s@WeIMA1yR^(Njibcr3bsLsqex0W4jo7i97%3cM-hz22d7rxf_SEm{N&;fn; z_e@og>VYP-`XxXcRlZE|a@o&SR#X+Pfx{q*X%K^BgTFofbDq3m`$C2Paaeyue7QIM1sV zMhoUH^!2kaFZu7!&l6_g@HNG=+zGdiDkJDceQoX37w{Yf}a11|wsS1MAcY=!CB9o1vOBf|z4elje)nDcR(p zra-eqv;KNJ;USE?wkom;RgnooN!UcJHYQEb(FT8HU|X9qqQ}#Ze5L1KxcpViLGE_bimS=ToA3-&lQg&$({p7i!PFzqRiZC?hWE{CBq3Ik2Si%#=mGjm7jVc_@HY zxFHuv9A~lUJfB+8EL=BOsbw!-gsdW8U0FK%(Gi-gIm&G9Nt2{mU`ho|Oo8Zr3chZG z6xr*P$cNrqk77D|6QWVL)7$6Vt<5-9t|?u@gPDs_N2q z=-NL`KSF-`mgmr`rTGZJVr80RUq$9V>zM4XmeQ>hxoy54as1GfXR^|AiyB-LSTQ3^ zv1iW?S-gth0FZ*Uv+^6!&Z64n4dv$-Q34=>Nr43mk75+hp&ah`p1oEtMY>{+a|lCr zB=5eRO#qAh>y%*|MxlK?2z+Oloy{^6>Y1o!KEDh*eRBR<(VS5Y ztcuuH3)^LGJ_x3W@qE455tlDdO3P`%b`mh-L^`)$OM*_YMA?;SEllPlXrNP=OyUZ1 zMRpBi_AmXlqV)%Z8Fu;P(t$$9POU73Sc7S!KFA_DX||UX=r{BZ;7XB$l}}Sxp>m1Z z^9!+WmphFsvev`6Zx8FjEqbYP;`IC9F>Z6R=sajED<>5iS7P3xO=a_JW%qV_o!6L7 z-{d%LH@C7gC)s{{;0x=mScoA@r1=V?r)J|bU&32FMnV%RQc^hLxx~{m+Ek?mqC0); z)?cBitb^83y2&0_AshRvA%c~mRfd;5FAg1+?p<*uif3f9Z2gfZJfm@TiKFkQSDE?& z4i$vis9goP5J_1tE8%;utlICiH?;0~T*qccFy*id=;Z$Ed zxqI1jiN1R&et7Ur+o8tsp$6P&Vm5Ibf=w-^WS=$(R!fOHjH|*p6X(Wz38|V>r+-qq zqxgK;Poz)VEI6_i5%G8i!`<*G;lSk4fkK|rphV?7@QS$Z{Qe~~&i0KbPfD}}5>`b- zN$!jArl7z{dDVolN`b(_T6U$g_%9lA-RRO%x7!4eA9wF9FY6E_JjgJ5i4|J~&02l_`h|1pU+a~R4)*gpRt zZyAM77*TtZ!DL&zf4g5rMrOYxOTbAYNXfg01PG!OC}X8dgU3 z_`@%y%<3?BTBJ-Z{g4k5Y=QRcVmXr>uXRBC2WVz z$bJu<FP4)u#G^Hv)k6w}yOPH;;3O$gN7EQNUn;k#4A9H)HM!Kl=A6YX_r0!G&pm zy;nkY2c2~Q7QzQACT89KXs#u*wCS#X?s($g4ZWHjw7A7)fj7|B;k*``^|aVbh)$h5 zjN0loG6Jv`F@(dh;GkUIph*A}1&fO>F5I-Ci8lZUCLar5#F}TZjzCBz>KzhJ$;o>a zZjs0aOWq>pZ5OOfK0;=+fI~Fv5YaWNR46-h(t;J8=HI1eLHvdkF-O!5OHz!Hj77~_ zYz)6*5#m1%`6)+@AZ47i4-uPaf3Zv~%UbDSX4Z90k?yTv`Uvj0PRwxgxxRj)VHHps z4AV+u7Nq)Ac4irLv*Pih0=VxEiLuB{9mzEX^=ReF^vvdcw`qFyr(`oJXQy7@2n*S9 z?I;$*5*r@MGN2>tGFpoSDtRj<#uQcF*L?o;c6`{=Ghq)OK2DO^sc}|cl|Wcrnq1b! z30dj+QWpyF7kg`UtQNNzAW5_zNPlU`D`p2Mx1-&|y4koB3!=s!3V-SwTIHNQeG}xs z5`I8KzkbPDh*PT2$z8i|Jt^{jlFB09gXXG!u5179fKybIaRYcS&P9M+-TUnX8Ypds zCA%S$sU!p@x7;JKucj_yj&dV)gFF{m#=OrrO0^Z~yo zEERSzB$HK%%h-?ff4nzHZO6d4{r5FLI#Z?MAxbNy@ z6Cj-}6XDXDzYRwrE1FVcA5v?ee_3;HTDmP`7zOw-?v3=aeYH}%Dm+gdy>PFc*=yN%_0k|&|Q9P&bFZW;i#^FhM z_l;F{am@(J^rHMn29lwkl)Y3FrgGC645=FeDv7N?30@cjd#L+Z$)`8q&_lyYIU35? zLruZ8kl#r6?u4Wi`zuRO5^bR@nAG@k&8)gjI76d^nlY-KMYS-Il9`VFMGUfNHjs9a z@qr5E<&WbV3F*qtmHt zandp_s#X6A@9EEqyFQ9ruG%v4l(|)MjCC1;cS%3 zBnomPHpX%(m1htn(~xNXDD!6vdR@pAv4_OyGOVDA;F@HyL^}q;F)JRK(zv)@FyyS= zs)&@SJOw9Zq+j?J6MT*u`r5@&!EhH92_tHS%} zzN%%+lNNVY&R#ltVAH_p^$8MBU8~>k#lo4Aj_EWr6zds33wo>j7P3hgk)2qZ&_$iq zE#1~dh!{#{!r@~`!)H;bysitlu(|EBwJ5JXe$Y^HsX|}l*k(P2S(G^>!84iaSt`k& zd#6`af#K)cn

;X|HE63+9Z8lvU6jvOC>fm(F0o}-$A9&$bEAd)-wUPlv1 zB`XYhIU}>y=r(2F{1ijI2toANYa2uXQm*1e0x*H6vGlTOCww z3eF)5UchPhT~GDRz7tnLV@&ok#iA~i-&@AaV8IQ_!Qi9l+msW7)f=^(77X)VH`J%5 zwcYwFseQ~f`0GuV=TkU}9+Y0}#xeG5om$ODj0Q{1P%2IOcJ+yRT)XvB0W z!E&J76=8@Iq`7lb{aKKnBnCM!PdJoEG(n6)2hB}LBDDjJ693#C?S<3YkOXp2>Y}k_ z59KRn^9_RNH}V)+OQE?n8yTm9&0s22Lb@XX-Rfsf zB-2})^qbXui(Ro9$A>E1OZHaHw_0lJxVf}l7)FlSV^vGi*g+6ZlBF)onfv7>Z=@fz z9(ki{!-8R`Lz)RZA*POSLZwzVDU&o`*h*?l%$2Z_Y_E)93@oz2zx9m_*;B9P5XRv% z&%U67jjh@|T#zX~Vjy7Z4lOXBR=G%Hi1XukOZk1ktSY-gBfwN@wXMo&_}S%JVVThY zljPSV*5MokDzrmn{_j_YW*}`0{=z)(W(4@Qr3y0IrTvd@hh3% zV=U+LtmXmY&>*$KcMKEX!Mmr@%};ocO$6?4%El)>P~cnpbh0jWG?T%+Zqrs9TQ`}V z#Q7z1Ii|u;1Hv&B>0!XGyhVL9WW1rR8_2*sz8};#F8 zQ&y6Tc3Q$4yJ2ZPt#9$d*?3Bk;CL|^<{Q*g778p?Zb8uJ{VE4Im7*Y3@XxTRt#@9* zG#bm!oyGZk6-PPNf7b153g!jdesijHbnA65)(#XiA@TYsv_OiTo&u1RmLDas6cuc# zG4X`P`FgrRw%PbVo0lBq@KlDiW<^w`MzJp*2N|-`-Z5l0bOyuS8}Fj`_XNc`>g4E> zQG}owwSs$DcL!d^@sN^q93AjbCDkD~F7=Sc)lqYxDU#>!Rj|misE6C)JvigD)}O=V zk5D;{yE4+)cE~)FJnA}uN{h5*BZ;7hRU~6t1?F$Bnfzi)E~&9XBn66*D}WUY2&>6E zy4C?#jRqLaZk*MTKy7v|&QV2pLr)7)_sbZyBeMeZ!brfG3_%_4pV*;!SHK5t?O8R=v<9Rf<^)%Ua7% zDZCMp4oDNdNV`o@OlPr(`@Ql)qRtz0(zJCdRG3C|0^%3q(6R*wEkogE++l4d>MRI; z0{7ZMwh9OT!GxA9k%F^~7sy1Q?p6L14=F%$fg7$=QV86latsQhK>_$9;!#yESQS{P z;A6OezB;Y20uSX0O$N-xYr8CU9B(r*y}^b}nZ3_d$%0{$cGguAEwa-bi=2z&_$nj+ zFp|~o!Yr^UwbjVXOFm4+DIv5ydP$eVxcv};H4c_l|&l`9=1w3 z39G0aBb`I){MzWOw9#TkWUbT>Ly^t8YZImmsya@|30TNH%qGNk_><8S-C9$QjkKA< zTa0Lo-J<_A+rsMWlpu6(qk%mueley;UGvs(7<)tg6j97r@oy}-rTERRz^g5W@-U_c zG$T)D0jX!t;*nfB82t<|^C|+X;>U>t*H!-wt$o@(Lx9=ZUc=*#%{Ul>@c+7RlcoirFZdxynDZ6ZW!AAR*8{dfT~FwRWl$(o-koRPa@mTKgey``Rx{Q& zvqRy6iS(b%rpwGM5ztX9#V~zjM0Pn1D_vAP=lY_oUYZW1cqo(_U7&ZMBOVP6yd%n#L*?$?U`56p2F%2MMTDQRt9~GY=gLLlK7g zGA;CGcmo45EM@(6v#}^q{Ta-rWk`FfR6DX5_y|Bo2S&=_13YbmZvB<+({3n~UAL=S z+YHT2f>9oL6NM+OT`2T3TSC19BjyD)O)3-^S)~}qnY9fSa{*hKk|kudPFv<%Tiy<6 z$JMojz)4!|ls*j`zgX(MCx^%w}6B{fsjh?K}Ve zTPL506E0r!m71iIE0riU#nu1_Z&tQl;ypAds3>pu!Wu*t%k8vKCZ3T&cbMWqohVt= zcA<}|)@4s1S7kHIPg;2^H^9MJAD{ed$c31Cg{m!Kc@87gpQ`%dC8Y-+?4VMGOvq^PeA1E@i4tZGFsz#*6=mRt5nW2HW2i%;8 z5CE+Uy{y2Dv|mYk0|rtb@+_K}SLi9`OPO&sMS@8XY{<+^ZpE(cJijpJ%LtVY`mqYl zurOL2mDafd@dLMv<CY$ihay!MF<6I|yg|R3iR04Y$`>+9B#)fq%VV_CTluQ8mN_#x0K&|8 zaaHxE6ZbTq9vFW)7`fsgBA}5`jF(0CO1W>%IGvawXxo zKpBft0|u6*7t9r3v?ypKR{HWI-|YLUR7aLUNL@z`c$SucjJd56tEfHH%RBsBu_ZW$ zFRcvGw%OtvFa=$ks#735F(d_K8L(`Y1%)tR3PQ8#mEjy^!`y@B^HX^^W!asH38p2` zI|KuEk_4A7rBh><+13ySnFS=MWn{5wfk#kXmnz&deF3N$QL;2&h(~~j!SF%utdGls z3+Eiv7kQhBV-j7zlphu$CS%>if*1p)LP6hQZAWL;8R(jV2(qL41~0CL&(sJ!iyg$^ z0y4^nGIT*c{5uI}-vrMsmeu@G8o=pRY_!FcB1deGvQ?D)0Rw&c3UeC{ zN|-zhP;9Tv;o>`(90rlIB-BKi*N$qK5yN>8^(OYa3lCFrry>rN{pU^yj&Lcz@o$yp z5Al)`XCf#HVrazLH(x}iOFDxx-l&TNiAkjepta|^S+`GHH9-|gO{uE!r0^cuoFt`b zgoRwz=og#|y+VohLnjjJ1mPs5`4cFf=!A@=KtZJ>iXI;3%1Avzt<5&j;7->)x#)J5 z1EO5MGwf9HUl1CU8bi%IG?(B7qG=fL6*O%OMh{>q(vMi5gx78c_~DlcY=mi;U4V<| zNHanW$*IT@Eu9+jL*xW{SM{+=B{!8gj9Q(7v!~#|cA{P46ug+IflN3CWU+aLu|mrz zGlielQcwI{NZF>o>#IUkZl_t8fHbNWlB?6Dr;5t6;(vzBoTg`=C#QjU&8JYGk&!R z8Pr^IMeN0_3(Vevhp3@<9&$qKRL=5b7-YVfmkRjHwm*ST7?D&RRPfzV@ z&9&vh;2=zcQP*2QMKOU{)l;nMhY8rI8Id?W^K zqz9mcBAST2u9c@@$j!pTOA%5j%Nysl+(@=hX(&LBQwF*ioaQgA)MUU3J88yCmc`f+ zd>9%VY$czA7PJCcq9RbMv4OI_l>%r)3BzNP%=e0B+xi;l7goxei5k=ca>TR6;gexE zm@Jz?)38@WaTjxn?4C`_otg1zQhD=tApF=rsjvs{hrw34yOP-%63kGuG_~B$NFT4zwiv(%X(uFPmZS^#w_ zry@ryQcrIwZuW!phpR4|Op$En9MvOW3cbVQZqAHqKHB#aBAjhF4EDuIPDA@H!rEOO zBxk7bcaf-tn3r;)7oU)Pffd%L9}$yfOD9=Ze!gt8Eu_T%vK$Z+V-Sfhy-uzw$@nrQ9_1BvM2d|uoTKeBy4DKyso3yS7gP^mU`94RC(N}0HGFbDP%x*ZPStaSje?-r#j$S2E}~nZHs2GUS{wzD1zH^W{;B7F@>kz zv$nYXKxLtx?IMYVx9n59VUsqAbABj(mt%{X?oB_0f3BRlfv8`)D_d`-%&kNo@fQPt z`IL5;HRXFWe6>jU1;Rt`s)iPdX54d_`$|6)I-iI-EIo!l6ZwJ=*coW;q|e$Bh^wra zHte0z*FotGP$QBWHl&(Xmm(flSE1*?S<@q6ZFiVjYk_i0k8bo3mhcZ9p61XLL`Yzu zgHR2Up#c_ca5_LN*NCKQ;Y*0Lp}H}7KWX%jESHM zJ`lg=^n?_MfpM+^Z^#BXZ(&e$xUZojH`Rdz3P=~?SC$tO4WyQWdwLMpZAHwy$K~aP zh zj28Y(zFobce`0>c16-m?E0~hv#s>#W!=mKTgKwLw^3@287XQdN9b-OGGzY1KWl~4L z3n!jzd^B#pG9)n$iY@VY+>Is^obj_zGaU*HN`qFK4x=MZ_BT#8s!0|%_RIVXM%UoR zW$xYNB(O2G2J?&<;@5+6v~hUFu3Z+u5~KgYdDVa8!%M~vM^;IyEPpP>_e;lcy^Vv5uwks>iUU!CoeN|*F8_O#euHKS-tSy|v zWGB7=bIeX(^!}(>DpSR#*+E*sW*W9)O{qeJ6vsUKLJVE(u&)`6M(1SlBOFd+5us5d z3x32ttVKj7RTZ5G?gwizMq{dga^oii5OWO2z|W?z{P1s$Vo( zkYq`-;vD>wyrJa``IB_=LgCT}EeIgi%tJyNQwMLR{D>fdvLY&X?EJa%^*Yi*!8}}K zj-4v@bw#xncA3`6N^xkx?&Y?Hxiay7K8G3b_&6?o$9#p^cif0h*R|C$%YXs(&Po6# zdZWq(S@<)vEiWQsL(z)ND`A2_sC~j^SFBxjI4q`xhRAuSsgN(gOxYipGNGAEJBCz& z8TL1JUW)1PE_4LKADHZhN)m(P=IhK=x(|~d(@P&j8|a=0pLVHZvE{SbMQ(~sH=wdb zej-qn86#mXLo+jkikDWL#@ddtPk^_>1+qFCbCj%v4Iv(iay|iOcny3w7bFG-`YF=L ztv-^-Z*s|=0h1vNCogAbm;)=HnY5iRt{uQqgPP}C_~kV{v0F0(Ev`#$4zd_*y-BVL zryv29%E}(&ILw`UH*5)INS0)A=n~undH-wxW9QcNBzlu}D3Zn}of|qoYdzraSpl># z)u4bk!$Ns@jg~5$@M$NFkg+Fxq0)Nz71d?%RmempJ?Y|!jghi5^bfw+0g+DTxtybx zJN0B7d=yTMHD_Zxv6t}(8NG0hSx{qb)e|ObXOip>bMS|=#DImxlku;$lln>t2-}Uy zkh8d))A>2UOLgRGSvR8A*4Ggw>A6thVmi%um1BFQ$kFpIh{CZT z6S7*nPB(~0)=B?7Q%GkXAqjT;2a&60VrQ7KdF93LpI$b+B=>oh@J@XF)b@Idm%RDy zNcv-521r(o{&URaNeeYph&OX-(Fw>AJtP5y;g$u?=->V~4WUFpEmUic!;L5hgb@^L z95symQIhQ8%6Fv*RvLoi<`f{dvE#&Y4rIBV>G*0{2t!K>aTQXF5;lxns1p`gdn}d0 ze3pK!d!yM__-m~L`RTu1`^XpTl^1bgMXe7 zqw39`%fM;J5+!H?B)q+lgxTsPF9nk2ip0J|`^yS2d&splk$%EN%YE^(B5-a-{9!PO z>VANmKxiJTGM>ZBcvywbFHvo6&MXC-ia1=9)q8*taW)5qyO=NH#gzvQx_x!c$JEeo zc2P`i4+On`m_PAL&{cgND9*RNt@M2o$4U-RVqxdgyGGZ6KIM`8oNViO@<45f)*FXv zGMnKFx2c(fA~3KK5s)`{vw`6#ZLliBjWxleTOo@3Gp8vO!|~`0i^AF*&tmm>FF<)N z>4so6=fqHzPd`8gvK#?gXC1qzphWQ6KSHdQ^G(Ss)U(6Z?Z(w>*x+BDeDn#&f zo7q1=eWnY=uSa{5+^%LSW1^sr-3|Uz;6Rb3GhF(VmDpo)aL|UKH8=z(YmUvo%MA1S zaQ%N4vwzrrn>`;|{vi7jdr`TzcvER`m93Jy^yQE=z?xW;u4Pfu@_7VVCIHzF4k&Gf*ebHa>31YF`wA^y_ zrh!rGi>Hr%R8hkPAJ^B!Y+(LWFs`HAiA{+jtBTlB+Nt$lHP`v?Q}mRB3(%FVjpptT zLa;MGiA*IhI2#)wdu#I7g?@)n4g{~VcQ#(ZB~VtxiBv1MXbV4 z80i~4B%ry$p33Pby+U+M9`o*YC^^kdSBAxwy^v29+OqTc@ndRuGz=Q|aVBW=y>-6V zpeb5OiJh{cD1b6CfS|K|vAt1+j?o?Mnjc^Zmuv*6?Ndw|P%q|Uwy0e0)+T~UVfuTT zUwFrz3K>9s{(MK(i9syv?xlzPQ$iPHFSp-D2R;n9bS575U&TmyeE;lRt$tus2A;qpTdD1V%(MYGy}yXHml@@p4FgpL3$TLbRbNyRvoIx##*8Vht+KZQ5dp%!rQN zdSD9}5j1rZwZ`f`Qb`h?L=TuzY8|sjO3adSZYyP?cWa4dr)bSwe^5IK3IaOkty@(% z&D?>P1KH+_6;w&h{p1bp%uIos%a`YHq?SF-%L^3<4D}FM+oISO5^DruaN}xpQ~W38u+;BlVH$sOj?pN&f@FAC`96gFtNNh`N4=+k{trPZeVgt7d+c>9LmFQK5w-KK#RGV8fEca`U|YQu!54&u5MN^J6fetv_*2FDP5v(K+LKr&aYKepyasm#n8C`% zQ<3`NXDqs=@kA86TSXk?R?Qf@zl;M>cRsjA4|%9haXOY;SQ9FZ*fPKxxBBhF1VW3l zvc=AIvLJQ-`cO~#@RYA{2Y`)-BYQbcyhmDCSwF5kvWowNx)R2WFAH3OonWE@NKXaw z$X!2Nf%}=Y%}mH~kP9A3AWXQU0_~F3lY|tfuZS7X!RQ5F!A8>Tcs`FV^Sw}50F&tm zJ#tG{*SHI;yFjUzTZm_2=PzCgA%vA0ss09fW!F@1)PhdkBt-z`2i;}y}xMo?oB(_5aQ>??` zRfU}27?+5N{gpoug=(2dhUJeCweB1;&@%6=i1CMV#JYo(Yk2Xdxo4ez0cNrXUChIQ zS>u-bgc8Vujg<79V$w(8ovz*u$|r=FD&4aq-E8qDUztAAGhMu*_Tm5sq|co=t8d{N z-1;j!yhPF$!a@X$v7I^zQ0cq#^}%Rhp&d*@8VU=(pZpL)+8GlG5Io~$pWSfy z9%?W}jqXctn-_G0hi&NT0nKetJ0Lk6KO))i7J(0E1~UM9x>Y(sOD(-$r;mvWEN`b$ zlc5k%O`)QZf-8=KZ>9_Hq4A$6zkD$IcHh;OEpOqAD)&I2zQ;!C*nfJSy{AKs9u&~k zHa>+xZx1XA8qKkYJoQhH{@yW1LN9C%J=G;=&2D%xd5VUg)2jR=rP(SeQsGN zpKBhdXV23!cy<7089UyVDH1XM!>i|o0zSaU#fC~hxtZrPjJd_^G*u!)0Mw>H50obY2q&x(97IXdFIm|BxdYt+-DHn6877J9E)be zb`BkB_ILU6z-C%T$&^{LV`bu?TVWVXt#VmrWwe~me#LjtPZrp2FqEFm23grpS|ZA~ z7ssiaTUen`?*cx~P1c2_Mp`b(CfX>r1EN6>jN^+|QI_GWtf@*dMrK|-%hVQ|r3*O9 z+c3;KFl^&IT{9KAyuqLB&j_^#{0AR136oNp7jpS#>z1#jJcqXEg z@d6}5!68gAdJoFf+mT-2@R41LsAD&&G5Rxj%}xs8sa~h3HGV3<1Uov#B_z0Uui=1; zG*&ETlH)D1y~kMvCBhEt@Y{gRaD|hN3<_>In0tNJAaGD3U#4Mf!*=II6b6iuS_?v2 zPnCj9@6mxt2@573j(_Onn7~Syn1-ZnrU&22I!T)rTf*aUh4C`RrlOzbd@S;^4}>rt z%Sr%Gm5L67^$fRBCl;Anmd(PhjA$0gM7!Et$- zPaAFyIy9_V76~As0SV5@bE+MU{&=sqWos?yGW7}a!YG+j14jwl(>8<&cCFxn6FNU> z73Gbx0@v;GdPd9BkbOJRN&Iz{juE)KzGpQ@&1%u{1D9Ip#=1CpLyix3v)OVIo-E-gQcSYo>)+uil+T6AJ zbevf>eGvE7ZIyaO$U6ac*ng9 zUKywCC~OSA&+;=Q2Di!?D9W^F?*#wAViYeF9wL4O%{2v4)W>8v%M!LUg-uC2zsC@VcC)c0|o0WoRg(sZKxSR7)S8< zIngX$EhpDHe}^bHiIM0w_YJbf4Ki%_6X)ZmY!b!GI$23pu*_N6llbxqUeTRI>^Mbh znwPNzYDM`3d^)Fd=3Fn5#VZZLQf}k2t}Qc`cueeL3_asdf5gveZ6EPR$#Uimj|;+p zO>yk^Y#V}?9?OsW`TbG|E8c`Sv@ke$So_38G)0Rxa6dfd!xHgfk2P$soiD-<*7u&pQ3WYH31@&*g};djtFW-b5Q3kiq6Pav z^42VEfpX8&+O*tS_~hfQI`s*{DVrqaSFeM`M$5$#g39G8wvz-%%M?FLBw!$pzs#`| z=TEO6Yn)*pJJfwfsdSV(DTs$YjvoVfR`SC!%^i>osydj07*r)Y}8vHys zwzY3mdYI*IuSgsxyAz(Ex~THxsj}@>?XzL#EqLahpF}qgTv9%jWE56TwGDPrPF{+5 zKD6km6|B8dI%w5H>_X6N; z+8(DQ$k`atR=z4SHliiYOASkpy90~i#$&`-k8@)CXj0iExFV1fpUEh%W3uC48jgp< z?GQ9sO~3gg(~Y1SER@9~!lABzazUOyG+N1Apm4p2FKiEf3p=TR*bI5*>t>W>n117z zQT=`qOXm#zqAdgEJr0J3z7zk>2pEUFs7~awa_Kx;D>6@WN_BtK+s(h;pPSSSQFxpE8fF?qVT_*$oz-NnUL+Yv?nwWcz9a0Cc4)T@r%E_!iNvQPV(&z{~Vh>p56z}0g z^vWp4hY>q=8nSSqpokfmYxLrdLOaCOjE`I}FWF%`%!`Mnm)IM4F1f-?6$`?sUTRKm zDR_F;eZ&J2JV)X&Q1g7E;E2<1lDAOcVPa~59{Zy1Q27ra=~Os) z%TYQ@R63=og8mP42ul;EqbF`B$9IKVo-n#o{UKvVj(Sq-IXx+ijnY66KC2Bdp8-GQ zbau>RX*YRi^rHpy;*|HX!RRj<$6Lf~Wz6c^VGb&5IPz_t{^~IUO%TJINYIF@O|eBP zqdO0YxE=`B(p!C4Ch>RZfaXLGW^#>;f|Z%7Ig|hgLQ`M?vofUM-O>(TqCcdAM%CXX z>RD)2WLc@)s+Ev)&f-RGnnct-9xR(#tvuE8AnUCj`!>%hPQCx<`(<8Grk%Fcp@uiV z|GsbR7cpkBgE1(|O(7fs>o-`m47eir*%_c|e1w5XESQi}Bv306Eq~K}yz5md2~g3F z%8zsgiSY1%N(n7jYZj9Ct#2c_hTlR~SiO=$v4&*GzgT6&8P@O3Hsf(*LNNV~OlbfK zE!SlbraZWvaWcZhT#3@(_gJ4Ztf;^Y*4RqJZnns^5BPY^Lz9Ua%!$>X%4 z50=EFxC?_L$D3JA;&>1eReH;(p@vAe_vY}631X8`A$i8f>)FegA+jWlBw#ytN)uYm z{ql9pYcxH{6C&AA6(A$Kz(jFSnF+OnJkeQRvPeKP)r6rZEO#J)_38Dm&oh$3kc2`} z=nXz7qtb<$8*4EQkW1r}oh9hBe1SN37PJ(0yLx}{OgMCMKo}LkN^;kM{^SDwf@z8s zEB=WI8eAhbpTw633S}hrFiFMx;;dDspuX2KEwnKbx0rWcZBqQBNrp7;L>6IANon^ zGW#Ye@1Uoy=^=*lDR{$kP&J40&_gmqs8&dr@lTD)fH-dqHMZF&&wg+WA+Z!Fe?_fM z2IA!8=-Txk**^ANOpg`h>Do|eD#p^{EpnD#1v%y_f~QbbGePV&Yk4`4PW+sBrAo^U zYr+ZXG1iI=t?r>4BI$IC74pb3Yz9;rRa~OzMxgI zS3~Q9AMqQ{Ewov=i?I_jG+9eXo6i#9io;P-k`g9%&{xX2=P{jJ&=fXg33OkkE7zq$ z-uYQW9G8;`X<15I7c(kSo|S7@M#JSv2Lpx_0N$m}uI9yo7$qy*#Yv5!pwAlY&_>6H z)7ntfESH8UcW_eKxlQ%Ylp+v*WwhcxtGeU>0cC1=N93*ggXpR?)M!loJlwOD(ODij z_SNR>qiKCJ&K~?Hnh&R%2$;|spvfFADI+KciCyHYmaW(N?3>?AE9{op@oIX!U>d1A zuNk3}=Tv`uE1a4MxmBgHIke(plKCF>kG8i~o^P>tcp3s5mpKAOiC{UL$6GnLNmX2B zn=r-yY2bJ({>1}JwQw$ z2<-=3`#uhqv4Z!3VO`V^9A5B17sOl*J;k{cadC;CZ+qoH?;RGn^OS7sfHv!>8B24i zf3wH^c(J%W8MTut*-j!)OR8Cy1SRhGw|Ztw53ckYb)Agp%E!>1XhrTkE&$S(ZBg%cR3HY1XWvvn-q?Z~E!kXZFvu3>r(PgE88yFyJwIUf2e0t))A4 zSgW3iP6yO%Z?Zd*VybMpo)#w(Tq#A$+&sx_Xb)Fe3M=%N2`gANk9R68MyI9M86+62f`@{( z4n?zL_A~}q{4_Ke+o<3CXIu<{?Zj-x>3W$V@QEyd7ieS^3s?{I#SF-0m}NmAVkhGc z*3~*&?RFs`%!Z1YnZ`5zO@C%lCPawW#j(nv;>qfsA?4$AnbVIMj}t4q6h3>Xo-}>v z%WbGF0R?)QOuvOwMz_YM+J8FK4x(bTt3amHIdRdLZpRDNGl9ejc)5&6FJ7t7y@Vyf z9KM4VjD<)DLoXsu2sP_JE*!Qc3Pgj)t~JPxe2LO3V{=z_{b*cjg;-^*3h8M0S*YI5 zx=g~-=F@3gN=QI_Fl6TyW*W`Nf2Ulls!oQc?F`BBA^YhGOjm|*@^6r8+CBUt`XWY) z97W}QoX2+EF8*M=>v(mKqSK@t|SIA>j=fyS$sR|qD2#p#qq;8hqKp`_ zu9AK#Tu}rl@&j!rg4Y9H{(YWn{?Wb5ct$0-=RS?BZSiTHlEkO$y0=GTOdCP#ob9L0 z42Q~^0vHjiHIyB`r>}rRIK(mR20WQ}ky$Abxn~SKj7$g5ll0bumczjRuMO6v+ca0qsChL#0#?(CUvrI9mOyH6hG?f7 zG+7yzGi#B(wpJqjn;2d(L&VP3Z4(q$Mp+|t9f+66)X<%yp(7<0pb*5tTZ|&3sj+7J4<^Z zll2I&No5Gt+&W6#czIlhg&6|;vvcbhQa`|7Wy&T@;KSDWkJzcxJ-cbNo)MpiYOl+z^2?k%{+7p%iU4W{2#NBs#Ok z0*(r!^FPwdJjjEtsLOe9P1qPRDA%K|o&^`)s=djcu0{EC7^h)DC(mMU72KS>TakJa z{q#^9tnZQ{D&-%Xu{Pk&VBI7s8nW^o;vJcy4511gh8;H&3%MhQXf&s_WQUqNb8AM0 z6QJt38?*t|Elc#mVuT1+R$QOT-M*TK00#Lr4{0_N`9?1@cH1M{IPS8#0F12*3-9S- zQUJRnUWoou0Y_Df`DtyOtmCCbEL-B-0L9l94qE0+Zjz>5$R~{^Y>N^zGUV||j}n)6 zcj#awjGUi@FoPnkBS(WoW5%$57MB>JojWiheSiHHmqH}b0uxt6yS{=s)!sGbGhL+`@*GN2kB&+(EK&AW}!av%&+A`^oIC4F&^8jk(u z&7!^(Ou6@Bh>28ns2615tD9K-J?HCRr39WH0ZZyrUSeT@_8Tsi|LXL6Jw)0b4t-Mo zhyw{FaCJN=WMcnw$z4yXq=d(=vjse$zH+vOd&bL#BGzH)M6#WSjj6F1LG!bTn2&;G zSuA4$w3+xgeA)_&ZbF)I_*1G*k;8|L%sxkfLuk!;FgGxqD(+-!>2F)BY6Crw+R_o$ zppStx=1VO(p)^07-eh*&ivEL?81rfsfeOd> ziOmc71O?rpjh0`BW^goz?CXP_qFJXVI3W_ss5eC(Fr-+}tS22(({0`?xF$<&d7lz^ zMpoA8PL&s63bwK@#>JXG!-a(-of@f9_L8a?7bpb+!+y64J`77QDlg(35;!r1Bbj51 zE?5M!w8(#}dunqT&AenW0M{=Urw5^2q!W?nu1y* z=<7C2PaaVBsD~Q@xBAR}<#OQ7qWpp1I(j0}DbVn>S>NVewp$N9(b9$|<(Y+J?d(X; zV;xsX_Ao$-WN`p7&uMO;Gn9)@Q{d!jT$z0?s~hvO3P8T43*c}vWM4T?e9oE3qdjWI zbc(Cu5l~F*t1P)ct&cI_J9Tv^7nv0pFgqyDN`Y)f+KgNlc`Dz-Nvhci@T0hjjbX?h zwQwB4c&XVVi%U3QA_Np>5>%aU!#7h%0HdO>;w^Yl&T3G_mzpo3iQQ&5mS0-^NP7D` z72Rm{wLCN?Xss->uM8Y&NtFC zofn}Cg@bT?Wtmt!~1Oou1b6Gh;t z+C0FP3FJ%$&K$#ni#8P%8yK{@@MK0=bX1S$*?2N$6J_V$78CoJ$|O1LmL@csd-QY^ zRZHlh@{#p0HNUKtFdDV5xIfKbS?8_toBZMpp(5QaAcqU{;(M6_c~|C_IJWD=Rb(UQ zmUB9V9?|?PLV5WLah7&gQm`O#HBwfnV&Sr%+3E;(UXu2ozYGov| zXg$%)MySf6D5#_wMUv;{Y+Rf~Pzk%G#Ldd}mb>>5^gweQc?I*m%Bn5?gNX!5fJ zd#ASNi?dQ)cOPsx{ML6b-;++#MvKUjV7 zor!nmwT;Wsgn!$5$sm z+;jC{)PIiD<&6k#$BX}X;+Km@&rM&_aO(VfS0!J(CfS)N&Hd%ebx&Wp`(fZ-@z(Zd z(icrBd%O7itMl(Ke&y8PC-x?e?cM$C$q}DeFh{i>#dDtT*;xFGSx@Kx%i-c>R}_5X z)#P#B?`uBv!yC7Cyi}b3?c~WT7WK@ZHT{34?mGWhfArI)fqz(Z$}IV5J~Zdnmi`|; za{HZq_ixZwWOM${<$rg_t%K3QYtg}P&RKoqn+a5M_xu*BcVD@GPs5V8ii^fTv`_}K zbYbmaRPbNVeP1Je=CuRY6sL|&?D_lOPL2A+J1wti-qj5!68~!}ZHM1O`&*Y(c9->k zY17+fECBUId4J;H+7mvpo}guJ7TTl|eNK?!NtK-_7SrAFgTcc==HJaOs}@ z>u+RWt4Gu)HsAF?`k&tEI4~H!G3uvhW}&6;Jg?JE|N7Bc$z3O(1^6#r_!#XgdDk}e zmG!@reDC7gu1o)Gk~^eYddfSt7r%b?wdCo(SzkPsc<<|bh7UBHxMu6+kI?D3;;H1X z>V9(B>i#ADm%X}i$t!5=Ju6q=wzhBL#}~}L_V#ODOkKZLb;VPDsBacL(A(0# z>db@bhu(hj3mxh0C0oCAs{ZKje7^I^5!;Iof|chCMvw0MZ@=7Aw_(N`Z@ppF6-tp{ zd_L+WOe;i{H4as%Y4Q28pO~M=2^t)_Zl_pn6*6$FLKfM2(|`IA0%^xZ&Wm$|iIvJ4 z#@bz@`bcB%V(gIp37k{pDx(2KA8eEbE`gG@!MWceT;pRgB%PrTFL?CV&h*3)wBg2p zdn`LIco-jOAe$!ZBT}5eP<0a+T!o_21g_0Vz=B#-G+L0rWA91rcV@%+s6rKV>nrrs z7z;+d!k&L;u!s(|C^bq1Y%`{l0Gj8dU>J{WzOcs83;$zu7-)K}1ISc?R1IexxCeu+ zlETKkW=(0x^=gmuN+_x9?jgWo{MO#iYI*ijB#wD4c?Q)TI{7` z?i+Zn>uTf0^z;wV7gVRa01#gj3rM-ZlGQzDU_L#GXUVl^_4H&zYU9zzT}Vf$&I=oi zn>HyDDWhL3Rhp`{6Sw2jbHH{Ap?Nsia8(Y2K!AE0JaCU16gk?E<2G2*!D$q{&5b<=02T2VJW2vt$-My5)NV6*nB0i@~M%*q6WvJ7h(MY@RU&@=ZY54Kp(}hpL{` z_3i1pstqYE0eaaTt;EKNS`+W-NzoI+NRo5)eyJuN*y2ss0*kgt5@YE?fA!Oh-sBhd zcV>l==w)w|J-=%r595A(ujnAehEUaGU!henQcQZe)(Y!#wgn2#b3hf5h@79pY2Qa% zWHfZ*e+Iyqxe$6m4^B~%UtRa?3wR1 z_0W+Ezw`h2@2pMxFrLcbvC3s%8QXN*TB>_*o%gXvFyysOuXpw|fBUt@C!cIsdAgvb z_JMQ(8*l_spZ8{R&$@LB25!4->bJ%;{kN9a168(fVozh@^r)@}-g{?Hdi=rSblzK2 ziW`<3D(-CVoO0x^2BT-jj@$Im#HPRb*Z03Q<;Ko}b6;+Jepk(&{r7+6SIJKfM*n*I zj_G$iGw}SBe@%}W_*vrVac$N24Xj%*7~Q#|qxYGX?_d1>ErZd7re_>dEl{eMZ^XzD-SOGOijm`h)xt)6vs?l12A-Q+Lt zC5`dI&hx9&BiGk}##%RQs5*FM`hVR1qgmZY4O`5u`}rUK>)B=7XZNM&%q;EuYRka& zZ%t12-&}TO(_dEKF|%Od+cz#+*wZz?Z{V{VruQ@U?;+8)v0I^_9=;Nqy$fsmg6Ave(=WEuI;OSWnS{d ze;tgT*bQuXU`6-9`dR^?e;@IUk*hyd)<{Floy~ho()&jB*46&(igBgYwdq+K?>Iht z-_e(!<=&UiEPeIKE|#PFc@D+-yOT`|dybbCRsZDe%~yOW|MZL92Uh>>!8^*HytMe; z9~CsETKdasi;}xm{^E{1fsdnaIMDMw!$emv`+8quV9|kJ4o2@iaISv$Ye&+bO4j`7 zwKKE#{o7t+F0~)&tqru66=aSCtkasrpi;)>5}|kW;420)kQt2!bJ9 zt)e2$r*q1x#dxf(oQ#|VgsdW@m1;PQvk-wHhY*e-goHePulxN3Yxn#7e&5&c4-|zY z&+~rXhwHlU>%Q*#L!B@5r3F?wH11W7Ey@x58ov#_fP-Yxv~ItqOK@y{kFLoSeKa$@ zq5M_6t9r6EQf_|Nw6N$Czd!Y-i4C6)???(N&o-}tIq}7Nes^)VXEY%J`m7`+Y$vK^ zZJBquy0C`M>t4SoaU6{w)l~X@0XVZ$bz}5F@jfoYQXJy_(B_#G^NMp3SF|L?4+Y z+FjPPG+7No8CN}^c}duU(USxpyj(R$kQd~y^%eC^;Dwo|0WX%}yWMXpmu6lSL_?hZ5gD4Qp$<Mcmrld^4$ls>lzENO~(@m(McdSc2()pXa-B@;pN(}t;er%G@9GtYQlD1vi_t3^5 zG9U)swzKk5Ppl+-Tzaj$S)aDJ%j;;CPFizg&o_a7+d79^*Q(}wHYnTTsx#Ji+z{6L z+wVnyG$7&cW}Q#{MBd$7|M7SBUOQ;}B-bHag46RqbGp+uws9FtwT&K&fw*5ZeiIhS zoen)8DVlPL5ubTzgH7z;5yDrSmM7&F^_BFtWL}v2g}~)V=fYo`_H>E>@V^}{u%vWJ z4@wm|PIwb@K=KdeuGojx7{B?d&2pLhq+v?jIV=;j*(f087~wLc|H$qH!GgqQRbs4a;hTjD#Fipkt+=`$UXXKb+|d%u-_ zHvdGC#XTUz?`py;h7pUxLl9|L6uGu=)J|#fY+Ca}?fCUGFH~*wE)-pJ^&DVPni1*G zrQdE^zDlDP9MQ~L;bPy-CF`7;aj|Rm|IGL~cc4>xxHY;})LSr^?;bSMQRSU+B{l`p;&L0K-q&H>UfaNYAOy6W!MvLu44U1C##A9zJ z9mZngRI(h4HBM1JRD)PY0HZ+I6TSt6By!QLf!BAiY9&Gpd<>98iqJ`j!;w-UzgjA5 z0Ls9v^Pq3m+fYJKohBKjvJm7GtmF&fJko99y75JP37-IXPPc&1V{$>f5$j`%YA|pr zz9|r8lMp`*g>JwU&PI@h!4T^LCZ>UFFg&(l5-w^3i+dtFV$z86eO-O57@EzN5NU5!<8eG*t6l~%U#oL9 z`PTCR^e378Kwx~1fD!>I9=Q`0Vg3vFlT62cMomg7Yf6D(`n5+%`93}@Of#1c6Jv57 zILjh7OlV5}q_4YEYLRyhg*gN%-C*OuUljvSK*OY=pePg{)CKr_M8sqz!WKhEj3Lpv z4-sPzWseEZnU5Sk*^Eq)w9llc+$!s7fi|5K@uZXhF(1|dgF$TDDefBMytF`G6zDbi zkeUpm6ICe=Ic1eq2?{pm*vM08bgE*`te7Y?zbT51(I?za1H~r|5kRCizgkR^dozt| z_*^I}(7@$Vbu6n-;C;Z^TN0Ef^#Qppg&|kE9PmGp)KAj09l??V%dr5N43s{W#U3#c zTM3ifQreV~Yy{%rV?cQaGF8HZ#RMeStw1gdjho~!tfe!{-cL~OI ztmOay6(~OnQ2w~n@bC2453Fzf2wLM~5VoYMB`h!O97rC`q%iS4zs+Dd{(mr!p9C6d zYvB7q&Mfx-{`;pvexBIOygUDD>n`!%FNB5Fk6IKO7Ju`ii5~y_+F>Og;d%-FIav_C zvoTR{NSEB+m89L}uHnuMJ5`teifa@6eE&1?2G?gT7jA)R>iaPr8`KjQ6W@v|dif~# zv;LK57MQ~lJ=SLJe|uz6&i;t5(~*f&;wrEL*DBYSq-TvwtJ>BTfzfRoOHw{B4hfd- zPB>lZ=jLE{=kO2SSE{c&IRFDE2(OPS!7y?w(7NlO@88xfe<@g68IdFX zPfyr@sp?=DcVLGkCMHPP<}B?oC?I3!;Q(=z$LDlQC&1}VD}u&HM9u2F}VJ(65qhjFGj&fz_hnW^3dk^iICwccZB8k z`3d5?P?=%LGM8;PqV~kUDZ_F(W?-aczSow_gbIrjC(0BKt4;}~mC}nreu8~?e^G%v zJ|PN`h^p4~MsF|k+*huGQH*mzQ8%0nJ8Sau$`P!2uPjGeq1kPAZOaYkn#-~H1p1xA zYj>rC`Ed@(9`q|RSN+cJ>9t*(WBg0Zl74JD4F$5ofP2zgV~9^WrI>e2PD>XI=Wm%f z8N2P;+OznOjYcFE?7=|cEP+|#f5_7W!n}? zt~Kdee>>T_OPhjEQ%{-tw@&VjN2Ibx;??-F+HzR-e*S3h^ecOn`cW@n#6oGctR?gN zp445;G-g7(A4hc=b=d*d}bvvwzne7sg$ z`0YL_AVlmKm5C%1YJDR1t*aRLy18`px1$)<&FAaJu?OB%a4izqRm*j3**G5@-sY}E zNpE`T5a2&48tvSHxgTX-l0lPPeL0t3!=@oQ%P0pP;o7DOM z1#QTAgMH__ObUhXa{1K`scP8dSr%$RN!LX@^8)WiZ_z-<`a5t0l&U8D%)!IcT%$zO zrtF!tyoR);>T+kW$Un@jAr=01I5iwsY|m_!l#fWy`5q*j`TpYG?HiGs0ty&VM1%zh zFwVUzGPu*&od4{&4~)uDJZ4$fp57+Xc4W4OYz^^SeX(hE_2GgaB#XV8JlsH)Ur;en z>zk5N{_6fek2IDUhK(C@c4cc>mH;>WsZix#u)IDj?@DO-O8h9 zo-8!SR}3EuNXH}XuNy0S@}_j;ec7=PTZ9@$FQpacMPu}(eGQ|c6lLMGTO7!gAyqo1 z&rTH`6KprHOB*)1StLz={Nao4?Fo^yYAi1q!OWhTWn9&5s~5}*meLUj4#X6^K1tmYek1a~Iz_^VUd1Q`?PYInoRF@cb?%&Uw=- z_3-^1mG)qV?6lZ83bKVhUw*x!3Qb#Rcgn{IOGMrHQw*;0y%WO%ThaZm<$a(R8TXSS z!L0IgPW-de_ROu3jyLBh^p$qi)1Hrb>40 zVQY(1Y`Bb zHPZJwuBnf3Fcb;^ALvRI3yF=jWXGzS=QX&5aw$fKFFrs|;F3zRt-c!nb$G%c(;ByD z{$T5RP=(0fIjv(-RWY>g+{gd((d8VqVS;jpe*{32O4=%%7vjE&2FMZ5YU7YIHm*AD*FD3w5pk0lekBh5K8) z6>ZVG19(G7!s7CfQ+ekG zR$a$OJCUYgdHyStd!_nrQaOPJA_!~EPm}E`GS8W?4+ACYt|R!2TsLDuASVPyS&D!u zaB{JKAZK!vpb$f)C;*#9TC_?cb|?lyCj$85;@24Gp_{?w^yg#BlCT0oWXcWEyJGUM zg<~ibnEfawS=bQ_RSbNYFeGFf+Lw2p#L;{wM>0@Na!0zEP^z;};-l&#yt~yOT!AN9 z06%VbI=6&ttn(S-1RN@Ye6)dx!|>L8O>KR|i?Xu9$#92_G1aj8SvAc7)O=ZuwU03t z6$Q1&h|$rMYQy@DrD!}=w@6r^-3A#5C>#Kn$>?^bKNlWU30};rdFUSkF1VkWZh!m& zOSCc>E(jXN10#bTQcuuz*iln>JTqW<;&iOL2NWLI==zR)rffsu*1@5V_#Y}D6qj$s)GJd-3S#lCDB+PyxwK@O{DI+`} z!l5aBe{GlgrTuRkqazK&^Aj>R*DJL{W2w#k{=Nn$wqDgb#ERYUjN3H&NU13)*A@W# zYV^|biV@eFjZJSzYXW_9-A2PIy$^T1;o#7*c+}^P8j&Zz48Zd9)d|rKslvl}$=f4q zJB!V>Z8>p^)^d{Nr`<;%+8h=aExr|5Exu}WT~k;KL_YSx4Uz9?PnvvMiPS(s1-u7) z45!8wZ=XzQ@44~9!GWm#2$*%t9Cylhj|z)pz6Qu_Zw~|1ksO2hhMy?D%Nq`{T!u{F zHZWSAFX?dj&9|NI^92B>U<~JAg^Ml7RoU-mCo^ai4{b~XsItew&T(2s8}kz zNAS|(V3`dxTodgg?KWi&XvS==U(hbzc(YkGYJ1E%O6xd$D;@JR{q#=7ML>f~LDE~F z_2TRvpWMnh=NV(}Tat8*eyt}j?1{rfPM41ALRKz*KFL4G<+iC%A;GGeGN2mj6)1AdG(L!gXQ6pr!#>ScVdk8Z|1=f!On?`N8zmizR7}@QhxC(B ztcKJNigsev&F>MW&OLNhl7sN&_Ih2@smsk#S5(b|(My%}S&^X|Y9HEkF2KI_KY|vn z{+)jYw^$I*&yd|CxKbd=P$)Q*G7uqSga*!UwoJWJDn8dZuf~G$tP-^J)*b} zDy07dO%yYYNw-e*9+()`sg3Jm+jnXPvBze@I$?p@IA-3=RwJlN$o&Z%&_9 zzAx}y{?wUW()vizV0;ti(6#^pp8GnjsxEJ6kfdUcewvnlyHpJP%Q-7bZQ3?sBv9 z(5~U!vC_th$mLu@5h=x@!9NZdiMZ<+D0WKm=HF4q(CdPHZ!*%2AF^1m44ztgdp^ zifCg;Xd`6#&(4LBTzcJQY7)1wCsjnx>IC5M<}CB3YS$@Ad3Ne1bzy*|{1vKdS8kHoHDhGQ;$?ET1WrqAl-JO1h2fUh%jjJWE45FtTUp9fH|85 zXjtHVNvdki=@5c9nHZB)70SOm~3}wA#TMN`J(wc#?4jH(HUX!uxzZjxKH!^*_iLwnz+081hxgLc@U40|>u3o!h>4;QP~ps8OGcrndaR z;pv~}zBA&b``hTBI)fXOhNZ>d1hzFDw_IZ>x=5Nrvetf6ZSVGvez1SlytXra+U_IM zxaG;JlPg=(l2U-WkKebIxg=L488Ym<9w=RhCQ=tvqa0{{&B){tlm_dyPObVkT0S3Z zNq=VBGSN`);QHeaZGKU7YKpm7Y#IY6`vI zJN@D{Q~f5c!gwGwrprk-WyTfl)PLCWcj24MI&XaNqUit~<9<2fPLVERB)Rqc)T6K1Fd*V|aGQ=6gW0N6ABjl@&RCxSOZgq+H-C;l@Qe?1Wlh>_$%xaKv_K-{3 ziwk>T?S&KE+SJ9EVBgN}(cAd4X%85TPP7~@efv==Db zVnC{i3|_3qFkyHVqSwSx0JjKT%z!8+3Id3D9JdpqB`nq;)!P~|lyj`~f+A@%DpVQ( zxIW|fr!p|4y;xLgQ4}sUQH?geLQ3X-~oiQ%%(;f zKbDYCCIn$r(pC^v_J>AiofD#E2Os5}EZ0MUsv zARVk7{aVaCoZ`3Ybv0C8)0XB(doX3x)zAD9#drXwJWgVJ#m zfTAs2aXyP0`WoG82%z-=n^H|8Tl9FeR@0+*o=)n};2EJh!Y5>$3-05EW>hrt5%4=) z%x5wug?K({S!+0mG0lToTs+f~;Wlf8-!m#o1dW3I42Tyqe*>byw$uCX5qE@-4FOR~ zet=EjciHP{B*!^9Q5iVN*`ezjuZ_n=xfT#Tvo%f?@wze8`yz2bvx+iZ@O+=Z`*~mn z)2)G#!c7-m-Ykkv)Ce?{YW$z#%~okOuqi#~8N4%7Cs82jF1(=uRaqVc!%87;tSMH9 z>7)IU&2rMql5{15S{qVLDP}kc&)6(twG1<}{?W|0`uFxc`hiujMx=G7JdId7=)|Lc z(`;bOodwG2Q)oIs7tfYG|6C}$$CnbYS%%?aJQ8&T7#gUe$X60qitAuPB({}+E?VWt z#Gi44{Qu4X>Yx1d5ySo|cK(rhv;T(+0Wu#}TiM5GIUmHIag)ND!EetjTD#CeX;eve z>gM?>FTk+IB12^2)5oLN3hTYfZ**ntY*qb@ThQk=zc($enqub3+V8n#T$5qt_U@ZX zocH;0wnhU7K$IRCFp^ZdYWVr!ph)=(`+Z`v1_b&^OoWEs?R!=VGo3qs2yr#`e!YJU zjJVa$%Z=-lVePhDS8G8=LYd~!a6xKjT1;2((VV#KwcMG3s@!f{g%roQs-Wyc?$@Ke z-ui)21i(!p8!9%xmVML2pdOAg%_ z&VEH7RP?I<`@&bt8=tD$sWM5Y5n#Rm#mnD@$}6}uYv8lOFz}#wFst~rIQ#UI!hg^z z$dbIKMhMvwV0TyGv;JUsaJWIVlhXY$$gA7Cd*+&=TQX&%ww0R}tjclyCU}v5P(hdM z52d5M2eXE^Ok&q*uK_d6^7&JlJnnQBvo=d z{2y1_=HzxuMs$cY6G#Rk55v(K^A6c`d2^#Mn@fo_XxrwG?^)pZS>Qp_H}d!-2&;=i z8a>+)2L^5gcxp{869hFby^j6K3*v+{P57j^&sPNfQf|ou;itI(ao+g>7?%18zMFXP z-wZ)gwR+lwFq4O87bo(M`)-urL7&3g zdn>o)d@}s)eW)Y~-9!`pKESRj?vwC2|0IkaQ8^JdfMCAgwA8nf%J1xCnbM|-S?>u# zMPv)p6Hb{-KrB(gtdK*(>@ct9a@@f{_MB4pc`^yfWU6R|g{qHq_G=cstvadRjt`&xfE>62q(0>y)5M?0{OZR=tsM)JI z^Ew9}^)_TJx;u_q%j^TEwXb)Bc-Sfc_vG??2%JdAoEH&>T!)}zan^VjL`qXnoIW-8 z2XmQ#^Q^Q#xqVDK_=@GVt}MwhOQvO|2}I1iP+DD>@y<)8GI=xRkq>={<3pQW*|v)2 z4qH7k5eu5hvh4P5uo`#Xp!GTFPsEU6vtQqA|`w zrB!~Oc}GK30Cx4}NojvUNS&G!UjfL+dpW+Mb6dW!Gv2TexZtw%0{qfa)kaHsF+w*} z__4ST5OaJmc3~&}0lZeklrDw-=6?f@mK!e7XI~cRrKa(-_AjYQZ`2G9Df>HMJvNnW zj*9{V%Y%?Ezn1PsP^9^wXo7WD#%pXF;Q_kEuu70vgX`=M3*BW%ocEEEFRMf=7yLdV zg8}xZ@KoOpsRvfRVfKSb*9bn)>r2goQuD|O7MJybkwqI*B;E=IC;cA9vTR*eP}nt zJ=4+BE-AIl2U@==u*d=hQRNV^3j0hoFX3UD1rKd<@VmB*j^ z!0RP`9;1Fg{Kok*yCXJr(ZZ(q070LR%uua$R~V{+v8VPGnq6YEHr&OmFPcswp`G)P zvOf7v03KN69)lJSV@jW8_P*M&&~oVGG-PN2)umbu}g6*_Cdr zsf&JE5&TxvpM*)j|Nf1H3lpK8>AEc=@m-fqwr$G10^2El?mMNmS;=>xnA0DfmW{}d zc=%jk@@nOvT;~j9a|oO z=NS;q!v13i&dx}zDL>DGSY%wxm-9Wc#$s>F-tMUxE8%ri5Qj@*fbVr$=L~TBwPC@@wMkvnXIl_qw*J!__90h(>HS{C-{;YQZj)Oz%H*uU5Hs~RzF;Qg%ZHlM#$#EA8iMs#X{~h zUkJ>i7a6l;@#)LQ`zxz}kQwr~mKI8y-m)ilwS_{|DsuXyHA`-BW7B9lL)?n+JqRCt zkyr+)WlSv`a{u7db&nN+vbl&Xiyh12Flde)gDKI#k!onB5|d{nfhRpI#K*4(s{@An z6BMzhwBV0L=A@ygN6m8rnj())d-|SD{h5KF!Juv`0RL7lN|^;~RGfv#p{c`C;;HF> zFX;cvzd@V$he$Jqeepz!KAeap3S0mC7w8u;h)k=58^NSlQh7<bZLZZg%$)ygSQFFF;gW86o+qOp&ntLf=lI2OE_5&m)+%U7b5-6UZt~V;rYqL zZ*RB>#zrx`8#ITicl#lrAG=DepCi&O>79rap%Wg|@hbWBbqjh#e+J#k`m4qIx1yHZ zys)-&_z(-FbN9-!Q%kE6w8zO^1W?^ORE%6gvc*X_GzNjlwwyO@GlK?QJOq+`SEQ=R zR*|+YZ+a^-fFkr63~lc-xO?WvB|9YC=NK6IUlDD^=U$eS>Q+wN75J`382vUMPdyi5 z7@qn_K|2f}tZevd&F-UylTvIG@8wifbS%*BcM^2!deY96?McD`&N4Q#O(jDv%cTm- zdDCt4Ahhk+jW+f^yS`GU7hD^7!eDB%EKv**ihse|_?5HM+6}l30?PB7^YLA}?NmZe z%y*{hWpuAiD?W>s)@F))^8jJXD|T?7W9;8Mq3F8v8zUmhlR>zjR!NFZ{FjF|>(6SQ z-D4_hQ(hZ#RrU@*3WLTWPFF|EdYgxo-71S?v1PI4_u4_xwwwbk6@&ICrK3*fxw_B+ zgy@GYX`yU1PSgP65U;ezEl-#HY&V6%Wfu2IPPnA}@6y5*!+MvG9&~iHW=TVvwGO@y zO41g4)eP#D#!*20oDXkB#m@IWq!&FOF6s-=sSPO>Pk@1jaLM1Rx&4`<3eyTKH`x-! z=>^LS0*g~lOz#0-+A1-6SCKvn3x)O6lIBt{gB<6AJ8nR%7TMZLm7$~95P9^;xy4vr zl_@QIg09!B0n|%KXRfL2+M`eS^57nnii6N!=yKcZ zZf~2t=W3)E#asQ`Ah>|njWN^yWPH;}<%BL^zPI7QRcXDnZmHARXOl|Rk*9i|7 zH7`X8=4wBI)T*f^J5#`UP?X)K)Eqx;yw{Mn*zzVIM=y4DzXnC8XXx6-rP+f8nF77c z6nnP!&4IE@J(HKB60Y67eOHS({pfH|nZ~rJDZr*VztK4gCF;pil`f2$GdIZi;_`R6 ztW-osg03jp5n59h`Q-y!qXi=|AkvOoev|LAJZZsgkpiVa%9^jA6qy(3zO)?5ch*bw zfl@sf8rCiDqq{5B`)W&jXh6AnMT{kUT(Rn;?IFdLKueKjKKQ``#f?nIUyNv`U$aM! zV>?y^##&yYh&O0gWu`6|&Xb<6ow!hSYx}zoZ3dItM8>fdato3W6oKz!D2R}oEPLTY zo2_qK0@tIacAwPfJbvP#w)xKp7X;=X`56_7-8@ z5C_qisNo#l0tidf#k^OJA*+E^?*nyq6ejWJoaCXqW=@KS;F_haso1`I))qP|vm)Hi z*flQI={?eVvNM8HX!gIXfc1l@{YwavdDEZuc&2pRA?A z@O6{NEJM;Fhe|&YjtluFrT)}ZNWBhaH@nSq!y`o78Yy+v+Euo27-am&Xa@MWM#@4vX#C4Y*q;v&}i^5T>jSXMlFFtPamU;Nx=hB({7D)qsA*{IJL}?%WO(9dK-RdS}x7A6dKyL6y?5;q|mC7 zcvZTu>E$Mud>vwENPA}1`X9e_tLL9T{H_LQ$?lv5$rT|M3t|*HQ`2Vks$E1R5F()H zHjf8L$w0}F+edi-iQ-e15rDqI=`vdAB9i=H?W76R@f-`zQ55k6WkJ>g-~MkNk|csA zEcInBQ{W>rvU;Y~G>gSQvSeYk3|32OVmFk2c?!B7Mlu+@ex!Gz5FOYWBm@9fZP@%{ zw=iiJ?t*>RS*EukTL|mB^W_@|e!hJZbnH=uGewel=LG;uzTpvXFaoNNG$(IR;>s#Q zx6oZ9<-ouwc|im8k!>$#-$vwsAn~?l7Od}%>=CB2tmLuJ0;heNnkw;a$DR4;_Y>%w zW#-#wv$taD2`1L31PAL;MA2@pg*#C%&nC$=Oxej!nrIv^x+ktojD=@L3UY$Q|MbGj zmpvB%PnPBQS;+99UZ!Z|KLUvXu!U?9>+cFUG>REW@5Iwlz2tnFL0A+1XMcf4&6e4JCo?YG)NOif^z8*^ zTg)g1%>-*D5cy!iewZ;Ej4|)O!JZ;rK5s`ubdrb%wKT`YF)L2=@_gzNvM=-_29KrK}fOilNis@_#D4X-m%k>i!enIK*(mpJ^Has=WI%QGP(|z z6&6_lznG+GROZ5#c0@g#mGzmpEB%*tr(Qa^0s+b0G z!bPI47f(b~R}%BM6(o6%Ss4evr14swZL%c9V|j-kD+^|FD=dUEix2iAb=HkI{dr zi%rtr?&V(F{}auTbFypEc9l*%KDx(ud4@nI?|X$4I^DgseW}E8fy+(1>9t`Ns;_MWIf=fClIuS4 z=xA`t;b@UUUovJ9`9;YV>Zsc2E3gL8~FTAPsX1bZrN8ev@3JLna>2zI{td}khCUB*+Xcc&^1@oqvATWYX=hrN8ZA99CdM~ z*tSp;o`c$BJH33CZqwj+5-P;ltmIG*5Zr<#lpU0uV<9}RRckisq4g*(`luWKQ zyEPs_vqO`h)p>JTfyz=2ITxOfW*b8O8f>Qhuxv4}D7)h~GE6vsr+n0%;lRr!2zWby zN|-}q5ZTS+4q}6wxR;Pr;VT-qG-f5Fe9PI@>V)k{p@!2(dj$}*6K29Lc5F*LSk`DJ z98)wqi_E)Z(-L2}+{90wx>TM5%(eU%x%bmR76jXfVpNbTU=`>Go_cFrL+YkgTG7;= z;=)|f^J>v^N8X%>T(zKDtaqA2+AvG`T~fLrIdoZ6&aZ~m+Ijp$*wD5S?xxPv9?lPPfM{62#sWM`L6TnlVB0hqm0Zt1dPvvHY zGd~%_&UrCw>Miq{pl@rC-&Gir2Gm@0lXFgvb8-IlI2SC4_YI+Uv+9E}N^y-wx_e2PZzgtj**@|yA%aL8uq&qawH0dm*ZrJ!g}9uSe8-ud3^uvtnsS6qsYr*QqT_3Mn(t zE^pR;IsU|u+Tt*{LLOmQcn>})r}F1M*d?vWbQ~TOjhRDNe{v6X+`42@cWWLtBc$kl zVL4T}bl53<^r@c1C1>@GRR=GFC!z{?AEu_DZ+XbpX6awrG?YQC&LxKKRhV<&7@mP^ z0j0zXuszXcC3D0@a-O2uy>Bi1ZIU~F4k?m-tl!&RT^mhGL(g!_aMlgL#N8*h-@R)V z>|VKCKBnbPZ;IGwKtk>8ediD;vN1NRC|uj#QuCBdCmV;ksCvq>LcXrOx6SGN{qTTD z!T5iqbu6yzuSaDxh|8O6#uSM8=mU^M(z@ryXZ6L)M@>t2W!6?~f2Vh7aG>gVlH>zK z`0@0$o`15;HnsNBfdR^iD0fI*{1Yn1cah=43ZD;9pS4w92%R4n^1F`%5-J9zVdZ6_ zu$&7O9CBMZ&JbOSFv(Y6F{GzA#`}~ncz)YN!EzUzXn&U0ol&``YW4xA5a59!}kJU3DEVHMtZlNE9ikR8YB!bDWL> z-MbQ~2`^H9kb6<+$la4inLTC426yE^1k^iJ_Uy{Pq7)}(Swd(B!df+^+O6CR#&1S? zRVT04##Kk>!(${WxNmZz-}#SWYL7Q>o)|#6mqRm-bf1utd4D<2RGSQMcGngE3;2k}7D|@Dt4RNFT?^9>tpp{h=>0_#st846yHNZF;a9zH(+q0^tyeF>_7iBOV{G7o$)L~KycFxbaeyt2LuFAZP5UPOC@($0Fe=1(cwE6V8@^9l$WF^(giLbl?Xw zU~4T6D~%r~H{io4%LDX@U@t4@v0+Im2&b&93QQCgPGbFC%9nr+;U^TZ`PgH|P)i#u7#VaX8fzyRc3RUU0@gJ#u1wQm4dLGy#TAGuM}rR3xvA$F0i` zQtbgf9IK8EwwY%i&%cs@&OVl@^{DQHckpJ_|3p98Vws^Gd`k!F%FLzK%fQQ?V);nW zWfG^c=fY>1$j|#MrGxm~4}xn{3C$u1#P6>d;_lkOQoFciyo@Q{tZFw33;g7Zcr?W|;A??vN4+<0iX;$h4N zUScJ3KlvNH4KHFkT|BYJEQCqsN98xx-Hg<7G9*()EP5W7vkYaH#$JsQ8Y ztem@Bi;Y3OVIoJ=6@5!)eAfeMMoG-q0|L_~e^Wv4^z~GP#YgwN{kcgqcO<}@PYa;gKhgI$7`{98^Z zy^s_W(_G*R5ZpLPAVbksn=;*<)htRx17|*^|}~C ztM~JxDSk)Q5Ht$hoE8Pq6)Jc3Qxg*}tUAlsD`=m|PtqZS=lY{h`u#opj1x zUz(~#Q9y0{2P@K+YJzIBcJIo83rd#syD-bF6*UQqJC#UN)kNo(V?EXt?@nAN9ZVX& zh7hC94Z_dnmhq%OheQ$Smv)5ubHC;)1fv!SbUiUh7gddO9$CAxL0zX$q2N0-h>~@i zFu>1~z9i>>Z|yJ)nQ97dtMf-BI=RIItNPNcb+syl-%*L1-cqhMsPk(L%M6*aLXB@_ zhb$Ba`*$bgREsh1lW)y?WmwT-qXX4d6}m`!dpQjnA(dD@mFGkAB?+rcH?6PDYUJV{oh&3fup=s6f>k zK5Bc!!}h-Aj~e*)HsG_6Ep19~t@;LhXIP&k*PohqY(=%>ufYvUijZFT)R*Y3Vj5)V z*XklT47G5`Z;MiQw`6Xu zY%A6-Q3GG$ucaWY(Sd!*bDD&;TnhJv4=j4-atBVKEnP2~BP4BFCAfCaznc?Ysnl2! zPzpp{BHf?G|1T0;N<(s~=L~ABuR_L&%yn33Kb&H;oT&Khysr9Nht!*jXk0hsh*W}Y zfR$}XphQAbmUDsfiATiEn?sSh%hNq!K<4j&dfD`IVTM&%r%(w)n$hYszq(i{23K_= zP7r0@kzFQhmez@oNagZly7nw~?b6IeL~=orRy{Qi+qh>)Mz~smXCe!juzhZ;Msnpk zWP@wLS>J2S3OCjE^;|#GJ1^QfxmsQ%`3fUk7=V>hxfk|PnE0y6L$29MZ9NwfDIB|N zx~tVS)J`0)HGMC`66jxt1ie$JdBI@t6$BnYV4iVm-D>$P3q@Vk7NLR53ZRlHk+i!b z)AHFZ)Y3QCJY9LYS$d6YHw>t8NmM3-tlVBV)EY?)JCQ}gl=>|UMgn^;Q0UY`z&>dGsEjK|fL^X+jDys!!;g z%^eM?rS2=hx6-*YOUm%E-6}5m>$iSLDnm>HqP30?sagk3A(f$haQ&f;y>xeo^z8d{ zK2oN-3RQ8HmN4QIREpC!f9W8ns?<$jxx@BCKO8*k;?=<#bC^NZq8Fje2E|I+QzB^9 zvdh(5x!;)Q_T;u@DQo+N{Z|DWI9x z6=e#ur~jSLgov^(owysb{llt5$CE6dM;hkWlp-&7DTezs$kImh9wf~Ukb=;by6*&c zh!cG!F=yf%zF>3+USUuMqj)6KKK&o~Ouro@Gde+%t*4p1qy3@Hea~1_rKn3u#8Z=H zzCB#AdkVcIPpR5OHPLw3c^m|dSM*o!G|!z=s-B*8(Htymy^_+FjPcU)$RW|` z`KJ$dU@nTNIc(t`L=^<64^-x9PgM^tPzbbx?cOIZ>5zV}k_a@7mCukW?k;mu@S|os~D5 z(ma$s9{iMC=Bn&5AVjwUOz^|)A6~AVzq$8doLs*slJmr*80I7hczr6W3|RpGv-IhA zV@+dH+l294Q`}z17e;$@R20~BUg3reB15@GR!P?&ve69GUEX zy7<1uZ_ur%X@!1Pg==d}P#%;g-opBjy9<6VbX(S`&J?`6B}n;+&?7-`OWrphf!iTb z|4`|iO{*5vLa1q+%-ym5xG?^Og9(2u!};0KQv2?M*0uP>dA@ zvhS@iX|N@hC?bqFqc-FL{Ue6N!(tNf!s>@25+Mi9A_c@!b&RlCL+4L`#qcE`g#^c41NHc@7Q)Y5Af8kS(`~{qo{Gn<(R9{`3ZZj` zsjc9Y?j9(a+w=f^CFu!UFZ9Q3*JB`-I>&&B`CuF69|ckZD8E2^Z2G5cqrzI2#Cj2v z^lYG#x+5qaF;x0AD*NYoDjWHCUY`7iwFxCTLxkaCVWoyYsRUmOMr>7e1yer;s6bg9 z3{k_&w<&euNrVwVnpjyLXc;wKV!)e_wv}3;Ip>cZg9%zcnlDI#AfW?4;8brYRnTGx zTevVEyoX8bFydk=4F{v*d;3u9Kjhn3yH_?H!O`;0XThb5G%u?#IL<0!dH6v^hr zU=*wp1+!+SSWtj31jEJZTNTWm9$qWcN`~k`s&OZevM*pq{5XSZq78tJu+o6j#)sYz z(%BG3&WI$iq#v6SI3>OUzoM-S|4(ynOuJ& z_E7+Vl4+RUc#AdX?;`5p>pZ}kGu0vqjI<`?f_!HOI0*D%2=0558?ksNc3sM3oiF}Z zE-k6;);L8h5;+ZQNbqdw2V$h!sMsHfq&}Q%0a#;ql5Mi&vvr=+Q~{l09PFpnK{MCM zFJDZtANd%FBjz3IhiD3blfCm_E1q7;ii>7qKkS`}zOgTs;*LY$fP|MwjDso7Fe?6O z(exrA>`XM_r+@#*9jyVN?bNu$I^iGto2Ap~Gm+5ZGpsEC&fN6qLQp!yOKJF=Ey)+} zLT<lD}yL|*Mu9295lw4w@xM_{-~KwXiQ06Aya?~sVsX8 z%Ki+e-aT)fmS5O*^($2?0J>o?FzBA2$e>;*RT&C$0s~oTb)k=}P%KHud7%lvp0zaT zt-l`So*C&ll0^nzXAHfT1&xU2BBq+UPLU=3B|wLGd^o_-5w#NtA4*CGDTNUgYo9J} zh97ZRuAA}TK1tTjbccpMN4G}T{7CUtON!3Hu9sr#)rS)u(zg{XBZUS;WIXK8Y2 zva)l*s?ERe4y?Tq87-PN1xWtq{aDG4g@`M+lPYs^QM~;r&oq2kbgQ)PiZUgCe><{f zWs*;BS6#hzCVBDm7+3?QW)tdLP!Z3qIcb$_WzOH6k=8q{WPvdhjtSgH1BUh($BH78Sa=IS8UM9TKaG_r-y`3O*=8$bI9Wi>SBiHh}$<<%0HCd9S3?)+MUJj0L)R ztj_CYkon7L6Ivd#a?Op zt~@_HVr9H$R9&8p+yz6KdPpa69tb&}TOUbg5c_`_LudTvC1(kJZQG)A zRB*qQHpdq=YQnN4fmH&CJmX~3`+cOgQdPFn&@{dO{TRtsf1>zYs$qa>3Wt<6%I>aW zGkgkA5chFu87P8m)% z`Wt;@@nL}u+f{?g@lj;)<)JwTF0PR3g@r3nJ~nTlD|OSi5Jl1S=3hu<7cjns{ zr8k|_?1sj9^LET-E;v)3-zcgTB9Y#$9yEwX$D>!@Ief3V6yjab{&vK6hfTgPgQK-r z&3S5t^?nLUM7L#r`M{+3cR-}$L?|ah2bp5%9%ITt(d-#@X`BGkX_5i6D-ZZX&?=1Z26TbG4CgxRx+B0>a|dysEo$WnH>@mxwH(BD~UTV5l;B{Q0@Oo(;qjj;hCd&|E1nL%TmRF7xR*&N1+vvM=x zHUa?w(*0SMKwYZ{ZBs0#N^Az4r#Gcu8SwDz#tK2HtkS$4*>_#zpJ`mmU+PH+l>Z?i zzKYzubD-(=Q0C!sWdl=Ed|#ed080vaTjKNS0N*q<_D?r zqJ8DZvtnR9$H@Zi&%8mY!dS=nhEwh!ym~QE=EV-!()YI_(&~bB4-51O$ad8Q2>;*Q zCb`faXgO;7I3e;(=Y8ohQ53=~2uR3|PTKxFx#mX#mimu$b?SAhqcv1c357%x5#1$?~fO+Jw$5E|D-fJ?1jNNDy>z!?6(~?ROC!2Qpef29!r~TxM_#ae_+X04|hLJf) z@IY?{@LB(URILtt=tn9&ikDN`17B31!V-3mSR833XF%QKg;|vE;~AVC>T8O({8UFp zl`V8?5haC46Jc2N&j3IL+j;EcQDTFhtU^{&=VcKkBj`aDILwe1nMsz%Azn##wLOm@iH zvCefE`hP(7O37Wc$fG2k4TCJH?n@E9%VgOUd=&4BVL;AL_z%X&wDHlY^Yu2)eYgNI z-S`tAolW5Ul$(P}lackabBNV6Ky*s2i9wwy#fqm(SwnW=2u0AqQrZu3jF%JcAIi#(@}IK=Z(aMM&HM1h*4M(u2ef|*8{?lrFyIt|KjGfMwC}7P zJO+i?xPY{?1~1^EtOFv;RL*|g3>#mn0@KmJvdt3Df!gBoER_FTT4N_7bvSW&7nsZ3fWxNPOyqRZN{4Y>WlKX)w z`J%XvCa;d!Am@QzzK*OpmZbFta3icuJ3GltNB z_Hl4@8X!iShgfy24eR3*Fn_&_q6K+F6B)K%n2Bvj#UqLj9`(I>4V>T3GW!Q`Lw!!v zd%xZ0y~(9|S%S)ZAhcVmO0)e}VsKT;d6cL4eb2eUt4$HN$IxcVMP*jxsmi9d+-P7T zCwu_%^!nDRAl2`S%|)|6JmcW(7q%KFw?MqRJac}I!r&B;8}WfNiSvTWIuwT6Q}^C` zt+%7|YHJ@)gYGdk2;MN+954 zJC*i(3%>u_4p}*8z>QpKUF6s1MY5J;M2&?7aN)?cD?6_^0PtwRuC)=h6^(MIFH@Ck)`jiE1er~yGqS@ zWIERC3{*mw7CftV+ZWXXts)a*2c^>b_ORB3-KbqdQXmBvUP~rckw;(mv^%P(22OCp zBq_8D-H?Z5Y;IKKT3>{NH^NtJ86c{w<^vc9*UB$q$D=xwHAuix_NMNN>XYTsipHvJ z1k5qgC-xb?+%Vpp<JB zjeRX<(${xod~kAoG+21EX%S`8CN0U5oD#SLLDqJBkAx;f4P3}KADv9Lb@$Bv7CFOp zN{hR1ayIgHwpUwx(!|_q`EC$Ix!MS`l0aQkkuP#=qQ1#LFN$9-RD~KclNPi#=W`V* z)2^^q>W&2~6NfoUWNb(=)}>-}*8d7u5l_BbY`Zt%zJ41tH;gV|7%$xAcTKPbtG4y0KHA`#1 zHVfKZBG)wem!msNbkwdZ<;V9SE9}43zsaj{YoDDy!_kQ99OVc06Y{uD7^K4i4g5OA z*TvQK#tOty{EQmT9^Ha}bTj8Ry@wQIKzV{iHNct}xs zSq*n5Iue;ApL66}zI5fE%dft4Y(pqMnOJXDB%U!Gr4!y8Z=1+Dk9pE;3a%1QxB*ihY?5RKf|%Br+LOeCUJ3r;Pa-XOhzq|_iJ0Jt#b`1t-u_b5<>@X~8C zh2>gf>L!Svv1HFSd=|a*>nG)-W%B8bI<1`Q{o(|_%yrGu!$r>3HA=58dOy)W{u^+o zq%Z_2aRAj;OnSgh?%7it=mJMSYgPlxs5tO0PDFI#;QU8;eWBR&fZ+n{mp@mf$SZ)%+W&^x z0q`%L`=YT}?HVH*+$$RGHkeO|;4N6zp7Eq*&Y=D!q!0PVf<8Kt)&v_r1lr&3-u_`F zHFLKX>VW*=&Y)UCt!M*EeDu>x_}D3&79|1d!FW($kiC*Qw|j<^{kN;2Exd0Bg@9{* zu%#rkr9)ahs>pq3jsUara;p$$gXpWg@)n5>`qEHX0b}b(&(;6ha0;-m*HG0vI054P zrcTf&`T#cHCP~hT;=3^r&g0jd28h;bEUU&5@%k1=1G)F{Q*Xn?AkOdC@Ra8RCzcBGHPyflG$<2O3#>pcBpx#7&6$*eW zlkrNL^Q#qSXB$qb%KYo1U6n=3t~mBVF#zG)upqQc)JV53iha8G>jdwvOOpo*#8(k9 zQk5lEUOu*5^b!(xdcLi>Llc{exR$KM1tDQ9 zxQ-fRf>rqm(pwqYtB58Py6ocGLTr(q*-)~D0k?n{r% zw>9)0TA|K|M!!^O@EH+(I=!JQca~POdqjLRCw4f`@85!1q9_pBu>s9Z@9)`|!z(_67_ zp-eA`ltJt)w{#!8l%KeK!ihX0esnnNn_;rlRvuYQU_E5|W^#9Yc^>1gjBH`%R6{ZV z49L`^|A(+Q4`}jC*T!i*mNRvfc3PkZluQ?^M^xIWXk8#Khg$6@_Ec7ZRB|i`D7AnH z7}C~JDikvvt4vi4idM^NWRo?mESZr?SjsL?V2~w52qYn9>+iasK--z`{J!rGDus}| z@AE$I^FG&oEw^Pe`7X_I+So3c!;cnJM@LGVSKvw_6d*ADr+qEZ5R;EuHs8QMtP{HB zjajoGNI%(Y&`}Z8&{pc>%_2~sBxHhA?3pc}Of=pszQ(^;LiUmrk5N3oIswZjIU0L% zuXyK63!Z0n81ULgt#gT&LWy` zmJVJ>36+HBEMiQcJr|u$HkECEF6d)t6M@wnKu_Bd#*VQ*rU5^qxeVHtAstsfWwC$J z&=T4B{1WI?2_q74MZyJQg)L|SZK&9HbE9qvoMF3i;P567d!fceuS76CI=y2@)RKy& z3y%)KG1Idi=@_Coz=LHlf?DccBVagZ8?u*vs|E5v_V^q?Bw$PBh(#ZgF)*3zf)Wg{ z7cQ*CV+f#l2}L#nuAqXubea*>fw=wqSfeF&gshM-1cCMjc-!AKMxEN{O&n+Y*V z72FagqdLcO92khjuv)q)kwy9|jt=92#srM^fisdqOMgL%v+0{SzGO=uEpU@ey*m5< z%uZ{$LqO$YmG2_)>}8ZD{x!4Ef+2skLj1g9(18C!e9xsD5_O1Mgu0i_7KWB%(pd^t zHY9HNmul9({6Fak{;#k2mCVzN<-E|Ege)Hn4NxGh&frZk7tw3sBOq9qP=zq8hSiZB z6ctMhgph=z(&fx7!?~ zkQ=wDZX58>Jk6mTOE21q|Uj>Y$Ldmqjr!>%ekeBIAo$qskmR0|iF%Cs_b zg_1iwRx!T&Raw!=`t|9_=9Q{dqH5yXv<9U^?fv6@z?48`d+)Tnc!VC`r5c(MxhkM| z6wa*r$oyj4N{F%U3Y*OJyp3N4dv+gt=Tw_*05VOsdv99SJ(-t?-A9_&mF%EC=$8J( z^N}A;OultpwD?#Z8KE|~z%u1xPDi`N10Fp?p~8$2Jqico0M(BJ}=yN1C2bV{ldOwP2I^S}`qA&2gu$Ai`2qXMW#8Q=4 zC=&$eR+PgFF)ctr|I)+i{dqpE`4N#+#K*}en7<6{6+&;8JvIY;ZitEIc$~U>vacI8 zaO4<;3eMb}9v}8Fcu>iOdt68dh>Nj{~hUPrY$J9o(aIAG~)qw_BNT zgB2vU{DAehdlk1a3z9?gImEm{ae_s`XMKZqEv2@_XdO8^WMEu&Xm>$Ep%+M6yfq)a zd}yN~>xo9ifNyX+86{qy9RJnKWry0Q!XYxaHHO2#O+u#@#8q;QQe17@hNynYx;GOL!r|8yrM-|J8v^?!hH}8id@@Lg z4NO3RYP<26>1rR+>PNSX_ovvQ*v{KuN!m{>IRUYm>S)c8Aa>dVIgO@MbtkS4+hiZ) z;6=Eq@n9lOV(aO|J(!y!RW}r&YEviw%W}8HYnN@U6&uSR%BJz45?{kU^)wERf&tUI z7A2f}G_K%!8*EOIUKrUa{`TiY*?tHX|LTkz+HGk2gS{B0EA z+<+&_I{PnJ8Yt|EN}msDt70HqSl_Odsbr`~Aqfq!ntS4~f^Y|^ z>PFi)Y{0NLLDCzyDgKBiV^}W802PYnHcQD1I{w55VW{w4_j=euJr_ zX@_DiHm&zu@Pz4&+|WQgSEh&jwBiwA=zZ$1^EanQ#h9YSun)Hhs3m}F<5P8K<3RJF z?Hu}7C2qt!COY3AtG!NFz47My#2mr1{oLRADy*R|uFKUTO%rS3HK<@jx&*%}d)|nT zX3XyLCKPbaw>?mJ!=qe8Yh;%*5CXNyt>ULM~)_K(SrX`N#1;~ch&J7+TOM{O~!6?wlyW; zvAF}2DQ#n!O0cZXmi{{L^R=(aQj+hA_Z)XLWx2|&dmwCIR%LF&%`?~yTdez`@X^#| zH5WLauPGNqd9<&}1KBwwA3ygPIaXXJ-(F!x`?ce4x^GmD{XxX1As&3=A9UaJ^HQLMu6P|OU8tdQF6`8+M<_=p_ z0w2t32nF50?l4Jbi7hWl2s04{R5X_pGRuE3$s%*C;XU)WG!i2DS@cGzgE$FL48;9G zpE;{YC0>n=mBF$C?K=}id!B^S6OfmBq%a(62QUOchxhv>DsWB?M{pGaOqNg&?ub5R zX!fIi{y2Iqu|%Y^tF%QW|K?n*1PIbR63&5J^|mm#3|q2@dDg~;d}QnsNd7tn^m_#y z$L9IIBuKyBq106JsE_W;#I5* zmgN;9792pA%ko7Nhf+x?35M3lKjFw2MBI$<0{BPpquKdufyY%%EW?zgocfUx8y3o) zE6|StHwTGa*)_RJ1S<3hSJi_G68W+#Vg<$(qe1f#plXOw>?}EyW+y-sQG+x=$f=1F z2T&!F8>GM$cp;WjI1fT$4uN6=i3ye&XGYFj%UO{JGYdf)0)j#W*y%+yRZD@)X>Q}1 z4+H13{!j&3YV-~JRXnBhCCZ@(^*CD%0RP6CPT_`_egCr;KZn@7G{qN5B=sSwHA^K_Ctc;kfKj(Ib`WYgu(PbpuR@g!c<_|Kta|=T;;}H zx^MVD{$!>5LdPHP>!OX@w_$5H+`1E)ds|lHa@vl&P?=bL6%P}IJ@>J55H|z@$H4at zQT~Qq_oJvrAAUiW%)MoFfq84m)?xEDhaxPp?MDv6paLo8<74?7pF1tlz8O%o{|mnI z1aehkCIHn^jC|r6Ass=iI+pjA+!)St$B#uI1<;&tkF;-~up@cG`pP22xa^p@wEi!` z@3*bQq34YueNUwdkk&ZYJ8qQ1y7hhxbe~G0VyvQ%)1}vdX~6xf8Gr+6tn;u#t;j}@ zv>hIY1V4QhXi14V9pjfO>Po!trHL$w3b1bKkWZpJk&iw^zbo|E?P=wO>+{Cf2E!-> zht1w4Pn{GScPKyf)O$ogrW;|WMjT1VxcG2oUiIC?dFBr0GD}Mcn?CY_IP3DO&d+J~ zOwOE|$QKUYgKMfh#dF~d!q`KxQ{lc3ubdkkZ`eB*;n|qKL3PANvAu2}%DF(KT9~d) z?N3PQhKvhuhPFSB8~v5$-;cFle!1+#e{ob3huUuXALQ^`+`A84neG}Zy+0{0l`Cm^ zSqUahTSyBfZOaV@a_f?Um%F{ZZAS&l^aI7Nk$80O6P&u2t&9IYR6oB~WULUJfe)(g zdx&pc8{!OKp*K`D-2dD4n$kp{;cr`>qYmpR|rG+Q{2&C&|rqNt|=RiPJRhn!UAv<(nWJC4S^g9{Ac-8#Kb<%Z-5-zRI- zVWRk)({%Bh_8yY{!}^dm0X1gafETkCPSzo2CUJ!-$#|+rUa$|-`TBLJX7x+b^E)D{ z?xQn7VFjKCY26O;*1~X((z9ewv}yk(Ge%Dd!Z^v~B%z%sPVE6~rh@YWIv*QcL^Nmu zdUSqc!kIfRR3khPN27JzATKm59A4ve=7oGq>oY}gi+%B(+lX4MC5lLXCQO=C)f2L% z6x(+4EOClNad99wKRG~g?Kq4sx#9hSR>;BlX3-X3!RNwfkk=wMS&t)arG6ASk!$ik z4iZL$cbx=mOO$KxK`sc&`SYmOxKhwrFft{ zbrbjNh4$44!*%(Qd8^L`w;7{Qkfd}4 zbm&lEG8C_KMS#b11SdMM)XVN$uXr$IbPExNXis3MxnBk;aEtaSr_jCOc` zQ}McT$0KqMhgDujTLPrBgy~g^mo&ux=qIed z59STd()+X)Oc{}OdE#@6?#Q1Hj-ZOG)(u2H7!zj@~0h>;a3J9y&Y}> zgiI0Ze{;jrScnicb96;jZX8M*x+#kA#Z9@%hntOB2~TCx##AgpA zKn~j%$$CZx8XgTQi~3H{oA9NWPR{7U4VE2zI3l;Q9V*jPFMks$$i*>V^G^Cqcx&QN zVh=y?_x7uT%tSBa8OGHaIlv1 zXl#DFpeP4jot5#|eVSuZ#ui0wtATL)0F6BoQc1yayA2xu59;K54GXQ%vKdwePLb(T zfJpuXGTlaHQ@y&f9YHv$m{MryQZy#LI{Id}x)SE4R( za9CDngldCeU+%XnYSTlSh=DF^bmSZwG;igMz-Q?#u#mQ40D$^cui2Uy*c{WDv9d)`83PRzJ&Nk0CZsD!c$GTlv9za*1u!ml3mYTD zsOeismf%y|Mc<*v0Sa~Y@iJ{K9Q;EAh*EUUz}24`La&OM_6xl39N|zG3UJe?yPChi zK%C>>O(dMj4Kf^tXai^arPZ+nDlH(@s>0{UHN|2m;6yroW(^m10o93j%nx)z!J!-B zcG9WQfS;_Aay0cDFQzW2fqp@CXv@&$PY%Yhj6WalwQ2K-1hhgogur~MqNjX=^_FYB zf`aQ54aCp0NPR;EP;DsN3!MaK;7=_>*JWfA2MMV)pJ(fpgE9)g*oO&A;is=y+VC zd4<0qX&;K@v7fC10&SMAGd`=@;|Ji2JnLJ>5@c!ZNuE`P*v>rsy5Uh(%7Ww{bl=yn zbe}9QU?GCIk$+sLT!1LnleyAQ%)jq5iTB{9+9-Xj&)-9$V8PQh4|ET9JC5bZujT~3 z`$rEzw;RUzMB>m;{RiAcldPZNre!}%R;6Du_vW_!1Z)VR0=~=!Wnjbl>%Aq}BkB3T z_PDcjrH5^B>9+`uB*O3Kuh~z=&UBt7-X3uef>w*05s(br2ZhPZ879$U0ZuyLGbaNy4g$sm7Z zRuTL2iQ+e;$V2;E!c5PTC7Z?huvkt6+5kEj@FB#P+Nf=AzpHh@<5_<*lXlHkkOzz+ zf|L0x=d`eN(?hLym3q7p*1%CT;?OL-C~o-Z*#iZ5^{}FC%?eAmGmAWP&fi#z17a{7 z5RR%v1o6?&A)P%*ehGRlDkPnchO%Z(>27&@78BX^i0Pk~^rEN(_ zt&-jTrZplr${{5suOU&1)U$imyGkE4@srh)_!&&-^g|nFvBWqLuLt_^W0HX)zIB*L zW6G*!MFDv0ya?2jpmR$Ddo8h5BZTjn))ekV*2;m#Ibl8q-5cgg=O$cTZ18BN5iXE*PpOI%@FPv?=Y9Tn3njhNKlvf? zk-wCOeu)>g)OpxMWp8e&x1bmUHrz5HX;S{5T7qBy@_+lDH5i91Caad3rsE_wTN6|7 zhq>TV2ukQiKT*7>P6wbv1cYYcYIPl5CWZYCR6<%gvPI$(Qk(dD`~8&u5{p?e4WSf*CEJB zi{bgGPR@Mp>N)QA95drTDii(Nu}Fw8M|dM`TB+)B(CTH+Y0f8}JpYfBl^&wsjOaPl z0+^wIFEQ)9s5`-eRjUW|`fOc69XFzNHFvnT3)ukm6TibA3{~^Y?rxv-(>3Dlr|OIF z&>L9zuykNof~O&}>rkO-H@th2*i7MFPKN2!y|R>Yu7uiv*|k%nF%efElgF)_`D@Xg z{Pdau&+>b%IcGSvmtUzW&q8+qe&eC>T|R~PAAV!LNY6P*?NE=*_FKV2$AwprcR^%B zdw1$gXO>ipM{{c(Hlhzxuys(6N&0tjlxQEEh+PTC{mUi$=67IHLi&n?Epvx~3#p;G zKG386lc&x$+!i{p&Fc-tGMBcB>K#xE^6-a2O+SjPfeTVhzr{u_p#7o_tZ1v|o#Q@@ zPD=_Z3yba_>$v10|GYuvAy_s}{Y@7t;VyNTZT~DrRD3LW&}5BFg-SY%?nq(t3h8Ij zf{&)fkjIvaQY$-$v3Uf`(B$=K!X_=UWJ!ex1$n>|oH{|lA6jR{Mt`IU{Z4pYa0(fSD1L>fwdRaR#sr05 zdae7f<`r?H5fk>|}OYnSF z$2UJ~Bk}WlU@GURG-uQcepkQtg8QkMcSOawX0ldn4*zu6upQaLh%LzxM))cm0xQoH znNx2HI2#oMc>^!7%}P80t0vspCch!M|mOWa{|dC!w03oT#}9zJjqnq1A`C@plQmE zJP@QYh*CL{F;aaP-SXZye|IM*aizLEzR|$xcvOZoRNq!Ky;jXz0C|H~t&eIEOaN%+-kE*eeR#_-`dN3ck*fP1AHCX)lpz2f^kFy4Vvg_=w2^uP z5{C7|ozRUyy^nd)T=@(PrV~}IwW*k!BZ9EnzV51a^T|BHiy#FCq`Kd;i3T-07q{wGx>A|deNgveGY5>B7j>x-fn54P zSqD&SI$GQG%y#_-o;8Gw2P&K zz|?xyn_Iln#aV0!F};~s7^twZ4?NQjRJ70<4MRp)NG+Iz%cAGXTG8Uru zDvt{G+Rgd2Ioiz1b zLB22>Es7$E@36M=<^GA96dV0x5Wlc@WK3#r;pQ5=SB6~My4wAR1yBn+Id<<$aGcQ1 zz%SyjKcW55bYo?VsKEH9E}gYUS{+|vH2DlmKiw3!KNHbgSoJF7=ifNJGCCm}G;dXD z>8~iLh-R zw)&PYKw)g2ne_J8$bMG1NvPKH#gdRpzh!GKv8eRX=)H%c0v2~*cQRfQ@krfC$kRBF z+NKk@{YC!EG>M4r#}d10Y0JU}zsthB z<3=b$;Pc%1!U@dmvkYmIv=-*MpXUNot+7f0U0Wi7>9o@4&dupN0@#2wbbT2}VSMq= z5SBbBgkiVqh18gu$^pbe5?^VO`Y)#mf~Q}R@aq98(=^OUZhx|R^>vFr7z8Gvd_>za zGy9xLHI8U=CbRaXf?X5t_uYGOV0JjhE11+Q6O~ls)LK57QPGx3fG0BgIkGy5rD#Lv zhLS}Spz_FGKzn*EYCnY-B+t7aFOWZu1zr1*rC5kT|GEkJ1X!w5;iqRWVYX!A&XGSs zAILaU%)HWfH{*0g7n6yp4y4vpB*n%{#*d>pE4bG=rMum7vNOzCno$v$=)P%*Fe2;@ zjy6lZ1PTF*YM%A9Sbt9wY>8D^P6@ws1=#1$O<_7-P^|3kWO83jVWQ0$PA0TWRR-#Q zK6k(V?8W}2j>X1q_J)|7D6_=~c02tnu3)pCtgYU-VScu7lG^vzepW)yxce>TGfOe+ z=aRv4s{&S$`PC1w3a~_F{FwwKHwnEF?@DzeOiVm}E%zRo$Y+bVA$phH?uA$oTZrdo zEIlp=(e6a@23n!UL35AJDlaO7wJTg9Wr=Q_A|6nsWD=nu(xE|ffO2_da<%ga2VDZM z=(z7-yTSP&|10y0$Npldv*#VIMCyNKjMF;R(it?jP$L{6FHvq@X79Vq@F)j+8uFZ$n}b$o*K5K7ma$o|?;ExE z5v%Zsa4&iYyj(3fc+tVrxO+9FhV-`O;6eI7&102)+NJ5ORVVvJ>J+HZ(DWFj>_T`B zp~?KP?RDj*9VWCA!xZ=7sQA8y50JM4b-yO>F7wz~D_j48bc}9$ujxCf^78uE{)W_5 zy?2r^4twn@UJ2!%<0;t*i}wkxK9nn6dESQDHEpuyRZr`SexzKF{H(^w%OU}`y>>-n zv`RpKHtMGUIPk~3!S5WnTK6<)Z1+AoT@(Bc1gfeDK@J{8`@EBql-8eVxDr2g769jX z+eC>MFS+Ex zs;@q+HXQEi#nvJ(zAf#uO>zAGa?fJ1Au9h1^Q*l)n`C>=vxz;3?}Y%Y?P!{CO0m7| z-RaV!j{lKd`#_VrZ+xv#t-_?j6Ws&%vXlEcalgV@aes9$nH9N1Q#n5CHKzr>khgx- zma@-f9OA@{3Yd0uMu3m~If(N$m7QqzKSu$L`8snRuQp8_0SQN=unBp9&8hngA5HJ2 z9W9CvjT2~PAZ!|O`+TjF%;unK@v*2s-f0|{4h`SvQm%l_RHzQ*j(e};jugi<9(124 zDJZ=Is@Buw)^-w&5EAPnP4HnwJ~n2I`2H!`{Ni``935qwED2r|c`54tY}v5pbr6bim> zC_rWl_F-_RA|?qyi>2#!-`ph69`kKJzT-+S)Np=H?)NFz_6pj9>r_pVRV}{SGPhK* zUf7+ReRDE>xEr%3^tW5zKstv6%?JD?WE;$cpKfX>0BnzDIC(X?-Z!DY9qeh%L})MD z!fM36$jyN#7Oqt=VyH~*9NsjEh+KDBmhg(;La9NDsO27E3&S~i;Slc!Uu~&AYDY%o zEme*S`P-o5Lpm|~C0$P^4!L$6032?ryc0VrCBWJMBRB3n6{1VRlO=7zihfNvi8a%V z$GuOsNkA=&)JTLLGIu6u^Q5ECJDqY|WDo8Ni_!Ce*yr8o_lA;D?GGbchL(6E1x9L< zmq=|W%a6mTaf^UB49cT`Qs_upH(!XHFdG(fI?jE#0I!AHR;zRx8#F!Dm5t(w0_{7r z@lTb!0g%yzyE_YAFS=K)`VQ9uOnZSL#8Z7SS*BWu8@dD=#P~&kz|9SH4GT3Vt+s z!@j%Eu&WZC8NE?DAT|tHx$*O!m58!*hpTVP>l5=*=N zg<4G-G>BC#0mHA0;`*hOACpq~Qr-8avSRlR9Ei5A3&vJ7lRuO@!f9J1oYhwddZ3zs zn>Z)2C2)REdMZjF&)|kD?&!qPDWfi)U$s)+8I^#6whbBs;TgaaW;R{xL22cNA66x> zqiVRKqt2Pz5GXM`oy{2`?XMaUejzOy4n1@RtH#tKI;L5K( z4>;MUx3ykhKh`&zj!r^r(P2oZH+q2F^w~1HU+aujLzsya7wrZM!6WOjzJ{Q(^lEbrxn6@bh9xA^61->o=(r?WV*Y-y0fi8mwELpjLtExd9QUv+3|DAd)RY;%}+^_&lEfx?vWPT*UmRR7Cmq!qB4$=g#US5W#=dR>1muOJblc$y&|W=DZA> zVqr^?!Xw@1YD1ZoB*foAVt>8}(~W8iBN}*zEwv5QA$JXH-C84pe7OFI{NraVfQTU+ zU0buT=5bHQE8m1o|INa-MUwDjNyz5^z=nWhjPpwIRM!Uq$;y>! zUSc3}Zb#n7YvvClzuVl6a`F#;t!ya8v=BL|pFnz&;C5Ti!40*ke?_`+4nuJJzg1z6v9&NXUrEYqCRKm#R!POhO;zETZ=+4%7|v{V9K2g{t+>)5(_ z4fi<^0Ih;$HAEgCzMXfXW}SHQawr%170(o5bK2R<6J%NJ&fII*fOK_t;GSPTZ)@`u z5+&Sa0ke%((U5id`i8-`0k_rd8OJ}kp5SC{uisjIyUF!3q8Q*3++F_Z&_eZr@8mks7uIa9;H zqFk?0#5t*bBX&&w_0qalKQZD<-+P7_EK9e03(B6uVsks{fBu*51=3^!5g7M39Kg7Ql*Dk%=oJAJnGt2ma+rkm}fY~DQo@xh_e>>|Ar4Sw9Vo@;$TNGwF z70l(`L|)Dzd<$_%c1r3z2}%-mOC`ur!XhZ!q4gsEM`ZkKEkjl?W}?(#X|%#fYZA?> z@peP({i3wI$6BiM4I<(ga3mSsCfh;%N-a$dhcG0B@+_Ceay%Gp9-3rHuQS#)@56tJ zMJ4P(XT#tyD%eSCcJ(2(#R=`CvPwS03oNDwP)cvhfE9BSbM!A+Xk}nzRy_I1>MxHW zFW`l;zr@?nz#y;zz3ya`U9r6AFq@`laz$x2f0c#O*0 z5D$#Dq0n!mq7+7w_Yj5!IXW8)Tqo!y93hCt>Usp_DS8&~gD(M={wK^|vMWJyXhiSe zLeCx}=>sq&X!6XuIaBUYq%MN$FnnyXSgIuWbNi#w`0uPKgwP^kMpm3)M@$L>dLMMK zv{84lNkYnIP^OHR1-~b3&Tb0XY>hWYkY6*^duUtMl^;iDXQT$BYc1&u*!ckoBYD>+ z6Xl!6sPnk-uo0)=DF=2j>58*JFU$GPCg~DUO@Hly&xLw|s9ek2(P(YA%nah!kt5}{ zqBa^8HPkrR7|o3p{VV*D4K6ib0T10c`+Js;h9hDwf%W&)bAN%^OjDdr4%9FPj3zY> zL^As(vzLk;6&=~^7aPT2Eih=rx+RE-B9^1RGmD&|LV%78xRO*ePmmuLtDN4QB>H6Z zved4Yvnfl<31$}*FAV9oVoJ&pKp)C_JZ0mV&_i-{s?w3^xo1dpN-Zr3x-h)KuH060 zC|LHLK8QOwFfdlZ8?YB~i#NTCp5YU^zk17RMNYiVE?8u-m^vV<6(Bx0&V(i(KW^kH z%bT?6TgNH_WWN(kEc}Of`q|H%;oZV>*;UeS;^6Tn*Tpv&d|SDV*%`W%1Mj$4k8tzke4wN=!i>WO`ml_Bn$M8pN`46hg1giYt{ z$cG&VPQI07cNBzO@4;Ay9^XQUl9yI0(bvQ6Lj*v5Qf|*d5b5?3=oQ*8o~}vv4OrLq zl}Ke%$LY2!;Y=wb0}{__0b(+1)>#~lnz$Q1`#HFn2&^)ybO!EAaY4-s4$SMOzciHFU>C$(pBVMuKyg|w9hS!-$ruMb`x9Wite5Lu;~d5R@mUQoN(3elS+!=avq(+|a%7l-niOTQ}v1o;<*3a>gbh=}ga5E4iwEf4lQB z&D@*^GFnyDT{4wDxRw3rbsiUPPDi*7k~HD`70Se&(x#igUAyePq;;Qyj@krK-S!S} zV5-s*eOr?8MePPRO6iiK^G1vEoV4Hcl6I;isL!{1PXgCZ^0zRp4+l4Tcl1#h<_s`rlRRe3-KxkwZ5 z2VKcVQwpG8pVoR)L4d=Jj-moBn$43DcjWei9d2zRzUih*6*Z7_LQ9}CqOS!>9&7tT zN^`Iy33qrXGM@ld0+Yh9wk)5F+Tw%*hhS)JUq5=$;a!PJa6`LPIn$}Jc^p-ais9kO zzIDaikQ~FtdIdna!w7dEE+-W1$&n+Qt&^~-7q2dT9{$OOHc@ft(WAvW0M)niK9z58 z)TFwYo`x6u#P>DljLS0nf&^Ff_Y5YdGW7Xe{7_p8u1DuyzrDT4#unU@P}Z(r*Xewd z*l6(zq}Bcoy-cvj^leK*3wC8tE;}teL8<&!8;R+0=heceA^PwUgI6BR&oxlSyIox34j59&M0I%Mt?0LS>X`R zMg-EhaI_9>)n&TDnhQhet(77z$5h$4X{{r4!!39;Lx{>cStB#7dsUcpZLE)d-;^q9 z%Xp4!GB_YRcgHfrZzKi}pYo>s{MA)G`<|N8PgfRIN=TcWeNH!jTUpYGkBgX6L-vbYcb%eXAAK5qKYZL8Dq;1lnhtsA- z=xiSZlW%cdf$=c6*!7Q4X_Oe_8eL1zY}%pa{>MJWdE9B*dyT3PJ+BKYnTG|?Zj*Uf zDIIC$o1c>_Q{Nl6mwte9THYJqMEFi>6aPM|Y{9JphKJLNZXLyl)523KlQ?n^>xaN( zZ80pgelZxbhS?GG>l4<)KhjF3`|Wlp4IA9R1c43&O7wO=o7pD} zW9o)?2OR4a&Yo)Q-7yu572^Lf#6pSsrlUgcf>;JIw5R!Mlk`Men*Q$RFp3LmP>hw#2A{>5jtSyQ!CQDrNTLHFmqLe z59ulwuEG#uQRr>lkIu4qP;4){BQ-3%e;=t_{yrRp?Sx-`rT9)&TZMB}W&nC3_QZ1d zRTbFc1WR%R{tY@jdM5O4*6xnaL=kkV`{oGlBd5Lt*>wE!HK@1?M_Xrg^pRLhQo?NZ z2k%A^4a@A1zxS%gkb_Oz$gJFn$1#hl->^Bn|+b#}4eaGK594WK6Ol6iP z3d?1!7^vp#G@!%^hMX7}(Lzm3`-sj0sIxS|Ljdcp&B6!#NHw2(Z% zfz)u{Hus#!e4xpe$n0kKKgscolNXqh(KQv@$3o-LnpkvO;81eG~JV#Ce_1cZ%c(=L#J%$?Wt~#V}pV*E}L*)GO;dv-F z3NDvfQA?X8xl(lNTA~>l%f!$js~Ub-U&xC44i{5%D_Ob022q6O zv)hx#&E1Fv*wPS91+Oa_%)||=#}#M`hDfPRy~3;L!Ya4`If=R+r6$b5+=ad=^;8f; z8&%5P&`kkh;ncc0Bdt7dO!?C{)~*Qxw5xB8Ui^?*e6zPcnvyjmc^ULA-|#*Pipk>E zFB&2QaN~s<0@IA3SX z*Oi-l%s&_w#=nJ)jg$1_JRW56Z6O!otaV^hYHQ^;yVM^ts6 znsT=wthLII2@z=XT%=7%*2S5&)Os5Z%oO>NrMT59G!Q97q>LzHA*FamgqTkp*o@B8 zYWH)E^ybldLetv9H5cz$HCBLfrz;m?59JJ68DEcPJ8-083P*~9L`A)rG)gO9Ote3a zB2tiuCTQioeChbDe6ZG^c58bZ+Z^N{-jiiNj|$H<7`o~IhSe9Ph)@X+ZYXVdW1BfB zu}2R-(ohu3CxT}PYZVeuGKSaKAT}S}e^IXJv%M06_IPwRKJmlBb?4@aH1*0&@2}K& zve2&DqYiJuem1gRzGqSks|ydQBz0$Vc%l01anhtBlmIW6FLd@b9LkM;74kY)`AzqK zH@+`zGRw{Byy)Z4!ltKsZwrhI=9VK^qR&0%!tgER_xLn({w;A+_+=ZjynW5S37r;G zV5j!OJ7<5Kt|&ANOl3Ds&d4rs>TI6s)@A<}eo4i9j50s}s(t8n2*jy7t zodu`=Cv63UVjIjk{ zmOaI40@jup@1F7kvy+6WjFXv*sp% zCM3cp1k&^HP-3;^SWyiizIgk(w7nJ!5i_zjvFiLBNFnMUMw=n_7m$`f!i{3ac0D0E z%b|d8C(OgTKe9pvMv}&mFd`-VS{rd2*Rz4che6iBc zmdgbEPzg{TGlq&FI(|1hehfQ3O*MJ*N$=$ZwJGgc3OX`~;4g($N5jPHF~pg|s^+u_ z@6B*Oxv8@VsG6K_aQ#v0^I#2TEx(-C0p|T`jt>cbhX<8~@Vg?L7hMi#_4-tLV1jG{ zm`F_`X#J!+E&+LhvGl?}145Xn153;m!@AfKj4+suOk`kfntH&X&yx!L=gJZE?!$xQ zd#M5g*>mx>#(w zJnC@0HRIpFIjAwxOk587Rm1|Y2oKJENg&K;2a33IK*bfvuq`Bc0}~NoRA=uko*9c@ zGk~^5{%s2_0}TMF+%Xxv;)$cDzQ%ODvXNeM4!0!kT$s#90FdGPU>a)xbIZ-K0+k-v zjfHB6Pl)}71&ZYI#GSKg`|EDTZ1DCx!8mpDNdI$hD-tRE|9)A*??^2GgOSK#3nVxEBSGE^b>-y?A2tGP0gj#g4p>UHQc0Cb+CZK3HW%hkzH9tQboyJ$3h*=)W3s~u?QHa~mlc^?rni%D09 zJMqH9n&4+g4|E>-$*Su=8lfn=+WxLdWKN0HcF86$pPAoUiS*%KUxj_F-CF-)y+?`O z_1K*g&0|E<84l%lz>M45R?%1c*NXDI>KZPBMI?C~@BB6T#Rb2$H1A8}3{#U!wWa7T z=~r^hZu032{VGJS)MG>0)egDvk|X-cVJ;Jvb*QQ&ravDVUzvm76{u4=W}%1m+Qp5N z9>X1)@dXa*6whaGPS%!IRzB>`jWR!nV!NvPFDAUpQtTWRP1ZQGxslu46ZPD zKqj1KOlXP$iG?1N;)3p`fMNN|EU?PAu*q~25dV*@uOA4U|20M{(E6>HfA`T(R-LT- z6jFOPbijEvb5uBG(mFg5{70}}$h7V&Y>S!rB(XK>qVr(nz6~&Z05ml!(prly{zzfA z8S2;cncUiVk28OENAcXn7MzA4L>mwNoaprGs_Iv!L(wa|dK9o_KOxr;64FSfLts>9kM&->7&Zs2v~9ISzUx$Ate%pa z@T90Of@BnODZSLR9;Kt)s8dPr9yBFUKAo2(HY%KhK19M0gKe}sSR7HNI%DuTFnCL^ z<6wC~2Zi8~G=Wd$qDM$tC&I(042JjoCZtHC3pV0Fq5T_S2?`(FG{h2sJG3t#jxcj7Oqk%irwdYfpYv zVmMriT8=&eE9f&9m7I65Peo-c4P-7qE=H@2z<3NHGRcm6pak3#r?4BheTjal^?z5} zKAM<+Jy+U+mqT&tGO&+^lFq=H`->k?xzi zjq$VeMQCHCkL?Iu-ypmXZrQDyZk89?W|b$^+B_>%2nBr zu_`_gd&&Cy@$-jG>%HA*a|64ln|l2{EG&4M*r0z+?bp!>`_6=+(QD%0Beh8J#aO0| zjcoWWFodCuJ4m&R972nDBaQu+OPr&`I5|oL#C{jJuz8>DqjBtb^e!D$)n*au(H#?{ zD&O1?1fi3=v$)(4156GY6(IvTiA^|Et(#^jSEg;9(ZOPkke;BJ(E)Ve0KeUKs{XxW9r`df-6Tx_@&-fXaa0 zOWKi-ve}>Atk++O9pHiGmZjb=>4HpUf;&B}y&x|)EPR+o7nvQOapDg-luHRuZC5J)K5|8<1uU(9O z2%8-_pjN}tzV@8xV2jU1LYA2`By

2@qSkJk=CnopI3G68XC@*dCXLS{mxO+Rt)Ko(fuuy}fY$%sL0*ez(`pw__xposFd?C_a;*e9Ei7E@?L z!3&<8r1Yq&QWs%T(1P}*17KeTmTxTl9OH`d1J>q-5|9bh5U8LQQiT1`nzND>4lKBZ zk@j;XfOO^`w-b-11%?4wAyyieY=+&fo`*euYJx#Dtc@&}n_(-`8Ni_nR0BainSNOl z88m?+91S!|Y$i^97B>9?7YVpoE*LWfSWX6_UPOO|m0&OQc+Y$>!XW4}LhVlksij6R zfmbe&gOO6Q`^o6#^QC{Mz(f-%Qif@a0=v(-ztl zHV1Cw>M@E5&+dJ9vr`iZ`&q{xJW6kDz|oNz06uamm9yc*>Cj#qi<$#DU#LDHz*S)v z3;5jrOGq?O<#>lyNcX)E{^F1XB9nbSXD7u^Q$*NRoGBiATz%~9H6H;Yv;b3F-jyTx z4;Lo3}yeHEwj=>&;#yKgT^uN#@AqMCyn|wNXZQrwvXMLqHJ-7E=sc z37*hyRNPz6o;$WER<@+WbU?8VRDz;H7n{SVY1sx(ee4WTD?4?Nv6zr})8t_*13&`* zF|BA+jiW2?;ke`pU+i=$_Co*-`V|?ufsknqfrA8R*=`dX2T=Xu{cvoZE+UL<^uQfn zF8{XzB0$M8h)0e+Z*<+cVSQ1w)5^B_N_b<_S|?UU;of6`uy0g|?+^DEV4)NNTqZF@oL}u?_Gysyx;Bs*+zd@AI z-O-kW=Tp-mg;*70)M3vi%Z&%JDIG!e*Muv1^Aez$_VxtGxcq;#x z&s|<{OGtTTm0^dg{LZHTeme>B>n7|!Bj6!!73rpdh<4?IAXA6QTeCB!&9^$mv04ZD zy1(KVTXSvHhMm~#OdQxUvj4&xnn;~yRC{3%3TKHa)AUs3JhmD{nO8z#eM4`X)~RT! zb~^vOtl2{a016r5GlJ3>#V6SIxV)^i{o&xzw34G=@wKOs>0D9u>(JCE*UfEl z=G;tEeh{`!8o4Q6aJ!J(4@vSG>G(#)lAXQ1?dG7dsRp`SsmXN)D9I@oj&H?q>a=~=MCN!?k_+I<|FfSlLAEM#dcRg zk^X2+?#o+SHJ*~tx_^Xm_=)=#yy}Q>`);+HFKekA=9iz4epIfI zR&zK6+q)|osN$b-3W)g z)V1;8hPso3^|{-&;d9Wj2$lj=*-{t5h_a^so=x~~fnOF5yGpKyYF}k47+T*m$-MU;fV^1vd=q5O!6d?(7gR zQoJYnH?*m6-#ek6c&y@vT%$eeIS}P=)$WJ+TGlEYRH$ruq8M;dK$z33Y#4xO~EwR;{gn(zUddOp#NSpR_hoY~@E|Q;xqtG_M#jvGu58ktV z($>EL&Zl$YkU;C55Aw4W$YH+V8V)l(6R1a?3!XM#0%g&GQ&0?)!OS2!h(|}HEf0$E zoYc^;u3A-@;X}$6FZTD@`XD?Clsyddlqwz8Du-16<{6D72z{m|NU;{u>yzjhl_5uC z2O zPom4hvEB6s{$8N!D6LpEs(bfZFAtvQ5gv>nhn+u&c6#x)gIjyY4)njmwS$;{OLjO1 zp=#|Yg0_oC_d2;dJ>=p+G&t}rrJ#$-I&uthbYnWfM_gKu`Thje@Z)eH`61!WC-2y; zMo@rOW~>^>5yH+z28Xs{D@GniBRm}KYRd(S?#WCqp}L_nO$c=A~FV%=Y)dMuL0 zR(#yX?09?#`m8BY{HFYLw!^NW8RIbpZ8mDQDXpd7y38cn=!rFChT_aZb+1GX$&!y zAjGk(5t{lHmA$Eg9ASGcoKby({PA^AA~i-minj*YuHF{~K7O}E8RU5lmJeD)@~U^X zxp#*%P!Gzyj)T??`Jt2I(-Z(dq0{AmVO%!AgEdV5a(|j=WrPwrIcFSycl_%ECrhEd zKW%thaI4$sthzn}I>s2+)81&m4UNLGr-jP2F1OsO(yM)f%Yy(+JMs&;@Uw4~4Sanf z@=7>w>KUw0Eb}dV;px)Jug*BguD$C`VKzhh?Dovkfs+LCX6@KS{;`%jY}3wgFp$RV zzrZM2QjopaLeB+yQ<|4G0=yaIxA1^bCqWkayopXp*q8tBV1b2DXWM@X%*UO4$@1I3BOFncKIR<_fiPt1kCI>TNSh$flThn%yX4=l#6D8#M@zZMwIp9QyGbN;pLzNJJ zC_s}TOAjd^X3A*8+riq8zsLt?j-+R*RQwa15oQSTaEjfykD>|YFp?Za14)QCgv9n= zJ|V_K3dbP9>JPG#E&!{LWN8}k5zGOPyfpSPID3Sx2UBT+5IeJFvvL5c7g@mibSOAV zOgGsK~grdQMr7!Wh{fJphCMJQ}D3Ytf z3FsREYPfcsXqqQ*S+Ht}OCj~nItZk`hQh+_qJiNT2l+SAIkL)#RQNQ@TFCp5XM%|Iv}>B}|*E&EozRVG6?imcW(>oKn)T53AOlJ=rZZ@iAVAs z&5)E{s=?>C<+0Y0A32yk8-uBAAYN(8V)%LHx(RJWXoQ=1OE@)Kh`;k z?QJ0gzn?xL3gHaTTGfrJGMW>LVYJ(RbTt@SEBrz4{FtbSs7c>H3Vt?m8$3Npt|6!Z#pC)|YFp3|J+-P|!3-~VEbwIHng zCC1q;EFrpHRlD_i_U=y(-`}Qm>&YIM49IkpLM7-M+q;&t%XpA8sde9R)Ayib)ZFhC z+RCejr%{EM_@LQ;=>5&ep*|GK6P5>gf0vNp>)V5k$X)WrQz?nr{d>EveD1_;dt_?a zUjQat9DXM@J<2>&3= zdHcqD0vtSBE2nknE^qB9A(Zg;We~sbReoT!JALzOeP%EZUiC=C1<5c;vmq*oe*y9e z^fapkikj!sh3-%2HuHnsjfX{fnp>mtcV{u@{jD$9)+&*qg=L_eX?p&%J)us06LV)r;yEE~ ztFN_6kUHxvc^}3`@2wpwA6@pIAg6OQ)37Qd{dT;a9e@zH1p{3tntncN<90Y@yg`1m zt+%r?0Dv(ZwgrC%e9_`W!lk}FQ){P$rjy<%wl$Jg<#NW$p|n_Tv_(Fx&M7fRO1 z)10_cT`f+H6-_5A&;v7B?nP2I6gO^1^_lU?W91xVF2myqI<=>E}QP4p!vPdu0@bK==g-(dcMH3C7- zu{P!@Q_=u)Deg{SC-~?YGv>E{{x@phbrTL~(dc2Z3@$}E^$*|2NM(VmO2pYv9JzL%rwrwCMq^s>HHX12?SV^Sj0OMBym!|KdlE!#D+wk#!JJM#_Xz130 zX__6P>^J&hQPjjRcK3Tet-$G3YTOr!UE?m^d!O2+VKER@df$h3iKGd(tI?05%mq@z zrM8>rhA>yDrr(!v+iF!f=`YWyXhe|C5>LNFPEY*r%DP&eL$uiKhf&3i+dx~9;Rg8^8zovnn-@= zIrTsioKojIgH$MdO6x~UtG3y?yqS;kjz9qwff5)NQzFut4BoD-V5-U3Cw4!L{#s-<4;aLLioNY=S`;aNvb;6sR`MHxPgt5eKPT|BaQ;j3>+^-lA@Gk6N1=d~}0dL$He1AeHd zR?xN;W-HWde&ZWyd>YK_^!V|K4^ocIPKEn6=lWz{(XNQ2v8!> zA?+OSa_9~#rW4Co|7yM;sqVO8U?ZRlD;w&5fg zQw>wt=#g7NvIrT75yt3g=-=^!@<0tS8ORE-D}Y~K0DZD$X(ytQZzFfH2a{;$c~S52 zK!EE%CjrWa+sY(BKN3nrp|!x?d6I7jnC(u2mQm!x0rlqe;^C+XOTgRs!fLAiiK1ad z8VKV#YiPl+@-Pxng2t@QLVKknV|(1bU1uw>oqu^Q?aJKctWL4ijfbLRGvuVUGrYb4hjexE8s7!U>SHu-); z4W45AoFNBaq$kdR_c%P{)9!*79(O}yS0r5Mxd$=I!sK(B?XOWN=ApYjV>>@uH@~Z| zYRuWm_~B1{iuJ^?tRR*%wFF%cf(Aq5Iyf5YwbmbPUFtDsFvgJ!{PTj`SjlVmXPN&{ z9qrrh+tYHYd0M(>sD)#BeD9%T_#a{#W`;SB&%ee0W#R=a+H%!KRQM;SFRM_!vV^oT zG|rS7B<&-~S^@V)c^P*ERnQHZKIul6Xa7*gxy*T__Vut}+wHc|g<<9DuX{q+!+;;uOOAtX%git7snmXI~p$~sS6C4bMHObn44M`$&ONx3{;S1T0Gv2^hdZ?kX zvSXHqO?GMK>pV=&W(yWG{Dz~j6hv=g-9rX|t5|?T03mDSbmT+d&?1lS{o!Ri{69L_ z3m<|vG)vi*?yzjO37Q2NIymcI$gw+M+_CM$R{8`9g!9BoYLESvPVx;!%|(LdM#?(^#7mz<_6gb+fv_y*KyLF-*TQYHhlMg)Z|_o#0quHlcKV9r`6Lg5h$j=tFl;#cE#Dgt%k zvC9Wqh7iJyrS79XfOUd*1ckS;be#o%2}&>rpvQAfC7(-T7&I&pSuX`@5)yRgJ4y;pb+#-XncjeY(8 ziiVW42^$dfiNB;k8NtOaa8)OVa-rQnaaY<1VZJ8b6FOD!a3XFPeJZbS2x7Hoi5=;R z;AN=}2`~B7D$y(LwmlyyEaRUyIfBi;Jn~plc1;!>pgiZF?W<~uJpWQ4|H>Dwfc6d|g3UkLgg3|E{pRByd~+hx<4E_Hx0uDr`J5`sgc(Mhw>vmiIXS8!vS=?l2u z`P9E>qRP&_eKk!3ea?e3<5@sNS+{1=U&WqUt?qLcb-O5n(x7fg?Niltn~pWqY2WB7 zbdwY$e05^Fdcri`keL;>c7sfcW15vHFG9b`FY}Fn@*mPe>s_|pzsanb+?QToG7F7( zm2zaIdRr+);e?dz{~GiXNrb@coqw&JsB==)%T(< z;}C`5EF1>&(HT1i{|Is-673msvV=c2F`A|Y zpQOB8i(&ipl_WQ%OCysS6g|0VH&B;MgD_?8`^F717n=Nh?sGpuaj;?5w4xF4un3lg9N$`u~m zyn*!IuKBji+3M5x`MKx|!zg?glpe>fvhpH>ym!CyhJzL;S&CAle9) zM20|1swu~{WaW`)xD&QSr<+%B=Vd|7A0ji)sgH}8>@jt>2&Xm}T*UZO&N(rV^T!lh zm}@Mn9S2C;PIXY1%9Sq7sZ@x*UR3GzqHEHeksISeF0}>%pp22Vt+)LQUB56`4&cFY z{_qwk5GGsAOauRIn-E!DkDXjohRCnNI&G>p7`|X4{B+ZqZghh;WpXD?r-!yoZa$mE z$fegRuN!R)o4N!;tw;k+#h2O-A%! zMKundTe8H8yKQeU^!LfK z2a~rcrs){$p3%0!J+ls1uFaegeTJ~VQ4tMSd3>eP1(D1d`W4GRa~$I6w@sA#D;xeU z7@YHhg`^0{{2z+`y05_;niH^;w1up*?fY`nTw0 z+A1eqHn$Z{-Dwl00z_yIE=dUN?464ay5mqP0afqv0aFoH_DP-eaa`b~5RMp~tCH_xwa;UDrsu>Ejo5T1#=S!O4P zhezV85M+E$YXp;nyu(=L1JiqrhsRTyFWkGGYQDp?z>07HSeerWr><$uabF1WxwK9t`z3`kQ+1$c$qux1n~L!fZH|?m$U>!6 zUw}D^7=F}Jhfs2UzA0F%%JRI(Qb1XJt=h3t@JZBd5ocsWW6xYPJu-Jd3x*SiHU@F< zC_e#u{VnDK)i0blNAS40v;&VHPw5GC|0IgGZb$3(#!+Ql$C-SW`eY9s*BECO8gwoD zoQ}xVg&7s?$3A3UEUvv)nV+O_a)7yI04(g1<^s>muM1L{EOWiZR4$0`(xkKpE?OU( zMlzXGQe?3fu>?gEMXI+bGeD@AGhs+XhPATp2zhF1x8rrr@J!HzX1#@95dVxy zgJ?hfiXz(iPLOzVZWhH=xMGtj#1U+Ku6g=6&|;#MI1Vh4Po*rsdv|1r**+8fVCE803ICw353;GPC@;T559zt#%}s)S`6*3Y zI$~UZgsX!#l=(UwD@t@t#TJiy_^CJbo4C56_@ImZt2j3*H=c+tOmM=6?r!9CO4f3R zl%tph!s&&Af&PzBVn_B(_)+3ZHPZ~0LMeh-J}qw*w4pYA2$5~1akRce)q?iZicMeRR(E3kMbUOY z-@G^lwbQ+laxZym=@+O6BZM+Z0j7U<9NvmEiV;$<1Ru)5Gq z6RFj2rR9w}Y9~F(aO2+oa@ zuN2g#7KFubvNmM%ANiNXe~isd!1+lm>}*nPYi^wv# z2$nx<@%1=<|0``6wSfh~KMD>2lrYx+^*R;-!P05plJ;-d;)7ogv_;5-1srt6Z1OSU zV(FWxi4V3GI*<4-DgPJ#0$>OQkC_nG${rL43v=`G*Xzf95L}04It%a#!xq)pq_9zd)rdEU*;5a|&YsE_X5&J0YCDPv3qN5AdYSa0|DV5z z6An{9#PYJg0STJ0vQM6RL^f^Qu@uDwFj^~Ead?kJVrVuA+6EWys40zQ-XOaSi3hOaPvVXkb?E-H_`CW0h1Po-=vC;uqKVE{ zS!fAhKh9cn0wBHld(unfSbjjy8D1Gk$Yyarp-=b#kmDeI>B4PrMo^T-dnlJ2O51dW^ zCZ~g#zp}*=tfygA0mui?c+eQ!!S>|oiWVv*EL-g|_Hq2(=rEiWo$>KI&|_jspzSQibY8AQFgXSJ7JM)d}X%l`_)wBX#0-Ue~7Vg*hfD&4Aoo@5NFsk zn-7-XNXa81W-GsH{AqGF;1|@Eh)z&V+9X>1-w;;=JRB59HU||0?vdf)5(w;)vVsl68VP8x&18v7`M) z&4$UZGhcgW=4avBYu6*tf*b%|w3`u9LPI4eXYy9*OeA+M1E(nAxk+J3T@=q$#20Iz zOZ3_voyZH#?e1h%%wYT)nZQxmCn@A$hofRvvj0{~w}YKR0vh3d-0mY}U9{n}EXOZ6 zaP*Iz>QI|6?dmmW>VEXfd@<$b;-mg6va`Rw{x6F^uAI#`I{&AwPOJGYly=n=k&t}Gg6`caKzLwS6-v-X#fF6q4Ux9=7yp#vv{bK#Io0Ta{;J|#b*hhNP1Mdp_{ZVD_(ARi zM&a<5Gw}m9H>xa8Ck1SMwYcDJcSpSOwWND4FC2g7{?#A-_oi_c_udtJE7HkIRdD!g z)8ejeoS)nf7rCSxTn_9~8x(V=ni{5tti8CQ^Y%QiYL+wm=dJgRCwqplB%7la_&O5U z1l*FCiX6sk>)jN8bh0#f0N?x02FiwdLCVyMm`Wn?Ds(G`wti6c4ItjEK3CoyhWc3W zd;(Jvh_jq}z>DQCOe+GR?pKrva$58ra~bVIjbIwORV8=m)(NSV_A&63f5hOx<2*j` zyf_(|J$l;eSRhX$8;%pg#<_H;bA6QnEzNqK3C3mt;%^S3 z8g&Ly>Ur{R2QChJZ4?;{c7+7r-q~Y1r9Y7aZ}kZo3V*=ffOm3u7$^HQWVsu_0#0d; zF@^plp+^;*c|V&ZVgR?D^ufb~cs_df;EYEl(vlV+Zbup7anNRGZO=2j%?H8!7&0-} zwc#{Bya|pQ$GC`+9xW5FL{69rG}_{m3RA`vNFapdHTGcq7Ne|)pDFGN3IVvlq1Fz` zUgs3#%Uuv_ccViZPHqbWOpra-w9TE)nz$Mm#N)pnHz=C|wgN8QaCDgdL0&rdW1;U{ z@P28H)cCf;m=JDGe7wG|kSgWD^akFKoP(3E%_n-hHi!9f%KS{%4iTmG$b7SN?a5M*culA1TA|GjfpOdvZc*EC;_5Ou;46sj z&^QXG!9Z?{7S+A!fWj0cQcs$&{y5LAdj2oBNlF)C=B&X;|6dlTgRpn;#aSiQMG!}o z>uk?kXC^?bU;{h1%uq1}tc zQsY{1b<0g#yt)0hTXjWpFAfj;CroeS6R|oovgYTMe1O!Z%kW5;i%IrnUIaMVLk@1o zYM}IX8B^WR0E*h1T|%m=#nuAdVBnIsE(n@yTVSKxq(p`AJIuM!!?gWCydDLd4mgn1 zfaA^|#Mc`A(4P}w%@<9(Hih=|!rT3p)9Th}2SF*fAAkcifW)u9c{7Mr5#~oIk8uxO zH!hn2#~iNjT;(o0^m&t{I?R`B$uAx`!fPS$T0%B)l}h<$+B2^9kwcwh9@xr3mZcDl zy$*UKo?G`8Ft|!T1v=(-@XVz+>!yf%nZg8h+LcY;M zS9gAlGrzO|Rv1~QL14@sn!gUa(uxH^KB(e=UZwIbS@T# z<)u~a@(yTO@E(H_T$8lwLKfs_Vgt*dsW{=R+#0s)OkEqQ@yu;_ii(56Qnnj+ zR}}>F&+mtyUf%ZC29{ZxUg}fa9J9^dhWg^hBnbR`!nblyM=>^cm)QhV+&Mfh;kK=H z#5Gh)@2hV{3g-&fR&R^OJHs(cFv1=29A4pZN^y#@8MRvj&%7>pl=P8N_8l>zCwi|3B|+5hxMDzNjg&0dj3wO zLc38AS)4Dh3^UkCBfs+rj?CH!qZ-l3#yAvrC%sQm+<4^fdqYMKX8ZW?A}Goj`J}tS zO4vEh9l_Bkk5bwb;i393i~FUN@`$JT!GEFhrYXeREI(`!*KCi3bU#EvLev}T+JuIG z27xjctR_e9G_jU2&Z_wY<;kmt(0LjET)!B)jOqE+k6nk`Aq<5pZ^<xFejY2f+y`Kg+C^)O(yX@b#s9efq9Hp_WoD z*$WE(6jAVx;=Da*H4M2B5b(p3@NfQ)eit2&sWk^s`cPm0YX~!os>vRkIj@J!@c0LF!5KDJIYCrKG###8sE zM%l0l{5TeJ{*Rw#X1#kk470sNd;(!Q!ZWdi`YCFR&hZb{{d(yC&8@;1{<{SzV7}*i z0#CSaI%lxN=EY{h742g<`#`fyUHUNsQULxaBH&Z0pN`>z2;PjWSTI7nI6BBwlU(@Q zyx9yud~n+>JX2|DvB7X+cDn>!K9=?pbDe-V{osSsYDNjZ#f???kY^h8CZpFcOoX?X zwaiC+`Q*q#8yC!W7>1#_M_Vq?`vRt8nF;15Iw4at*pV=Q)h`S&+P`h%!$fn&y+c$^ z+ItLLwpG)T5?}N+Ym!iYz+9Or4nRj}#v@Y?f_4;}rA)REp*Gl3+2nIL!=o6Rjn_Mi zjPSKn@PyFu3m44RCN8m36o5V{N;N1IQScxB3!I#GOBzV@3Hxsd zh#yE~{{ZVr@Hg;KkmCY^{>&WsX7lnka||dOjvbHx{eRY;{$ULUA`Z4b#y;H`bJlQ@ z>M$4x9KaAE+DteVc^T*`z!aOf95!?iYjIOWKbU0|`p7w?$_6cO8fk~&hrjK@0)(X< z@(7@WF(+QI&w!cBf%*!&ml6wst})wUWoHk{18zhnrJ@hM8$%5c909rp)nLemfCO=S zZ&ASj^8pV5YW4%aWx;3?WeHDY%E3qw=8zgRZK>YE~7iE%l%)n3#Z1uu!3bRWF!K2;r_ zsrGX2#oNHe^Zxa3-ZC(Oczplj5ELX(XhY}J%*%$yt-Utz>hxJ%HSP5?;9j1S&z-~t zIzC#VaDS3F7mqJlh9g^$4ZEPxn+$*7L3>yefs-gB553|l3bN19XYKm@em2?$$9k6q zt{mkauDJGN;qEp@kT1@;|MoIP_rAp)xd|QG>a~f<363t^ht5PUhKe%_2Vx9G#K?q` z&6}m=IFZQzQMe1?L6z5Qk!yXwtC$O?yvlw;_3-rf^{S`103ZtW5gdfe&sko)vC`B7 z_t;R;0|x%5uwhxlUsBhl`sA?VvRskqyhn-ev#y%JSB|ai{UX_PQeS27RN^|~2Row< zg_ZD^Y>q$mmittWDL4h&VPFdD-ZEM~xAy0CFMd0$-{{S1hv9m*w^Gp>nbR6IdUS~* zWYadwCn9Fmn9sw7AGf8ygTAk#Q(2!_=JLk6H#^AAXtNlu;JB(TFyaMQ5sFiUS;N84 z(smbbL0TaJH&f2n1{f~9M?D&4@jq776gt4w1`%9i+tAHJTNYwNA+sGX_sZq@E)6M= zJ1{5Egd?SQ91PjTyR5kp_#J!$xlxt>q=j)IdGq00hH|Lj2sz6J(EJ#!>6IR2gvYsq z_i(Sx+<_ZHo#i!eXT_m|UtEN@_8BGHh1=Q^PL3d)Qc-;3j(Wcfp#TkaPj>Ie~=&N8oNKyN;MT1Pc1D7?JT%3$d& z)YWaY-P$~9iMRwx1NL_6x7{}8yoU1doQIuPu7m%UX^l`C7OY!kB{IRrcaOZf@QvYi z4Sa?)u}7J6E^Xk&WuTkHYsL9I7y~rpB}W^t5&cXdBB=BdWJ=MITGmjuK~U7$rQ)*X zk1@oXuUdTe5V`-Sx?R-|f&zJ%H|K$Hp`@!F`=TOxUp875$h#xIxp(tCZQW#0lBLab znNft>h`Kp6K1^!f{`Vfnc6Q=GrDgvBDhn*$02blPTUC4;0qfoy%~072g$Hd6NnLP3 zqr+QY??w=$Syb(c0ttq=TUkHI>+)?*+I;xqHq&xtZ+?WK9x-Zl`W1xs`Gxc%RSyLG zQoateE+}=W0}a#p7vn0s(SF0N22ASQG)vZWvTF8AAkzK%RaQR>Kpqg>z&7-_XO=1X zK%(9Wn!fRAW5_5L4Lno!hSa9W;iy@O1ZQoQS3lVMYE7IR;qR#CQil1Pn9c`}b*p{% zFL6cDt~fVz4i)S*wP#^39TlWL(61;*n@;K3ba3V`JX2vKBZ%bIr@SmJHae$gb=I~3 zBPusEZu3v$JPEH5^n?^W34-~TFKn)E9pCPW80InSO8I<^>TO$IOovN%A*y6>sYGV< z7`Ohy>#FVUdHibrISi}G<|@bUuCC~(W=6#}FT|!a)QIvFAVGP*ol+u*611Is%tvY_ zbjPDJjXDY%$Bo)0t`}+NM*JP5*N9gq$mV8@vj7=1nCcipw6t(y&S(=A0CNg1b+^Nt z+H%v=MG;*Msh0&;6GQ>VOjw#((V59b-ihI4wWX2sB_!uh?r zZBSdCWl{>ow!dTw`Ioy!R}5+~23V(dg1_V?By?`O@LwY~f+N-UNZhST_gT1f_={p! z3A!U(79%1*_`$gj6<3UrP>dKi9J3M8mbwNss!)D%oajM(;WsYw9O)8HB3$I)@6l{N z4EoGwXb+zGTN7$AkTY;m05jOX#Dc!f`U`+qhP7b`hNpF8{#D>2?%zy$b$@aXGE1 z&P}~L^v3W)BspnJ#i$WWv@@-X>#ugpir1Sygc+D}mOD9wli#~=omHIM?wlW4`?Ap` zH5rj`wMx7z9*L--R}qdr6F&9r^`R{&K{My|E1Jo3A=h6mP~2X!CD^nvZL0$*#j&Y; z@i~zMQw=FFrSJ-_a5!8Et zl?{zTVCpCEkvV;CR=)}_kTC_*mDCGmNqe%#^D##+_NrqJ2QGQxPT*W!eTpsK?{I}Af>QwNqgF;DR$wceX3cq~9R|8luXG_^unmd7}-iiD?(8xWMce{pJ>!;At7 zzffF^FfIAqjHPA5bpJ+a&M>pNB^nLs;KQN%XwA6?|FU47u8Hw2k#`p!aRGRA%z)i( zh|#?gkAO6~P=HXDDG1N~rd?KFr_DtuEF&HL#a@bXo1FBVi7pz0KV4Zc*;Fh5HhL8b>}gtvAof}Zb)RA9N=HN4KbZ9y9*McU7DhXAC+}zQaY9QaN-4k zKtF9@%*}%3t3$W*7Dx!LNOLO*@I!VTQv1eK*4*i_-r{*~K4~b7aZw4W%xf#?jk4r| z;!TvTj_71XmiA`8}}v4HI0++ zFjlgLy~&8!oXZ*$4XZ@Dv0(e2xwFhuPA>qc9j-O*lXZ;U;r?@7PJjSA%H;ro{C0NI z=D?QeuQNHjbeFyI4ShH;VU=X5pHKPSh+%NZu=^t0@vcv{=MC6>1_|^#zlEp2{vN=5 z!Kogg`8PRc!@v0o9_+f<7Wlu1>BTvZvIbXr>w9`1x{y-?|BxEQ?vyLHyl1J5@j&?Qa$?1nuzZ$1!=9!8gh?#te_*OobYAl{$zhi;? zurN-S$V&xzX3lKr9$XAif$8S9R>qip^r-^DlBxLAm!uaW+d^l#&Qx45>%$YV!%?~3Fc-ii$xby0x(uU+8p+v7K)<%(gFpP^1OxXG@?B_l^=xJaBXH+@A41W@I9}G zT=pBMC2Xr3s_f@5#$>ezM@Z^>H_;3OUegT|VV|8CQcbXu34*2|jHAW@gS(k#n47Y( zk7zBoFz2a;g{(R#HbMi5u+hHYG076!bu=*i_R|vfD?BkU>@$oZB4Pi3dsDK7A6i~C z5dZeqf&c}Ah9q1%@l5DP(O|Rk1h(2^u9)B&sVc!PPxe^Q3`7l1I-Xyke)i7DlAfv? z_(dIWB77hgfz5m%8^Qeq!An?^LG#07cBCKBY_1UaE-X^nzU3vKTVA)9$_ag&5l=+4`*mWY;zG|iHroBQZ7Cg~O$b>qJ z?dkqL#>4Fm6%fZ(OTGmAeWb$9cJ%rm(HT}~c?t`dCWPdG@{H>Mey*%Y=vt2IO$MLAwtFYk6+QhMs{z_>1k zWQBBxNs^CB18sjWJgLfxihI2NYNvLsyWe!dt02bxSq&NSo$aq(HWlx=81};7J(lo< z#l^Kor*2ZIYDH6o;c-S1=TzPwoVjY+?tjOy_zPbAiPOu23&Pf3J!Mp4-hxAl(U|#T zJ%=Y{8jW#yPTwqyk>Bh;!X6nb$DnaL|6}yQWa<XhZrlia&iU-+gVM2)W;zM4>g;@LUUnGQJ4c+jAd0*`DTK6VZA{!24@-boXh? z?1W_775j4U!|)2bVR4z z_HuH@#y=`P-)Qfoo0xl$GwS!kYzBPkac&1vq1Y}OQP2pO)VOGV(EBgX+`l6DBp(t~ zPwjev&ZC^n6=&z?nqBJPp0i7k0qw^cga`!7zeY=E6IbD)VS(u_e4ZJY?MWFSmwf{y-y-Z zXYWQwjV2m^g33D>(W_{Qz<#EF5^42RK5^Ky<-RJ4a;Kq?)ok23qy=|{#dU)?AL1M- zh)zDl4}2^;aY<3pPmUU-KZkCN#%N z<#TUO2P)SdagqpH3YbnN)!z;-gngGWZVxLV?kW@fcj-KO!zpr5m~ve5W1o{nVPHgye3Yc_oHU`Y zV9{Q3+|N|m$OJ`-Caq8!)+Za0B41H>i6XlNIOhMf#I#(lX>X40D_e)gKDQ;WM$!#D z;!^K9iLyvhZwG{$=*=0tDb28Dc+$e)pI-O)k{=Vl10SA{6m8#gL-1kx^ z?p6GtfJYDOPn{VX*t`F++~Kv4om}a+VLD`Aa)pDlxg;{`zyA3 zu%uvxebi7on3TY8Vy&D|@|&E|uW+(=K|);cNmP8A(lW+1C|xnv?l zZHTwd5`|P=mv;$T!#0VgERA9K_Xwo6_V0xs08$4lF?82VRo3cIsJw&tsm6Ci#e9+W zOA*9r=l928MR78@p<;^7LUrdC6!B15H+UPOO||%9{a5OGx!`+g~}BZfxn!d)Ni$1 zIZ6aI{3%4Te~zEOiUSOtF_Y*F;|Zq8{yNy=Es;6XM_>CcLYc(*1X#KkKTF;YS72;0 z3Bj68vT&mtNK=xv_S_rZi1`Wbh~#?R7Z^P2XS)kiT4Dm1H1!J$xd(-xa_;|iemA+o zd?ol)22YLA+8;3CLh#vF%**Ev_3!Q8<`3o-xN!TjQ@%XzJb6-hQ<%C{a5`TwINDUR zecPP}f092L1`H+nySq$Kla)qWuc5$eHy|MZAQn^*_lA zGK*dKIz+7#c^3UY>M0T;rr%>M@L8q50s`y;@SMO6tbSCEr@MrI&DceBYm5eaN(VT7 z=*&hViV4~SS8i}u*>NSGEc(Ne1z5=95f?d=rouC_gmEq;*nlAdyaSt@E9Z9x%Fur4iU}ae%@C;rsp11~VXI z6=1|(@Luo(%7~#Z!Kve_PbMnPw|Cwfoi0?!fBQgf-xr-shap$Nzvxv|%YatJ< zdm?Z!tywz?2UzI+)7$RA4r><`)kyGfSfpSBBw@XbHQ(F&g48FYBQChIEWEdJ>Jq%jm-^@b`-YknmRT7<>QO?CTT4h#$kPErXg^gi!q zJ~MQnzgv2FF60;0H-16EN!}ilr}cApyNAmH$G$2l&KYhzwpH+iDNQ39;YTh4FAoS= z9HL@&SkI(OOlK7DlwWK4!5XiAt!sPg+$Z+D8^;8b>t_oE0GWe<6DQ}_q*=6UxBj@( z+j`BOHJbANs_O8}dlh6l81X>xy8@0INe&OXf{>x~0!1hrRlJZrHI(wf^-Q`qqWaFW`+A}NrKwVwF?U?W_rrUsEN zkNI`!5=g<>RO}M zMLi1ehQi}-4-Xs4Qk%&ms1gNkJ&)ulH-vIz9)t^uE8>)To z`0h3hY<9il%VqDM&?ejt>nxr4>v4~xmlD1l^0*@#Xd16{xmLw_wW#CgSDoJ3*``&A zhBp_$7&FWe+$|fphv%((%=sP7&U`{_H`|~&ZW+ry^3I)(sw$@AzDz%unHqEWD5g3q z?%r$?QnzaKwmnavds1T&s1EYyr)ky0Got=gzrWez-(f22>L2L+?CqakhtfNLfT`Dm zf$51oirgkjc{evAuonjAChOYxXPKYDP}s}1$NQ{KTS$;U^=jEIbSy`EJLw1aCG*@69Y5jW@!ygkN%GpZcUB4TO&6dro?)~F_=BW3G+T5mKBpe2*^U+YS&v-0)jk& z)1C$*KhDgEdGl_Knv8@UMj`*Ib_4A9>V^JJb903$ZBV0A<9WrfrTDR)zDr~hoMx}~TtAW*Um>DS1P8FQ_4NJ#cYRm8MS3EIiYIQ@$C zB3%dfPPe>Z{*BP*03)`c(w&kt}i^AK^2{z0H01cq4m{# zs4#8SsSd(f({$o!p|_Jb(nghP+Qc}6Td^e>hBm~Jf)h?4{2QYp)Ve`m^zk^6=(%C* z%q%5&Ph@N5GJlgLvmNCt4d>9sBe5%MdIqgUN2>d=hE3PTOV+LzNL79v!^<7Ch$2G~ ztQL^x0NJLsMSgzorY+~N5m=&7+zZr4Rp(}K1Qwt8hz;){IwVm(!ML4v9n7RN(ZnPy zdH-(Ilm?sWy55^#02`yal492o^F{?z%m}Ce!&|6_ZY?`fxBvt-8zKe}D(+G`a&UH$ z-4slIFudTeRy5<4iOT`S>=u-|MxZA0KXNt zNI?uH+Zg)8UxjaXv4l( z3w}|Mzu*eSrZf~>mH2mq_~mHcgIFGDL2c%1r0Hl3*1bf2VsMHhD%@XiuB6TI5=iWT z^fDc{*mQ>EYaZo9g_G{EW{dTx25nt`P6Ce`oJl(;G3UG*46E=ZzvsMx;;TTr*mM0h z4lJ-Oi-8OfF*8K|d_s4?b9{mbt9fcv1-?VU4%IKm4b3s`MQ&Yc^+A4qxL|Utw5(vM zKay{`*ev5ie>oZ_N>zGO33YlT9x0ehu?24`u?X@M%z^DB7|F&ixD0LqV0?kcPeB(T z^H606c3?5P$ihkbt&;E43MPB2$Gq%`4ir-Q!+i`M^GSF4YwA(a5acjc&50dnTVZgY z7#ZzPwMb!b5!B<(JS{R4Y6?$iQ#B|Xg2(+@4!PnKQytJ4it=aHoQKaQDDNaYTG3YM zHB?$|d+N-Iw~Wg!)Lcb@N7cqG@OuD*IC4T797Kd|>VL53{)ZTV zj^;oJfOT7L&9LrOL^%F$&a+}GVkOE_6xl>jVtKID4>IjE|A=*e4A$TPIN^rM9_gr?AQG#5z8vD(}{V|}(EGeygsYK05jba<9IW1y`?(*x%) zM-S%}|5wkWR6}D9WJC=~AR1z<(}%I6hJPw1Su3uK8tKGFUhoRRxw)D%t;MXcWrsoB zEL$YNFi@wJ4!ux+@r5RP33tz$3jA%_gz_T*8kdc`^GQn~wie9{?6B+e91#qTnKoX< zs|05Y!v|!({YbFs3A5UlMu|2LkAGR@(msKCBL6KG*g`lRa^tkcMr!lZam8&!t|Wpl zg^>|A7~mQ=LcVM$otSSAaE~V69s#b&49%i-EDlQGoS_I$yAG{zOK_4=Ib;_Zm}wbkng5^%9z^UofQjGmF|-2@t%@za#E9o*oZlAiHjW$5_8} zPzz96qHhKnhEz_)SVrV|W=4+ek|v`-06@ZdfX#y-$uIwyH0Q!S5|FtU$_4Z&vEKZw zC&5GeK^9U<-S9dBh94uIH{11Qry6)Un0_SggNWlZY&D1VBw(LD^q>R8WcKg%f20?f zBp}&`Yt8(BR`UFtzZ`l3L(Eok7PJ_cLogIDRzSxj>Zhi2k!WfXPYJ&zPK2^+ODJjE za~{R(2g8mVVoNAkCF~k`3@K+h0=Wr;MXi7K+t@EK>-Ew1W)`*J+fY_xx0RK>ff!X{ z%hR}=UXAk|?AJ2P4+A?c>>XTwZ)ufUlN?;`eHOxFo(NLQL?O*rYvg|7$u(WZJ*xMvT~q#K+;?S5fzy8o=Q0H$cI{34{|LPQ z{pAa)nM3Vy|Cn%;??FdG+SQ)!)zr}T?)7V%t_)~;;Tj5beWLY-_@$iIXctv3e3MSZ z;q?D@JMXQ_UOA-|qVUk{KlfJ6lqgHi2YkFtHg8Zp(x(ZWyVP4{6jya$)IZ&NLimT; zla|(ki_mwqhemXv2L88Dqs?xq(Ybcr-hpI=t}2=HKu=aMQSjRi7`V|_36U;(xMT2w zPEkcOC$RRWpD8@?R&_nR;c`Z9t|C7YJ@p$JYi{Eochc!#4RuiPYU5!iE_*Cn)$!f- zeNC6fC$vd62m%ZCQ+ej_~fwiwyaaTr8IAa2gL*B-~hQlHh;s+h8ga)grmDZ{l(b) zpA-5#Mf2JXRoNddvuj~8cNV9U{+RyIH9*Uc|>Z#hg`Qr>d+gq!N(`}*{# zn1r%v#c1T)Z7?J14v140I+i*FMP@2*{PR8U1G)YQm+q-dHg4BNqv2f>yBcoRdM5s1 zX$?;_$CMe}yzey)F}pI_!bM9b;@%VVy;4;6v8Ah*V}9)6SKC&&i45+M{%8FK?{@}F zGZlY3()i9-FhLIbx;_hxQ|vIFUwr#T=oOEG^PJ)#hP$R-bw(4H{Ee;s`Q+qtOZkOE z8h1_m$$ap--rhcDqbi5h7YY)=lS96S@@~mm$92>jm*$OFzZ|hxKdXTfE<+UFzjvwg z5^Z~bh)47?X;AC5(GD+o@@|~*8=Y#n@2~bf5dVHu!C^$+1~lArL9<%|$d2|TI$>^%DPYWZ`S7MrRSKlep9)6#9BQ$6cl{Fk#Q+>g7CU|9bO)zL)$ft%XqA3%<^dWE_)ug+qxWY`$GFv z)=eg4;%4Sn=W_TtyH0g{_C`g}G_^`V-*z{+g(f(TH@-F5^hH5$q$qrocc|4augO*V zLPtcrABhyU#tPWBNb`Wf z+musYLqWZyBXY>&vQOk!hJAx)q;R-(KRv&Rcexv58|=;1D>VH5KBt@8;c9(*joP%$ z^!pPnXI4tEmlGaT06I5~0iw5uO!yc%+=CTH6oogHA#boLDg1?52!%zHgI>^0Bd7TG zihbD`gg`ApQfIA?g{LZLj&4@bXP5JR>$7v~Huz17UJp#g$$;@Nx~T<^t# zE1*pdF^kZu@r4VQNeY?_=Z>YNRQPSLSE{~dsK%R-Q{=U{IWChVsi=SFqcV6EK9`Gj z-ALkguV1x8AeSLYy-kBIlnI{I;1+1SoO=e-@1eB`Mrw}3B!&PVPbiz*>4g!^#zX1Z zC}qMF{TEXV+P(4fRn1F3tf5+vY6F?8zmmJc3$*GP$OrS77f~ycL*X7Lw$hygmBQYSnNG0tz_QGtP&ZdVe&0 z?Y>9XbzH-Ax;A}?a}KWQGr<~4A=#wqv4@r4EfhWcnb)kEV_%rXQukJe<(vH7$>4CO zjF{6l@Wud;p`O+78(IgbMeRz~ly^+j}wZ+*gm^26E>4W&Tdz?+3gM38tat7!6WmqfXdB;k@681w2az4Z?y z#&8O9$>2dF(-|SNK2`!TswuC(j!6msE8FTWo{4}3xfHCoJXm#guLXYuUS_Dw6lSsWyzTbL5qR**1)zk8dn4?;8?q5&DB zB0&XTuF8%3>jmym!%r>Yp?t8K@1Pb9cce@h&<}O9IY_}p9W7#9SQc{uGEnvW9HOUY zM2P?*Bex950&7BtHG?64z?0d~T>xfY*1%L@N{jX+9#Q1ni{|Did;&swEUMTSW-j(Q zbC_qBN{Y^**mrRz-{{{sv8=~m9mXbCWxJpzA8H zFQCo=ch1C8FhQFepMpP<70pu~&W$*^Syvdn4TmhksEY)LBR^kt60syBVWr@I8o?Rd zR`D)vygoj77?L0mf0Rh5#m)|G(rxQl)-ZM~XJ|czg+(g!ebHNgtcRfanK8|Ck6Z-B zpa%s~gC}vG{!}`)!%FLfmJq--R)VmLJ>SNn&DNH|9VnQ0MH}-*C%-PCB$C zC&9|7NzZ`J-KRB9sLPWxKgp^qjVWmt@7`?}SVA-pi@Dle*waJQK>FxAOI?rUU*>ESEdywh9jw`#$)omT#9`#nUpD)&1e`Pu{&N9o9 z5uc>shGNV1cI_K$v~^Az*ahI+bTbo;GwuuFv#4=R#0C8onA)@k16iLL-?e3k35*0y zUN)bse!_a_zsBwCT95<-*PR04} zRBHU~Hrn%ayk9nb#_f9@7GiRPTWOrtv`phZ`TK_ZBl_P1LytDqenO4~@qNPdo8vg| zN&{be{Qwbt&4*-Sy8`Drp)I^}5NA~C`=QW9eLS2?^F5Q%9YTW{zv%dF#?;ubuWntv zbns>H9{FBrJjPf^s5;mv}U=rESY9#BpP5u#1DVPCS)28ITG zZq{+IPr}_@y+!3UkV}EFSU+LH==2LBU}}29?92=-ytg!)aUf3DEEdr_hH*j>Zt(;f}!{U^wA($ zC06$WWQ}E|rEESlT_uH|(Z8|Y+@u%(mHur8E#VoZC#kD3RaZNbWuk@Y84k0QtcNV>nL?v_N|&x5EP1lY=*S8ib65d zQM5%76mesT0V2wlR)ln*5*8sW0s=*rBF`}@OUC4?vUbKm!K zpYu7Nb2LY&PKSCs;daJ@%24K)G?fC#4q$sB9y4i%Eu;I_(d18)1CJe)A;()Vut@a6 zBkf`f1h=s=uF3Z`#gF`qArXu4zLd> z9DX$0oY!9+PYUz2SJ2Q2>uz$Loax;LY@+*1ziTB9b2CO#=+DNzCvJ|=sxb}oLL*Al z0_-L4YQ~@HRY;b?fXK*q9p#*V&Tuz|xlPsGnORIT87g3FJU@FLffd0^sj9seCx|7# z158MOh_Im9$J`d&8sjFYpMG)19A{W1vI0X%87-ttp!-b6^Q4Q_09>^&8|b<6ce$f0 z+77^i7(%`PWbc%?!Eh3DtJe;+xGnLfe@iksd zZ2-~cAR^-4((D0LPnb;IqSgRc2I2owN`K%;0# zTf@EQ7yf=9FnFM><4tKJXyhdhIY=^aI{Vv$j5D>%+`Kk43jIs~6k(`Z2>fK?p0dAB zmxu1ok$JtSm~bARP#k+$S-RlEM**=#7dlICpIFgg?I^mrsHz2%>z^K;32VIVYxE8E zn_d`s$3G9}O+^TE1xr;Q6lVXOXF4l$AoWNw3)>YZ)8ch4e1BV^hb)0Jc@VpPKv^Nw zoqHA;TM;W&CH&naz$9=$^)NlTA*({zZOIqHkO*hMirYBZcJ!%4CC@C^^qLO|IqeV3 z#p-iyqrBR|V28(uW&24LizDLcI2d06XPb$~I=xW8v#gTa#{d$IPT}l~NhYd}!EkEd#;QFK<;h%-E4x!} z*SB($a3%U7@8;dp9_fQHw|1!bZw^m`?}+2Xzx`Ez$y*Yab@F+UN+H|Yy}UV z&;`>?cN~v=`gnc?+GWA&=xWW-u5T)ToyFs;C$jj91~hFwZKY>8s^IY;abS-2VPT3a zE`L^6=swktgS=XOgZE-ghq(CiEz1}D+O6F3*O58eU@c3#II4pqPaZgK*;%FYkt%%N z_q@bA+&FkMBVeBHeC*NyReiz<{RO8@i}LP7hI%xJEaz-~_qRE*L4%n$wd;Q!mOoFt zW->eSSfw1J#y3HhOjSce%(1x67;D$RYmem2-6&X@THdleF~_s-LZC`}qUqR8WHQ}6 zzO{4cwekbc?5!lX)=rtN&z^pFnPqcXmW8R6y11u1B>dfp@E1l#ebWcJf{oPMUp)|D zRHCrnVO!}F$^Rei2c;#s!10fxBFdcJIQ~B)Mg?$l9SmA~NR=QKVY3zrJ}n$RTR`4) z{vG|SW^w%*&G7`2Fm^>f#K8Il)qK=NJFvy`!o0KY=0o6jeYkiVI^(v*vqR`_RK&)3-0N=4R<7oxB2U zW=NX;LTi0gkCMk3hZ7V^Mv{3v2gaoA1^Oq|-_*Ma^9r+G82u^ooOZt)7W&9+hCTRD zfmcJ~3Z02_|3{@+bU?kwy;h+jO}eP9@l+Dl6IvWTb&j7rcFYcD?5Gnp#Wvt+hWw>! z&FDafEtK^g{w)*bi9^#xt_f#0l+-x(;Bv<78zTpH0ZoD$=OvKc1|acw>u z8opQVyIal^qLY{Um$jyR6@MJnL5mkbxa(LgDVSmbgCqe=7~cp!e@dGb15Kuf zY-`oT2?rTveg8d^HT(lBmwT!*vjrR4VQ*ZjF9!IOzJd*Rx$(E$8iXoA0ZO9=oQ8Ko zH#&ONSwZQSs_q!yMtm!Uywj;(7==U4$mG`gQ&vcf+a=XT-a$ZNA3oHM=_p=R;%rkk zHm@#MmDXmQFWRevNmC%sYH&aY-zwghJ#WPhtshdYJ~d87%=8#WVo~b#=7Xpn6q=Z~ zyEUWWis-a=7_0A=US)@W&gC^VuwI}kDqkhZ%Q*qg!Ox?2M*a{KW@^=!*4zbV3~a}Z za4f6%`A17`QvR8DbHmqu2{;M@E5l&%)}~)I?0R}QXK@n{l6*%9q3pU#>~Rm!WcnmJ zD(OzaW!*utwFeFZpaf!5$D=!?<(pY=MOL8Iw8i!;&X#X=OvU6}8)7&VJg97?cru)Y zf?_}I**DG}1rkO%6m)K;(W6km$(Wxl`6Jo602k!_oYx( za(UIJhogA|kZAXgH}DvreeXY`=0ircINGxs3uo=$gQcZKhF)8{iSiR1FfERW_t7lH zfvSgtW^oi1vlCtQ_PkkMx+V|+;bEP8_JS|92Xsk|HvV|!D)==lJ2=plDckn+d0Bf8 z>a<{a@jP_C8U=<1$g1{DOgOZL15m@Lu;Hwg66Ha7*N3Net)AGQIUpX9tKYKJ-7`F0EGo+!oEC_EQpjPyr{(CkWCiii@?d0;^f4!6 zIF|9GKc>yqdAtNqaJ+X`Q7s7OEXB|O)>UWWXijB0RwH^Y-^Q_m4A@iQ))fUgU5b&v zd#&nM4;Uf1VblPNu*?Oz=$hF)^)$vA&U$b|v_myItrsTUdWkdLev$q^9Mzp~3g?Hi z$wLOR%Osy6%*tso?9Gw4q1NY|_|Dn}E|vKXty~qlee6@ouFy3kGcu?``mAI6eGi-F z9BXn;WA`_HwMmPv2Ip~_Ni|xQj;%ir7k9T_Gm+0Agr7qc!)DpSEmHc9578M9W~9F3 z4oOV)1}AsucjlQU|5Q6Dq@_tA9DA@xFh2i-xq@S_%H%|H#mRK_DC5B~m=E!o3`t&p zEPX@Gl3e*K)K1~jJkOS9_!9JFM~V>@$wD>8mz!L~H;F8JSZY!?F~__^{ln9MW4zJH zC`_9t7LndKpbaOsz^YQ;sx%exvCCv8ETIU7EN?Gv!oz!w4XLVjS;3Q8SFr`Mn;>d} zn1hdV51A}6j0H}t?kE|pFv$nz5dD+jF+U0LNBGhnjd zGtQ&M)D46#BAn423$o`ehqtA7d8jINWH~J;(`OMD3qZEWvWGJJ_-Di;|2S^*#G#x6 zC0B6n(`MPPdW(_etMRRG9NyC&ya7P;rr#pjQ)ge<;I*0_0$ykk5j##5>u7t}xk zvsMB6VRXzl86E`1w%p;4@+Fd!PAx&}w3nn5K-Omr%X5f6hRc`$j?{6x@ed@G2RDYM z0rJ;w`IJ^0YH)~9x(38L|4S;VGa_<8azb7(vLY}`n1`S?Q_4hD&CX!_O(X+;j0vHB ziQ@m)SCZROU{~6!OzoHeB7n5P;{ec@%SZkVAQ&NYLqmndd8+v$B-;?zREruv(31g< z&OjX#O{@-DbPoBO8Z5_WUO{?W051k7jT+bwgIP6Nnn}m}JJy@c6z>R$8o=TV7htei zX2!!HAjUs1AD1yJW1^R~8e*4?NgV(t1sx33YfWNaU?lux3#7e-{R#PxCREaYLEQab z?FV9~u3EYf08AeS&N4cCdb|?1F^+#{{#%Xe?G9S+fV)n-6}1Fqu?M z1eO3d$gq;JA8B?>T&q{C0|xDxnPNL;W)jAMxF?30p`QnVlR`@8DCYP3VHV;Y@hM1C zKu8!gCJ8V=H*v6lYnfL$qM-{rIJPtF;F#{jL*S{28B2e2q@eD=%dfyaLfU@~Jxc!6 zM$|(0d=*_JBLtX=#z!CY&r2s|KK>Q#+KhKtJmfbY&4$~_IIccrAv2*Er)&rC2@ja| z>6r&c4>}=;v726+CfbxZA5;|#cOJLD4L=w~ViY3VV5W8x z&qsXrO?ib-$20UatsXE245`AffiDRKYqng&tO+AuVUxBF>&xJ}mXA+oh^)M*zLdqw z@Z#O%{qkXT(Hy%`o;jZWIQQMenX-rRPjfIYVgw;zPe8xLCj&-e8r3FZi2=``MQHH(w}hRCu!e?>~w2kn781bm_q{st&f|nd~pmiY_g)^txYM z&90H(5*2CR&zfw)oc^S5)0)4IS#Y~8=h!lZT>;PD&uBu;PSd(c>yfcF{-j5 z=fA2;UuSS_2rY3}?t5!*uGOIpNl!d_$&y;iI5&DO**(;h&8rvO^1QtG9B1cn2UFpn zB;T%OM;i;v6Mxu<*$|Wbb#cVL>vOsHo_Dk~7Bxx|&XnyR+5W;PvDhc=XTK;;h@Ri? zRgq(Jcw!H`Hq%1xRx^+$+f|_-0g9(V>{!!1K!&td%Ec{AB9FYsL|AeDaX~oV&*koW+PRT5!iT&+daJp99A~+06Z` z@M>a!N}_wG$$6gV&c@;$fx*#MIqbSULJZ$6Gm z?sp5$wphR7!-xBVDp~W2*RKmNFg+FlhH3<-vrK!nfsc|KSJX%A?$j?9Y_++Q-Xr`l>T-^s`e3jAPu2mwdrCI` z=fQ(~(R84L{GnM@SD-Gq>>~VozUd3~*x+C;%pI|Z&5PAgUdT~U!sna|bMqhEFBQpE zPI)rRXp4#^t0z4#vrmeg3p^8pcib%~(C*B-w43#D<7)BUp~FFeAG_de^lh;2waUEd zzaHXt+0RrxFs=BN!xlM`x>%9T9MN`3vE?c0&tV>(5?OCaZ+ORYuV}A z#tn&K36@(`=4D_9#@aNd`p2Hak(*t6IzcruRHdH|3RfuJSTzi1!GP-5r2RR2p zjeI;hUq4d*!f5)q7q2QM6sg0A6x5&a`5FV#wz^BAfYp1M`rW?)XX(n%I!KYa-Qi-N zwSL8etB2A%8q05<8WNrLujAFO)|Yu>I*fA{?5N4HEjc);FR@;!5%%s6Surc>ncRGH zjzhi6YV4cc?f14g;+ebxqJ6eYUTGehgloigK`S+dv(Ky@#0$`l9h9B8VuvtSqzxgM zdq=Z}&sF(qBC`|N>WKo4?1TMlmfP20i<7VF9YnBDP-$BXa6?j4z)$WijrPCF1k{-9*oP2jC{|c?sU?*HR zHQh5&CI)LO;MraDTmwD&WbPdmK#eZO+jv}3kdsGcI?xL581{H;&Av}*o!Lyi0nU<96~yn9@}-pfB<(vmHmj`6Hni-re> z31|^k7QvR7joP^XE3kP)oxi3o77sWJTVI8rk{6t9qD-CA3Df4YyB7VBTZs8Pl;`ES z_fThERK`vd9!`q2k>`{ZqK9zd@<=cuR2PJjtOvtuY z`@^YMWgn`H(!r`eVlqzIF*@X8!|tyvG-?uo#p;LfG(45zC+O<1?jTM5gX_k|&WI>1 zYJI!Q$sTGL|0%w|Y{BU@N*mbYq0+sifHm5Uk#6vv`As+A5sC2>kn*0RD}btWXPPCYGQcHlX-~|q zs#ZH8nYv-$<7;-iYTwSIDLc@&C0Eh{ixU@X8)KDsIiIw7(hLOSMNCK5GCys!WVgRJ z@>IyTNl5Bxgdc{ZNoOr#3q;^(LgkR!SV1H2+HmU?aVyx^6L2VqCK}aYQ@VoJ?;WLa z#$mrI35DxROLc`r#px_joYy**xaO}%Gm)(!_hx=5qkiGlpveTP3cr4QON4JZyf9i~ zR0EqRY;REA?60c^nNpnub)3#LpL!I$jS#h=ci`7$Mds-pO4}%|%w4lvc@gK`y%^Xq zi(`KtH~9lO_q7MYR3eAfseGoj7K5|ehrI)WV7%fzydv_P-d+)WSgah(n|6mzM&Jby z1A18f&ZK_%1vN%2y3gS+Sbm3jk4B-6Tj_Z@GZdrD-z+G2Q^h|K`<`}Lbt>1#BqGaK z7BU=)pXT#5;l(+kyCZM*X}YzXkMkO=hs3=DrT@~M4Ns6)d2=2X6m*P_`C3lfk1T(X z73tF@fyYOyqN9J2l9jvSyLqYQ%N(@&SaGeuGTr3C$b$>ZQa>jHt4!zUai_}wI43SM znB34|%~)*!#WV1lYGS`P@G|%t3<7#wsVe`!{MM*058VQ8J=7>n*%oiWU?Cs_Lu&wm zq2G~_<01-sVtVEG?$Gd>(Ls13ZPiS37DYqjhv-)@Ad$cl;4u}0@vi!lK;`+x*NALv zGbj|$@5?Y6RF!`TbPOKWgqN5&F>s@_gB#v;5eEbw!JnCHqhCN^oxwC=Yf!WB2H1-|Hb=pHnTq3H#`jG^fmkWXI`0aERT z!|PAcppDNhPA4_Q9t@PBe+NmMHDLsQ4ZIBRD*QAcI>Y@mX~`Y|P%o|ZV`9T<0E8{h zV-=-?@i(5n452LEQklBM`wj$a4^M?I?U^? z0K-EXgIAy8V^bWb6>LA~PJ+;4B;H0=efZfHg0^JMPop#5Vzjaq7azv!)A;84axZ+f zfEiNYa!1Ozhr9wj5q<-)N3co5uoSZNM0YaYaYXMh=4l3s2W%3PE{s_D0vLArtDe0{;j4gAw;Yh^jeayb<@C7!rRPB?ALTvIvIA zAOi!eD+2V)%mvB9ycAo&fBCxjm87IVU;r9o(1byZ9O-lzw*q1Y4YQy6U1%C&KEb>2 zQ^x5o0IPxtUaR@0O;i~riP6#oUUyxG+{YeqHhJY{W({LMHohb2D_(J`rfK*GEybnBwx($s zXJ_;zUb(2h#KkTFnYL&7$($5b;gr|biC|&sowkI)qk-RJ`9taFssD4KXU=S7>9sje_VC6vME78Oz=aP;vXKPM3c_LhGWhZl_gD1}ApYc|1rF!t>HESpCUH3v zp6F0)o*FxQU3MAzP=^oi3)!Q}D4&}#U0U(%$n9ODza9){^?g(9RHiB@yjKzu%ju;7^@8V?}!1ee)uzu&vI1i)Bgk)$6>Vz$LKD z64z$SZ`7>2B+ty$%~xra>skNo8n_#l0;=n9=Eo0{%O^4#%FhTNAQF8=Xljx6^#}Kb zH~~aApd}4PI)-*Za*4f|S&0B#1v?}3%2_iG2Pf3G(IYBn_4D?j3d$*6GT-VJMER79 zHZ&EPwr)L+70vkEJ?$hx*X4++&#zXOyp1}C*6|A@wd>#7ZKJazgRIaSZo`vJhbO2d zK_YF*EU2^P29}9!pWR0p0V+Pz^pa5M{uI0dY-vJY*K1GvdLKEgzn5#esld#)KfLzj z)8Vl#PTQlmpI^sau?K-_?{ukp$JUkpwIR0t#v(I7?o<(HAmXCT<({TSe%o7j1<*w* z5(TzLMnBrh>k9l?^<7K!F~L<=5nmd%M=>Ud6VDq88Z7O|?(iDez#5I8No?$Sm0@Ix6jWn2t6oW z_k-gbkBpdfa!lr?j_cS-g8vyQ?^W3g7WI@`jWnL|^S3Qoth=e*rL(ARl$C^bxz}EF z>`IRHdlz(9?ET|WK~`C@(wn+-x(@~N_IH+uekvAj(tdQu^sB@V+ka{1-Hfj;%r6-p zyt#P}TJcoIUT(>x#W>`ERwV zuP8X*y*Z9985fW357^Ddn)w;i^3-{H)!XL+jX7ygiZ=xwy1f;q^L(RxrJF{7JMzM? zJ2q3v&pG09J0nr;yta#by&4dpCLNc{SyQ2c;02-2hkrPoU)a>EWOp5P=i!t zk;&@w#WD0|{M6H(DmSk^acs!FuOd`hB|(^wTYRSs`8^LZ1+v*Ll_bz z_!dwrKxUxndI)omz=wV*)#&)Zx#KZIuxOOMwqkDdM!#brlf2>c>T?w?`c);RXot5> zZO&36+gFOy;-pM`=s~uL0`Yg&R(gq-(0keC zgSbMq}J2{GDS;SjyCiAywtL$!QlChxZU%Jv4pXLpFb1 zOlP3}q$+tzvB&(grn3D(;raz5jiN(W`E#CeznXjSq6@Ph?a5(3jFyml)O3t4H&6eU z$cKX=rVm;YD^xeW~b=dIAK zb(VYJ+2t%}`sT18)l(Gg*3_|I>DZPs(tzxW7e-$tA@d-I7Yq+PN?q8vRG;nV#ymP? zd++Gtpc24(YYyMq*ZEyck(~eq_bQf%`z=v1IidAdK!;|^4$V$XAA>hh&IwevSQv(z ztNKuo&nvARypLNMa|c}(pmch&#hP#s0%?77m{^9Ao}@{j23~rtA)(-7m)shK5(4?y z`i~Y*^7=27_=5((Syz}RB2R28eZ9(g31{oKqWkhrw-Oz~;oh^+WRa2$t8(VB;?K>I zC`8gWY?jWx$S52Fx~&&y8~Fz`@(znAw@-1bVA0ixte8e_2b2u9HDUPZQThaTSj!*I zIq_&iN>*Yhej(=gsy@>7#uQ1%Jp#X6M;U!ia1^5$D~=q^Mbpu(x>e%!$HZe}-hb;g zQSXH1s6IK6<1zwC9(Lp2_WOovN`CJrT9c;0>mzG69@!1+M6_?gsOs+0x;LM-_iK>LaK9j-6#KvRkVzH)L1h`In$d2xxC6VUQ$%d4k!?6O*Hn04=q>o(%RvJ;HR>W?y4x9YgK)stfic< z@@)!$rUIjo>Sw4w+hop$gM4>IruvjrZ;z8N3+Hs&QEql_t^ERH5?l}~O7+0mz-^+g zV3YF8qGFg7q%M&n>Vm}oIT?4kZVXXm>jU%Wi`~xh>#qzcY<*B$dn)o6Z_3W(;JXg`ET%T1G7c>TGwWsklF7WrMh8T&#vt^UJjm1KbNp-B$vO2sIRS zIy-1uSnyiKQIuC7TkpcY7o3F=ecn#K`M%TQW*m_sqBuyX-@2c=b{xvUSOB%*P6SW#sxaTAwaD({ z!5kKO0JC{QUmu<%37MLiEZlnjcRV;t_tvZe_SItqYeG(Dq%ab39HZDlI)C!B;z@d; zwD5*{ck(>!Bcf5s?%|wq}+eh8e5cX;G@5qw%JzhosZ@b;peDbh(ffe?oyC87e!^pxxz z>_g!W@^P6AsrE;tt>T{1LKCwE4>)BV2tnuzZz7XP-TM1lA2Pl3o}E@&EYi-ieb8(D zEANlk%;2dISCHB?>B>5i&g>VYO#L3^)`x``BrX>m($r4fXby{R*LurZD<(ly{*Y?%g? z!maSrK>FHCk42{Co=Lcx@xnHo&I@r7X}!ciq#2s>fpIBp91vrsU)TVEfj|g_Fn_2# z4++T{hFq08sq0Rol^2+{c|MRx->6 zLjxFe3&ds^Lf05;F+s;erq7?D0|q2i$Rzw=;7c%E4EhFyVi*I$SQUZT5efOB> z8|fF^nb=RwY=Cxj!9LYMOV|{Wd`Rb=Pd@ilDx~#Fy%kXT1jDEt8z$`CaA)w21OjK$ z(u81M>jT8ou^^Zxr~?D2D8UYt!dE8KuU{L4>Vi>`7}-GNL3a{6^XkvVqZ9f|A1%65 z&a?&?G50ab5cbZRvPmWzb={kA+D1-U<&07T#p#>+Kj1Q1;XbHQ<7?{J1f0xfYR5px zuatLSe+Mk1dxN-~mm&-r2m+ym)?kA!ZDrR$7zRrT^m|J+N9kqAO{d2ZQ3`!1vCmi` z%#190?{RR4XSjh3GDL_oS4Q)cJ?guTqTfijU=RinklRaw4jNJ-gNQQ3JXvVu!E1dy zW8#k}KTIV92wAw-V9ZZB&Fubl%uvHaV0=bdF%t`5#_Qu^TLTk8L(25#0CLZ%#Al!FR;{38%0M;kqQ(j3B z74m$L0i$r2ptqcvMk}$0P>*XIv3~=Ri_5a5ornltfKN6SF!{~1q3I_K8 zv_5`2Ba|>ADS;vL5A7Lq7P7`Wzz~rvfuV+J5rBh%(*;B#u-u{9?Mw^IIht(zu=u6= z%eRQidYsY#Hc*m*&wvRShKdVF(@z+p0&{_xQU;Y#UfO(KSCCL;>B-ZC=LuMK!D|t* zJ#LaXiBOXLCaA_=YT;dN(cO<%7x(FS-wt;uPL(D-1A^IUt7LnXut(<>Yj-qT=8Vkl zXkMr0y_xYWK^T?s{Km+Qtb2Zni^u$ur^cEB)Q^s8CZ6f*OxdcDPa{3#+{fkpcl3XK z0JwgtJ7~AKF{__VQDCoUS@heKtl*7)_!yr$oDTOZ8mF?sNh3GftF_rZddx#} z;oP-;SC=L4r$H<(XjxsF^x%nb*W7<^;(cf48Me~?aR}Qa3+CTmzC39|7;Ev6%?=ee z@fnT-r+oLJwub2$@bh3yr1uA>Bng}FxDy17#gu%}+{U{h3qd>=D-Svg*GuYe6E>qh znl0V2dNcnGiYj`9lhFH?X(Ci=PEMErI!MaVG5dzV>iEu3mE6(>sHHw5dP96zk@=DN ziz*yy{I@8|Grs%Vn2D+S%Rary6_sjoQYAaq@lVmME%y4jBl;g4YF)M*$?^?cqAWj| zk|=Ff@_Gg{b##OZt$8@TBILT9HDZZLy+lBs%lXCZ1Cr^OTH0S~zAUMnj4q$_w6shX zMnz%6TJP+uOy#ipTpJKJ&4~(E4a_Th$Q!jN&>o4NNNMOjY5Vr$G-=C&@+khd<2-e; z7f;p`bF{CcrNwJ;t_3%Gv9~lqy5UmVQUv7>$5T6!&e5b#vdLmfb05bF7C-TQQH56xRIsN8bNY+426+?4yo&XsUAU7#W@&=D_NA zQ$_GvTCNyt5(4_x6?vNHgtV7zSu@s@VtV76mxu1e`nbuq%$=S~y}k9?#_zM&@l0BD z_vBy8y-dIF(k+nKoo*c8rD#nk9Eh=4>e}cjtW(PriJesm>x(5Vj*%TrOG+ap69WvaHD)5kq!m*R`;vjj;i4sr%cmtxXIeEiL%QrtiB2rg9* zcOV)U5}IR^5vRuf{^Wz$S~hlGIl96hr^pb?&R8=ib?r~#=TxnpTxb1yLwQM3(%IGP zq~pQ42TNGbCVH+2LSIYh`e-g|^!dD~*92YKdGGXULqVhP`JZSOsP^Su82Sk6aJSaf zyi|v}fxA^}*d7&g&iFQb;A*!lsD8(=D&4sG)I5v7)a{7R8XDL7jjlcPU1JLYyErBdf-4uRfJeqBR z3BR^Y?K1gO)4=5V$$Q?bdEJAslj8XKTAq1yLwc`lU)G88WVnIISg<`uEkzch%Tu*a zNYx1?QTO9Q>&o`ehZTi&3}Qy{HOD{~O+6Iq450^NIwvLD7p-z60R~5B=4- zR(U?WE^}~evb&M(P>lMvW={)| zO!EB0+k0(LMzYqoIxB4x7Nor}+NTri9d3=x>*1?rL$MU1N^Om5?EgDT^}Y;PfXAA zC6|HJRnoldIZp7|6jg_IMOcyW2tMCj3OI;N2FM%FxvZo9gVMYPcDF?D;Qt3eio;0g z+K}3$nM6upjBGEW_rmKQAl8m-4qq58_qSiV{%6$fqx30UDsxUy-f<1gH$8P9%3aI}Z63uz3t2`mN9EO?M3QhOol>zEU}HY@#hF`Jt{#V;IvF(1vQXwxjN zr(#P3vd^e_eOG0_=hu8g-OwxvJ^gr;;YXO7XO>%gsQ2Gq8AYQ@?f?^aipF}qeQcZe zpApVDFI5w+w8MaWOnEE`Jr=h%#b&t{)4{yoWn*4ZK^Lr|4lnY3v;i52-fmey zCQaZeCu|@1x~zfqou%L|;4=9g39DTZHpiwnyHVHHT>pv~g%fPKJk1W2I55mf zW=Ze?0O(+>|B{d}r$f~q!@I=;KY|yOz=jugfy@X&|7qC}QwcD!=^@1u5%lPBEzUWS zCZs|3-G>>#obDg|BrU!(aWx95@1Z9%di*`w;gEoOc}5Q!{Fwkab`78uDQFkE z<7{0X1~37xhsMQdWnK6s;L(wweZFn*TfTv&;*riHOx^U;&;{tL^oe<5b}XXfKvzZnmd#L!kXd7*$7sdq zz=_L}d@Me=`CxiJfpI2ByWLo0Z+0HXvCtF<20-%juDVECW3l?=*q_kyAQi&(&`Id7 zOIO}Vu&Y5UyC3$i&Q-aku({OpgkUym4*j{g%{kFW3d1_2j6$D*_60i9_IiHJDSPA& zQB`HhyPyj3#L1M7U)ntMiDJu&)?vX`s*lfM=YyZo9K0CpNMZ$OYqlXi zmgpJ~tP+{7l$)Gy$!5^xXRp>dShTZ5p0c}#>?EyWMKO5bUFYT&vwmjv@%lDgD=B*z zaj^r;-!?=$uG}&$ImoTm{sn1iZ#l)W`&?m0(=fm=FfyFK?gpQ6kO%@Sj7xCU$(nZz z`2a&@tT)a)B}iPj=L-f&F-Ltm?h}gA0l6`u$iE|KGH$1r1Gv&|!ocRW{qA{qcW4$REE8vf4}?HtIx6alI-w&-5H8vz9w2{mJ8WjK*j zU5`LDVw7G2B&NUG;4;7Jr6GXfdxQfSd|*Ymicm)j6#*H1$y^l*&nmHoOGQ<-;h(+R z(RgQzP-zDl29quF)v=XfgheSX3=Pz7cw{=A$A{9NM4i!9>&dGZ8G(tpxN1F?ONyb^ z$rPLEN>N&+8@i4_En|t2(=j&k5mLnAHUP-Mtj96bSW&#BCIa9D{3cge_r>GsfsS`e zl7iP79whz%1N3_sNDq_^jQW)oCNjXkkGNZsETlp=W55TqG$y+owu1(K0Dytsj(|1d zsR|jyiz{*rz1-=Csv-sydIIPQuwCGF=7vLFMZWor1u^?)WJBbO*`1omz+wt<1h|TQx>UB&81&~t_gc&eB8TZx>4rNr zbEAG=n+cT+4kw-j-5xXpiJmYf9Tadm3f-8-@l@$I7y>Rc5Ef*^|6c_D@=BPoz`)4$ zTcT*Qp)VZdph#NEP%Z0--<#?ALllD6Ok5thmNW8y@QjM_-Q4-Hn;`p)@bFjo0} zr?~0+jz~B2U0b0D6@6S$;I^PwDJd;5!x5>M*C3D_fk z8U$DIdb;(o@;XRTIessU{*tPm6gZ3+tKNTp6V0Z<`B4j(E%vTH@odlef+gPvdRm!y z^p|c5_{pu+-K48nyD2B|VMIacAJoMzN1lVYUr;B{JH-3En{IA6a zs$+-BA{sA(&t|LNK!{nOIeRKZ;8?$9o?*V1pH+xlSw*RiklRanIp6>qvz`5wr zI?&?v-2KYEful@|?>^F(DyB+dWm^^@7+C&c#b z+R-=Z+!Pt}H?SJC?d`#(1vCv$qHQx$wtexMjVbfLX(aBXt5Px|;0%A)vu@K6Hlo_+ z63yzsDu=}wpTm(TN$z`G@;F*mYMW@jO&f65F~-gNPgB)Vr~mQqv`{wZRiAHO7y&Aq z+49T^cS~DiB5oHu9nfVTnO&m@khF}11|QIu#>Dq|UWu^nzFpACo{B);MX_vOV7l(? zb7Ap<>plsY$CKOCx4S%TUhQKC<(ooi`2a*^o^fNz7^#uwVhh?zicYWt+meAf|t5IE;h+--cSHaDzq^_+e%>t@pPc|o_}6g($ji8_44Xp@ZzP@n@UFh zn5spoLm~TvE?pt&HS;kyO-`ve6LTU2$B)%X3XMqCRQe7|P=||0tp^Y5t7ot6vMEd0 zEM4FxPu`PoiJzM)!lwSqxZujKM?12zP0}Ae=fCk@>SsOTGo>anw0xJD+V-`1uDR=pbw}d~~J2^g? z4(5dGlE#j9K-q5g$WlD@~Pj0@d^l#x*~KTUeP1 zq*0I)D!zJ^0tp(eW6NSVssPD|7}e@~>#7ou7Q!KKwyp@$cs>}f#kKm{1KFtTq|3It zefC(nWv5Cy=c)gGiIUZyM0t=}i`&6~*EVLhGIdAUwZTR>N3l(Y${)e&P-SiT%HXJ^ z93l~rF{yfQviUDQzIS5dL02Zxsr-PR?`=H&-bHZ}yD7P0vc*~{?o~J^dj5=-+Ovm_ zO*VS;wgg^8>cXBADm31zbFf#Hd|2Mj=jv_;Q&hzzJ4ndQH+x3+j;Frm>l1K6vkYQe zcNsjNTDneTEG=o)9%z77qArRl|8h^=vkgQKb2oZxeJme83X0x%6-Y0Pcq@n`f2Fp7e*>l5gbW; z%o%9;xzPkG8z1y_S?eZvG5U^T0&hzj|FU%vNR=d6YR1gXCa8K1LVLGMg7KC<``Hip zndCrtIPz4LBEolK0$WpzxfCvLC+a=%=|EX}k@>&B zj0d6|+vFA(E~&F~GK~;_K3cC^p9u?>gk3mQ%Ha7Xfkk;@*etZSjZ9!<9VQ5v=g5)9 zhejXm7Z`(hNq()uGHuS1JvQtb#|bwCmq;+J8$hU|5JkzokPsf`-o;ar2n%6LE$q2H zzXlI`xLetP#)BcXKGWmU)C3rSaTpE@wd7ESm;|FYh9 zREMOsU7X7pbn?x(n2P;yYl$#R?xfb<*EnC=Rr!S(kMYOuNuAa=B)(_9`A*-aN+?FO zYBY@^Cr$SnHkBrTPFyccn$23-y3bKk;SPym0-<_gBtldAwE8)dm&5CZH~SWca+MS@ z8#pX_S}e%ZH>FD~Ot3KoevWkk`Onnx{IvN#)l4*baARgWYCDkT^Y7`fCSgB_KGY~s zHUba&xaaA}vxhy3JZw5SqOGz5T!67W*}k9^W_vL)cOV8Zq+O|u+@az7MLu|!i|oy2K8lWEtnJECnO;z-i}&b z5cLW zgeqC$iu`q#Dtk9onWpyn0zM6e8_vsX+bgL#rwf)G;Gpu0XS@RrDe>?UUB!fP%AZvO z!5=&!e3hAZhli-4Gfq6#9Y4XKf~8I9XDnV)4lhIHVKP{}uUkGQL!jpzHlh|au3LY? zzw$C%PJtuE;5sd;YDaQIn1w~d<5c~*PeYO|78Gk2$#gNUcO#wLP;uXY-#n)Du{jni z>9#z}?nZJP=UHN^N^){~FDDu)tHJ8A8wFdYT_g>xyyf`(uXj}JhR)u5Vlz8mCNqmd zf=E+l=L@3~$zsf%PGgxd-n{j-vGpKe6`f9rk1jB$XH0q6ZVhbO1xkTBWotsuH6Z<~ z&!9Yq${c}nfTcaC@G=1kdkMAYe7SBVPDvya=P=?R!r{BA1=*P~w4ggrVr$0g9A*KA z@;5C29M-u7@I5bN_z0%_Y$NhvUP(wD+F&Y_7vqG@Tpa^NGJ0MD5;}(icaypvx?=`; z5MaX}gTvk^AuzO{4Lc87gp4=s!_c&&wPoe=>H)HLH^h1Kt4gf`#K_2^0KOUiguu=#MR>lQ=95~Gk#fe=7x0-Fj{ ziVcl7?BRD{Bb2KvTF1ava6MwGT|+q!eX|$@C}ZouyV*j~fufGCV*nmS6{zJs@GO!PVs$xKzfo(uhH}#Qc*g5DT6$ z=r=59WU(YMrOPqfJ6{I z=oBDj585YiTv-JwgIkMryXSMJZ6ow6eC)T;TsDuPjaJ|S^mPsBY``;;%msj#q-XsB zjcJNNK&gS*{p5LPnoAH#RGMhaU}r!W8VDVP3C5E&F_WQ3Y77#YV<*S^`l|I^v?}OS1;bCaYLt}pP#=+w zp1Fs3R|q|r&I|+GhJ1p7zr|Pd%?#Jxvh?QOBblg4PXAWax>1#+Z^q*iSS$(6~u1yk&{QVn_*QC8O(WCYEC* zklWg9kTpQLiD?61aCu}&VTOCoz6C zzVOg4a|ADQz_@zdqCLT@cPpMs0VUKyKtJ{mU9P14-*`)AhGE^})wCT@yam%`2`fz8 zrcb$e*-ZmS6aO+S6B8DDqt@`R&?=58qUZfojxvrpc14fYCfWKG9D$W<#k-PexV0m! zy@AUu{m<>z=0?fsFy1&0;1v&w%wks*>}42@{tF!ge%{7S*F~P_IflB<%3Saf)FM>_lMH6rW=1(8zUMPOL6i zsP!|8iP^Fy!MakX=7AdpU6tDl>Ouc*56ShXIsbmQsL1T1>mnreozQ;v zX=w?pgZ;d8jqs4ljBgrFp~LOz{;l<=%Xqmv^T zGv~0^%oj|29;&x$+~S-bxxY(aiNkkG-t_xffywJn1ncr*LT^3gRrR#B^+X%1?30w) z6=$-Vx!2mC|GW2R{bINh#>mmW&I6N?t~(#HwQ|Hx5v=z8Twf`AZB#P3B=JVjfcftI z71b@zn%!Cv7k^a$m*N%9TH|R=ZA4T{WlHmJTxTEEI+!BqlY%)1^yi$>;W{)m@_-D_ zIN8kFKbvti^$% z3tDZBO|QDf#VJk{)4Eo7rr+oE31xK9YO!p|K-uBQrmPJO2V$Qm-uXiJ@8?-+UfGRb zom9dnPb^GXho9_Lc4UOrx4C=#{ljQkdfb>p=z`OSniu9M6o;qY5f*P%*@!#`b(RNd z5Bj{gE-dsnZdO=7{J0a}I6CkpYf16!m~QdxfhFMy+$&wr>veApp6j$zr*Bs~-|Q=0 zE{3P$-RA@Hl4CxW4MB=*=T^H75;Mk zqr%!Pj^U~WSiz`Q>nz_H9IcSFl@=qy zVobpe+y>|HYA@B@3LJ@HWYvIBIjL?{$HHar$9r@1TX~7TMR~%BJ8C?%!i`^f+6%4) zr3(d~%!srs$h5E;S~~J_B;`N$S;G zjl7@OlU<&qglTv5js%cLxen4CZV{xy|Iz=&$KA)9ch%KVy2JWOukyGrXE<-0g;{M@ zY;EvuURB#s(00`Gt93cqi?r86Dp)Y)x_Q2vR@6BA`NiP)?&*&K=gf`+M9Lp_{|x3L!T%o5UtCz!|2b^y#(jYqt4cyu7Y<5c z+!k9KvU9(CJ?AbBM~182hUw!W-j1*jlmrIj$hXL+eg{+c-hCrf=MOFiM4o?s-eW;) z=B*JEa5SCfBJGzijCP{P_;`++s9zP(^T@dIJ3Qo3OvmHy6PWr@d&T2WO+iyIySuWu zzNRc79`{6hAlobgP*(X+rf_i~eg~m;Ee^L4V}fV-I*YIS(V%c773G!q9;Hq2kadU( zv7K8{d8}7P^`bMw?b?vCW3-2>ie4qzaPf1C5T8MnMymJCyjTR)`HqR4^hb7*CS`0H z$157qy0#DcI=6xTsiVNRdh2NH1e|_UX#u~t!#yScn=ys&sMa0!IKV7>Cay;K>4X_5 zqoaNjI(fdadrlFE9qimz1F+-jFPlI>Qh%y{X|C;)v)`0==y)z{P}cn_upsE79xA#0 zwM1WYA$C>Xdd>5M7!^^L1$I9=jeI}y2((z#xQ?Z;S2u$@qF14@xOAY)XUchF3c5+Kpc>zx+wh3dzy?^PDGtBM#n+1+UPy+jR^ z#68q(uyW%Lxwh-Mu$y zaH_;Yycumob{Bh``P;LhPnYA+514{lptl?MD+Vb=;gfq%brVjYdUocmSJ@n#xwe%8 zdyPyoC3^!jn`P+caAmj{q+MV}@)D`KNz~TH$I$8UGBU`J;NwC$J~z1$fD*E%woWM0 zJys8`L|y&>!Zn-fJ@`ocKlu2%4_p9_9r9$E z+OTVpj^V{^reia6ZX*y*upWIN_x9pYp5QDuoICR5*v)EY^X~b&H=fEKdDm;i;v@jB zXbAp=atHJ!6CwG&KX8EAO*D3(aoUPx8DTiGkttZ{B@*59Kby!hVQdUCUuJFLv_@z* z%ZJR0DD9c(GAt7dnq++?ku=Fj2kqnD!M#IPc1#v^@0-L^7{ASdz^BBm6?Y-rXGpPb zmRCFOK&j0OBRipgY;&Nh)A0s;Ly<*7-3uJbVbauu{-`QJ5wHt~{2L8~s!$6z1UiEN zMau>H3mCFeTmkPkD4KS$?|Hb`yXmoLHj?`Sn9+PohdLw<7Ha1)=?AFI!w=mhPGj6` z$x^2Fm2A{BBrYG7D3WT7_ag9S8-T12pJ+k{=SRowYo)52t(o`fB-mf3h-^xHSg&{E z-Yc5-9)87vr#xwScF06c9F?0&D3O^v_B6F62KY$S{IptgfTtY`4L3<%Ib48HBDXzUPe?G0jW$7 z8Lw`mxJgX?muOBK4EikG2l8|xWwYNz#-uU9U>vUhPg}$Fn9^|cq>*F(YG~=Fvp#mx z8`P3Q;${*_h=4a3++aA*;}{S6>y>y0$fIj{M*a@G@Deis%7Y4_wi(6(V1`715E~#L zk`7$@?>}6!aA^->NgVQ>qY2AdanB;=9tz^Nuxs(Nf>_9@#59d^oOQZsW?qpO0@PcE zkDJ;g?y02ah68cfR4M=m>(FUUQy9Vu1Aqmbg5;SIhl9uq#;nkVc_U8-Z-Z*M-7EMH zR7RM&)tJ5vU^l{rWbtld@}G2_`GFa}2d~03&ICF`Ai$R}6UOEe?t@lY&QAsHh3QWzc}=f(uPyr_X74u}nGNV{`FFGCv#eiAFRK4~ErAtR&R-o{(O zcn}e|s^sN**h>JOvQD(KFZw?i@Ftlqo;L#D^fHIRpbQ{?25qqZRh>Tw#lIKI!3OvK zQZs7@V_ZiP4M2Z5?|9uQ0md}Y5xoHW{7-OS8OTob3v3e&-&(0<1*=tjs$A4^Gthhvt&Cch@^jo*Dg zP4G73cFHv1g{6gEk%7 zELP6^l#3lI*DF*4psBL3_92M}72Cb5z4THQM|9{&s{x`4j^6v~+3oe#| zIhp2}1Un)rR)0?W=xp#K?ltHwp{)ShE2JveZm@#&m)Rb@2$S);g-$J;a+1*pqxZRG zpkuL#GPET^C>Y96$JVjn)JnUvEtVzpLLg&M@E8!%9iG`vX-ob6&$g$1Z}v{RGrRX7vAl@?qe}HT#WP5P{WzC&Z4&=8rj!Ubu#a&n{4m1OJcS9 zXRRYzuwkeq^Btb%D_XU!tLA^V`@-Cx)F=Ip+wO|##7eTg+Q}AjFY%|w?&z&pqb8+T z8u2w!)yBt3#v7&|#x?27@sw%k__Z{5a4!_TFq-Z0^$qS_)>Dh5 zQuENqkBZ*G83Q4Yv=NC?;qt=^;C@qCAPU^IdG`W1N8BlNvV#yl8?Y<7OH-LQNJIiM zO18Me`#V3xOMlLft129m2G2WH)3@#t?6I;FmrEg@&SXK~zQ4K;BdZ4W zKgo|Mt;!3UE)0F(`lAQ1R>f^FGDCU&DpB9kL$4I;2gkMzxBsZJY#mZh6>E1#-9IOo zn`~$Dhd$WsHEMdNr<-Nz%LV&wuU@@U@XmeSse8AvLKjpyO-1G04huAYe5}KZJu2V6 zc&@euy3;RUw4=7A$g~w~MM$Skt3MGKN5QI)RQ(~X)>5a1uv4YVs;GdhA-0ZM zp_q1DXjP1gTiIe{lf4x&trWr{VG|H2J7EnWBqV*$bAvPUdq1D|{X<2CKhC$PXc{dMPKYAUd;}zlNLX(GXwm?;^66)wOOMaP~X3_|obQ&t|XIZj&w_+ws@hqDH!u zV_<_CwPq$A?NQ$ZdZhfcp{!ro&|~pdHfQXdm(d&eUt}ehhHZW*`cdd|zSGyXSo*BbCktDbd6y(N>7<8IVYNj)B0L9R zGODwTN_aH%==GtaoQVm`uNxGxLzuD%K59Fx4N(5wdoim*INo=7iqcz*dNNA_qV7#d z9wz(tyMq67WxpBqdWFaK6G;KzJVnis(iSERoDwRxT`j7u7GTe^g|dpc0L`H+q_eF= z3Dm6=Lly!hlSP32>}EuQmh62b{VZQ|s31?XE5suz#4w$266NQm*@d@?k#5VzlTR*a z3EnD$Aidwtm9sx9#Ln!~W_9(+N9f_RhP$S9Z0mF(whD4+9oUw z6Ki<)G;X;tR;<I{c0NT0aO@NSo*mRu4sU*a3 z)EI?bIr6j*J6T>tDaIf01j-(@y!}e-dj}Gjy-dc}Yv==ILMcfaq7{A4#H5r#yk@v3K1!pe(l^Xh2=j`nrsGt7%Qc)pDm#< zkGnGd*Zgh_t4xeb0AW zZG;kKDI)rOFt0OYfuO>n%<#GJWh(hROfT?pO+Lm<0yA}h#H{OyJ2#JdpqK|^Vcy0q zhAysPH#Ph3HN;j{nsFFi?#2H3_lvRNAMkJC--03sQ!Tt(lfwxrEzRPi_5I2V7J5)8sL z2X^=nXCRye@)Dd2^v(+pX)ww{ouLtgQ-Z5lfacqQWnw(}h3cVAm_S8QkBDszTz%@( z%9JodxLI;09}hv0Pcuw};?Se)p%#1hF;S=QvDpQoS-ir&-S-Rd>Kw)6@z^p&5s0Y% zjg_!$*hc}AXqo=69+fZOm$-TJXfDzYK%;Pi2#(s!Mj=ldvj13_7fHGrEs2o@v4!b zbH2}Egm{i-ybcGeYSPOD`y>=9c^+mwSfwE8eHU3?u4KCCkI&R1Q>ag_K%HJ1Ac z)!=9vUh=)$NbNR~`4&T~(c*)u5i(bPOS&IrtW@W$A%5`KE0~d4nBKA%xi{KP8N6@W zVN{c}a(KmOD9tY#ObK4Xg?K1x$wS#7zGBHKmw_cgn|4ojZ=rF@!-K&%$0D^!0MM;K z(UHyN+-QejqC$k2&FGet1&HHh)*yXUsgjUEVesLFXHj8U+YYjfL&G8xS5L%Zdl(dc zP8HIW@u`k(?;VfcI6h$X?xfBt{+e*+DEM5_RnF%&uer6yy?eh#hmFTZ7?>?e(Y?uk z*uC?9jdb?m@JMDlnB=bxsUB#YH0)Ecy0f8Z9|aAtKYfk&Wwaw2l!eAuVlT`A`&CyR zf!#1r0p!RahyQl&Gto;M0QS&7MmLWt+x1M%O_=MyU=0SKKoyjv0LL*DsEMAYqcHL3 z?*o9q;6oDE6aV3>U4&=+69%h7EaHqx92V(le(G}-3HUL^IMv74)!2GZ18PCe!4|A7 zX3n3+D;)bEbB(W=M-Uof2{>6~&lw1U6#;r& zlx$bE46y^mrA+x8TfesD@grg*@Z1%!c0hfFeNI>pUlLfU)y_?9Tbx5<;r1py#>0Tr zUPcyj_6=w!79_wflv(tQ9Y5aF(E$F5|CT=81MCA`V8GCF;Uo;-mA z8-yUXZU>Q50CmqX&xAoOeV?4cU6_=>7?ng|tG|APf9U)rulA)xfL&=%o?lE z#eWTA_k;$993NA-bRMCH%D^BMUp+xjU3AZfE*rjt?*V$lf6<)z^E3fw4j+^vIRSLH z13i+DEE$)A(=`DpUW<@Iq7_Io=7NnIqn6!EFyw_T$#)Ot)}iaOS*5?}f1;yo!D}<8 zWDZifWv3~Kabl(g?7g31V_HEykl=8RM5F$d;Jn zxUl8X?*gf6!OOXpnRGJ@9B&q+Z&b zM(bEuwqx#_1k-$ZH8E%ZMe;EjDKtI-1W&)_XdQ?Np!T8Nc@;vI@tO229L~~nlM3XR= zE&*R`#Xu90*o?ZtvB&M|%TrkESP|iK`YLcZSe5!0V$O-REhCqsZ32@>V3hQUlO#>S zU*@(p(x<@Y;6L&O=IgoTNaynZ{-Oi=w^PRR6G{69rLWSrf^%C1rf?va#zLem{2>|c zR~k{o+fs1L1X^Y64Y!z+i_TO(E{Ow{Ib-I5Y%DrQ^FZRlMFK{k4GhbLoLJbcfSbip z6N@!iK@CQ%2a;*bF)88yW(!h|O!N0(prJJDD)y$_io?_AdwczJ%Y~!-t$|5)cWPh<4#eug#A)cpkaz?&VfAFwS`ZM(Y6DtW~n#T|Q&?$pgzwM=`BS|sNZt$$&a#J758inJdy5vuy zvLM;lg!3QRsv5WSt-L1oR9?KBQy(wczcFLpQQOaLLN~o!Q^%>dKFF@n2|~T!I8Yt3 zqqGNe(pr(uzN~K||JP-blkv@RcNA&+e^i=TE-SmyX1qAScdxW%?Ur7L#Va0i1}@(A zfMo)%6C9_rsw>@08SSxCnzGO#8M%o?NSlue`S>L7X3A8bIwFaoK0ThfdwBBB)I%}8 z;zPL0q5cW~OUdr1QSbgXgsu1Zn8;TZd!xzS)3AJ>CNXGo^q;2u)}^f0$V6qF!sMw> zRGa;#%KaEksVli`E$Q3Xec_#A`;RfV|Mjd%^mW_sV3*{8C;QIatd&2zy4oevD1U|j zK%tBL;q<1Bf07=K7Up`tJTSZAreUx^+!i?{+Ud2Upli?Fu?L;Y4+adiIG(@$)aSdZ zs^|r`1X;n$GAqvHUTgL8MB$Rs7AyKjzWsdX_a&KYeO!W-Lz_Kghg$j`rW>z$*k zFU>oyV=oN8S7%_?@kQ3UPLSXu>y?JuuLJ(B%73Tn_qc5+%SntZ z6sl%L8;Zu?nd*F< zj@EqB4LLkG*bg*ZQ?6Bt?lL}T{ER?8`4|dfR3GhpqFx_!d$>(<#@c?Z;Hg*O+rP(0 zgmszxP=~PzcnLso)BA_PpNf5XW8D?3&+V3O;Pgj4iOPPowEeFz3zPF)y}$3|+zW|o zh_klxU)mowecl)QGe9Hz*GM>im}8*cAcH0MjgQ*wzZxXYsNZeKj7b;63x8GWd$ZyeLBnuW$aIqHy2Ke5-!+8jO}== zfs$YMB;VjWoXtNBbZLqs4O2Io-@EMjbN6c_ahJ#rWzjUEzXc{y6S8*QwARcSJyCOi z)yZnf!;A0+EB#p~aKf;LjLFIq#Ks0${3UhtP$usaaaUz4+yYM)#1?HHs`Q0vUY_-6 zd~<|ys09cZkgq9NIP}UuIU)k3*d(tuqsJeBvE`oq{F^0l1&f67=kd1#BWH$@b z-8=X<_bUqFct~`pv}wfhVoeJMVM?dmByGqQEb6(&f|}(Eh*vfM8UwGJR^wqh5dH6KyczV*HcOfXP|69DVD`>Jr@rkRU5T< za(e?(2R`aw&&gv;0-aiMw(!)DC&3GWf72xq05QxPhx*m%Fbl`MHPhJGr~j)V-AafD zBHXd_jqdYvnL4mcT{qaS2EHoIWHeSRj59E`nj2OMxP|0Fvh%2tgZ1EJjziL#4jOL{ zYXHqvkEQ1YU_Bzy(T04hfC}Ok>rXA;n%t%HWGEbGXii;HK%RYpAw*iPM%mYBoLQHf*9U$3`t$lZno&Nr2w$F=yB72JQDV#>m37qQ2H99FT_L^6# z=Tq!67Lj%;H~`;=M+D%prHHdb9Epo=&B1sNn664uKCcj#31G7H%y3KTTK0wDU?d?pPmWq^G@jX)F zf3hHkqd-3mM_9{16Sz~5m~9EaKDqS{3bxf106e-A2fLQQFcR z3U%bNA+4xJ=olqiDAU^jrLb+q}jdbW#Rm}kpI2EhlNG9SZ0kztU5X)cIVI9AHALasVnyn160jft~x8jOOKAD zZ<4K(6lz|=?vNm@i{JywEfW#oZQL^1sg#NbBK138))lGF;eampAx41S_mH%5y1R!X zp2RsI`gp8V7$4yIWfT84FL@XY@A#0voARP$_WsLF$c$9G9GP~Fl*O-lvQe`^y)Pp3 z%(|`BlRDp|q!_32w!OHwhjG!zPCoT@#}Lm^mT`QAGAlUw)5~hVPPSO-@Q?PA70Ufs zE}dqo+&gZ~_{6E84tHmZjK>`NV(Wdz$Ark_9ls#tK}CPQ{v4TA?oe zq7P;sPzRAvw67b}O#B?k%qHV+pmYQob2H_5VkXuLGXDmu0niu_O8OmxvN7PnI=}?B zGuZpr^Hi1-MM3cCJe_JN6ZFpXgo_v{04(3|&9F+O+TT#`l|}@GlWr%j(Z2-|bWd@5 zTH(1(t_zG}M+DbA^aPi8UQu$NFw+D*%nCGM_zgfzLMKkt+^qsZG28A9urQym=$O&o zyiWFo?=Qkzz+eEKVLe2lmZ;to@;`@zjw+h4AYGmxAFu@0*qjOhQlNf1vQ_%`1aA9x zua$=7X&i_~O2~zunH_PaTtuQK2&kTi^el*E-=@jucHQifol2?{_H^nD2=1D@e`X3n7)hAny$$ds>ORO`UFV{hYv z;D*Oc6re0M_yM>_(Y1#s0tnAXkLkSZ#1`Z`kn2I0Wf&h%PzVwctU;dvjX&yXRPZ;j zBvPQSI$Nwg1>lU6_;0zqW8Z8M7NPZL=-%Rj!@ve-h0-N}`Mukr6Chiq0NBn@tJVS1 z2`mJ}Z^C4d;jQa+jf@_kzJx*fy%R$aUO@Vx(5iR4$lt^HW+@WZFkB&Q0pZ9;Gh);e zw#GKCA;ixxTwqM*3`yWiNsoGrnuL@pfzzZ@{n%tMil#O(fPWOgUZore7yAw#F)-{i z#<45VqlX_Pge8D|O6iPtJ)|>3mE(932;uq(Lt?0Z0P;Z!9Kt@~n?$RPh&l5`#V!!Q zrzyBs@u8vO6A)tv?H&0-&ZCQjF^CsngpYGuOYegnO(|6$!YR>rt|8`_i=L!!hvNW4 zgpi8B6fPW48qv%GllZWhraHqBP5qpcgLSPQva+xh8sk4B{T> zG~n?e7b-f{pwM6f63mbggq5GWLmDfL@9lOpji`n}C2j(sY9xJ#V3epI`OuSFG#mu6 z5|fm;_*gnPLRdE7MiwYmW9X|IJ}h_|`100%<|ZcbCHadMh2*$nVM_y zUoR^dV#)e5a2L!K-wJ6eGbqC>2Id4&tm^APcXUhA-3OqLy+c2p0F44NK|u_yA-Mvx zeK0f&J{xwkVwwxVx+Z!^zul0M0XhTRxQsGuPEBKgd-VC4f)?pNC+7c6UFiP~na&-& z8<7rk-+;UKIf08i6k~9K{=6YPi15I|eT3VEmePDZsff=Y*a3pl0H6U;95~f*mIN_~ zktnb?_?QVuO46g<_8B`LEt_5WwDWIp)u%Ql?fmmXGRu_v|0 z@;9=|QFd>uIcO;z<hU_3mhIw^8$FV>jf+?SD%fW>x!c<)$l{+RUI`-{H6q<|4zgYe-j`zIW7#NQ3w9 zl@`G2O!v;Rlv0YEYj(3}H`}(YGo`66F5+-(>@W3>!ExdDYVDnV9GL#6_Da{YvEX^3 zoX)?$HX}}wX6kZ>{v*&gHb)7?-4IN_gr+i;@Yk5VsFg?((Q~G*qvrk8S#86(O zDc9pKIT}{Wy1;r_$@0I~KAjn^TvETw!X+|Q;}Xj;+CNxiSYn}S^Fj*L+wCW7Gcu+& zAC6XW*Oi-FuC;IV{p-+I<=a{JE39q_yQtXWuMO)pEX5^$whwg6Wp4x@^;l3cX282$S2TJ`t`!9z0GR*yddWA| zcV>!R-~8+NqC0^bE=DHFGtq(SWd*x!%5RO^5TVVF;lDgkx}?8+hr8sH;FH?Bxs@HK z*nhe|?s%eirRI}k@wW0-?YF9r)|a)GWE6fVD*6@J3B1H9M; z(}wTHoE|M>owfo+AP$w6N|hXyBCF%!(W9<@^7QtC=}r8$wS2Dlw?NLs5|Zm(s&M@x zD#)yr18T#p==}qkqvBt(QUi+W*^gFj9Oo9U9790h@we2jNjE#1R2g#X`r*BX@38LA z7Y!ag5Uu3j^Xb0mf9dO7Nz`$!#yrn4tnor}Vcw4i!*qfJo*RztpJ)V7gDZHU$<46> z(0t~1xFLU2CKL#}J-29+563F)f9G8(-ah*CwE8_nAEx(fV?TW5`fo3H&T?-U?)0&I zP(9I+xuT_V?WzgmSKWm(i&2VIaqDsgc`dS2Ux#3>=lo$@B#mi_?}feG2KTA&4}5%` ze8*g=&S;jMJX_J&6dY&$Xdx#_!Ot5!?^BOXD4sR@OYX2{kVgy?M6H)9rTX|*K#QFk zgVH>>;v;OV#vAqWGlwvBZgT6vvc9eYv6LKn7Ff&bx(V; zfd6MaU5CH zNf$RHvkg@N@j%6vfQ^Wg=Z&}nPkt(QLxVKf>R~^S)IT}8cYF^rAS#RRs^a=(1aJ;w zmHkv$(;%{yz7})=cw;m)*mG&}Db2!;ZRDrFuEj0WnNu{zpM8tpzahVaK$S&g1hU#U zz>8)oAgaAu<5qKEIACn5=L`%t%vz0S{{&lCb@d4d*A*U;23#2KVcqfQ;7c}Ğ~ zI649(m}0pq?&I6ZgDspAuI?8+(pj=1BVQ-YKL566?U}mb?B?>lXV^3lB3$9hK^fyQ z=Ms<+xZa%OZnV;=RLTj|IEDmna+q!r>&`l+gamTPl>IYVU&`q<-YPccGdLOXQZJn- z6|?c8*(g--ji1{l9RUuV(8z7YH`)VYB+bKouwE`t^6W=5cy42wqTetpf$h55A)wX6{7aUCW;{PyzI5+yY4b7k2nJ?Chox|tV zpz`wtk?=4j?a^6u<+Y3C4bnjw{|oeqZpZX(9A-rD!WMRGt1kWysXmhjr2x@wREF~F zsN97w#Qc)B9uuMRH|-TnzQf7udGC+39gM*p;BpfjHT%}{7*`ijpza66jZn*tEV`qg zOP?`^h^h8thDFRGDlM;K$P8K?_XHVW~LKuD-EEoLMr zHcp%lB_&`1Sq(_xHSw(_s)B1%xH=O(|~{K$}o)Why*~Vgh7} z)-J=tXE(z{AO+!>Q{C}zYQ{squ_@lcE_a*qqxntDW5cw7AAm)gAZ-}65#dF<)i`yS zG6x}n431;Q!@MJtmn_XLVd-l=FU|!Oh4U7q5d10bb8+qLqQjk@t!9^E6DIMT-zucb z6o&Lk{Uw3WJdP*OZo>2@ve1F~ul8ADYdB9D^IDv)a~{)p=NN-U>d@xFIdgs(sdG@m z!twL*Tn$dphUw-)0n2NMSsb<%T_ad;Q~j%iuZ(1pO0M~3YFKTOZP~^L3|uO~`T(JJ zyVDi@$m8EYT*OhN!6IPhNT%Uq4BmuVm$rE?x}qO}M|w-eO<~g@&S30nOf}=+$_~=@ zVX}7w_6B>>aJvUz0K!GKI#r@B39DXU*GWy2SqEVB_^Og69|e|-0mV{HE*{nj-ZxM| z79PKhXv6@uc~3)$5)4u3c_9)nI6ajn!f0q7(+sS2PxjWCNP=@VG-niL`L|PcTCK?FD@|=ciOwR zHQL;0=Vgr)CjIqNWt#uSA&VDfOv+VX7rZR_2a5~yL474B%#?NZ&GNi>!To~c$tRVA z$0ffx=ULeb^2jgfGY=nE{#T&jpSAK82tZ0VsfqhxZ^E1~n-N05-cV)ijbb3ck!hzdu$RDErGZN;61k9J{e-sO(nYzhBn%kbE4XC$X+7Mgx*a6)A zMaB_<#wZNj^b2^4vXX^Hpx@~GuLJetobI3>fNF)55T7hK1K`-L6S&X>5VJL0KenEF zoZ4(T+yN-@!8BrTc0W-N@0&6-|f^=?^CKZUN&M8&rbN1d9xStpw|AC>%GT8+?b}0XzZ& zeaN&Gz1MJtq+zln@kj+=BDP>o!3}^;@D0TYfgnAnumnkfZTf~M^AF6?GmTMzqB*2K z$8=zd2tNk2i4%yE1$bwpUZl0J`4a(b$1Ox9^nZ4aGBfYs`p_%^L%l2@2np(j(XXuH z&qU7I-_zhY0WZ)p2859yGU3;sVj?e{nsCBm#ufi94j={tY8X!Z`8lBr0cJQ7NRnso zW4E#X49AAUB8r0`75+xQI@IQkO`-f{6cu8C_fC}yCw~v19fg1-ouI{s=yyu=AeNN& zUZ7EWv+H%FIlu=6l}qO>woI)<)&XorP#@D=m2xI^*79&r=$$`?IQyX)bwkn`d zT2zQtCism^oSHk(x_}zfflhCr(fwgQO0$7lUK z#eyZfeU9Nbe9r#QKB1ZlGy~&ioU@zZASmmD8D(MQ75I3rh@pi~1buoe9%vxwl)U2iS!PU>4w>l<(fsTC2BD z!1~cAXR>f^R^i?!c?#JN7=R=J<$`M<6y(*+V|X(L1w;KYb5vst=?5ZsZg=Vx9QZ(s z@VSBa;Z%!R3lPx9j1^!KF&hQO%5W1*J!!b&5%VTQ&%w{}WyYlhkE3#OgJAqX9K%wu z$_H>>7!d|BA9I2iya0_W;Rxyakd$Wbqmp+7PLsc0BcXp4+NnU|;4IUbW`2=K#bEa~ zJR_f5-x420@O}>bJHp|@vH@&`ez*tc@W1#7=!$T1HsSsM`=6gnEcAK)He5M24>G`RZ^}EOz_|2Kuq1G0S`x^I1|8D?RLJj*V9Gf+eh(@e)SB6|Y@g zvh*CEGz$MbEwzwbFK4xE7wFzhA%vR{W^Ws#*{(K`~>B5PkMA1xC z7OUdIQML`M>4I>8wXeAMcx~l&)uM-+W%HEA+2guok7aq^!nnG%Ibgc!e!+6tE=*`D zMe@Riu^8V0{-4)GT>4oVdt}q69yw=*eG4zA7C1fhu+F~`B>Wr(T}dmcy|Y#1sG1&s z{ef=vxp#i-ZOpQ)&g9hSq)MBa=pP>{EiE5*jk|5`GD%9Z;z&eoVfG%m|*$Pz?%eqjBJR=G*G^lN_i z6J9XqL(NeE$h=go#c|t0E= z#-SoN?2626FUd+}R++jMj=Zjf^&m58aqMqA;qX)+>eP!j%bLpSf@{_cWAi^S9FOvj zWjnCvOi1Mc%XuHuqHX zP<3%=s+(%G%TCw@KW|~SyY@=SWhc#b&ZY;Q7prvQ8t3NJ^1PSAlp0G`c55SCFYA=X zul)?N1l6(M&uQ&%jP+bOb9aI7is9~rx@Bo?NjYO3BVYY?*k);x@|TLT!q&qVa|{~f z?`1{X*k*b@Y&6|))E9FJqeI^D$u|vs6?gw% zom117aFB3Xv_5&{mBg}qUK+1seAKHf;im?LDDO+y#PKp4il`|p3@PgQ!K`r&ib!RHbn241uj}My!(A)n7xzgL z5YO=)>UV33UjIv?uVS_RO0-m8VzsOZ5rG2oXsQc`pgyf{<8N>=O&q<~3t(0syv2;?g6k|?pT`=n(4bWvD+Owl%2p%0#=TvjExpGTm-VpJFkfL`A1{~#J_v5qLg0) z8GkvyNm+KVBrB}U=xc=MNyQ$mjfZiBmA?6VK~NdHqf5G13BiDJv&Z7N4+Rnae#$h- z`2G;Ne)hFD+TEIKkfnB&Dh@G9`J?ReeRw@W zpngA_0_dL?@+C>!OhZIPOamc3Hcs_i zn28d}4udF|*eGnuh^^$z!0Iy9s6c-?RO1<#&EQum*;UG*^-PA~Kz>L6l&@-i0iyrKcty%9A*9ec7f zVxPrl#ENQjHa9hj!$P)6~0e)8iCE z3W&s{QwtaH+}wnUq1B94T}D&_s=D|#cLfb(-{oD%9+?t|8xkZC&Htyfk5eo>fzb2g zm$AMuGpnqXRchZ&F9XZBr5X-^zA=RHBkCC2hw)T-8QyxTW}dP`4030P#g(xo9!n!H z*r7@8rY>s=wFLO7q!YFw%zB;spveWgI;nt^=f_?xkD&WsIj3DEWCEF0nf+(j6 zKb|hONF+6f8#80OOiF7{aY3>LA4fT~?l3Bwo^_YeP#=;vb`03=t($~;%{~ZD1X}wi zID9r-7<@2{k86E&jitA7uU?};$pR`3f-}oxP5oXSMDSyHcuRf~y2IC0^Ma6pO z`ZOhlL}KYxnvtXH;xLZRULN0}m%3ONx**&V=a)Z_=K}G{zhIz>uEk)QQaII(b9TxN zI$HL!X~W0TBD{3jqdWlN&FNroDQyHl_nG-Z9*7#$CabPbOzQXnjJAOaEl^?1cxhS( z8kUz_QQ8lI`)HwB)O8h{FNr{;@$&ZUGF6pe`_a@YJoNb{AqrYXe?@=|{Sj@jYbzUKcWSb>guC^*My!6NLsOWs9UH9hydq z59x6TRuR*Sx5;7n6x5Cr8LoQw38h0`p8XKSiCXDpjvblQ zy(>I;D{AKS^rCW0Ae9#nl!Ty6J_30mw$*2w`Rb#^g?U+sL^<~`gaO}FAZH!_o+@t#dsz1t_-WFWMnQxH#-u;cMaKs()C;d^!e9UD0Qi{O24nBzrD^11XXefB;ZK)PsI;5+GSK zbb0=nkdZr-B|=Qz18T1VFHo!8>yE+P=cE}C!aYn@9!MPg7sHN>(wh+|!w)T)^V}rR z!)$*LQs^-i&9_4(4~l?+G~}-zq$hgFSh0ruCNXHhB;YW>cr|_M7Z{=lM*{ekgLm}i zcWV+jV-|?r!QF+)1iU%2Y`C7W<;z@Vki%aVg>p~BoY1#-kqp01TCu=Srq!k z_{$D4eVVGC2BZ*}_!0=G{TkCYbZKhoYXgGIuFMn-Vg$fm`3C5JeonN6Pey1KIR4jf zXn7dU3218t>_r9~ngy6m9x-17k>q zBf~M^BO8dLDer`Vg!3!5;MmX^;(%S{&wh`Q{=ImdfstXX4_q0CQ|glkA_pQ=WEDil zt&HVKIKmL{nR_rvHizFZh;|MOn>8`1WYKT~DxQx9oMtsmx^wdsguV5=1N0Fw5Q;IW zDMT(Y;FCX4O?Qm`5|O8aJ|L8MAAM^a;(r3_Qr(p>g3-DY2*_qaM*%x&D`l`fLw3;c zzven6(=ZN>dXorzKU6Mukr*=3Q&do6K1+fJ^aXgp1&=I7SgLocgjMP%WS; zz$L+&ga#_`xtB1BobV*355$;2;hS;vSR2952WOb2!(_kESICuwa}xQb(s>Ive{TAL zn2_hMNk$UHNa)E78sVWQ(gStE{@+=*^oW@wR+zhsgbTQBo^KgT zc9VJJzsU;RypT!Im4`gzzb7L8zwkWs@3exkM9^bFERV>Q*p)Pe0raja&1^uLf`-!= z;sLA?&IHIyX4(dB9}8|W7yCA*An`$9_CT?`fy!k#+hBnl7?#+P;V|GjEkHd6MUNZ& zQqyZ0BUC7hn2*iu$&5(i`QI1&bxDv^q06m|iXTq-7z6j?i8==>WJuNB=Nq@X^MjB$ zS>5a3U%XM*>#u6B)=kK|O;S3wKEDlo>+933!_qjPI_}FsHN34Pi}UlF#bae!d)~cL z?yCjrci;3~=r(TD*MvJX$$#K|Gv9o7L3ZoY+Hpfz=_q&H1>OH*L12-+Pb3GQVl%4-gb&4z2Ro)lNLdst5$S&6SBpZ~l2A0}|SAVmk zi2I)LpUIt}OTz($C44OE9FnR&Tx45@IzPDEO#9j}?&#ZU^H1u~l#V!zhl|4E8=q5K zf&&HFXPqF2CDIyAAGH(4FLWoiKK)6-KY8@}=(eR7{B;gP_X`(}v^>yY@*R*nLK4R= zT(JMNCR2A^`5=Mev{DA z5J8L8lx5|m1exk==Vr4}BVK0KH+>e4Tb?~dXeHerD)7)UaX7qgx zCrG+b;*>Y=rDyxnBF%x@t-Ib9Z+g4oW>2tZyL(IKBP;tE^a5K76!~`fZ?*()uRZqO zdW)lPfAz1zy1gs`<{Gyzc^R_>|Hpz;!UmP)g>OURrfDG|6AOy+DGDR zUT%xgU-B{Byi=e)n$hu^r@QLZuFF>KTNGJdJ4#P&fAdb)NL+W9)JTx)#jh;!YXg|R zP_nJdRNxmIp=1Hz+~BezQ5jxu?1^so^bZ?7bpO(2t?*=3?{RGbD&8kiCt-GuiF%Ho z#xd!oQM-%1(c>Y!A5nQh^&$0==T7-ZdeoZ>d6-IQ8v+=c4M-9V-MY4EF zgo{F`y6jzx*0aJh0k5?P8d`Z{VV}0}Q_mEw5v2Z`IGAK8(%uu~`SQ}`Ee+GkqhLBX z5sipTc>tMZ*63*NrB{>IalQa=;QO^G>5Ksq2bir8?B3<5lrV3)O|sgJhA7i1CRE z)mjd3YKU1hy8?sE`K9+tHIC>Q_W~Z}eyMmbWXJZXV)0mms3{~QSXi8m+Ep;LdeDB` zGtKC5nOr&(nFS?9$e;pQY7NkCka|OeG}=bQ}V`Sev|Ly5ZZ=4 z;~Gb05gi zH{6;jjh7P{t%YGNt}x>4_uTD;WumgY3U)C~NTnrFK}IZ?sdUU4^dHigZqztyV;Z+~ zZmxvihN4AU2X<4(#)Sf((d6CY273r4fFungbkRm-i<9}%%GuZ^zNCcJ$~iFV<)1X= z$9ZVyr?Uq!g(CkU=W}|(DH78m&4f2*IVDq67v7a*U+BjSo-i8jc}Ll~LaLPu1=q_L zwZ`MK458D5dRoxJe6YqccrsEK^w=K;9*fUX%;!{w22rE=rJ2ZJu$S>}3e~58m$SdMF4y*O znZOaiSdL0X6(e^`O+X+Aj+%g~6H8`SWv82-LTt#vCG*%Q-B`X~-&qS!EJl)me>H(A zkI~*Cbl6-*Vf77RhA!>|mOc<^8}`cUvUvQEMcJ`TV+5?hG2;#NPF+AbuctZ!giYn> z4*iq`JBN4I9gZF#=<;ktR?&DsgO!NPfP19w3g~}VL`7aFyPlpHg4J-=8@0UplL$F` zgrA)sq$3Fem$iNql5Q-q;HS5^g zp{>?Rd48^B(BVGihm-6Qn&%sfsz1)5 zvrL}!=`a`+NDSSv!9CcJ#!FD<*stu`IchX0JV>`lVvuy)|8!{ahiQCi2|_*?D5fuW z0ptp4ay0|<*FY(lQ1ahF=nS1m0Rtme>kDOcgM zeH$8{`wcu`E*e^n=G{7uNRsqztyuEOV976 zP>uy$eFX@M3m7XD8p~r@s_pD@rLiJmaTp7D1fI?jdyl=$>7IZgXadY!l;&4{RLb|O zkFCRnLz$v{l-}+tNwirLYU`|Ew& zaKRU-p@iX&c4&;kEsnmGMW#PF>jjGH9|Cgg3y5b+NSg+EoRL0U6$H6e{jOUxU z$+};Xg&|`HB-(iH4ObOUdj4TZz;66su8@WkpkTeo^o@%%aQ~o@|H;BrbIJt9BUk@u z22h7vj6TgU#SOxJD)rxA#G34mVAvcg)2?DL=L%Y+G(do*S4a(K3Y3o?CxkI7ePy({SUzn)%;BnSMC4dxu_{@eL(x4;l|>kLIc{ONT|Wnl5*m1_&EK9l~=wskbsn70Dku z`zRd8#_U2p6QS3Li`0m=d z+yutX6MLOLS7$DbQym}5>5l-YcahZS{xf4r)&rI$f04@L0nIo-|EwV zhrrX~_G6e1V($rh1jnTmy>^CPh#`z<(-FZ&Sz9(#4bJ?GiYQ{(e^NIZ+o$nW=KjF8 zf<3t81w!Aoz^{uK%M>sS40*)dK@j8D0G~3K}`E)Lpx)Thxi9$f`DdBodSN(Uutsazv=m6gZz>RyBiWjzo zC&z7qc?TzqY2tX-(<=b&>tEqxN=w3*bFw8d7}x6=F7zNE>kQ%`@FSl=Q-r^O6?6jK z`21@eZVys2^euVf;)oZjCDQHr>oepXZe5X_a?m8o=q8A@_jC z|1g*b4vo=4Fvms5#{6Qa7n;|=^8fcgKL1bHPQZwyYXx2t#N?FrxqB|NhDF z5_2!UVClKJcf;({Zhtrz?M-iSuia9EKQRzNJjR?83Z6mM3akrzz@f%W(+J|F7&Dh8 z2WcH)2SD#I94`jlrJhN}Mh2+}@g}%2m(bCh;z&V5Kq|!OrRdhjzTr>OivK#qx!sW& z1-u#@maO^}+tH)@2P}1eRbR0&p5<}<#$O)uY4dt{0%~U{M+Zwk{X6HQPU-aDC67!^ zkGUs28k_29QgFmr*=-@SZaGVZ{v}eiLzSN@QoBwx8^@LI-N2qS)cwv?{nN4UcortI zx>~gs1%DX4Y}oZy$%c-A%s&jw+F$QX@)BsmvtB?hd4JA|=E|}*_b1-&Zr)G5eA@Xp z48QYiUtE`wWjp^T?z-q`9Hf%-xG>n;r}D!wX{$i3N&HCgxbV+=&*pwtK9ql=bfe+x zdo+n4G6ZRL;|qkdyB2b~yRwV_C(tjt+iI(dmBYJOonQRvjds)HZbrXv;dsf6y?52` zzE^PG)!w)TF=C&N{>Q4i*h%xlfbQk&Y*Yw>#|q<@q7I?oy@!e={*Foh0*|J) z;4(jj$oaC5xy|rblW&Ur49A2&_&c|Y?XEljEBX$q@yS*!gWEu8TUe;DjOP*;#h*QPQh z9BF5z*-#MgD#`jP%p}4uOI)}9faUIA-`HT=6y?JiPXEL>NK&&VW7MjwZv4>Z#%({g z@V?um>ngF|@ZFgEnZIx%F5Vg$8EhTSi#!x>`@!=5FL>EK$v5 zQA+iK3%tSD{Y(A++PFpV17{Yn{eNdISL~=;6Ov{0Ex4n?;`N4=TwP<1%-P|HCFdUv z#p}9S&-&fKO9kX9ZXfJDd?9bUW8bMK!5$|>kidvuRs9gi>#T}gp>xnR`;?`r3Kdb0 zI<^*@2VWU~bguO6u?DnQVh8f~x0gC^dY}#dg7>axLKOGMUb_pzardXZghxppzo|k$ z%P%dtd%gbQg-z__!&yE}TF^7TR+bILD?2;>6jk2oQg*uH%lrAQm$qTwG8q>(3zo#X zU9Q>_67Bre8ij)eHY?oar%zOo>cUY`)#lM1F4?ba{~z116}nep78g6zp_(pCP2B962pf`tk$*j^EO%XI18S)w z_b?)Gd7VYR`N4Rxo2Sc-_PPbNNVM-UNdg!k>KK=MF3X>NyG7DGRkakv*{Sg|2(CAX zJVF0SDoUb9x7{{6`WSk*rYYyUvWi=ReW_ZZ*S%6a0v=0qipl@Y+NP|#rGpvqxa6=% zq_uC{*nb%yx6KdXxf0g+lvl~kj#jG!RsO2{hsoq>VUj!>+Wp(nb;UH_c_$ zH=FypDm;B&<1Nw{Yu|0@DlS8RJx>WYGz;{+Znh5nfV=nNIL3FNabIr+@uomzMnF6B^g&}|~oC&%6)Nm%ag$0n6?a;L)^o@+NB^B~;EEu_$wr5>!Ick`^ zMa}U)dtgJsN78PV#go`XL1 zn3iZIK7bkib>(~=5hG^0^j@Wz8L^*)Yu&O!*G9lEazxkWGJ|>!Dyzl9^tS%x6K!%Zfx-`<1|NbHE`)nGH^dSzj|qg? znD{00geaM`ND76I4k7w&x%ac;Mt zWrUeVK*>SFlC{*R$>Y?pe?=(g)mJnatbm>$|5j)$ZqlCB+KESo`#rxDy6EsQ{@CRj z#sJ?)9X`sQlI6X?`orLzN%V`Rn~`Rjqjz6G+Qyz+3*i<-s3%B5ip4pfvv4pacrKS# z>MkM90#@0W$Mt_jWz1nREx=mvYt9A#J5_Ixlrd}1v8v9b^I{833E#r2K z|BY_z{`c0dG>*ih_hVb~>a0k?YWBw&xw&Yq4?6fvZPm|=jO$+t(518{?1H{aCHhI|3EKu{K9A|I5P5_F^XE;N;!T-^ZX?HR*aZH8a zrmJ7?h*r( z72+;)xZM<@$`ly8YqPa~;=Z?x>?C&Nr$Z_rKmu=BPcO0vZt+`l0|M>*HUH?A=hlRE z?SjPzI|rGhYI8UY!Xj7?7#1HwDobrZT`2559zVh;kgarJh0e=qR`L*-2Vzq_l!@DoY0H=8B2YD_FwcW?DAw28%%Y=8n)<D-uypZFdJra z7~hggXhaX^Q2^sRAsGOc=s`1>>7>JN4{!+0THN7)T=bvlAk(*`gfH&_Lbf5ut+%QJ zaAj&`{E4QdVm0H%LAvNvG_GqMd%}*n*P;^TkM}sDB_;A$MZ?(F>lt*Rhvj4~sWO`fFT70+nqWAPbgn7BFLq^1cLEeClp$J+CH*yfr3l4~@V7F3StvA;G4BqN9G*?~7 z1B}QA8efM#3LA(1Q~;25JOJg@(+5KdtX^6j&^;I`xPELgcwQgnLBE5jqY!Sb{0>S|7A?Wd1tpTJB zKy?bHtp76{F!gy2Xb)*1FhiIdBM|sBbVm;9I0dx1&#y=!`Fb$OCCI*+V+FjyR0P3& zMH0sZ|a-=;XQ_tg9Vg7L*As5U&JS43XKt8~5NCv4w z)cMj}&>6%&UF|c%a2+xNIa6}So<;9Ne3mO|#bIyNlU|H}0s7~u!$L(5*42`E3P|3t zenjv085328<7D^(TsV3?W_gjq1nbr}0P9vS($`OkP^yUD6A-5bWIU-?@EzdB$QThA z8v|$q%oRr312%o&UU&e~u`mw_K>ax`gSmAWg!r;P;Dr23XeNbWbMVLf5;sO+U;@*E z6s8NlI~u^6Bn|GhSg+KxCn5dPb6~FY$lTE_>yTwFQ3&{dd@7=EmO^g7Ko3zVxgix+ zSKiU;-FlV;&!*tZ^CuGMd4RQ|zwzg&Lgog=Z@3Ma{}N9%f8kAy!E zw15WGuY1I)?A(O4MUExRM?HnF$vlKhn*qM-dEx|-UXq)Xef*rVV<;MNG?O{a!G! zU-%RlDm~0}?1QVz{)rMc&C9JO-{IXq44RGoRW3Id;ORFKTzjxEetz+2=Twul z3DvsC=s~%lIlab{_5JNm4erBy}^REbqa{fSiIKUr=8 z7ktSSipAeyykf{#9{;-v)9}cF3CpFDi@V=7+Fw1n zAo!N#+Npp28|z#7#x{*qSc{Y(Cz+s?}FJUt%SwTG8#akwh{yVJ!#4C+i9GUV%@g>FV) zGAuH%i}T_otEHYAd!<%uS?zdpK*KhQvHtRapMS9OX12x0mUZRpI@j6%{o~$UxxLn( z-zys0{$sFT(7J@`@yrx$@9odmhbiOv-6L03Uq-}6UNhTlqf+Iiz1Y^Ypa;D^gd8RX zw=|zk+??$tIkEndIL#|~yRNHS?vQ?a%(?uM`7&AeaAZDbV1qF5npb~??MbISfx-K3 z27c>}C)G60_bJ`mB{Hilqo#s4Pzole)zn;k+!O!MW}Pyq)GG5s6!u$YxLYRf1{awD@&T&r^W^G$2l z(<2MdMjn{_JYj?TC5)JOx6bI!f6yk#X+5qJ>|&eMa&kGHtBS-Gk5+Bk7#&^^EkCfS zO)Hd&)*|9C;_S6SozcCaAA=q}d%gd#&g5y-t?KHL-VTqP-xD&^9L^+%>3)dSbPsvm zKOc76Fm^CnkT7dLHp#8Gj&Iwi6&}9T!oN8#$Y~6IX8eNN%5s<5HD|TUAuRCljBz#n z@fjE|mEe(GOIv4NeqJs$qU6q?z z!zEiEDJ43GP)mWz8NJWX?wk50D2#mS%H0Z|tjUnf@-P9zdX`vew%Ieuc$njVU$Zan z%DxkMYo^AkRonbrF$U~74Yl=IRkrRm(QAQ6)#h@+Y$-Rxhs%ld_pw`Pp4xwuKOqmSh%SXjSXScq&@RTLN7O0`{uS{L%FXElF*ou10G_GG+5MoVKDKpM z=?cI@#5!~|6@NGuDVH7fEz>MD>1Leao%6Xj0@;6E#yRna!QIi&vhLpE;kEVOcNQ(@ zpDJDioL<85D#2-pz1UP#JIwESTrde;13(})hS<=V@T~Ikjy<}{Y6P%-Z`mYi$QX|Q z)vKiaOvu!zV)Y;bF6D`7eY0+%8FGFNBr*lnm+h`~3-Wyi_yc7Vav$e(UXjyS8TuP> z-rU3KnD79U+XamQe8Xz^cZW$dAGT%6JxDDU-0<-kT6RmgC({=Oj)OV&?yz(pT|V)MdU` zG7Q$CI!yZ+yBSCxHUo(ABTSKwF?>N{%i<0;e#z}1iAB~>r*(eY);!!tD0APbT zoOM&&aMFpD&c79r!34jH@g@IiBL|fhu%$#EzGs?!>Sgk&bMc{sN8Fybyu;H?FGS~R zB3sjM3>cQ|)i~kmtd^3J1hd)7;2_&zPHvlw!#u|SAIjc5s;P608mD^ITdC6C3RqOq zxmKoHMJfVuxaf7(s}Mkj3bhDfPyv|?vE?cjs;RX=#UVH#R)!dvWUd3IS1JLNDF{?3 zGhs{uCM5ms_dU?wZ>?{wZ~gwTR)*x{ocFxv+56eg-t*z=;exjc^XY2~(s-$tptfFr z3kHe-sXeO+ZOngq`Yw+R0=JCwdybZxIiU5G zv3^i0PZZ*uLhx;ts@N?g)B!4&3{p>_#;ueGIEOb5@HWXDe$dMgC72cAi=LeI&tKC# zmX<&h@fex~pMDRn&qXN-t0x81dSXk;gl%k_JAO30H4h>Z({PW$zkzO=MPW@fP+%ZwFX$9#Xr4Z*X)WLw*zv^ zZ^uh%Woyjuy>yTnQ#B+K;hME?F%Ak6y4t85IEn;0sKt)tvbT{sAL-}7Z&C2#%f$@x z0_PLyCx?bs7AN$>>m=GA5Q;>mp^U;vIqpnpF)IL6_#8$xb%BM@QV@+f&N>k1&U``^B0y ztUD0RtVGXUajTKMBiu;KG#vKGked-2>c!cyRSz!_^J!@CgHmxvScY}$2uv3o_Sh)G zES3KAgZwe2?zDT^Io*k{V4rngRby#$igLsnrzz=)0K*^0b8-JWf@uKfKB-F)tBAFy zr($szG$6QEK1gv(gaMDer9-ZpCxim2wW0ipYe81X2QD_4kGq?9xe!{UR$jq(?ea&Z8(y4#fp=Y(?O52|zVjygKn9Xl zOhGo{c+Yvhzy;15;}2Bgxzuhg(b~iArpB26xn`%@Fe<^P^Y8}%piIfXdMhL#bVDPh zvEh_1)+vg1%0j;Ko{owDxB~&yy}A-VPQMolP>2;~mFEpXG%bc189V7{(OGGx_$Q?? z^M~}pNpb)g6#&IxF!^_;+;`+)+v{wl;{a%#GZ=j?vRlPU>KcG5UL;jet211hqA4$S z(ln!;b^HS4?hGSE@E+J9<}I;c{s{~N6*mQWly?BlL@njf%Y*|y{xLPjJqL|VM`FN% za-V{0Y8;7>JNO0?oajkt0xFeuM6-~CPA@y=2ikMjz^RV28qHFme~_MxiHC@cauX6^ zx|zI;_WP;`Mj!@#A#wEN<6IDB0JQYe)5;(m{&X<}P=mWZUXt|n2_%8Mo&5h|R~cz3 zlOsiQo{UA4%ur&yKzoYhek6#ytOhTJE-NF1PlQi^W&&@;%V71^N4?lpJ}ObOfxLM5;fV|8x)8*a~$Swke*%r&PiQoK1iCB$$Lp9s8#A!x}KdKnNb zLuFLB(pU@RLPBJLF2MLObxI&n zi-ey@q>}=K%$UVeJ20Kz^{Icm`czkb6VD%3eH{4|OcS{TD~$wBE|;kgya4qC0q^i* zAh6ProL34GnQM~PNl`HahOs+5IOb8x}?C^wZ?Iw{D2Q7wssMJ1)O~H zWShG0Nc)|fNus);(J0v=skZ#)I%ci{(;#?IP(p>QXtKB%B8fKKnV|$i&(Wm|W|^=a zWXyoBPd!vAk~Yi^;8aj^j#~T==(RubI+Ua7w&b=arw%tL)v#0m)g#UlJSDz>{yA8S zL~=_|>H!hU`q*+n94sK;`hWbv;SwT|t`J8T4Quk{Ts_fKUF%IK;d^*X@kvQEG82YL zr8)zB4^Rb^>dn#G$QcSOhSkPi}QQAwd0P*8sI6&e#fec+Sp;5uN{*MDA zt^ko8VSB_8KNJgGPqsRktd(&URDz*P%RmZgi+$tfv5#D4Sfc@0x*Q>bb8;vN_D?#g7`K4 zRSw^H|ND-eu;P49ZZ`kk962#v6500I1vv`Uyd%0dHTLhw74?tUZOIPC7jSf z#0%SR{*~UN@Z*9ab^u(TKYWVRR%%22kn4&eJ&)GrgYS;S4e9>-B*fvJ(wTcTM*La_ ziHGE#g=r;`N^Yw-T)reU0g{B!%T=|SY=NwFmU#5XV>=AL)IIoV69J3d_#VXB1^pax z`8T10#K&8e-{#()Kug^I-|g_g>c12e_NCkY`7iC(XwUwX&|>m1a^dfs-D#eDf7$QO zUf*O7W!c$2@n-YSEF1hGK&*PhGogA?mviKFQ`U`^rsjRp2&;%e?)SqzWp#OLP6v_L zI&Sp(5|-@4N14xM+H)73O>$1We?#-ltRe``FYTNxzW8OKz@({F=66(nAX1RS*?FV) z-A4_EzhaOX-8SRImg=7SlV2V^cdO0Qttu?R+zF$_xPIxLAWof!^Vq&|oxQHQmBW`* zd1>;T#ZScpA3GXlA+Beb)Q2D@H4(iaPm7NOOFqu@GeL3HSaAPU(grha8yn_cs~$OQ z;kC!Vfm3Gh9~jl>uR1AZm&x4)ag&>(qa0d}mHXbEGcX=@1=HB&?`c=1dB5gxYbYdj zSlP6ytv4z3^4s~a8`m62dtvCaF~1|lZbQ)Fl)h(OMe6#Kv+B|kI}6lHhK70t*Dg7C z=~rHPQu@!LhNfem+kIlb^F5tI#!VwGH}E2dz2~ZvE<1OHQ-J(f|B7DcV{C1L_8Zk3 zz7+-jIpJwmO=S(2xyMHipf`PMUt=s8hP_a-;*+iY9-AkfuD|)y%xf;+*3W7?Il#}e z7!n2WYt8s;OI9U+I4Uo%j%iRGc57Weoc8TGzK@J|Ojw*<56|C|z5ib1-Wkvo{FVzqJ~ZE1oW`$en=W zy<@~5dZ)o=A&k#MLZ;rs4!aZVZkaW(bF!54sOzJq&L6!cak)r?YY~)^eqp)ZHp-#S$t5&eJ0Ik0>brJ*%poC8MY&hI5owx{z341?o z|9!SEkT9r{=DNiHtj^4@5WKF98z?a8l#i?JkEkxchg$io;F}``%>!~`ZN`GZ0k)OA zyJ94*=T(H7);HaFqp&JXTun`QT;U4PYgUZ;Ua{?SdZvQwFSZ!y`pvnDnm9nk67U;g8X-0B=-qEVQMSCQb{(y4SD@*c`nkCR~uG_ zRA^ml&6Q7lbZjoWA_>zLFgyE){u*~L7wyuI-vmzoNuf5B+eo|0f&<;tTtip`~ap8mED-~0Y(FmKFB*G(RqQ(+Sa|i4^syoBdDfvI|1Xb=$V>5 zMLQ8g3M4pT5a0l=0nD!gIuoiS&l5vuTEPXGc@GJwAcZCL1bA4(p$uYv=~cix6n8k6 zY@NLszCvohHOXrA`8uw8*;`0CGVQCM>dg8HRE7WSB4I97T%_s#hkp z;FXhiQ944{+BX*#cZyf8z`o4a4Ly=>0|*nK*d&5qg#(TUo#UR*?qmL9J(BkzpCFQ7 z95_9*aF;+i43BS^r4#Z-rj3)=F$3VFnAL+FnPv&-g>cL@z5F-XG2FD}3vhj}D6?oI z{uQqW4axPGEWtxp5HCiyl?L|}Kz?6F9UzXh7X#c>nUTNpdviJ0Go@uXEQkLt2=wQaB`DQ6YsBFV9)m?Y-6TvN@u3%DcNVePQvk66<3&9p+Y{>dU>Q zpkI7N4Aadp%;w!0)4k8uMoDt(9VDqpLI8u-fUc|NKsuV!0e&_}#YAcZhDb8CA4(xU z%>KG6<$3oiM61SeRVGB{uMnwK*fiOzP&M23V&gJj; zLbCsu5#8VHgxtM{+!v%Pi(Ve#BZBz`I!p#6Z&{=Fc z!B2m=F8Z-vHTXl$j+k6JC;!7P5@AOWCg3KzcwPrrS5kP!{7@c2;J9L2h|eJ|D{)g3 zlUHKi8~-3*hW{C@t(VL`!GXP>ynJLt(9H<%soFaOqs?$A&JbiC-T~@-dLarz7<22`wVf%MqXhi41X>7fj%sV?#jYM^7%=ujNEYv4?tX&>4C%r420F zF{X7kG^GC~q5z8LAqFRS5mD)AlOEqq4e^QAo9NDwrvPC-zxFRS<|5yXpr=)Y6aWY_ z0*FG+0-zmu)JsUTtHEX$3V#EfDpsfy7?ZisT9Gj)W+;g=F4(vZ0^*Y0&8rS=Q`&<~ zcuXECQytlvSVE+Eik2#Mok`ePzzhLg&GgBWpXSiMEMpw7tXy{rsO-p^ zd*LIHECayKP9Q?UJwUS#b5(*Y=~wX|2EpQD(UV7m5Wun!mno}fROZg)9uUY!l>OVM zIst?vG5!G*N+b*b$(1H@H^QtlsS;Df5xE}3`ogljH_sEQiQ3pyd`z$gSD%9ErL0Hn zMK5uqrtS>C`%!&{6UJat4X)GEi@QXyh0%*3^`S@4iC})>4@$^Vl9vfm1)K5X1O(Yk z5AkCl=ZD?CSRybWT!cvsMs*JU#7H<{02Zhkv;-nDbV60p6q_(DYM>w7kYl`UCtCt@lvkxXPjZyZ+yV3u zG~AJj<%yUh85F||axzHYy`u__Z5XP@qql)9NIXka>@?(%yh5ZJgj9n#sFRLHV0qJ} zO-VeE5QK%o!{B2gUz&T4l7Emy;3}uL5H1RY4o*7i1#B8awXlC6`y&}mpoSQP{-D_& zk1_v4Ny%vqK6L~Ai?N($E~$!+vIp?zB`KAor(_MeYqUw zaY}R}gbX4!4uq3Q!?_VDWb+Rbev}C_fhdHJW5+H+4bw4~Tm%zcI2RF9ea3&yH=+QLxiKkBHVe zWa;v%ViJeBk@O6P}j&ZhWl&$|rN8>Xz`oKDIB? z@5Olq7o(?lQEZw`guB5^uwem;KdIXK^JDgK^!rJ7frmBNkcfNW_1C$kCaYjsGgz@k zxaJ!#r$;UC{Wz>mp4@6_o})Tb_w?8@Ze_WBp!rB}P(&u{Rhef3nxeu9m1yIzVSMk_ z{MyPHNyZ@;Ry-`@_}qB*{pgV6s^ejXw=Dt`S!?74$=AlUvW1bh@={j@K>D6!Yg!2FFU6o%@^ytLl$mnN^1pzzW9nC3} z%FbrEU0C7xBsC8rgfNRbVbiR8cUUJl72{1Y4^%^v1p+^>P_N<9uiJe7i=e1Yb;|9l zV2=LI;aTPD`rS{SjaDuH-l68ON5P4bO_^@JIc|%jZ=X3?G~DbT8JSXH;r*Rvt67_T z&7mkK&DJY^VJfTJcn+RX#6IMIaaZHfIMyxk&pFVMVOET~{6|Z(pIG*%BJ%Ssj)jf{ z#|6abd-T_5&)t_Q(;ITPx~Ey;>vx!+*(%KSV7-jOaLG3*_VS_sY(FI~D6~D@CbOFL zg&*hZ0qFtX=sl;x_b2U-JhapE#*?&50`(@;x*e^3IaztZCNP)27ZG5&JlO9SWd^&+ zxWH#sc%5~tC_HWAa@&TS83wts!`tVj-HwQPT49uu-fR^xHv3vL#;rUY({0GsvwQPT z9v)48!=HHNZfMQV3dqd<4~xs%($?2~;nc(4qrQHg@6LHxV6vp;H;@vBrJrg3IVr1S ziJN5oUO`63E-R9^X1e#SAFWmB9}DsD+>}!H`}vKHtxeoRE+}(nbn86(lvSmXA>k^+ zJD=fMl=-a4_mm=#<94?upm|)?79|Y#4@z$zzNff)@7If^onq;}3E9v#uKeBF``x?l z8uMm3-ohR#QyW=r8XAPkLnIRnWCb1J=p6W+eW^ut6&NY#ROW@`hi&|LOYh=1S+3Q* z=6w_Ixs_=v>e^%u4Q_2FlRo_Gy(!LlzxBToa`?z0hs0ke*K5Kj`L6Ix%kmmdKC>xd z(mLcC_$*09f!mnFoiIDIqKAt&owAN*8Bbh(4iC8muZhXb;8P}!(1Te2cPMwi>Xc=X z%(+VRqS!#bRbc4q#lP-`yEx7C?UJ%OOa~9$)(ZQ*K0d!+nsGl%U_-?As8O~XK0uQY zWwZajHea1R5#ggf9OTg4TGFOrX_s|q*9_FX?D8obKObb$BX4-fxok0VYGk80N^i-m zP__?}(t@m?_#=!A;H}ql=kT~Id-zP1h2C6Vyk>ydC~{3N zFMKhHG3`Z9_F!|Kh#N5C%UJvI*oQ9La?;wzH|$VAMlyU3)D|H^$NxNkp$zoVeRvU8 z14hC|XDC!rEJ5%NOFvHYO)v_=fn7Xr$An@K0mwcqe>w7mGcsSC4c~{leHtHGS8Q-b z%BtjKne~%WW%6WE-RXS*&2<&S8f!YjpWI)wF)P|95_{!dLvN|fE+Nm}R%&APIMUimJXI-KR^HU6E{C@^6=WD<8v%d$h#`_csR1Pwaj zDzxu2MDVbpDCEJQt#Q2MnT4i&+e4Mbq+FD?I&rmP4s8hlysR_C}FF zEZHvluvtY4PK#hF+>~%!YR3Du64QPT&xi2IRU70l5T*-ZGu)d3pLMU8iaHkrLmk!p*2odqp zInTkKy0ZJ2-ilf#Aw$zw!hc(l>BkQ$O&uFqc#fr=qp^}VWuiX@TL$>Qsplav6Gafj zA2lZ$#hWH$_!2n>S%ynyOVAmrR&GrgDsUq1is{<9C}1#|iaHKmf1qS9CZMtopoO6I zr?*N8nyyAq9tHzgBOSE_u`?D@O4T2F1^miOhzJ`ZGYj5ZFep#r&tY6~n_XQdrAWNz zljV2|l|alB{GnF|q62ixRY+DKh=rL^I0U`ML%`a%s-GH&$%eO9aM!9>EqptfMO^$Mcy z)adK?OdP)W7nn70rSKQQ%@ENl}346Ina9Lo~?Eg}fXvR+7T$jU}LYPDvoG@qY7 zw>l?tOcxh+7}vz1?g~+*l1>s;$xh{;&~^Tb*ObVW`~>a0;qYM3=n%gQlwt&MB%GUQ z4JAe&cpRl-0B*kbzq{Z9+;*L@|(fFF(ves`P3!N0WA zrS>qK{Fh<819_ss7!Kkq9#>y`q&n)bs3B`0KvdpY)$oVjjGM?bt+=9LspkbyrqxBg zJ=7?yA?-{^YHc*QyN2D@8>}#|@Qkd5(CiK0_Svl=*T07qAg;;cN);&s*1^PYU5hPF zQa5Mz`zk>!85lW{_d?S~mVI3wZx_2m)>^h{;yvtN9c>0}ykhOU^<}pa(}Gd92Iqw@ zf8EhJtyXYo;>kTWr4VKL9wka$hOUTDC2NngLSp#X#fClI-6sGdMms$S=`5ScrEeG=+**x?fQgz9mrLR;G3o?L z5`3wcfGRyqdN`5Y>bN=K-LT9+rZD|!{Q4+%3PRQpqY$D65p0TDM?HEkUXfEBN+GMuB0(cbtz zz8hl?QM)6Z+L#<13RrM6Zotu$GW<~{Mgyu}3HQa@J_CT~0l%JbgeV{la~IYig14h8F0Dvj0`#`JTLO5^g38t zR7o}szscX9<>V2#%zPMfwe@h@HcUp0G3>@%oH+7rXeMwPDFv}M7myluj$j=aeuSP$ z;B0y-E%?<_BM(plVJ$`B$kM4ojm*)!Tuy!mp)VwR7o-w1@DGlBJ-J**aH7Fp0zJp3 zT49_63yuoy=*X*WiQqNJTT-zGb`HE2c|H8W#tE|lDo6l@R>9mVR3L)YO}@h1 z-aSNqK%6ruh zhiyEKWU%1Ookn59k;$4D>Km?zS)TE9 zAms+{mV|ZSZmITTJ@)VqXV{%pg4bodoDFv|&BQJ}v}AK%HD5fSXQsqAgX(|lEZkxO)`S{C%%WE`*qZH(FELkB6yTbCHQXJkPj4P?vr9CF4 zgWs=eGst6Yq8v7KTjMOxz%2H)b|*#{wR<0OujwqQ9y-> z_BUPaLM)v9do73Npjq}W#qfCHWj^Lco{Dv6J;F~a*M$Yn8uI4OI&nJn=Pdp#PZt?l z-ri^GYEO@BwzJSI$;(q-suEhav^Ajvrm54;} z+*&<)Y>b?b`bggu=dxaZ)#C7r^4#;~-F<^6GM@E5c#&o7QY zR-ErA|NHOGv&+i;3k&LlBtt*rdb&MWd;dk6H1aLo+@kstSW6h^yL-4Qvo(+Yo#0`+ z=qyjgw^QCy)wDD>HAS2r2wAE3wf$+|3C%;JJu=>%F)gffcUN0i`0Vlebf=mJQLN(; zbwRVpvK(Out(ffJ7;IsoYuX>;TITS>MDzxmeD=u5;fnP7LV3v32i2RKzUyy))YlbV z8{%TZncS?@2*fV+k|ryDj>XVsJT`jU4&2ok3e=8~5wSI!-R=QfK;wX0$k<5&yr!$f zM?jNIl&`(jro3L12RZGyc1^>jZytCoaXEUn!_-DIpWh)FPyix8y!@C zbSlpbHVQe-m<<{c#Oq$yd{`V4kjbtY@p|?Af&6Ehl5=lXW5~U+sUO031DS=KD`Wd2 zp;Y*W0@E5*kGi!cWqi_w%&IuUt~Kl3*=Y69)-su2z$rBv_Hb1|J?sART|AS-JtH3s z7jCNmN1@CuBKZE*zK#uLVv!9rsl%)j%7lI?k?9GxD2Gm#Eh#T-u#U}qt+|hrY@^4Q z=0!?RW2#~@QhR0ickKnaiSMKqmCY004xi)?vyXZ1i!b|BVg(;n`I$smJlSh~%pDkxPf0$hHyGLNkKq zpeu;HAS&P>bw#)-=^{VD%0Uu_GQHCUNfOPr;2Su(HBT-ltNK|J6ITLwNehu10in_r z^ixS+@Tp5#NyorSbeZoIK7NL#A2F3e)$4`s9L?Gxnb@?d&GcIpKA!BgAMq2!G!XQV zpsxpR=LelRUO@=L=&UNN3fS0fm2q$!9o$}c=((4l!W8qZ=Ef$3mqyKrSAt3a=y;*z zRxUB^AFwI!wOKHGQ3#tuOj9ZZy6yFB0$b_4bsam$ta zDiKs6t$;?u2Wf>+CdSnBn*kAG*Y)fV-H0#^b6ulDi^>ETQ_?&~8f%`Qv$OW#$0piId5n*leZU)D01V_ft6tw+vPQOEKo6 zN|7`UmbIw*`Yg%P2}UOF1WF4@U53L`?Q7a1Rlb!O+ZmR1^kv6JRLi6q-)75<4fX?~ z3{!-`hKHf!fYW=vHV2h@|N0_5%wjVZmlGr^^N$+)$5j;?{Ha+?lk zqWPX#sCi85m0koT%vn7>jUgn`Q4c@{h~uR#h+AN1W~3X4t9-GsU(pJ!z0v&h^Z=sn z|Lp!?l^7ZYBI^in=5<0aLuMR+2x>gu1{;$|=MnsTrzR4X7Hj4mB<2#ZaBu_Y?`Iu= zSm)YLVENBVNv}PP8PWcxCNd3N#B*_=Y%SwLaNk&2SIuTO4#ZTd2Hb0;lZ7tN>raY& zeWJOpDn5)=l9!=A1ZC8)r3dN>B|fD{;oprpEwwWk-|~+7w%&EpZ)`8vir_eJ6y}4@ zp52i$3`synrn{vQ?$fIT{-V&~TS zmyakX;=PFfKQa}_joF*e_RG?@U7eF#&;?(a%Iw@%#Gntr6sg!R4?~~;^9Df{o*OQ? zIb|hvs4ky%uYpiil2xkW-W(pE(q`SxQvN z7*Zn;jQaW4kc9|B64`?wN5sQe(r?L8Em{P)r zLyHp5Ar$D5DQw0?2MZIJhUDbo%Fn1E>h`)Q+-lY8>RFVdz~kwqNtW&cu3v60ZAs&- z;#Ki0%qaFu=ne?;Ljn$PQ)bg|!i2#iTn{Q5O-U*83Q~QLSnh&5h+fB|kx!~jifGy*A#{8g10vn9@icSKq zr~m`YjGaXWR0IkGg#d;DL|{M3dj?UQ?!+FgfFSHGXo?R)G%!fW$_; z;u&>>L4rbowD_rXg|*Fzc-zD)NY9;q7}?Ss;@46A$|j7SFkS*szjSugCp@0w%_xio zQILI6i$#khQ5cQD6bEP)>jH~?>e%Vi+Ca+!%Ta@~Ow=oY+|=@cvOQw3y5x5-EP}T| z4VG*|4q$EWHL^-nQuQH-Qi$#Z3%>|A53V$MarWp44;lzuws2oZ$YZ9=Bv})xyV=L* z$Dnp~Ae>YbZgvs)4aD%Qoz-%Qjz@sT>3uL|zZq{uhy`2nUrO{$nM5$>8g>_CRm9-p zoW`UOxm5TD82S!icszXQtI?aBRB6@BYz#<%aI91CELfK{!~%l;&3b8-0{caJgMhRU z^HBNJ(Ww+dMg&c929&>|9K_DHG}aeljRpQemO5ST%ozmA$5GLZjWSULzGKLW;VQ7a ziJ>W-(=Y^GG4&IF@(ko20KfV_ulm<3gLom|{_71vih!ju3MGCmtWbPlyfH?ma7J-r zDNm)qO~`Dqx(j%CX1t0QptHCTge;iKyTlL|?%qHD&|f8UB9JJ7%=_yb|MNq;0?L}W zDpmpSg4}C#+|SV^=eo+|`G}@`UR#7gg~`dd=0ur>Q@sdmM{%Uf-Rf9U;w z)t{wQee4J;GIXE%N6U;qeXiYO{)gU?joK&a`-aq^nFg=7By6{Am(&&o?g>}9v%bjQ z)NYvGs91D}-#uEg|AohhZ}LP!=@GS4WAH+4z7)@?yLFkmCa$}Ei%{k8Q!sCx`b)R$ z(*~#5K{O7VH6jN9KE>~-Z`BxzGq}b;Rk-alH5qF#G^%J*Poyet7kyfG4!3W4^Vh{% zw_(qoU9NoT6QRs3X=3lI#>PbzLgB2Vw)yA0L)-FQ|IoXc+V!i)NYZNk(nT8YzUSx( zWJQgi2+kbL3Y1P3h~62#VjOiNpua4qv?RCM*Emk3{_M)Z$ik#W8=lIo=7}y{C_*$j zcf`I_c|JR;YE

Izdo8A!r3u{yA+-{)Fl9Pq2rTB>oq| zHEVwk-u=l~9ozX+6r3=`>OrbjLE zpep9ua9q>fE#x841%N?L$Oq10zJZGY0;t5ZbTtL}vqPhg*XE)z9$rU;qzBz-6W#$f$TEgc z0IMh@XoJUijf_W-7j|rL)Uj)@!*Z=&X(nsoGF@tiOqxy0!HAYgvA_je^Y9b&@>PZa zL40t|yn=fLk_L&1Lv?o{d+rSv@@B#KkcxcoCaQM>n0tmNiKXA5qg9ixaTw2RATN5x4v!)s=xLJPp5YPj7JiG{r&UEOn^*&s z>p`CPm?Vq=AyyILn)oW+AYg1J&9ER-q&6G$yt_ZSn%F^w>cb~^>fNKP$$y>r?5Pzb#vkRXcXK|q zdLP|4-<1pg&D5s-UG)R*XnT=xjmxS@CIUjfLsJ1S3ekdll(IvhniqUL*A<}Q^>FoD z+YE9!ceHHtpJfIeb_YX63(;+obDT@YpEib!g&9JI^f;Y@xCU+3xz< z6Gy!UQ_sTjNveQw8dVA)6|r0=7C>g{lF zSTZHsHaemqFKl2SGM@>pJ^Lku_F;VC_D5rFF@#qb?YN9c58xqrnCR*{nT1y5Vt!50 z@}mpzNdhDB>c$x!cw-#kOoEqk)21M~LL5-M#n7?`m_0g5SpceSsL1F9hDC>)HA_k* z$G#TJ@-8JS&$GP+O7_V5uM6!OwQv25kpOO>t+Bp7Qpj3z0u~@!eszD z68z<$*d3P+djSn;YmBW7a7utKHdvx{2t-F2YH*-wdF&ojCe#7u1HoaeN7qe-jTAsBTVlbnGL*W~$FC`f7FpKw%dubZsd^prkaD+z@#JkkA(I+%?k}?ew7Z zF@G6wxh;k=#&_9dH}DG>HUfW|as8mBKfzXbD2fABVTNiiS20pg{GqUHBe)FWqnd$a z3+2iY1aN3u`^GkSxX+f+c6p3sA^BMK$aiNGz#2dO5E>6!Vph$*E>(Qh@5+fgsF{ax9_Ra?@djQi_ z;4%f#gG;wTnqy^d)+Cc$14#vEa}m$54mu{Zoy%sBC1@;3c7qIBd2F@6+H4e@;~d}2 zCBtSbbA_cqWufiLI%JDPv%Pb2A}jbliWYUO&kzBH#-mXy(nmy_>LcSG^C7;dp4o?uaoBhUdM`duF#4-mebp$}cKq zeuHWQZP#7Zz4CjtuzCFwpbsZujH}HM-fV-=rO>DKY3sol#$hEXmC|msb{S=SjYIvC zLs4K>xRx}~1_OuH$mkkbvhU337E_bmNj$tj=(IsXFZdw&!JZgBJE&w7_4-!7BvSJGJVRufS_Emv!+H3H|HGbt657P6&a&{=Mbh3$XCSS^Rr|T@d z!Br8C3xVxCfditZU!pq9usiUpepvd;U*b92kbwu8iGbbZFwXdD`V%z5OE8)6H2W&m zsa4w+?nqjavuG_TZWnH98Q^sBRnA0mLdb`Bp}3{WHKz*JCIEIu$E;HWV@2Vi%0K?9 z=bPly#8*trM}kjNS6Dd?&qD;`f&5bCn%6K<0lM=L>u`VvD6ZL~$<3u7aso2E0wbhT z(*Zu`2gi?N$M^YF4`(>Pz(Q$2VG&MYTfK0_R^>HHbEzU8Dxz0!XwJeu6DF%CSa=Bl zIOfFUaIT!bB6SX6_1OE%uH35PEtGn?9sm_jIOikXZo>}1$u=jpaZ2=vg`@F5{h@5_ zn;I`%c(teDQ1F3X{yF2`Zhiio!Z^I)jM2O4^(S8KsTkw@Me2nVxyM%qkcePv&dO`y zF1+LKrnbM75ToS*b)3>yBlOl>;>of+Km9(ei-myE3OQGDWfl`z$8K8Dr>-Rs@1*` z$;PPhCRVme?gh1$8YyjTthtrNj!A;^xiS8Sk80zvccv9}ztJlXNI&~H%gkW=H~EvT zB~N9FT^9OqdpIU=n%r1zuNDe%nui^qMcF8RI7g%r`_9S+};DnJwRcN;N(yZ!Zr1#HMDN z`$N^f#<3sKY@8l9j|~?V=htxW3Q|=S`(e{@sq6CN$Wzrn)meoe+jV%GRC_)1cB7qR zKt}t;;6wl3J-#lfE9K0fYQ^Ux*IAz1tU|DYJ8t>qtgzUfXr9fIB_l2~SL@@@f%|K2 zKVdGnOun#zRp_0-E$rZ?cP|-r^Ra2P+TC2eBKVx`m%r>)2bkOas%KoBFYbsrp26~8 z?sFYnGqMgSQI>Xgx^wSd@6Md_t5Lt#9+TcFNj$tacOdV>QtwTiSM+Q4D$avq6fdvI zWY6)v^gCG9WhGlos&i90`El=+#`T8tmF%x$yOeKwcDOp2e-gdP=-ofrE{EaNTMyKG zSHPtLJ0HZS_Sq3;SUKDAz(&9d;#nys=H>;)D~nV%rYloyDu^SM=1&e3 zWq#duH8?0YqNikZ&)FhvP9?>5rn?cQ;+8krHPh8x(aO^ zou0>&73cqeZRDpyR$!iC!;F8@z?F_4#0x8$zY-SRXIDlkk9vtkhyQeV#!}+qZkGRa zZm(1Q*m#aZ6?U49{6jg&QY|dEfWv`x;7+wI~NECU~0<4>kq`(d+7=3&Y5fQ3`zRy?H5oLB=N%;qZP zx{5>vS2!GD=>H?$hjaB|e(bJ2eUI5wuNRi5e+Z-Uis9aoTqtY{ygeII-wZxJaJpJm zmQ(Sq8P=CCCHib>~|zpa{ZUHHd}u2j5(JO)nK zXZej3CzN`(5584cdEMwOo2t&9aZ=SvwodZh)Mh_~(q2|G((+knC+D0?TxAoOWwkNx zM+P0D;>v%kdSPOW6+|szw&mf8j51DR>%w~;f2MQQ+O+RaF+F#t9qzu!)22)b^u*&) zz%VRps+9yx5F-Q%92h1rTza2Oka5x!^;`M!agn9p=K0K#z|Ie><5`4}R`0uZz zwzvk&0)ImS`ZTJiaSC0u!z8K&(_Exa*1;w?qcTAYdiWZ^Jj>wzhoi?af&t2DD7Xk! zj-1w2kjz2yzcdR579k-6+#DG(2Py}gALuXWSWT9mRApiqX_)E2x)hCH2CP9n25e_} zhgB9+MlcnHPSt^7(tkqLGkp#r0Cbw9x|-p~6ShR(%Z%I-9fusZYJs1^E7r3BLPISw zee`*7`I2pV>AZPuaT+$yWJGV`Cj#VQFoQ)Z4W$=GfKj5mTZY*|WCZXjfL=ULy3j>1 zB##|;Q0RR}kA*q`IGkfTP$wFXOZTYgn)t&N)BU?hWr4c&j;)T#fkPEl1P>=c9w481 zvM+`9kY1_G-Q$%!A-}N<7m4%ZUYy(sd0~S?h!McI?!?2S`VXgUGN{M7Z&*4bV_L}# z^cRcf4ER!5&|na8FeSeLvh&2Qx+mNh0qnX7Ce$6L7pIw9Zo0-6{9fWrQ~&u#>@;d* zc{2GL_&p&Cq~ig~Lki_0RdY0qAbi!~484F%9hgS94>-d7yMzu9E1x-buT`@`=jLff z=VH>+-8eiCaMAJpk?fG0)}%cV=8T3EZ>Il z12d;{r!QMtb9-rj)f1Q9gw`Fpasu9knm5TS+uKFn%6 zhiGk+J0M6j=x}n<$X0|Yy@+Lju|VMg_5y(l;MwCq)q{#~Z$4AvfSu97?ym-A1(bkA zk5W99e19#NFoT}gMqz6Qt^(m8kY9m>o;QO&* z<}koH;YD=>9FU$#pGXn%9%t&-F=&8J&ON9HnG&rFNR((RA|3?T_>=+)XVE6vQ{9)X zC3I0&sV^(?CqmGU*f#BHD7~oQlU-s3VQ!hw1S?)k-C##!z8_5?3^kd*qQC>%K~x3c zc%-4ir(vDh{<&kBRCQ3YQ>cV>07GbyAYTCgj(N)1Q2;lf%VQeY1P?H*2WC86(_n&% z-H|{AOsygMOM)+E{7zq^#kp1oaS&p~Y_tq=N{T4m?nA$l`3OVT(8KAMUcdu9nc!y{&>O0+d(%9u5{Xp@82h!mI>v`p}U2j&vs;-DiJP&kN(4-D|d@Csps6QJ?a zGUq$$F~+fb=~4m1Vvp0oGmBt~PJcv*0Zni{@Rv2bXIX z8$VsXurO*kBa&J7?w#AA{avuJR4Ma)8^;NLC4Wwy_ck*D;oSXl6OWdX#r)K+;fPq139Vz-vpdTI2-eh68EY|nYu zYbSQ2CzPz&EV+rH`~q!}yfDoV%8|t!s|T4}#R9wfYRn9#g;u?|#LdmQqAi81%z z2Kzo`O$JI<418BNS>7K&EQRHxaq-ykBz}|=rMVgxFSh!6yLdu6GB0%#Mqtd4sW z1MgQ}K?^$(q!#O4y!qKI22b>0!E;aYfn}{eVmV5MS~Pj@#KSr8E4*}+tOGZOko(}Y zP^u}J!Hs=}e_V&gy>c_67I0_F5WOMs5Jm({%;bSq1@euq^h`C>wnEIRqkjRVpV6Mt z$rx@7?Ev8B{I%x(gvLfCB+Uc>Bb;68!m|?-HrKH3H!zR69roqy3~XHWf2j(IjDT=~ zPwzTpu28*C=7q_x-ZM4k9#Fe3a5+HhH5VjWJM3KWQHo%I722Gh9ir1SsvIth^L>1?Ds_T7WbXI7-xsT8bWGFqF0yquX*P%&+Oc?h&Yj6a5iN zMg`aKg-*m@7_=Q8LpG{wBfH5#41+8BK!T9!Vao{JvIp5Y<%%j7+G`P^ctfURhkOI26DzlNvf;}s=@pnXqNEW#>G{xXZyTb1hIHU7QhT<(lBwLB2+0= zgsRknEhQ~>=IKN1R-aXvs)Cy0Yg@l!)HvA?9%*y9V-elWsfVK6zVe#_6Uh`g6YZFO z^;$6}8Xz%yxN_#k3LJic7XS?TYRNMGBs6LhSg!0V5%yMCUp$&mcJ%f z%LSZODpcH90h3E?SGc(mORG=;yI=KS7Gn`TW@iuIH_6z>=E3Xlt@#vMq=j-RPBKCq z^IdODpURwzlfeSKe@8ZUh3XIDpO;E}Z69>6?Yuw6Puo@JsIVEnU z(9~DxgN1w3HM(^|n0j-RHSCN#UhmAisW~gwvV(6RWoEEVzWWF$DCS*ly%OX=V1=fk zZ#{4(ypp?OlotU}yJQ&~nAZGlYa5agQ3I+P(_jtb;16R(OIY zx9s_6yw;2aJI9T#Jj`tz|Em|6d8W1h$nx&?8y(xwG6?PnOS74bd;TBB-aM?yGi@J6 zt=j4!(^k+zCDYZ8fYd5j5fhh}+G;I2E&GxRwX8xdAe$jti&!bm+o{M@#h{d8Wizs` zsUoB!m9PkeuoWn>hDAcagrwhjJ;BZ#-|-#a@An7QA|y}l=YH<{zRv4hgy7FqaDAs7 zceB^D2FFQ5K|!nQ^6i3H3Sd_5mn^5UQb21{uO+4CR3;~*8jD%}B^HZ~Ib$~ISp2+` z2mUZH3bn+-*c{Yqzur%qr%DP_^h_zdzmUy!O7^fI=-E<&Q%lcOzY4mP?jT^ZM|_>( z$mm!Eew?h0(;8tvIutyFC1K@!z=m3!2vVJ`EFqk4*t?h%Gl-@HA?AZ`03qC3RTUU)rRyBp>$7g?g0~7h|f|?eQ?*(i)L? z;JI~QeqtWvCTZW&iM1)$crfmzmWfa!)%u)hS<rznW9_1cFc0P}bvFi||HB|`yOD{zk>UVbjmajtpXyRj zZW8INjJgH%w)?I!I~K-Q`2V$sZ}yBOcv_)g3He`#yUV+;GYe0P`B#0H3>VGp-+UzO z_dt`?8~FE-vL zzgNN1T4-%6Un~!+|5e;&^zq~j{58#vA$n5xG>*-MC8C^jnu<+x&Y$Fb4dkKhgM*mC z8{WWO-$#%Ez7p*g{&EhL0$aIg2M+jo*0pHg9@=R<#2lqbF@gdeGG7!?j2*%`Zjq~| zVKLz&z!K2QgsB8WN_hM^6L-nOuEi3fra4N|bVTkU@(_|))Z_%NKq8JjOdu2hA{KBD zlkg`12)efaRR_Q`;J=FkwC+iDp@$>>>rZ|E4ZLhLXP+2c;-5D_Edv_Ce>fw0J^zz3 zo&*&C^&(^#L9hHoQ$kz(8%Pe(|H(vSWCJrYW(+DolTq{AQT_t`&_{icB4@$)0u|k2 zz)1*qgl&Tz{!6j+d0Gc&aoL=(pa&w9s(M=GZd&& zag8c36p_F!42kB!m`iv)y%^=-k_8L~Scwz$?pG+O_Vq1PB!SgH3Jde|7+P1{@iKic z7#QHxeJVVm2GZwGKoi8^LCh^Yxq>bM_Ybrl2rZ0#njVp%fl$}IkZGcSMf|&XN?QYr z3Cb9fM2k?oGNzia3!qnMd)wS-FZ}}=p~-E+3~+R)(()$rObC^zsMjM1Pb5xnFrlM* zU~M&FbAWc}U4{#uAXE%^LBrwB67{1v6am?(A;=Ph zv}I}*KV*p^!BZ!&ejEqDXmJhmIf3K(;?~}n@uTF>>x&iur7T=P7xM#9@rWSmA&Lsf zA|~M^v%okE;ECb?UKJ=&8w$b1c$lRHxx-&yGA{s;6vz6!jw5Ec4FBy=aS{&@`F>(! z5O`Z2y-uScuFYy9_ODa{eu$L0WtJnAc%PjnxD7-)qEEM6=WQ)^2WrZRJMWIryaHT2 z@iRb}DZhuO5-+sRfD}n2Kg!q7YI@o#7JNJaLo$RC`~uZk_>cnDzD;sfEy(upTy;q^ zTw%dRDNlD6vm$*65d_e!;m+cf%P8UvcN~IgeQ*gbZB(i?2Og1$jy6VmgP39N>n9ED zH5PXg?g;LK6x0v<>rA1>ufrCp$HzbogEa`uRS8*R#2tFHi*jul%mK!M1eXDNWFl-q zW+9qHx^_8dxcC7h4byW!$dnXaiwt0NdeQ!q{{`P7dRINUhT!r3a}8pkAL3jA6vIrV zCyk)CrC29v;bV|Df@tsyxHtxBA)KUf0gNNub>(We7I+};Y#mJ0^;Eqo%AY$BdR1Gl za}1uA4$zA z3==xUcdoSL2Io82-r6yBFSsH>X{7mY8f`aSIV^e`fk(9+J3= z+#6zCk2dzrMuEVpCRdheGj(WtKgu6j0rp)~kp$dun zrJ`1LPBu4aalbB(0n^-O`5_vYj;b#FFlzC9l`S>yd7d+22SF;+ArG^hb4s^HXZF7_Bp(@uxDZJ$#|SGxp*O*+V1wK% zz2ABpri8IKc<91<2iEj-WY6UoonhOHCi&+!TIcZxrDH9~@Z;rwR8oZzyJrBSX#SiL zbq{d&#G^<=TC`jW*Dyf=Nig43!s&s`TW_e}6AfzFuuXEKppE6gO;mObz;msT(zjq3;R~}2mO8Q{ z>$_(~^)4z%r{Ng37n2$nuwU1SyZufv?n`;uAPJw+CWQi<`>g%CYEf10$D64qHM2hn zY=pS`(a=+EWlUi(G$NTcOVOUQVsLBqxWAjWm^i2~7T7q-ZhBM;5`uU@4qpiI7!=kQ za1?`$d7|CMv5Cr{%S)J5f?@hNq`JcRfu)WxD7JtzMBP^IaG?+l$B5P#jy|_ADyJCq8v`E$kOE- z@g;4f6)&LeT{UsDVHr3Q9acdgD~KA=;j5}vT^tyL8>gb}LcxnY&bI7!Ui6>YC&Qe> z`LIJ(q&U?VZ}K=pLBpYVP@G2hP+KjeXe*pJ4-6!ww)lHN4mMJ%*>Bf?t=BR6R6T7a zsJn?G0mqM&DWk90|5nACS(uJ&oJnLoXH@JhR6@58pu!Fa4qMWMdu`W1XSnH2>%ad> zHqL2?t5c1^+6Jng#<=RevI%jMWzhE_QMJ5dx|eWhE-Mk`UYk&;e34c$69o8#TkNuX z$BUaJa&L%*dN@Obt-wr}hyzYZua}ufekihql}aPOIW!kW8%I>jJVOi?jgFlqueNgJeEQjJoY}IG z{u=M~ZS4D@Futuxdic{Z)w5w?s=)D$5Yp*R$NH)+N~a93Bt%CNcOu`#_7%gQhIoFz zG)$@lxS9Q44nKYq0gZk(-^iPPH{qPD>BM`ag9D`7?_-{6)X%@!F7I^WKZ9*LRoq%; z^i|bUQ_gu;eH~u0?U{NU=`_DS+PAkHPP44J$3Mof+_)dvlE*AUk6rTTH&^H+P%KtS zREbHBkeU3lCCOaqj&EPV53Yl5+R)%zYD$!en+!)$_1`dd|xeK!+mKdn>qMMrPcf zuAHKr6NijcY?<>V?amU|ng)gPVz~}yWPKjUxyJg9U~6#8F#jUkQoO9Hr1)toET>FY zQwINx_8`@Wy&D#mr@N4FVkm7N56T=-xonHekM!l}_Lzug6nhiK$FgU($7(vP3!I@G zo7kHmZVsuczWAo>(mJD6@*PWVd%x_#UDwgwF39ib;-~vm7S(p&llQF6yHdgVl;8VC zvFR_G9hJrE12L7tv$a`2b({j5C5(IRlcobB@0i=8=<{zb*_Jj&0rs%ly_9^^B|JAb*&x-Zpx_Pqx= zHEF-Du{XW!@XgrfUyE}pOOpy>treq5VLD#+XvtC2TB%n_+j0bqTTE?e z0Y)1giPLz?v@BJSQO0EAoH|DQCB95O3 z<~L61zW2Cdev5lH?pe_T%edyyr;LYRy|~TnSblCa_pbHwp6(fJUVC%nD!13Ro_@F4 zZ)>`ir8wTxRI<*!saC49Pf+iO`q$-6s&?51LEv$RrK58I2v4<43no6t?Z^-7YQS3C|6Bq^`ZZNye}+DObHP=jiKA^%MVt z13*t;w%Gw6jpoC#HHAar_Wv{ftJZxQ$;bR=@}!05Il#H%OPqb9^Fe{DzE|Dhe`eSA zQUo-8?ho%~ZEm_#o;>fSvd#}9dobaos(x-HWese>vwS%nOSTpYdpG@I@X|`~w6cvH z{xB%JWNwSIf#8HNzs>rGZvNe7t!-@?|LjxzPi5ZSZ7*R;;_$Kj9W&1t+UCy8;`+Y) zWR!oj??J?MK(btKf7=;>?ldTtur-S2Mc=0|fsb!y0=4rRjJIaD^?uABepn%eM$>Pf zqKegDUFKBQ?bgZ1PA$pU67~Hx*_rWK-TKzqmd>b{`VyYU7)M?-BMab!XsjZ8BiWIg zWMlRqnE;nSkeN11lk`uds9)8vSu=Vtt0{mBUP z85{!D4ZO`;n2&=OMhGe12H%9tp3#9b_yzvo7Oqq{jk!CoxahV=1s@5k$9DW+gQfm~|R}U^`dinHg=s6pN zfrd-!QsRsp+R5l162*^okl7&V^q({iiTB9HWQ;%kJMa&jGOr2d|AqSrjDrC#aHmQS zsn|VJ(?H%|*WY?V?VD%x_|BjA!3Y``hL*E}ZkwFo5C$Jvgd&6Z7%o}s@IY%&>EWNC zw-D)Bn+%sl!5W^8s9$~>Bx}qXmu(FY+u*HGS>ezQfa~;Sj7N+U83BO(z?^ZJiO00I zv!Mrvqc`TF{y^sVIEr{RR7jF)=@jV(b_KphimodeT4>A;ucZH>(5Uy zXI7?IWk81>EgubgMusx^@<~f?eseNYM-wCDII(WfYzrq4hu)U)?O-e}F&6Zc!-rJg zV{F90)MwKiqT-lr^yub*mjJxr4cNed;XNWDE`#tQy1ppo7VH`r_R|FMbchUd6%PV| zjO4v?jvjR8(&+MNuIu4LXvETit&qP2d5th?FmMCt@wcSuSg2>lJoygO0fmVs4GZBb z7`X%JfW5wn9)p;JPn1aIjG^eZ{4)4S5Pv}+QelV#@zgSt2rE$rv!)%oGF$wBxsrNj z-jEiF0VuY(K{*f3eOVse)kQeSe+SM5>FvY|1_R7{qpi1aKk&(!OCt0|*+0sRmv#1=7!pA? zvyRbUQ(Bqem4BJg3m*y+;6Mi=%hJ+Ee1#1L13TXA{T=({*O-Je1C114b8eIXtyIJG~C=0Vos@+=VBch#VNG7 zJUdxVfCK>D<-53J{GrMywNs*h=qI$mX zxRU?GN#vkbH3lSZ!qc?H4K$!c+LWR1KkDyBHihwnFJP9y6!hMp-Ls+qI$Cn3CJ@TS zV8tlS-hk_Vqr&81M6G#hO-kg%hxU>o5pM7kvT?1ynI2iGy6N08AoX< zLtd5MXruN!RA`w0ElAZ+)dye4OJW|UjI0W~Ib}G0HbmUFA48@B0-`d)yZqR~TmMc~ zd09yn!ZjyieLY9RE(6+$4-cJtBudv{1dv~>GGhnAheYiJJV7mLK-!C|Dkb@Y13}wI%C9eBejLRFjb5cI1z;2+Mw$#;^r-8d=audMydmR>+j|m?JOX!h`;S@ zPY90D*_!)4wU0`8?vHfSyYM9AZY8<{-0FR`jonf+Q&iK+QU6hXJ?YNYP_)|+;gCKn=G5sIUUh%kS?QlxQ_1tHyr`O~Lqb_~WF zw{k3`sSXn2xI3 zNb9utYrW_GqQ9z)o|y~Py~>frSkHe6OGCCdM4kuSEI$bCp$%V@DxiS@bu4 zvl!XB6($oVF`2Ts9ulHir-XmHf8;0D^7xvHPdDY(B)u*L$<;pS`Xrms4i-8z>0O9t zkp(O8y>~>E2?H~U=n|LX-@AV2I|l#6E>GS$xW9lt#R!M<`qrVlaq7mPcC(g)(4Zmb z^?Tw&UhMY5f)523ZpJZsfaT+(S%zP|GwR#QHg8Y$9_g+{z=^K?BTyF1PaF&F{W`9% z=e@1#r?bV+%-zvcGlRwL4^5n?dSxzsZ1J6k9okiumnCoA;eR5VygBE|Rn>IYasaUE zQXMh|W>t#Hq-6NR>I502y4eBj0s^$dxAKzjR61tJhOF#K!NYH^ydMJ~qQ6-oo>t%) ziufH{pXNvo2!bxx16ZLIP5ACW&sb}mcxsv({cY#tUw=_@R&;xf6g7Bbnfp0c;LwzE z;KBOOq7HlVznJlOz33_PmUg%QjWN|ezp6!W_E(<=LZ73ai;k-H^`G#s^Gi4Dz)oh~ zZ__w;+WFA2f^g&e^Dw@;3k@Xrbt_A4HV)*o2C*p-vnZWSlz3_QGP?-J>oH5_T; zmJ@1_8Zy0Wjl5-6{>+5i))jNPKg** zuln}L@uJzZu%n(!+dA^BLaej1T28zWXv6(W-9*5-s3T$NoA1ikFX{KNFe|^5*wKBb zJT#&El}V>7XD#Cg^4iDus$YYl<9GYkY=ud-V%W1bBg4BxdNlpy|EOla*En6zP5n%s z_>?U{|Ig`uk(+X8P zB&x29+Vph}(Iu&Z(3d~2wn_MQfCKV1R_xRn>>)F*EmhUgpclIT?&j6O`(^_3sFw4N z`-?z|;c*Xq_3B7!P+D2z7zE)8S(f?g{9BDRk`9QPQbR01c(v^*e;pblg&#+~EFBY4 zXfr1{FU!m5*&(ivw0-)cEkfmv4{=bkX(%%fwC)3u-HI5SElvg?x>U>T`~0V||T{xok95p&`I z>2O&rz;F+`PTyTyF#3ab-3u-+TyAO%EUl=&2Fzwep_nTl{2jO zUlR5iDerIWZ_S)L5|D!v)>w(8v8U3>4c6!%~8*4-t*^Ta{x z91E>0OqbfbT+$u$yhc?wFI^3``mOMh6}xcol`&hNZlEJZ^2Bm~>rlH>7j5iM)U7o8 zv8KEtH@#Xe>9EqQ`5V%~SmhJ5ETOZvL0u1iAN7QYYxusP&mc7TPDo1qa2T%eidhrRgV>ByPJbJ>-fJy{zkoun$ zlyT6~^7N;CQCw&}uCziAUYs+azAuZOG3y717!OYX8c#2@a-ke}QeiSxIw49zF{q#V z^Er)mRy(8g-m(a274^L^@NjOJx>hl11jU5DhHs=}34J1u_=Ut1U=B=4?^l^Tv)TZg zcji6Lz@W~{pW)W~!c>7_*E28;o@olh#fN1+H*$y8;1=yUJuD%=(rXE>$}R(w>4&diWp)y7~TVz!88wH7s&4 z!fz@UCOF}F^kgv);=(eZPZK3(6i`k5%mvSeTyz|8x4?!VtOam`o`!=$9>+yS(}E=d z4xT^>@#BpBOr@*dmImXv8w!dh)`Jx`3J->03&5ZA&t2DBDiwwvqfSUm?)Z+#RdAB6p@UhY~ zJf(DsVQLz^>xBk%>Id#bCxhlj^(PYXCwD^289NUU&JLIH7{`3NP^O~AU8_rGd`f}n z#2jK+5I~D4Vuvsd+>kjV29Ti}@ACH2u7Nc%5}C3*aa5k?G#?tR;w#QtnO3->2a1VF`eR*Y=SvkcA^H}eP3Nv+@q z>dpMZgu9DSvm$6Bgr%AWf1m@tG;5BB8$eOn#o7b%MOCX-RInva%MwNT6^mMn^#^0|y%342)$F;-Nx{fQtj9S`C+@ypF1BoxlMs z@|rxE|C@KmQ3l~o0^-ajw}UL$Yz4zIiMZyTH<)seh{y;U=puv7i_KF6i@0 zJ#oerZw9Z=3G7fdB{WU4mTOl+QG~<|FiT($NUIkKRM<*tH(&*LhhaxEls{O3z`;td zrrWwpb=!}L95p>e2SU62n^ESVHgulCtnV0ZAdum3&7A3&jNfyPM9 zf%6Gk>ZmOD3}dyEA!igRgwB?uLU;o_DpfA=$#z>X*03D?dR011<&Dw4)+KssGla6y z-PE!k`~onCNd#8`XoEo}T(*_T*m|R92to26V&4V5WC%5K2;@Q43cIL5yC30BFbOT3{K(_;oYX3dZW@y_DJ0((e_e_sMR5btCndeV^0s!;= z?orAfsb^2>{`N z7EEbDMO*N=t~yrttuv>H-G(GVe=J%x{C>$f5k49YkQTd7u5?sx&rSB#E*?iH$a>KC zw$&juC;wrQmG$Nb-6dp<_(ppRC!6n>Kiz7xJuEs_sN~G4%X0sx(Ek0p=Z}8i`)=dR zeIE#LupFlqO!8(1=c8oWYh_99(|a0qjaa%_tRvZB*+Cch!>y4$pXO$W`gy*ids$ij zvprxA9`GHJ?@&c)9OV`_fhcT*zQFuZsnK4;s}h_hw7FSbLvRIG2c@;#5X-9Yy^%C! z+$|T~FU(m3k|0iHusiH5n+mohvBttl^sw5sX$xNjN|8`)<=S-24>ycsEP{w9=u(d| zHb!|tA~<`u>++5sSuhsnk-UK4uxdMU8iLV8ZiA~TqCU4?u*1?3oXwfs2v5u&f3;yv zl1-_&HQaE0uh%z%G;LWOjH*VlkblY1y$0x^3J_zKNo0a%S^pkP`W*mME)JfVfz~Rj zxD5FNkk4&aUfRxj&dcYXM!}hW>?KL4VEJq;$=1Fu1$Z&!Y|=AqgI4F%D9*i{;nh{S zQKc7NMGWd9_$J=!R@b3vE5!KDezmcx`iS?p$Jy-}+Ao0!18WA*rn)%%?v(fIb9-Pa z(Y9w&dgrT&Egn6Om$lF2U9>iwRkKTD=~mb%^M1bXTb*)c$&$OT^3tnviF3d z2bnPNd#=nBK33;m-q`P9hGma1szHtM)vPeG->t}*+Wtyvh)YX^K;+&cFme+L;-(r8b`EI&-ebelFEgM(y8^?pLPU`Yhb-@>F+KyT$NwsfI zxI16?w7zZ3mOnM&J$`-anBjY`yOx(j43OGq^mv)0GE05rlAAVN5n1~1df)|N@bm3& zYHzPzX%r-@pUnB5ZF*5Pt@950zD4}|-*xd`XBq>3e?7ODB zFNM;`uDmG9h&nTI$JG34054W|D)4O^QE%OLi`%MJe%}3!*AYu&u0}}e^fs-JY(8>vvu$I zCU;J76@ctQ-)cFhV(czf-JXZ@f3U*cp*|OJN}fn$LXUF+K%-fc)7Nzb>&-jZA-HJp`{du z7!Ue{vlCW+i%;qIG9ZM5&z-8v{v5P^!fOU8MTs*(zU-UDZ+Nvo9Mx3@NtNwnd|!=F0d-|SYnTo zU*sJBqd+Utor>j!c@8?iF8!Qu01UO(?u^lV8N#2B+Z~vgKVXquT5xMp(G{<_h&u_a z<(8(+0i=wzM_=gs)Gq7Fw|+1eInj0)r-Q$h^E0BgLN%*>+WQoE89##4p>hk6{C?uS zGb8A0#_p{p^ZB$crapdBMGa7K17`U0ZCp5EhzB16l@RSTCYC-5CPEV|@eNAz1PK=9 zl)lIvrc*PHYd!rAU8s3yQMoY0Y=bq&BfH^arz3qrF;m=7aROMN9=dV7?htBw29?nJ zHKL%TV*mgAH3%69^HWUO09gcM--*l0T1_Yl{z-%*+R|7%ikM3^0QdR7OZt`fn3}5q zr@R&9SNP=p=o zL#CD{?}lx6>ggY8kW}yj!!uY2JcrZ3{2P%>4~$>q5Gm~C6RA(iBGfoc~gc*<%Z+YQ(? z)YFL7O%e5jsy72>gdFs-pem{QGm0ez5yuh}gaH)@AZZe4Ni+rXPW`O~#)T8|2Qx6e zBcehwG1DLG&Kb>Dxa!X10^SOjSfTH+vXIFX8nZjAv^QGR?d!~MhHHB>9))uNQs8JtlfC=rZGpMr{K z(xtejE!B9<%73Ccf%*@&r1bF`7CB+1Ml>joaHWQ&2{b~sxQR^Y32KV8m0*eJO%?D; zuzW!+cpMG%VXBhK+3$zX>5SU)^ zNhK5u^Za9Qa>OI2=kE}N2D}N|S2931)O0G;?>$9)_0R$%0G|p=0(NzJNBRP;hL4tt zpC&O1s6{k|RSYhnCj>DBgGIi~=^bNDun##t892t)znfv%0NCKVAccr36^!F=PzQ7kgd z3v^*wBQ%A)U5VNdyznU*9Ci6NkgxRgD5797ViJ`6xUqU?2%tI(XpB<~wZokh9Q5J| zsNGd&MBu<7vhRiI!JmqCc#&S(RYsaCS|9S%#(*M;^-;P|G!i{&{B$@omdwhsdaYF* z7A_H9kb;{CZlechT%aWMms#TuE{fK}qD)=M3<63ex`#omK2kw`Y1>z1kuui>U}8}x zK&Jr+F;ZQ9`~NWbI#LWTYE*uKll-$TMO%{E?wADeZfGQMdhbusMz7LFI7dF{lw=@Ti zrJj}Qe-(|o(AEO@0g|7j{#ybVYAW4j6|iihF8jx6CWD+Gy179n#V_EKCVBuPf`X&d zvUFVW;7EK4-U}8EdH@iYNQ%1TGMUN4cs$4r zTC^(*ESBJsnTL>h)E7}z$w)D1QJ16ns@h(LC7wzC^`kk9kLaq;af~3z+em9TY=Ez| zz1{k-1~n@GTBGOSk2K@SE*D#E@WT2_(|e21u*cgS&K#wh9XWp8@RX|? z#x-B4;UyFBC{=dIMSi;)<7FaZbr2C0A`>J!qJ&`UDfuxg!}jP=X}F6{JpNo`Z?hQ7A;dUAX9hGwI7aTp{o@5zmguF_o{kWSdMB3mK$QehtP8xEaTZ%C zAF|)kUpKK_jTC9eaT&G1- zL_;qQfwu3}_qY2i}=_NGq%@LplBTPB&DW0$Pdm zg8y_C0f*EiN02TyMV<>QVQ047t{WflpiSvjRQkBdE@=O{c*mWGSLfi? zcjA8(sgMz(S>7Hb%+Jw7r&lPiyUF1!mg356b;Bfh1tr?D(%T`lh3zX5T^p$fH^}!c zar}iN(v}=1vTDddf83x!+rtT|%!WtFBSe`a`Cw3uVS5l2^GKrMJ8F&!a-C0W4{K^F$P=f` zb!{E@wZQ-^DzZ&}!ys^{G|rau@1bXJwgLE-E{qDoK1Y05r-mXusvjqj$2Gen>!kuG zB7v!-DY}J&)k^z{j=P~Y$4|B6L!{$rX%%6VC2(%kod^vwZ<6XxWci#HK40RLv|iB@ zNWTk%5YU^1Q&XOU+aR+*eQa*)G-Up~2CSPnEM!P-LBvqZ-cu9o_Owp!!4nTZ31aOy{(wybiuFW^_tmMO^KV<-l;J zOhjAl!B|&c4q#Q5d)_-a@G}6!g2}XO@Nlnvx4UyqPO3nj#kvzo3n?x0yE~_*-p6f) ztiD6!$p+-x;c;$^I3@0qDMO*!*)l6aBt6rK6ci~Qvbv;TcYer9?%WjU{cxE8)T@a| z4nFaI%UIsV0-jN|ty{5FedHgVJ`D2euT9o4$_tU=9HNSpyt$A$g1u?;)fzp<2 zCA)|_Y~$-pIHOnwqWviQ(dj`%=2lI1<*kc zmk0Qdm*C+=5L5TJsvU3j)KDVJyQR4FeIj5^Z7EKIzBGIC7w6lhSAU55 zPI2C=+g0ttf&ykQJ_b_CKabUwN)yK!enM_xs%7PqWlw%y>&Yd`~9VgFJEr+cIs!R z>qOSiB&npnr?g+QU-gen*=qjut%h+casG~=X5FUlM6b7EJ2uHH5^kH?2u`tI2{hk; z?dKiM0b%=S4v)D>r-z>+xPm|nfvzrSf8-;A-hp^ZaT@swW?8( z%f8D|G*vEsB-NVqNfcx~!jIIY-0$|064!0+xLxsUBlnQ*7t(1~C&e2W^7htF@5M zB)$@myDP>rr$SKJx;C|O_(<7W0=5fFG@ButLi)i>iOKbF<=jh z+KOLy%|spPl)%;*);n#JP(qscF@Zc+RX~;P^1JZxhI9s+FSR!?ej7ZDf{$_ zS=b5b^NDwUn@DhZb4%*q0@$yW+-kMXKNkEpmU>oHf=@mBw(1*B;|3f9pZmk$qq$SO z>~=wc$El`UxysmyLjKW~9b48fpLp%>p+1s3iMO+cOdi&kbFy8rhLo7ya)h9fxcuzOt zk9tbgc6oRa8TnnppF9%9C-i(w7*fRVmHnW&~l(ep+hLDKnb+v zpsjlGoBf&2GsB@Upg={O-bjZ|PzA*c7A*44#xIk90cIUU0t=m8D)?d2fFhu(k?0k` zE~x53wYE?T;ZGEr^i6y%oYD3F!~7(ZPfAtEf3X_y&tQe zsi~Kb0l;mS|A~n(j10#9Qb2J9I9k#d3eJV9a-krj!VnGbg{+YSWS)licg6>KgHE<+ zWH2v`{TSw%dLv7wP^BzIICfB_Rf|XxFB}m~G z$Dl;a0q4)lCyR#T5Dw@$z6*{6FY>Za9Kx{jx-VG2sB+EMnUt~NM`Q~JaL zqCj37XivfBWc9)2)<3k*^=o3s))e;yG>_h74RQUbNME=S@C~E_nF{-*tNpP5*bw6+ z8nx}3^VHl2hJ$u5{z@?vhn%h0)eKYv0t5{tLmgla`2<(cwWTcqpX7OjTmCr*jbP3N z;s@T-x5^f{s83cmh`1)0NAwC!(hi-Nqy&-saGX+#9=!cePSU^34eGWj2blG3!WSJ!-VZL^B&4L(f^l=fMt?X$K}6W?B|76c9g<7oq2U0Lo+lVIzQ@jH(9Ap!vt9jK@YHSMYeM zER9&wdMtx1Y-pv1tZFqZKiETfrU+bm!r34U2@{?i=U~-KZT4(6P27XaNFU$^Q}uy4 z@IovE+N;(a(Y)DZoY@8JhmXIBrlGFE8o*ZDrIBi#=YdG*;bn>->HHpWmQTy8 zUzimR3;k^|EUJL*#mep8 zfYp{j4-$uuaMxa{LVa+S<1=>V|ua|!82Fkq;p0n7~mKG|ZwKAprx zH+bg|)sRrlm*FghD!ngj&hc(>w^<8ku&==0qi_GjbAiGT8m7dHgG<;2=;X7LQ;AoX zPrDD&F@Oa?D@z_dus~W?=Tqa#nbyXD0!br{R92Xm$i_lxo8ExWW`5rM3rlym77&gw z#*T$oQ388c@ZPZdD_yY??-PUd0svyVBBkeuX4Qtuu$=nBo8U9}GiR1!55tg!q!h3+ z9%-EyFsXl^AX~-R42C|L`xz5s+P=n0%eQML-hejl`3PFeb=|2WjM?|Y{L0eb0F+rKF95hM3?IF=5*JZU?9RB zg3;sX8ib_Ychu%ZPk0V)dN6GFb0c9@z6jInPEjdLN)Y#&dz!)*;40wnaZ1{Sc?K3Y})ytzLNe&e*lK}5&RhWA2g{b%_(EA6DT zZn>LWILTxb#F|FAz9gAgRUcQ*9 zH9>U9&_`C%D|dVAu{UoJpAA2OP#+*rK&^5FM(9B%_JBwBW!RYSTr5a#d>MCo#@Qy+ zkc)*lb0rHEpcIUke4VUiGm0)0@sZId*iv3Lj691y76}6U2ykmFY%r^1Jts#1NiA$t zl+prTqSm~-Okfnn)@ZCXVLGqj%O#f&eB(V)&Bv{6gh^-cL^eo6(lsDuAR+c0xG5kR zgr|xbv>*H6wat6*`~+%_C-SuEnYe^5Nb(~D%AUuOnj?NuAKrDTda+?9)}mLH$lIS0 zvpgYq-Wom;<99Ugh}J$J?6A06xczMCU+Q0RMiX{N7dSg{?GbJ|%Sk?YxgN>ocVxIu zf^=_H`Ej_?4g`oI_dYB;?LFRMu>|yN&x4-`d`)?oTHDN#vFUp62~Z4XrkuB2;uHm5 zn0*kHU+vNw$?dqnotmI;KAze#DLcATF~3J;W!~DYuCG?Gh9Z6@L*@6|Fxd&UYIzXF zciWoc9-lNZA1iOiiqXdD_w|jw0ViGWjoG6OcQ4C$zk<5%&G%Jao38#;x=Ytdnlotu z%a`>r)}V8;0#2}IN0MUgk+4P!IZ&AkMfU4^MJn9zs4SC<9JSSPmQgsoPx5Xeyz=EL z!K-y4(yhi{iH2E!DL9rsy&T z3s&_6xDR$@Pvrh(o$f(^?EC|vMdvS5jtK{HE0-$%rLFllTzd&g2af!{(iTxOYAKv&3&2p zUt?SGpn8dGJnWyD$=d$xXUCc<`lj*VI!NX#T)z5>t+9kPq@CgGQWGo|I5P;a7&99Q z*}hs~p&S0_LbZ(o324H{s+^=Z+PznPz2axUz}D!_&E!lhldXHrZ)?J((-(un>T9;I zw%_yV3&GZ*j_nF%M_tglwwo=%^B#TMW>U@nFt9bbmNi$$nG}o|h*6|rFgEjo={)^P|f1&_q4Xp`h00UqYjHh7> zO0y?t5{4R%)*_7Rw!Aoiq>!dZ`Uq0kjXkLj{{R2fmpNpsNwE`5?eL_QAVEC!Ln*U! zH410FVT_Z}l@($|9H#G6E39 zhSI?v<CM@DWi_&iFzW>zrv)v100 zicn}Hri&639g<>vM1lk{SqX6N-Y6)MeqcCgrE>1*n%-GWC@iDs*v?&dBjdw z&?C?R#2*0<1Z3f|;T1qH31o>H+RUpz5oIiz7L!vbgLNq7)Ac7w=Ym2F`wo4Gg_)>F z1ZaGAp=7yeuuWq|8p4 z=f@iFpG-uSgJH=dHH?=8375>zAJR`=YULrV;hbRu%G@+iDE=@Ax<*UOX#L7ys-5fgm8f3z!(K`cZ}ts z4E7Qaz-t3ZwOHCwd!W3>Y+77n-C@ZaDFh6FX$$QI^ipEkMCFYdD$~%*rO8al&)^s* zgW)9*MFjUtFN~tu#3is&FozOSAkQT{%$f9GD4v-}PlvFR3_$|R4p~|N0l`IWYaKY3 zN0cF|$$a!A^K$&Y2q{A024_rq=zbpZSPoVLz6)vsW)Av};2!CoB9snx z0vd+_>Qw*eK9jbD(Nd-2@m(le0E-cU0>qDD2Rt2wm(fGb_?GJJTq)2s1{8W~dgBSC zuCy?hcCAoS=Gb(eUmi>nZG4n)7m754WbThK4R*2>X~@lkj|n2oGPtx5h2iiM5q zacyY@w?y!fH$;m+&mm9kw+~2_Qaq>ATEM>GEhuzOZsWPvOHd(*A1N3E1w_;MysBQT;obIv_KpkHXC26=Z3KkWRz!=dFAhr>Q%N6tdrfDad zne4b~Vg?bE08`&L3~ZyRvr4Eb$_0>)Q8hvT2Q`u#Mz0cQSGj0s>qAs63;+U@CLhO( zOg+KMH9t2j9LxopVTIn!y6!-9!Y<+?tx5KNbc$>cL6gCDA%^xk2}!&t{&6Y+F$^{o z1QJ@^d6Xzd)xMf^V{jzit6S2r$eBuCrx;l6sF_GmPBAY_I5!9PUh-q5(tuHOFw=(c zJ!VOAO$+*r{+$FItIcwfR)_*{pFgkx_X;)2uPDlXY%|xNfrYtl{5+k&bH?h4C^=^UmdrX(r)I z*j0jg-$=%AoT**Lp>NaH)*4Sh(N4{42EHifaSEY99b8&^)z6tNvm7VS&%M{`@f46z zrwXxpNKK4Em6_hdOJNASmBbHG=3IV32p0Hsf1F<#izFnWF5rBW+?5ZV-J|a zHBd3rz`Lb`4QJB*<+iW+`y-_mTz};RCUM7-G4aO$lc`4zL@jB#9XL19s(Q<;v;qtb z7|0?|ZECpb2O`BzAG@XgOjb)PL@80J{xOh%bV!XUFb`SX|Zb z>tPakfpa8p&0kN9zbcBv*jV>WO##GhBqf`T&x9+O`?fX~zV(~NarY;tvNiYe2^HrQ zx{uditd#Eki%W$j6kFI2-bfNukX9;ZWu@8J@xlj1TPYdk$8h{V3!h`SXseCzH(@ak z4ZX6J2Yhr=-Ib%mSJ&=ZxxY?|Asu$>>$2od^qb~e#f97;rpa^Cib!ShMWHAdRk zANE!7$N0V)4zghTDo_7rGYFksQW$4S;n6EFvva<)j-A!pQusjmhrvp(cD|oBQ~S!f zhc-byV~Ke;TN1_RguV>*+`+etNM*u~3(hvuv+yLaT{p9`Tr84znvSqv2x9p*DdsR= z-1{+J&0DI(!^Ye@-mikm(8g2`U$Y0_S-(o~E)re~1qGnH^gMd(3e2g)#_?v`p@2_A zx0s8S)1KO{0m13{L!mrum4G&VbFXAx31?w-!<`oQKZKA)qJ7)N`ZvK^Zq#?)B_h0I zY|Qu;;haMOzhLxY+k9->Zu_wV&8w`fj*f)}`-8d{s@>QJUC0gv+*n(!E8YuIDc0zA z&2-2!k|xgNDY+uPL=vccQ}#)i{r-0!0@>8&Y4+e?d&PoPtBNrf9fjo$|Lz_6Bh2@? zz00dp|Iw|U?BHE_DAm4In&`Ml@LXWA`QID)zq3^L_)g_FZ_fRYd{2Gg`>EG|I>tM4 z*r6+P-uXDnR8gMulwZP$XMg*{Al}7%z3sB@o{$hJ3fl8sbN`gMHYsN6VEx&NhHpj; zr(Nj$7u!!o?<7=j-#_u}XekF;#e1qzj?qu81sbmC7>uD$DKjP~&Scx~@A*f$$xOqr z_r^;0rjmwi$v%z!cPnSSq-)-)v3LG;#K!C8TGelEtjC%h{^P^Cl(|;hOIcBVcU)eU zI0xt&!$#eg@2n`}D`G1@nQC5KC9*QQbiVLG#7&J={SJp+`tzjY=XV-DG5w--vfJ~w zcHi!AuN96v9a4FGedVRB-D8VR^7mbN^s8s?&TKH<{FbeCu4RUN1u&l&AEJmWv|OzoNF9c<}@i$hmePH}ta#^2^G zE^7Sby84ibZCPtCgOshp1=?-}}6@wzPG^PS4i$98*Qy?I;~NhMfWm6)I2Z5s)mAL}6S)N;5NqVVy}xS%nmk&0$Nl8h$E!w+o zJea87b4^w)+PBx^$H5pQv#~!Kq>-h-@r=~5eQJxWEHpI)R#Tyi{-{&`7VvKLq^@CW zbL)`AV>;uwL;20`B;n;(LwkzZsYaGXX{yz#jffQPF}{V}N&8Y=-4+n?YWl#)v)o=} zGX7h0jP4Ux(Hsvr+?nhvztj&cDw4l85dG`8ThayL*V)SM$ZFXYEAJJRKfAqkOufD? zEqz++zQfY=*fU9G#ZsZ-y&S#b;M5kMj#rgmqPGy>eQB2ct|y+(#`-J-^WDi6{}7tS zU0~f`qO=Y&*8L!BEV#IDuE-U|K$B38m=1mHlmEtfvFpNXLS;n8(JVc`wWbj9NH5b< zbzmeXQupn`#gE|4BdJmF%gg+Zug=!zdlgqJti`4q`_7-QU+g<58GrBb_#u8zW1;Xg ze|*^Fccf6mD`;9bD)-dG!6)q6-ZtUi@5iXV9;R*d=s303m2+OApIwZ9M1A zu=!k0@;cCBGAd}F?679j!>RcS0g}>Kq#ZI*2WS(Lc7=EnOcR?}``{sA~s;duVFbn`%nDK)Yh~&XLe?^+ky)u`kESD@yl8WSU}~0 zipUzKmBdJ3cJmI5%NV+F^iE6%lZXveexvCB8bosd5vWZnYrrHa&@deY1&8Rjk!7w; zpacll_i)KT$lXU>?y|s>9B4Oaw~c5+oxrR3#H;*X>|gs`w5Xz{;0`_X+NFpdBdh_o z6ZrNbnpT7&5LQE~2OK#LpSFoq?f)@fk<&E_^bEGuBCa|N2YATl#KoEqPK9uWJ^+7k z6{yTxhzQm&{0hP~2qy9k#|D=W8K%kt5X3+T_`xnM?9?c-koHtaYvuPx<9bm4mC^~< zeYRkd2W~#?XR&KZDGTe{>c-R*WbTNROgt&BQ)SLrHBO-PLf;65CsUZL@`n?ztDN`uP%Nc+N?goJG>@iKW?D_EXBrybo3}K)Gqe{TNvDG{E1Q(9!d15?Y zBqvLADt1wMX&C8ivf$v#gpvY1%O^>N2EXyo*>sP&jnI_*8yEp&uBRa-(cmZ1!F7yy zk}J@80Dwdz|7yFRvEmIJD$WiO2wYPAFmM{2Bbig7u}54ZQ~oWK#CQqB9uU$jp-L5j z8E;Tgj=(r$XNDevikgvMkW&Q+OoR{e=boRh+?o4~U+n)6b%8!q^M}FJ2r7Yx2E8SN z2B2)5P;P&s^RfnHgtcfHKK@sKI9|@Xtm3-$yXZ5M&R)ida!%39!%c$kFxH!-lt`n4 znTNQrEsbv-`%9}e4zBv;m#qbJ3OK1YWjpguOIZWrNr}_~j$Pdr5|tG(w1W?X*hS5G z6oLL)CJlvGBG^45-$t9&aPgZmoKQ+;7pX0J)2mbN4S%VAgC3{0M-(}NzhD)Nkj^6Z zfmAHJ<>jo2Sv+rI%x%iL!iQXpqM!8QP}RQ^F^GH}p{xMYDp%3;`at^EJm=JQ4DTBa zf}G{miWyu2Y?+0twwgVZQ4^Q?+n9`+ZZC!h|GuSge9WX~12oC<9YK?V*+7u%;31x2 z>U+D++@`!Mf~TwI@oR5ydPLE|u(d`oTQ-$dAx<+PXrlEXlfkm|6-6#cLlfhUGqE+H zcdN>Wi%sHjtweWD%P#l1Tssfo;9a$=?~Y->`YDQ!1sNP2KK7+$4ZVIot*p7%fx@FI z6HrWNpivFl&glMUn*{=2hrJn!F;r^874`62#4-hkA*?mN+eb@7WZ`b`>x62wQp>X; zeBCI_UFVgAlqcbcwyMZB@XE~{jFf2Jgn!wdfKu+Dr zqk;`ko0?I=z{&l$VWT{r0XT7q#8EUI$xbZ@~O zz?2)v@l+=aB^G97lEb<$3!@eYR;7Oqa$8a6XOyD#Kh=X5PW8$RYhVWPdx+zL8uBOS zDh~QNqgKQN29-EcD*!9J5a>?EiXw$UONK9z-y2yyvV5}7iOu|}MfKgt>8ng)DC&N| zEbZ8Bx69y0ppFWzJ53gok^vXPy#(+~iC_Ua0hIx6G>#|3Zzn}N-qBnEsL9|n3U#wZ zHG)Hj`2+WI)=dk-f0-TL`9K3%Xly<;3qogh1A#KO`4zD3z|*{3pyy6FXe?J6XBqoA z2VfilpJAbAKWx(cLL-#0Mtuc6;_~Jp zR%vUQpvB+Rr>xPYLLCM9o#J-r$5rL`VT{qn?`_PQj;Lri{_L^o2A3gWR%4QFwAUE3*|!9Wn+fONnP2HnYl^_lKM)iMa=R_ZRHD! z^5Gf%hR3L!=XOzl%qHX=uG2=d4S``I;A;C|2dDZjAM}YULlxX9gpiO-LtS2Q-Pq)O zt-jd?P>*;F4^RBDByo%74R{OP+vn|ZE*{nvvdy## zSJ-nGwevEUN~N*I#pSJq4zjE-{IhT5gp6-fHxw_wI-R-V;_oZ=-q(GR(=Tg$TH|`P z9oISC%dNG=r{Iej{-E{XY>rr4ZLQt=nMLE>9B$_FN4#74Rkw>jwKEPrw9`+j+vl`+ zLQyr?@$lgT@!a9Dr{zvCRyruHR#^14`u;G^xpXXeH_wav#6@*Q9B91%jPF0w18>(! z{rt}!v8{aS?|Cz=5TjSVbVr-ny|r#FwM8E0eM83)x?jbsGtHyyjS zV(i@AMzJJ&?~%?%4C|K%8$-g6Y~-m-_}m=aW95D)!D_v}opn>Zn{Oq*bm!m?U$j3u z%x2%60KCrXREbWymi*jR>-+Pc4rlIcy!Mu5>+gLl?4_Un>H7DKUk5HvhS*H%gciQ) zY@gVWad+=Rgvjy)s>NyE?6Y-Es<#^=BzM3wYN*OJ*}tQ7=EkT>XV@`s)I~o&ZsYOi zoUT1rckbOG%lIG7TpT1GMfrKZq)V?qb^q7Zv`+x3_UT17oj4M~R=?aH-()_uyK@$> zkzjAqzVSof3SM$uZepI*Qyh3Lyo_rwymU85cv55~Pfc2k=gEjkNp=qFkj1aZw-vss z)ax7AGZ;SmY0kSlL;upcKs-ucA=oy}BdH!`$Poe`#OEHyePwZ2`QYD0KgZ9#lp3?= zGp`WI#H9CS?s6}g9ah2qEMvTKG)*$pAqrJ)b&2p=)Pln~eR|tdH&eI5*r}X-U4=u1 zZ6^HJWYZaMjXEmz+RNKg`3?=kX>;pkET2EO?C|vHS!zD~hN4fZ2gq>h%Vyt=6rpqw zB5C$d$E~qW*Q8^gJ5ki>pgblB4mjXee#+vA>P*2DzoM^o&H0N5Z!PN_->WdX$!ZJd zrLnhlxl~kGzwnoqb%X;yx=e6xZapD|=K4*s6~{NofA&ZVy~!Tib3vM`FiUO?&G_p0 zrII&R+jhMx+no9}zoIf6j+kkY^#L3%6p>BczK=J2XF9hJjg!Qtta$(9xe##_th9ZZ zV;`4Uaq z{AVaUK`cPoLS=C;b;;;2>UJ@Xg2ZzC{df%t7O0i2*Z*{4FgkPl6G9|Rf)RgpBbq7= zDy06eKa4d4)%cpeC5iY!^GBVaX{%|G2@(b4K?y|`~avy68wSDh~Oj5lvP2mQE8J!TmuS= zqyMCiJO^A1`Y)$i=Kru0wGCQz75q`A1f^_yG%0RG_iqKhfEty-G{|TG2dzN`=-O$H zlr`Z&ln49=Awk$eL#BXV(I^5K&rq0G(l!cRE{3xK`2*?UQTzg)U|Nf^f@Rvo7QvfP ztx4AgdtDe*qRD>)I0KbO%(cOo%|>Hyp46W7@`A~7)?|+_nuQ*t`!O}!*`h5}Sko_% zPNEQURj);P=<8xLM*Th*17HV78+|X@0d1=SrGZ}_KbJ=wjzJ($kbl?=N{~t)HHC7R zf2$UD42%_OH1@4)82btA+3^9Ij7##sU{0=f+o>nu-+M460id88(Wd<dl$|ODrVk40PNYe{06wGYUOg39R&Zt^)n-T(PuYH0tM(u|ONlif>n&2>_ z(az9(&)lKXqnAbnc1B0b$oK;M`WyMb`fI{E2rI{4fxMGl!HsCgC5=~R4V9f544y+G-?To7`javp$g=nPrblkDD zWoXIYFauu%{S5HX@>uv~l6!<;R$iteD-E}jc0~gZrwFw{-xM-G_bSwM;s_V{4=o{O z=6i+#g}VTZ3vdPC2Fxu4+`yo+gl5fY@^V3=EuwF>A$$R`T8L!9d@|#Dc@5HS`e6Ll zie8KaMzltWrAa}3LmIdU#2Disn(bI_x8IzX(NmDg370sGRRXBQ9A=RsB#D7#L^Q^< z9I?^a3xS(8Eo^k_nEL>V5|e$({Kf)YSsR2?V8vmeNp0rWnb$HF7n-X}oEvxu^fFfn z94b_`D4UOuDz|fRO!rzJn+9^1EyV?2bwx%=fMy&BAlhOkC z4fTd=$GjWn&EcnV_(|-`xi-lx73mIai>sPV1U)ud8L)WTaP!IRy~9vAW^i|W6`C2+1lPgWESY4cxNW3M6A zZM39GeZ@(UtMGJQI#$j~eKE`z??UnAJJ693NbYo?Sa;yT2kQN1gzFIl$A}BnyZzh_ zBx4~Iaa1nHn8Di%S2=(gqjOZE2*&n*{6qB>oS*$oU-gf5StvrS$o*skA{Sa{4l@-p zE(YpEnyQRT41mByUIx5P5az+3R6HYmy>RA^Q{x%Z5K_h`VE`OZZ2V7Y5L@tTGW{4= zRu~U2&NXSN#lsY-yTt~Km|`KLXIuxuaz$#y^3K`to6oe_+d$?Te8KgD(5RiLrNEl( z6_bNESP2axbJ05NdInl!thCSOq4z+K)@=i3r5d$RtUFDv8`C`lu21_#XzC;SIm{N= z*rvOf9eRUhNeS~kd>p4mzI(Jyo2>dTEjQFyGxDZQ1*YH`m=2aDvnG1f0&vH4$ofJ) zabskPmnv<7WMZ@N?9oi6H8D{Dw`?^lWtMsr{J3dTQ28`slLRu%y7XZ$t7Xo(I6{?8 z0|+h3((q*2N(7zsCHQ(EzI6sSC3~ zuPK$5iiy-#&LO~eSjmIH%a|v(F_h&MT|YsiKJK>YR;D@zj)*V>WkJ0WBGY*n6g)%y za0y%~q6VA|q3O-*XpFoBA2Lbqazl{a;1jF8={gn9L+v2<2cV>OxeOSz_hEtV-(aIL zggak(?nz(S%Ecnj3@_{-dhO@71Xq4O$%nJ;_-jStvI~O7%(>N`JWod>>x(y)?ux3W zXwgsdcs1Xa#@k2YA>6$h%*0;BsmL_4z4gOQ##5zFYHnU^0s~T&{s~29rnhXUYE4m^ za?j`e)9!FD5yC=bx&8*YF0?y990$RpuuElBem{KJ@bzLPtJM}ZQ%1{`JENdJfTlI^ zPnXzHkSXyDy!RpY_BNIq8(iV0JEVHP5zZO`o38r@n?=t=u2LG$ofW0th7csOA|S(w z%fH#gO66Z~ZEIGYZG8$QQiOCSPJ*tezz1$cQQN%3U;vkJ&@Jk#VRcIqJbuf=d*-Y$ zShJchk8D|ucu?M_-Q~kfj8UK9hvsHfOe&T`Sv$N9(u*%3h;$qm^ep5HGQ+_wFLt|( zzT(C!97eWViE&`>`u9BYHhK&rR9AK-oG1`~{#)+wAhvtA)c7+)BUU#z(9=;NjteRj z1Wa8X*fPBSdWffoXOgPmH#Vz#rIBKCTV4~Zw$1D8c(3|J-`y{M zxh}l*^3(hNL4IF^t^%e4dF-&!OGV(I0j?OGiXK<)m0xOib;RZ{qvsyqzqlY*v}4_t zxeDFu0}PVw5OvDj`~n0x(hHudyAP{>Er@EC^mG^Ko((xW@w8JKB{iCsh508xa1Yvf zCi(0zU%bvdEe(_s{=I^2hTmFMo~!f?-Fz=@eWeO4HGxgz-C%@c95pvvuF{tBJuPHE z_?b5ZZ?TJ3)c0B;l!*`fRd?Q3E#rZ$sqeJ3hi&vgpyc1d9x+TRJ8#MI6lbWLc5$$B^aA|JtO=ZB&yO#Q} zJMFVh+i0wF=5bSY<*oCwiVdp2wB{6bgSmKA<0Cpd8)EjWV`LS-%EEtZnmQSgqBD$T zTdXAp@G@|$6RJrH04fmQ5!WaGayn`AR>&FClgkw`{wBuVT9(IzhqE?iyqo%uE8eSq zP_B$u*rGcVk2M7=Dq{D~xi~dU73+J+wNM_S`;Csuzo&3sEi5S<8x)VJ z)SJ@l^4oWQyuM&de0RXvVcql6-$1E@A2%+ZZ}_0iTP>hG=9b{3F)K&SC3k$Q1|v_# zbq+>9jfxU&Yp^L3P9bt^Pmi*&E#Yq+YjF8n-pkd})Pyx=9UULa=s zyZ~H7k6_9nDv9)iJ_(FscD87S3Mc>cLI|4jAWCY`olh*$268_LH9FeKgA!ss-hmC1 z{pcEa?NMuT2&q?McXFf3Hv8q-UL34SWG{(GrcJxbH8-F)VE7IIS^n>TvHM{h`U%m{ zKpqa;Iz*0{fQqQM{mFcZ0rZL4Jb`BsOV^c(Q^G80kI$6tm?pabj^Nrr zr@oIlc|w3Re`sTBM|cL+x|E8F3rJfr4{_`TO&Rc}rE4&t@t>Djv?vrgEvVyCoY9kU`d?ffdY}vS8TELB9 z!9a{g%3wHQqT$x#YTTf-T99mVmuO(NZXkE>@K#9Tdm5JQNkiMFtq;jDmF`+%+zuq|q> zYy4nk$vsjiTn_KnfOWVZ*sbsV9@}fo5B!b7TqGawBPt9tTU>{}#F&BVf*B{|i-2Mp zfB@(V3|E5;8@q7Lt)48LVD9@3h$)8PZc4Y4!M||iM9^UH9k_rA>FZrYQXNny$bbF%G7FX#}`sSN2^abI?3s9ap)xb11buMMM0;ffi!-s$%%KkQ${tS@w zOwd9OrQ{r%sPk+9M~fKB4ZbI=f<{|FE?|cfrcO9Q9KL=$?k<*Ew9a*BkK%X-7~zfv z*rp9tvo+wtmUR9{pZ#riliBY)jSWd(A;!Elu zQ+Q`f_R8BMJUn|kmvMQ8sy7B9x=sRM5g@Brd8n}nOhoQfuiM#6vLSdeWg5$;GguB8 z`}^+{t|&=IDhJNn;;|cQmgx-ceYb)1YFq!KGkgk_ux`PqSQ=yyTRqs&kaRKIqmkFC zSeZ^)TdaDM?eLm!mJ|E8m>%q#MRQ9sc3k)K=-^*h-PGfv{Tm;ind$-kg_u)yVk3#r zXb_5~U{-b$8-pF<`q4DOTz)O8D!swtUg+-yVt#ccyCDr!jTI7#3C!6bnQBFH9Qf4f zf@J_Z?q}pHbl*24o8giM{uH(PL1s8$0AdCX!Y)yZvwS_q$Ob3;&=@JK?y#I3-314O zW*Vkq+)WrdEKh`4F|Nn*hgu?3Rauoy;F^Gm!T13Xb!v^JEr!3;98nrDzsv@RyF~&g zM@=dRVmtt`=fltd!7pHf!Hn7-{U#6@fNfK3>_iMDY`|dxL!I%q#0-Y^kOA3>fU3|sonx70DfQ9$@EgJp=Ct6z{xw9Ty57k9on9yCJ-&g3X3(I z1c{esf_c_77dnZeR@e;3g(wBgbAj!yZKgQ|Kg0*Qx;ZO7R8}%f`!#7ey_nr3?lFTo zgM&559cqhA!hrIo4U6xMLBH?vRFKIN{H4|4KugR9XQ!r3jTbH$tGB^rf|fUePFZ;{ zviZ#DR5MIR5w*%MPtdqhQA+^vA92NAT1o@RN~>*0{?LB+XdnP!-I6h_Wq** zrzwi)=r@Cpm~g69L{V&?b||F{w02+^uU(w0%Lvuhn%GpOLH47IbrtN6VM*>GtVl3S zf^nm?wOp|@_QK*<5>9`pd|s?v`pr$1OClb7`YJoRzS?Xm2$BC*&+>BhV5d zQeSbnDHwJfx-aU@xd1(iTo9>_!&N_|7)D(adm@C9QaZg5HrS{&0loPABh{K$llA(u633 zAr}KrO0e9p0GXl$-(|v*g`i0`3}`(!+I!Ft)51}P;L$9Q#wbBfI3p8Er~JH?Vsc4$9D?CpSQLR>G<1NY|{N1ZMN$#hqV?KnRM&l<#(-q zY+V3O zH<%p>kXId6m$lxh$hCUcbYs-bIaZzdoL$B?UYa25SKqq2oa3z2X8Df$GYjP)55n3q z!N|C%qz)!Hqjyv{2X=njRj4c7&DY~}^6}G(q{yhauIjJK%&I?@|GQ_o%H^YmV9_hh zI=(R;r>D|{VlU3Wa`MJ@#_*IeE|P3s#-*~7O-9Sc8gedx%VKO`y*8?}GnZdJJvw*X zjooPa@o%%XXS@sg0+KKRTMYG0qei9RY9!^2i#u*LTG_UrsSB~2d1_T7>4+D@sau{oDaJsk8 zVb7;y)zi0pB3IRBr~7o;76o)ZQMLB(-t)0b5-Zr>`2_D;q?xa_{rEDYH*;&G=Gux6 zQe~-WEa&+veRJQ@mdq=MZ}O zW2sO+E813Y_pbOQ)i*p}gFOX>#r1ve63-~{QI*@ZIOCU+YVSW8?66R`UmJ6Db?>e} zIpA!0>=+AYPxCm!`S2}KNJ{9no*hLC`B~0^p8)&dwEm-m>##cM<(%~;Z#!M7*d3uV zoKoF*jH#R+!d*D$($zM4s`2u4{fYSJLl$msTAtFOvtv2t!%F;EKnLW^J}ZauZJ9rP$hqiG3!5Z$J9EY4C#r^ z^!1cw`Ulu`xklZdtE&IVYG-;^SGZ}^pz{Sx7GhDNYD2dqgRcd8!MBXDU+5xS~wb$#h!LqkHKYhHFCr1>+Mk@e(<&Recuso%lgq}9N2KM3hINPx57&G%d4U#t9x#LY2&dUWQ568`8 zUDL;sOA8tri*54C)z*^Ql=2HjE@2^EKWy-=mdkU8+_X!l3|VK-{EUS%CW~`qdK}ZFzrp=jnXK@fE5r_8$LX6p%huQk z-j6m2iLE*Q?x+(Jv6d=G514x=Ynqj$6L|WazqEp^Pw;WXZy53p14 z111aGe^Bg62sSdtleEE1Wg<*RnkY}e&CSfXnj2X8=r64&KMok6f5-LRfe$1%G9It4 zdG6;CJ+y#7qJ}lPj7*&eML#SWL{u3-Qf^ZF|H-$|Fuq}k$?V}Jio;m$;<*rgP{E8% zH+Jug$oW%j*TDiu!(WqP@k#Ecdk7f?7y-e<|4rpUm!1(rP)P@%gQOmWYV1S(9C!k9 zU=6r|a!*6loiG3Y`$$oQ!}JY@G5<$HQ?Tt**-*nfnyDdU3z~oR$DRibW=#*24cy&5o1N{1~4twxST*x zOC}n@F%R4QfEq=Xlh-2hPWUyqZ~OpZ{61{CfTO_r6GRKD^vQ~l==b;@qK|@QxBw_@ z)=d#;LDez}Bw|ZQwdl5C&%OwoHyop^Kx;liHRd6bL21)S#Wxe1Nd^i`sSJW_^n8v+ zFD@{70nA?bM@O|Y?LFukFhYGo5s2xMxUdTHfM;bAy(7xhdMUN5wbg&a zy#JOb8w>DeT?taNRJuzn{Oks1i;0RrWTJvBMX{JJLWyBvS!u zL8%s~B5@Jpof#nkQdpocz{&5%{~5E5ccl|*gwfAP$ zsjpz(zz+^X5%nPf6ku0?ZGfX8NQi7d;ktoS#`Z7~p9X7PEg@+fP?xgO(QxJ81bjoL z3nUK#JhKM4gAmLJ0k(Zf!F`l@!C{JTeDb{Q20vN#9o<9{98uq#3)M^s5k*KB!FY7e z!&3xoVCaDV84t*Yk?Y`H=6`64=hq&b3&bjLBgM%aX6)jbE1tNJAzt9-5{Zi-Et)n9 zI1{*dOua6wC)CS=a|x_kzSqKPa#*oP7F`uCKl5VBQzc#(UBzdP8(RBtjBAJm1s1$! zCO39rB8l zG~N;?Hqcq))4~6teYP!9cQFYmYX#g3TtP<5PsA6*GUAq3NC{&?FYQ(>6ZJ|EFTNm5 zj5rON*-L{C-_hO}5FnlG|Ey93LI9QeAK3|MPQWgMr{{dYDKnoi9>Jv`%Zxjqx6!X? zeyTBu(0oIqj3Bjnq8NixM2j%MYHmNbodFy;%t<~&IHu6YCv;Awu_oRA#5ZtzliCI| ztgpL-rMe|DPgAe*F`k;_x_WPEcflxR|AzLp`P}!1A>UOQ!Io_R0u$k!H3TMTevQt} zHkp%RB+1p`!6ira>v$wwM^04#mxU|%=lDI)(#MbM&V@hW9x4#L7?Q!I9s&mu?A!X& ztDq(zd77j8jZtUhE;WYZVaJBAe47;>0zd8hx$$D;M3$6=WKi`P51CI5I;J}chU_fk z;#bswjubg6Ewxwbl3loTLuiPYEHLW4x#QFF%UZRCO5N7fXb78BE^2H3P?O3qUn!ZY$cUA616;{dI4tS}(L$HI zgx#m+`uTe)R|;Ap+hZ<68AAi{`cF>HEzUJTuQGNKwnOE0sVz<0VD=6pwF8aWo3HR^ z!_vb0Tu!=I!$?)>0JTs!7-vTbPONyDR%}`gWCSr%@QXGa!aa!cctV$vN;FC4l(qK# zhEHm6-3kCyz%T*iF>)#aB##Nd&b}%bqBKCOOdFWoD0Ko^>PVY5o-2}doE zPmyykaTFXtx$tahJae?$5|R)5QC7W8Em15w+tNl-UxGRvbe6-F+D+^rtJdv$fWS&3T#_GB6z;7Kbsc7b`v1Gr9Uy|T?IxV}PuPNQ{FUmHRgtw#e=ui{m2s|p@wZ@4V_Nicgv@4Bb! z9{^5}@BwB|fYRR2nX)u`q_8}6E$+yaRfuwV1*fmCwDodo!uIL!Z~moqdu|9}bj5)Q z&o?t55g}7Vsxjq#&ATL>_5h}UbwsnGuN$?_fKw@#ouzz58Pe=i?8UpoKlsFvJS-V( z$5ZxjLu$n+P&LYSMg3R9XqzxF*1iA{^S+;kw3X(=7J}4$zUnAoyjXdRBA4slhJd2} z0&dHM%6_(}AmdbBwe)+Vlfs`ag$27XG*m{D|^M$6PVmhiWgxP_)nb* z?j2r~-<$n=mBL1CY@xY=&|dV{o)4fdO`zjJDQD)9wu zR!PoMlPXl>ct1=-GR5dF(N;r18+|YH?mn2v8bS(9Ii5i@+B2DOk`C-`-Wp=5vH7SS zEKce}*Aowh))Q8`WbJ4VfD&t~&)|jxBgMoJR2kMo!ny#{U>Ga3@Nc@Zk96(vP#O^# zQmakCc?cs1{SD(;s6>ekSZXr=3@5OE9NZ-v~-~7?YgIJ20~(6RY@9AOtv; zY|MwbVrh-ylGK-YfXD(dulxLOl+YB6Jdc$dJJ*#~DoPpO)-cwjl3t<9O z-X3n-d3J8CfBMws<%T_TFRG&cU=9DJ6}4)((+f3ZUp+EBYfIh?J*IARt*yGaKHye| zi|dQH-4;_$$6N*9Kgx8?t1#N3+#Xu$DnF7mAhV3KWDQt!lgC0)Q%SqbIIFFlT@u-G z?e*H=-KqlH{=8F-)!3g1Q)V-;j>qN5-h)hE6j#-xJp8Tf_=amHs}-dVo(GcnzwJk8 z^Bcvde>FZ(Jr}CFkq6nl(K02c@bdsaR_!>ay35-5{jrfG7(2)!1Oslb`n#l8vo@FM zAPz%m4ug?9K#6vUvG>3gjd9-X;$kdz#E!9SUAnAO>>3$6?Y?uj->MO#G?g!uIb0{0t1FJ z^!3<3lJ8r+Lh!R7CUTuxoRlK}x~0{Xck$lS;`gsbC7esVlG!TXxGy+)&(VOA+CPoN zzsMVhoX55ABNy_BuV-UHfR_B!L!G@jcqrb+zc|ueHFMwF<9xE#f?;50Vb^2^ zhHWNt%lllvg^!Mj7sJ8CYOu9O8r^cbE^p<6$cn%tX|1}Q-{Q!`CD+y#=6jiG{aNpz zZ?bfqZ~0U2<@TWX4696D@yWfMp6TI!SS^+;!ed&nzVVK+M8$Jx*N&2lT~^G@trE64 ze_LD^%(^h@d9F`c zUYgO&OPk5ashe7980Z;x(JJ~6i_vAzOOJH#%|4H4D&R?*eZ#yHe?1s>Xv>(1z@z2< z0g+xHr*Wp#YUh=Jh=|ZF!@-r@{F1Dk`mVxkB}-u!Uao$G=T*GR^04B{PYT=)jf{Xr zSK%A6+*dNS`xC4B{L1N|R`1xk4^wnpyhkIRb3KO=6noxsg>8`9qA0U4UtMNB^t$bZ z=3jr3*<5=5yHZY>>Yc~!u7}3#V?-~U4*VvmA!4JYII+Xt&Pz{kBa7wvf_<8oxA#v2YAY=5H;c|x?k)v?EJ5dKVp7aOv6}-%*s1^rbe+PH$rc2TWZH( z>EUxq+p_9i$e2<`VasP?T-p?88P7KEw`h_(KlaJpq&yZ`>Gu3Une&>b7Kc5R+|QJH zu@|aa%>t0Kbh682eOHCcakeltqtb6KL{_Vo7<*;(eOOddRG`!%Szy8s64{RC_mpPE zqXYJp$*t2LDL+#iWh;%M{K2LK^vN3g{u-;ds(`&?^1(FXMe)r3^)BLu3)pi}$62TW zvg5z^-pKU>PKs}0`wvP+JvVx)pH@@&k04c$Tv4_R-vp|b95@n$Iho_XX z6$SWJd>Cme{zIw&W;cOwGfk`bfBuE0Wxkx$kSb{34m16$*xOSZ`hU4iV1I4gIInhD zLRi4FUuOPg2=9=UKs&-TRKv&|0=x#&XgKHu2H#=m`_T}NFM@#@X;2L)M3}OUJn=ryg%F>V8}L4(|22OQ-3P6RC*nMNv8q8W!hkx&ABR)G zK3MNbCZ^SAK13YAXXL!WIA_+NV*76P1`0hbY|!CtMdfmgl?FDzgl}a^Eynbnlq5JT z+H==1fQp@BQ9E;hX!-!d091f4c(pjvd9w+on?Iicf%fgzMJv#KruUft!&KQCc*6W+ z8?nYMFj0r)mpPNt1obi%{Y)852gU3k8F`16^5BdZ%{!{r?GU)~JyS>N{-`cj$D){j zqC%JLM$u#<^~$H#-r@H>m}&_BhkloxS^(vZV+**<(a6AD7>%^ahtmM%LtuHsO#u60C%R$@sCU)q zP#AY6!M5X8Lo}+>6DE^k!DQS~m_@9En1|RKp;*TwmB<+6A^`-2!8#}lh0u`kE*u*e zFpP@mFRe?j%!4;{GidUld_>m9vt(W3r4T&9Xnjzolcy;SvU#HkTr%u=LFCyAyMt*^ z{Ux`Vfdk)199${}1|`riQxh+mXWnRzkfP$^AtR1pAqakg4v}}Gioq&?&LloZAc?14 z8-HxoyhwotGXp2MApisSGPMO2JHHOR2P&Mns^^B88$VyTYv2(rs>~THS5WkU69J@S z{>&4vm=QG*t!2g{PaOOxYu;o1zegqBVP1HH0R!C+?ur(bET98u{5TgzGr%K$ou>D^ zn+O9HU=2%|h5!ywXR1bi<%p?;xw-(IT@7JF12e1vU_rx$R6JW3&3j8Q14whE8pjfv zTtq~m)5cI|7zz*vG5_MWgKEU2x_*zIH{5-EKz4wK*%1d?Md0?KnuDW;Ce?y@=i8Xd z(hWk~Zo$`lE21}m!vYo%C?vRQ3@s@-7tXFfRM;hIehIfHyHb5ZAca!u4tR5qNs z5Mw?YCc15AzCOjJ=@D9J2##P0HYg!6Rv!F#l}FvZ5y5KBdID)h0i2>jGhFVo#s_W0 z4|y*bczn~4Dr`u-tx4;w@R6YEhi&K$l<{n}jpRiqLve4Ap$K*W3-fs{Fd7IR@JN3Z zM2H!ABY)$u_!SOO@C4*FD-UM)1*j`sV?Yf+@l)Q;=n|++SR2`Hn-H9SV#|)3L-Ch& zn}fJnkV+19=6E^GG{H$59A}P$bsx}&C_K}5KXO)`&DTQjTOE7kGLZ#dlRQY{S5&u2 zoJC<6ZBq*b-Nx{3lr~g%un&M|(3r1su`5X&E%gW49V}y|c3Ub-WmT@Yd&}#G>G9Etm2Bb;~V(!E@`;-$SY&mw{+wS6Cm>Al$c#PV1w%w*86Ph1D zaAd52;R)!O<~1-x)BePEL;WC7Lz<>Fu9HmAtR5#>xpGFVWv0iem!i{{YJ9jP%lJTI z+QxsODko^sZRqMCI+MM!Y!eot{w!dV+YpY+tWgLdHA!YUV?F^*E>BwAPv8x(!;<5L!H(2c@}$5DlzO$Fs2L=`pyNeKOxgRg7$w~ti z8=(Od>31Nvd~Oy$!`PRA-NJ#J8)iC+ZSH~9ZXzM-nPZO!fU;?QO*C!%95#@30tk}|ehYIwX`ZYt6E%}Y2TS8L7dC+fK=!%*|PSvdkIm8lH2Bvw_IbZ808RMv{{me6o0`fzTvUqzBmb$WtZGKr#nKEsT5|DX9wHjJTKJj-fGx z;~H8k!qRFDl)7ApV7@!gxlu;?yMsoQ6h8n}nX@dtzr4)?t!U zrbG4R&5}EbO5@#JxiVXHz0~jc(>Hzmi&Z(iN;13a~W7l9FPFFhdK(LHHuF@FBq-f_XNeZ!Kr zc!_M*n-k#e>25I;G4wjuPCm4YyUcalP3M+w-UnY?o>a1)4`u&y`D&lDev^T=ddshR zJ?=;QrVEAaxTpU*4=yMGnttpu6+MZ*pc)oKli*mK_PyZ+d7>P?it}&QmnLV zmPjXxSG<$8#dYYj@I|*oFhIdl+Q1p+x?RS z(%@-FSBvU1_cJ4Z=Y{#Agq*gCKV*AfpPl2Vjy*Iw%<{<{OaCT)PvH9nW0JGiiyAKa z8~iM?J#bvGzPtN!O#6?9OD|l!lwK8dX5zIxah1vL`}#MUKC&!{{U)+4Bsf53)v<8W z{%x;Wj387D8Q0x;jX!elodj^St!Pzsxkb%@b8&1^W!Z)HxYv{`J=UZ6-gh!ReNp+( zQ|YsFRs}x(LvDX#-|w=I|MWuaog7S)G~E|7A)#+C4J{btag1Xv z0?#*y1>*K#qWKGV7TOyhE-Jpia_Gc3=i6k}ExwMY%4V=Y5G5HON^Tyq>>SXmXe<|> z&uv)8EowB)JaX`hb=VUO431=FC#-!qG9+!hT$`t0A0F54Y7m|)de49AY8Y5izR&$) zXm+yJQ=c8VHK%aZPt5}t~iTca}f?Zygg?y(fZ!HmXrB=)F(%u`tCCwo!Xf%w5<)0j;yZd*N#YmlSeq5 z$yEMZ>xlh`|I(@ixKUmUw4d9b${E;rC9Ukh`P^+kb$s1gmr*t#w7%e_`o?L1f4F!{ zll7QgSZTLEqH%FjLPB+$nbqT|Enn~OeRWuOlhU|l%ibfrP#>4V(%?V4 zN3J)CY?CrAW2-wjoP>9=TG%xYRo{0HH{O4?^sR4Q%Giy^a&ooRE=}LGdUZC>u%AQt zN@___aZzE-=lY(>icJ~XWnVe_{8%Ki9^~C}iu%w$NMZlS-I`?VAB4WFZ1#dwez{22 zC9{ja49Jf(IF{<#wz`a~9 z&MHy=3RV&d$mp87;F!_7ZCEEi33d-Q8O%L!+(Xt@Jv*6kv$Qxho(bJqXP%k||Dtl= zZBL(W+*STlX!x+(z%iMPSaR6APN_~X`9U7xEvsDohwkiLM&U>8s(4mwKkMDH&F+t2 zVaCdKM;y=6&d|v+!^o(E8BYcv*zx{L>;3f8cog{}p7baAK3M#}v=_&d#i?@2%VN#E zRAU?8ECp3)G+Py~=8W{G!jHRTz%SBahAR3~R2}~JpZQ{4Q$8ZJ{{Qtu%?pbX`?RsM z3Kuz;;;DsH#{aPCshdu_dT6YfE;6=ZE+;rt^-^Gh5y%34RBgw%{+AC0N^Yk1Euq@? z8Kgn~QZtxb4=7p>hmlE8!heo56LuMz=OGi?$$c6R3dZ(FeQc65#KHnu^lpgr>RgFywKSIbs&<9QMQNfR3K1@CD7}UmXpr%W^_f z(764GiQ}XylFAvTtoHqqTGf;|(Nj)F(12*^P2 z1+&-fSIo;8p7orGa6yHlDYnYpCrqf)#x-DcxDXQPSRvh9&9U~nsgcSzZP!9w9q<_q z2vobzs1XDy;C-R_0se5p#uIF4qSS|*R9ijAX#Z>Bk3l14k+Gjd$({>EZ;;{u;CsM_ zW zqVS+k$xH#1yGS3U8UWF+^U@tHB8&hQ(1fPYy^p}o=P}7tU^vL?f$o&28Jo{M3{)@9 zOGwZOtRk01+>mDhqtq8IVTyAkgHG8HGllLxQ?_Dm0-B>BQTl?+59WbBea6KDCR3ye zHU#{~6sh>SfEl24%5<-hnW90ENHjypkpaJ-=LNe)*iz6Khz0M%9vLfS6gh%g`@bd_ zQwI}|pSU5oPMFpU$V-!KLgudwWDx)&z@wf=5%75wL!+^Sq^r1O3LgoqiDn`Z5AgHa zdGt#21Gf&%Bz=Y@&U-d2VgtFNBnJjffayYd^)uVk#mQlffgH zkAfP<_4a>o28c7nMAV^bgH{&iomJ@h(kt*7S+(tPjw!%Hc49Ah3-iB5um}A=lJ=-j zoOkG?-~Z(8BA9v57XabFTc5|2LRr!al)R_g$29qaKbR0d z_r_dvlTa}At}qv>=0u++;rQ++>?;W#>bHakiV=jN-R2hnJ}~NnzMf(G>%o}jdcv^n zPX^x|86Hg=na9=AJXLFZc7SydvCrGn&G&OtqFVtf&=>kMmW;mN{6oReNoK;!VMiD` zLpa!?d?9kZhsBVc8&AayR%-&YMd}1kU-0(73S@AJTA*~n^UfEH$4P{ayQCQvT=@7@ z`V!QC0gW&xn72}42qJ9844*#$IAU~bV;2!Skjs#;xIYuBBy^dV)PO2auqb62LcCz` zJp~~f2YWCG4d9LnDdZ3Y;4h6gcvY$IMBdJb2c81;KXuHUvi0q><4ncUhP{sI@K@@RAIG!ZhKXlL*ylAnwTb6 zJBMBmbJov|TfHq80E>JMqiGN5h0r=UAx$Kpq*BJ6jYzb)dk?IIk?P@K3N>7tUv+h3 z7cMW>!P;##xu$Rd7_hH766Wh#9KEJuwtbqi$3G}s@`4lI&;Q;E>Qp)GhTBw&a4_);=T9)AD)}h17A`XB8*Rmsf zGXMjD!kS1%izf}Z{hv9URBFrY%M~FC+arS(7@oYl*WwtFei@QSon-N`j2mTxmLmg6AQgn!WTe7kk^m3{izEe}6nMI6p9>)O*NElj za(O|T!kpkM+?5v9MbLnNx29_*{>Z0F^}%3~j0! z?YRz()+r$6@Y+dnHD9J_n-=v2gDO<@r%O}f*Zc?GHM z@&GkgjnOX$PDcTB=Nu^ah%)f9%uxAm1~^SKks#hO^O2<599Q z_%OuIv_@9A^3MgBiv?K?h|qMD`y+DaK|$Q={g1+YGKX>Y1oOz?S=o>L5Iw=z%zei5 z4v8oblcex(VWGO&bT?hw%9(q2?}eI*1=9wrPP*OmX}l~sZ}Y3~!N3Kmb>j>E zhaf|la2)6QbTG@tN&z3B0cwln7A+jato%hlLCcvWNf}PhgUdr&K z>)vxh$C`hi50lot?0i8t@8ik+j}tRpOi%ePnRIY*IcKh`m{_Y$4O%ynK6(Q8&7<;jADIVR ztUUI$ZFytKk-|bD|EuU-?Muf~b&B7S58>|MSvZzUhk_OD{xt}Zv9!teahbl?x@;3zFgc;!(etHq|FTY1{Mb@XzVo9^{e zw>?jq3eZZ7KDt24@#tROCiRf&TOt-p$~g&g!41_}B!cbMsf-wAO-_cn7{wXv*xG*j z7Pmjn!P<1NB=%-^W5<2$Fe<%#E`LxL+&R28^zl&2xWSq1gm)TsB)U&#Yc7I^QO&Lw#kLtnD`s`6|5p0cgEUD8`y zUNcZz%hQzyWUQro`T|a#SRJ240bBh(ryS>!m9EU)Zf26+vflO z`j^Umrh1#VO4NSu|Do*7dVol2E1iY-*~Ozo&6AnjD70uq;pGS#YOS{7L= zR6)a11!Oa%)~YR(rq&`;Ee02~0wG2=Wvv2cs+9mr*aRvJvIb-en2_{$eb0?;=Y5~g z`+k0Z&?-W5=iKL<`&{4ewYbGW+}dPrNc(|O637NHi(y;@Ip9PO(S5ZTG64#!17-gI zC1r3iyc}$HnQ0IreG* zXVZlOyjJ2g=O1UGRM3!T;l#>O1PCny=#%>-{TO-rGc9~BXV8%6N-Y?JEh8nkI)4~e zFrS?1fKatg06NeO0BTex`ag+7b?5@U6o?Gu1s@73Md0jdCcfn?5E&>m0}Waz9gi%~@rAjLrVQOyb-u$UvsfhWJWJeXlh8c{%6 zVgXPy*qJvvbXtqs$r{M~x}u12#%bw63yA$KQ&T=N39Z(F1*B3F35 zI2UX!0~Unj1Z~0nK*MgVOhQv@BYli%q>RtS>O+*c33@+T`NWrDqf4j+R&r#pu@$uq z241k0rLOGp2GGSGlfrvqbBf3{97YEG2a5e@#1Wju!Kic_ZZ%PntMsB0LOR^ywi`fK_P~W2OHxVXG!l3 zmkFnzSc%X;KZ2UWj^9k{j;Udb{VCFQ%jEq;f7as<`_k`L}v=Ty>-)sXdFk>~?{S#-Vjrf$S zP-1|Y+^Vt%!CYy86cyRRLbb$D#BpVw)@s;Zl*JtP5bXJnMii#b9Xi4&5#UN(L+Ap# z+Y9&3a1NqM12Zw@Mq@>BNv0-J^$RCC?!g-k_xf%D9>X%^S*X$iP zq0jY*(F~?`n2S}{=RaYIABrM<^ z6QYrY*U&EqNXuf{l54t9vwU}M_990!4rzi%bUz_bpWj`?Iz+%GoIY()5%v5mNGm#2hT zh6Ln!v~cBlUE-WV_>Fv|furn*>gf37&cYwbe z>tge$ZUZ6g#yk~Y-En=`;nN3=^Z`*o>4t$K>M-tAvZ-RamBzPa?yxTxYWxeE+~T(v z?igJ^JNOkSeMFahY7zIDgw&;gc;1ZG(2ByqraDqbI5|w!+(s`{R zs3$Xm1$b}n%+_Tk#76v@aCD8yOW6bdKia9SM!}QFjeu~uH0LnlhWIc9<4x(P6vlt? zdLVW~?cUNZY&)#9=Fi>Ynhy(Mgeh-E`JhVHl4X$xrG~ksAJQR@Trb?T2}+dV)rA<* z@VqA2evW8w_qAyY4Q7)?w;L+2_{E&6Zi}%#Hm5IrvMTnRpWvMC?KO(+IY;E(`kif6 z!eyFUL<_ogqGOA>gY3#FW!~e~i7%|n9@vyA!<{dRHS7Ly_`;a+Rtv>I!Gy}={$5p7 zg$Up7N_j z&{xRT(^0GZ)-@4ob2`s_U6eXG+QQ;-d*fhPYrNmlm!7;^b;|S<2rc!u1U&cJA@Nk~ zyn(k$iVGiQ^)>TL;}$!HXa7xfG%o+>H|3%H;bGscm!`HM&>}>D3da1j`U4ggbk^i` z`JFU>A-3t8w~wa81_sHzfyYXN_ojX}c`9DblU{jr9p#1zSn+D?+Iz+&l-SX)!f~g* z{%B=iY`J?~_xUyf|M6Eb{)QJr1xv+~p7nVZ??~V+2rpsemd*HHMI|IP3h9iB}c~{OZh0Zp6lfR`WaW zeU>+V(9mB63B+@Sw%mG{znU!9eDJvi-fM{VC0 zFPR+8UsxEV=c!X3O~jRUe?Dks6CN(g&9Rp8YX)<&aD@8ogmYbJwpF-ILi;`(#?DJ4 zg#q2)KUU5Q4?dOu-KT;rS#8eN^1QmxvZm;`poxrv+^dWEA$hqYCOIoUz2EA+Jy_J> zwrqc1{_eZgvR$!zG%ar)_qFc!nG`-H6f7N9w!4P#We73~Z`$3gWR!HF9ljrX5 z&W+mozS+JImFjY&5kGmqS^3mK)Y!i`(5U%1>(t%v2U9P!^%Zs92sDU&ovYO1V>#AC zN_~92*$mWopZNSsv+tRXZH9)*ol>|!h(r%O#FOow$hUk|cy#2}FUEWOr_8h4+XSzZ zt5y4=L!IJ6ugNnA4n^&$Q|jpaWV^$vitJUk=a;(i1lNDGy<=Xix~>aN8?U-Hu}H2d zZYkZ?p;3%nPQFr9cq6Z0P*>8b%&o{@bUEdr1Ak0b=y9>vETdJGG;7C1&fIsvT)%YO zAP)Cc91ERuBICwK;r4O&yetmvK9PSnUbuO%@nHRl$k}a|)(zzgyjz~tUroRJXlrXz zVI*)fxjq-*E_wBGR`^HxI!oHwA4HcjA%(q^Xo6WZ%fcHi4wKnwn{L*IG|3&oFS8rA>{wKr_`?YRe!^5d9vJjEF zE=l>qqrOXZGOKOHnxQ9}+zY~$6~Vy*t+Zo4-`iEE?OQGNw%?sl<%37oSNs}Tc~^Cw z(@#Qus^*oNh=5mfj-S=YqazNv+lF5O6QhMW^R61Nf$FKv*SjY6TAJA7ajJItXRDz- z4URdiE`R)`=Sx zQ1{iY$YTs$hyxQ^*_2?<&|MmM`ghanpVHZ13Prg=j7Fz`K?TNV{zKOu7sSv9&M@d7 z0u-a$%rz^4`G7LA=-iAW5qgB_z)BEAvi2_C8O;Zj%YSb=QWJFg%NX!bYrqRd$x)o3 zCpVK3$UM!@DV~xBV0cJxj2gu`gr&_O4L_mKYyhXHl?j}O$leh)dYnk&yJ`Zx_@aNn5f}D1OO0FD2`R?!pdE5j zcqBm55A!~Nky?;%lQz^GCTmK=p*;tvJ3VrC|>T)IQ~q{ z_31Mjj;ySR^_!4~axYo6i<@<1(07~l*T+%jz+@SrrR-RXPZY`Aw(#4(hrR-%FZ6he z#b1z&pPf)w%$&9xX`WUN(WgVT37kF-b`m6iDzk?doDp&%ppwQUxzB^Xvnz(t1oc{4b-{9 zXkhuh6YBe*?Ax%#=!^gcvHtm|VUV-DVOz|L+q{w*ONQ0v=QmRQLt2&Wk$XPr4myeW zL6i(g3BmzqxO0G3!-@rH`<`4AX+$SubOhN3GP8=P5PK16F|pnhWi`6e6xu{UjvyC& zhRH!=$}2gKg3NobPY(-lCJJ%vV-W=*I@X0P=g#RJXu9+Nls-c8530}UPRUqxDiO{K zQN16cj5lHM{{caZ9wj;Jt++IHC=?Q~U-qL8HC^Yy4r3|SpC7o}eWNr0mU;bGO0q?t zyNg=*&qRskB4yf~$x!9-@$kuU9Is_7yQt-#+2^=X)K%GF)3Iudx9Lb@_mCxr;cM!% zc!mJU&h#<&M+s8R5Xf)o4Lq_hxRV@hmY)Vca!lMyeGBpeTj7kKy|Ky(|Vy^AXE=~s)+t8Sgxg^t>o*@Djk5jGLnAGkvkRkOEI=b;S+WW1w>WE}IH zDpADNF`}muZMSnMn*{bC$bN(&-5}c9z$FMz)Y!`nN5BC<*e*~KTnLD01-%1k%NyZ5 z>R(3m9jfaw4b-#n%|(rfE6#woQDo%`)&_)X&a)7(2FF8KB}zjU1&m-}k;Py&Mg@)~ zBJ@W+*20;6Bj<4fZ^Z@Qphp@14+x4-0(E8if!866rW&l9+NsqYJQFG;Ecnb5?tpF* zcd}ifWrnSrN(4|RQ2%fTl<+eL(HRfkflVA+&}tOnK7;TDj9@bBXSI2d`~v}NQ6BjL z0{P`FJ&(uJk_RuJZZ?v;!v_aBBIR&w7k*}jR%QR~1=(-hXZCb^W|_fFGMB?^{kI0UCfLY*_hg7jB}nVsqa+W?Si&Y_k7$H-0ayvr<= z=-0X*!|{)J9gtD%iLqH*DmAuONarz?KTZp4krt?ec^&AIW#dw@3Dgpeg6hf{_l3FY z)DHmR$8t*a2#GK#_83lk&hURnH(%0h6~0$t8Xf&gE#e)o_GOd2G6RmqIC~j#z>im_ z+-U-K0)g|sKG<<;+ao2Oi`U(zO*g@Y98x!Y-*7UfF}cFMgyfN!M=+=Gom<&HS8&Fp z#(Jo%HFTd3xa&ci`zn0K{?>^|4d~>Vvjo7trbzq|oqOt&wimkM%t{S-RFVj6;9;fqr$BXTL#nS}V>JXM7rhX{^pf?zoEn6hOlhnaFMzygG7*Wfv#nKhu^qmS zZ>SNY2u4qyHNU33QhAz<+ITpTPWE> zD*i74uh)au;-+jy&aPV{={JlcPQMUo`ke~3_F$YYol-R*Dp1^QuO{W}9*sHHb2B`F zEh8-Mxj-m^6F%lbVq=|^0bz_sC*V}TLj;8K1JG?C1Gse}n`O>^`i40?3?2BBDLVJ6 zpvi3`S|v^km*Ox!sDl(%gdN=d8pciDTrcz+(dlgT^K=HMOqel7C54ZX7_R(^WX#-> zoM2DPLV0anLWR)R<+HIQHE#oud&KztnC&}sewXHaAs&lAHTLn(U6rSJEzByye~n#v zP}$XN<>zIlrH{+bYTQ&`PU+9jIa}`-729D6U?D%G_DfZ*_5I4T=KaKP4+KR7L(UKf z=|lPBrM+%b%kdG4jEf!2@7kyvb?f4v>G}AzduxZY{bRx*BdBp)7a!QM&c^Us;dqz9 zszh$MT@}T9c4$r`GQ~9s=qMC@Jb8TgTfQdii*tFqt^ugxMiew+>MGQ3!x)8K#tHn7 zE$6zI)Ql&=#jRajRfc>3K$373s+1_|#N)S*tWqxpX$>E!4UaPgX z{_oPI;gaJ{^aVsbpfj80nx?N4u{5V z{RTCfzV7A>`ycJ?nn!P0{@32_%B2R!pLt$=J+apgIusW*mglTrojaqrL_dC`D#Fg= z$KC=n<o!k5pxXRE!j zoqVwy+Or3ui;+^}-DdeYp?`X@;jc}PBF{X3U%k52+PSt@up_SQcwp@18^Tz(J8i8k z+ckXn$z=;IE29ffeQ(M~*;2eyS^9ouui%yA6|16OGm~ZvH7Bdn0{1i|M_=~Y(K-$b zf^*iPN;H$YIM~A{Sx4Xcq%M98O8L{8$NEFW`S^p>{ZBK&IN2GD?Eh;o! z_NX{{+0R~k4ruJkdskZ|eB@WBwvuaFEPrmnLoDMj^>;5dJzCqdvc5~T>choRr#tpr zTL(GaSH8I*r!HtwW}8s)J z_voD6htGT+YAY8B43`YPPWd%4A{X86OPc=LyW5ACy6~Ic)>K!hUsUnjiq=Jck@t;5 z_|c-*>k@c(;)H$HffeqrHVo|L?|XOh(=O#CX&-a_r-TYWpmSQ5#D1NM$mlmzX z9^SBJd!Nhgof(N*-S+r1`z{(US?c{N>Y*3RbA2GYneU|;>>jX;?e5NxoKOYoKJFL( zOSZOtTwEY%--(izi_HA2c8&MZZ621G#P`(j2U)$2_bxTrQ}N8a!-I*j=)8LZDl@Cf zTCqzm6bCm>#wQiaxR6-#_S>8zx(Qzz?D8Ve<`27-Ru^kln(!9}6M@Y#@WyhT|1B?J zs&;#)TVA^L>ZTEghE0zf3~vWJeamzB&Tf27{A_9ZtktK=rw)Jq!)n>2W~$FO9(+l& z8@w}upT&P0-qk*J&_AG9{E12g3Z%au?$q5_l+E~^&83?jrbrs6=7O4>qIvmcM;F7t zVHoX~XxXRy`O`l_umAr)b5!Q14c2ZGIPv0qPMt<}!dxr7d)^w1)pTCm@8~XZ z#rgT5Z$oyz2u)SyS@*ON_lN(}u?Wxxgykv^m|!txbK-w=cG>ZXns9&o0<0l4`v3f* zo+JaQz|eru=zU}96p=UnigdxpbgCgzI;WPWA7QUa*k}5+&O?U)LK^)bb-5`=@wYem zEB~IT`}RpRR>5Ru)LziviNuZ+2d6xOJroOPgr@4fA94v;6UHVqZ?pnu7Fi#=6wrr> z-cIieFMkFsK<6eTixHjfPBIulAI++Va1OvVNDTl-y!!-vwXc$Boyy%bif9j`rL2~W zJ=B7dj;j)$XMVv9^#UZ~*zamMLN4pvxPTk}6~Fsd-zNlSaFOt2U(Mh%V|c=Ekgmd& z!voBi5gpek2qJhk^@la)H};_on?n6*oT`jj8`0H8yNJt|=$@%S_9M^-!Ae4_Gr~|m z!GRWMT@DO_W0yJRh0aYRxisKJyn;#I(Kps5;*Y>G(Z0dI-3qjTY3xm)m{mW6%L1}Q z-vrH4F2I6N^%2Z3)CK^Usg%Op5^>9Bf}b66C}5fc39w{LDV!r{r~~QAsjE%8YpmYw z-{SMw6sqncB_mfkh2WX`z39s0`24@<$NzKw{oeRyhbHz(2zap@hKd56rl}GE2uskF z)lq!)+z+RdPs8mqTAlr{g#IZ8= z;|fQ1d4Sa7NiWEC$ex;?zGT{MdP>J#hwai>WAvZ>X)|j61ODLoB z4=9o?6(cwClUQAJzZ+W@_$>tv+Z!1hpBNfs|LE3JLLUbVXxKOOdC%@*_nO1Kv*w|; za*MHSjEsNAA+weh_nvC#Q~ZB5n$d&j-nPj1kjq%;77-IZcd7)p1;rtZo-?f}#3(Q` zKz2h>!g*>40Le!W4IROoOi=UDT_U?p2sc)MW+CeSS@er93w}9A%Yx<~AbOVabT%qo z`^qYXP3#Ws_NlMD@Bk2l*$ZXt$UUw|L-xtoO#76u9 z2^k)1+mfh0i@M+8v)F7dHHx~m>@LzGus6MR>?zGBRT7vv?gx(eY8rZHf`ZB{)5s;k zjeyTQ?fhWcKCcVZx8hnB_+E14W-60&$ z7A_mzlv~g5h%y`x_eOQCr05!`3-JwSBL5pU_PnMl!LLSww)lGTXrq@7p$Z(gEK$62 z8$yx*+xTE10NwVIzz+&_p%YDT-3V{|)kr_Bc!;#HdMr8;DazF+YMUQ*LTQ)^Fi-0} zBMlt%)}W>~e_>`G9ESI3aGjz~XrXa39Vu{JM}Rhr@*|?mki&KNmLSl;hsG&)O#CDo zA2G3}6l*25zi(9O2&#NzLj7HIzA2J0+skAi-v7xH%z~b{j2{(KLexj)`mfOcEm$qN zqR){NN?nzeo|h{h&xttw;&r*2 zHCng|aN z*MX`?gS3__VQldo=n4w zh=-R6%c_gq|NNk*@?mPdNpP{sw$x8$Y*Xm%6Q>GFGFbFmSk;7Cupr*m{YCil#e++0 z^SuM5!4n7m?IRw`@;(!)eCX%)(8jc}X)iy-%)0c)@tpQ8U~jz;k=^EfIwDq+^sYz7 z5p9|>NtsamMU}tF+2ju8k@hBIr=*8j?M{Yv{nEK8rYvTHqO$~yR8 zPMleG@4wM=U7%@x#?W9$JyU<^f>-ja%INv0(>Ct^Ye#5J&rPeK`ts(jCpsLK_&f+s zjV^o{+mctG)!=~CI(N^#x@-O7=i00lEvxe0?-p;`qEUi7@e5TR+6u~dHr#CW=^qb& zPMUUMqA+cCggt^Q%KiQ^|JB>dcMlIlm)iTPoF6WR^Xt{@Hx^qgo)6Hr`}SZrKTMe~ z=v}7MZR_|ME1>iv{KDhiHTAkJTg+uI`t0C6iOs#d;S2lSc~=d$zKi#G9uuRZsjtvt zSuRHL@7L)WIdrD~?>c^?>fYn3efILJ_4(y;FTYXs_Dv^Ng?oR!_oROaT=wv^dpp~> z7)4W18L!U1gko~rzRV3D**Cn=_fWpZD&`+}|C?LG!af;!!;tE0xV-0l$1H((Z|Bt7 zjNRR90vFwH@e+P>Po0_;UlS91qHNW~-{yBu#p??)C$qveJ9WFl``{(eJ|G(TvEj{5 zpI?o3n_Jq?{j4~6#Khdo-%HRef;MX1-EQ0R%tO(kvnA393#$^H-{o*k`;({=-PTB6 zNU6#?B1o$;ju<@o(}^8kvp!xOf#;e>RJm;OpjS&E?|P=61)j*xQgz(JDDS|J98_A* zQTbb6Sb5325YUO6>!OH$xfED?UFJj;Z*j7BWkgfpgjw)UuU;)%)`aa|@f8Xb!uBt$ z66%xpmP~D1zJ=#<^p5FYv){Sby0w=L8HZ?(UnL7O!s4i0!WJ%Dm-t;`Cjc_d3`^F+geXTUq`&nh%L4A2mMKw>Qlw z9WRlF&u&{1SXpRS7-U#?ZPAGfP3?YrB%$cen4BDGsIVJw(Qd7JsPmnhmRN6YdJ2!l zAfI4`Az|3>@MDExWBsFvls?`w13gaO_E&FO9A^i5!GveQC*7B&#%GT8@6hVo<~Ek6 z1q$B$<)rkLIpug>*Z1*E`2XHmD6^|a*{nkt9$-@r2p?FX7)Y33D3{6prR04+yyhlg zp1=FS-;DB}wvqoY{*U}H8#>DqEerPWsA#8mlczoT^Q=IcICbPm&PMnVplUXnldT-6 z`XlUq+U7*h2-axh_%Ac+CRJ=VvE%=M>VN+;188)5=DsMVjd2qH6Hd`f4OWxR9w#*D z>xp_;o%q*e(&|P?@2b!UCGT`RNMUrw`Ry}h`-g>g8=M!4l;te~nCLBZ^)!Nh)ijd0 z=!1Z9Ox8{!0HO-BlRYlPgidZVA21d=;Zh;A}h|NaYzQ7jk50*pfToREkC)eibb|0D)&=26)`yWLg=$`S}P{1&vq6bEn7fGFa7@fOr zs9rRE){wbSmjMFb=~h303(=padEczY0h~265v3uJzF8}ViUD{v+%Eiy?ko58yZst^ zoCukJI{O%4W{q$R{qO(h8qdgl0Muhhir5I$tV9Kb>=eM(U+Bju`=k!A_*TdPW!7=0 z;Wc_IHy`gjvosACTSNJLuzT#{fF@o$=VKG9y#OxGROtB$DSj;AiQ+kxFyp;gM=%w9 zUwqnfgBYl6D~>QLjdC^ZlM~9(Oa`+t9Z0K9s1rYpF<@YuVMG&hDp+NJ!#)fXac4dt z=EVqU8_}Gmh_om_uIuz8j;`qZvTbK zP#Eta3d0XG{@dsu6G>Te4`NiRtsoZAcr)6L-~+(q3NGZBCj@5Q;yUe5-hhaJz3EFd zwp@=N5qYeI%-}Yu(78&`bX|j5jD(td=%#WvlmT~I+ss0%xyOje2D90#K}7*23%coS z<7pXD}w@B;gib;rc}ao-L8g4=U^Cgr$A*)O?^; zo~W}vo!&+wKdtv^{KsyuU{y9>eGw}T&Y_wg zw$$0z+(i24-B%5*EW>x`N7C5mann%I7Vo$oFF%i&KT5UBYYgl*9^Oair&-i- za@53;c}VGQP|^<}_#HL6b7P%wd$bB)@vPHBcbZO^>h@a@e*>*_@VV>Mj3!c3i<2S> zTpf!~SCeB2H%pW3yCKd7euFmNLuw)y;@na1W~kD@jMfbl73H~+Y(IBlYuYUDW1M}$ zo>+JOLvwj<4J`-z+Ot9KmQHme)Re9jz{@VlzNSuKQJqH?yLi>rOa9bnZH4Mp8`#+W zPStsqckG`-QV9eSAaH1x$ToM|Kj#a4o`uQ<#_nAVpJktn%mGn9!j79f&)TV7lgT0z zHv?(Ii!Iw<2pkp$91b%8;|LJjeaX>}Q%B%boMVX;0aLskx}=zJ)O>$bIXKr9U`SEE z=&Z)l9LoIcP)nF3zUaG^eAP-ne&waKS_)<$kL``wx-H5AF9>?sf{=O1FZ4`Q82HXJ zz0J(FrcRZ+b3^ge4X%6b#OgPiZ{N!5(<&PKd6i~W<7ZMe{VNrDDx3V%{)K)uJU1)f zHeqfO%7QHW?Yw@@sKN1v;yg?fm%9^Fac%Dv;x=4p!z<$f$*^MgEiHuaqH_%!ss4Z- z0M}7oh~n?rvGf~Z!Ps(8lEh3{O&|3hY?;|6D^66hV+zYNgzejd>3v;EbHL5eqQO7= zA`3673$k-@GsI94v^-Lq8Kv&&cZHAn&qt|_Jtl3gX19Y+uFs2_sLw;pcp(b8lj z)TZ6GqTF_oGT&_b)DiwY;Azlednez2 zu~8#FYKQRhob#1Vjs^v9r_63LK3kZmDw^*-U|-eHlKlhUv1(^|%lxGB7d-ptWU2TF z;_2U{#_y!}czy(11YJ-@>49I3o}Kc4C=9&WdDHM0o!yeG>Qz|8tIBT9_spvu2m!l$ zmqr5X`>wdRTmwFC_i}OV3~AdNO`H-}lpNp$7#{?c6Q;KImqj&&ueYp64#8kq1*c{x zeZ1z_L)qtTFRqa^y4oBmZ&X$ zFq-Bys<#}l^@rr+Wk~bK#gp&YZlC<7@j<K2%po7LYp~HfKpa7>`zMF=7Yi!_w)t!`>vqi$*mobD+I6WX`lN^5nhF1r z3BThHVbdls`%V{{ysvr2Vnh_G5*A;VJZRW(VUky6ilU*$t1G{}Ij*^*H2be3-z6U} z?NH83N;f$=WL>FgA0aU;Pn~92snY&psQuJ{Q{l2KnZW<^x0R2|O-I2wI^H43^0ZC% zn^3>hJ+tTy`>YcuAGQv(*5(IwIJ7_C8Ze?8Q2lB&I^i3BvvtnsKYUej-OZsok6DlJ zRB1d;wg+6T4K>t;`jrOBjMrKDH8-_XtW$2!eg9_5by8eV+Nx$T&}{`1Hj z@2c0A9~x`43HvH+)q-ba>k9!OZ-4qEd^vy5lDo0|s-fp%E_VA~sa?ILHZ*Lzc2{eC zc!tT2XF7%kHv0n#HRJ(C(~_(DIgq<>mT#cQ+0 z3tB~$*G|9LEPXHL+GlO62Bg9RPWLqSol|ZH)^|T?ct2K>C&?eL+Sm5y;wpsJT3t)m z?5~eqTJ5+gUZ}sU`Lmbm)VlFYN6RXL;tgNVopasTy|j4pi#bWVdJW(CNdl~L8orNR z(*Num8qd=^zp;whc2D>IHeRo!yl`mfe+nCtJ`OpZE=|e1n)gb>+74ZDsg*MJS0gLu zNDFHd&G+xW#e2D}ZB3!ZA?M_jYSv`*_Tl+AasxGoBHR<2GFvjGXzJGPI?%p2-=X|DK>_MpQ}Hc3 z^cNbIqI8!P*In6idQ;8q`(+tJVZkv+M=JiFubY!-d5li%vO>QG*Q_1gYNYeHjb&}n zU%u3HRH5eWIU5ou8oajY((uJ^9RG4be>}Wg)O=|&{(@foa7c5v2e-9EWm0+5CTzcn z);T}2!0TL7%rZsAhb=8PLS|R3GgLGJQkx{JJpKza z0;Qh9bM1yR{k&Tfhjnw+_pj(A=0~^xvm5@S#pQ(_c{{L^_pEs$<2^=xMf@oZSXZ*5 zpAI-hB$6XrX>Nz=26sxQlluSH7s+vvfArZOer)<%N3P5!H};5>9uVn12ARwrj~Da= z*=fw^l^xBsLC&O!{$w67EumIdZ|Nt$bzMKi@8T%R9g#Ws>4B4{+?ikr(XQp}9Lvd0 ziJJGne&L7C)s5zZ1&sy*Z}NG9D0HHKZ${96e5{N#{6LBk>r(>b;>S1#cNd^1G{139 zpT;)R2WrBn{!gL_^d>mcGXfo(`H_U^Qa>a zNDne2-X%hM6M%%HD`OIEsK&n&4vAU2p4#0+H=URSO66W30#+dk|l6?HuIZGoe zbalyI!1NbWFt-t5hzbr2(o0{T$DzZBPXK7A&HA~k`>nm6Rjb)z^JL_AM97dKToK@` zgV7M;ypTtHZZ@7y4LFhIO%sb3`IAflqKrkf3bNWTFgAG*KquYC(tl#n$rdbm zx{CLh6&;Rh3PsCabb)QJ!3LUMrWtt>7s3b!wSH}?PQ!f9MQqY_Vc?h9n~vjY`$X7(q@Bg46~h*CfuU&<8fqO4}6%16>OR+ z3E|-VdR6z&zZw;$d}nAYYzQf@m135*sH6UEc(KILn1789tB*PdqK0?{$={Xa>0o>F zsJblrv2%ChlxtY@_S>P#^kO-Qh-WQ%iWZ zr=wZ8J5|%CsQ-?mH30H}M)qvyRUxRJ@n^xH-Z%1wHdg}^; z?HSx&c$;t>D#Kob~m(HjA0?hqr14>fjxuai$ zKDnu6T<;EbTi)_yjhH|7dMPq#g0r4M#v5>CzmQn!)EHtEoO2&LlqF8?D@rqy<8geCtHJ-u`e6 zGf2}~qly8o^PV8+&p`>lWB>Y%ATM4Z?<1WsNPkt)mbb!wo{n~j)_#jIENv9ZU~w?L z8TyT6H4zZcv+Cy=B;_#nK233L0ogn1leY4jG0H9DGRK*y05B`HkUh3{zj(0VF!0V_ zGAO`aHibwmG=ck3GCdgKGxrR12%dG}PEB=LtvDl%XJb46)dBd_sf;8BPFd3&A z0Ta=M4R9yVC5rtIW)(|+KzBwwAR6xSJ$5W#TmZDLn9w@MmS%K$)*XBY+k!Sa-Sf*g zddv#oM(3$5M6jX^glmh`LRd=s4S8Z;*!ZB^GxS|lIWfhunh`RnXc25IGF9Xhz}bOW zNH*f7gy&m5Y&17C<%LWWUtIL;zU8jbv7%T*n!DrjSd9b74)wz2d{hNCo*(dN7l6`| zCOC6^4^yAJ#pj}uWY(>Ad`B27z?lZ&eagv^vtbc>Po~a?LqB31e9B0ViO>fcSZSp7 zk;HX@HxtZs7Kg*>!Lry5E7u~$zd|x%+2RHw4&otYKX^-?NqmqK{szTl%GGSNv`Hg*LZ?`ds}SB8N~x(PNu=V3b_GCsr>8!;$9 zC@i~sz^w3#)o*6B=#!A;CUbc>!g=@c#k0>r07us}4J`EYaW@Q`y@JxLNY$&HT589Nq z8jgFLo~jZb9p7FER;^M3P2$nYq`%!Uwb|63XJeY=?Yi5>P&-?(X@|wi+JW(;L*;sx zxDm0*L4QMILs#F)*&2F9UZ*60KEwGi_uF5(1XsMOqcC^ce3%xU|``1L(>Z7_hyG$$4Hfu%Dn??erW?>?Om2$H`s_}*Lud1r``hKcC2OgQ1e@Om_Ip? z?Vf!)_{86$^j`32dS)uPF2~SxQJ6R(b=eoj&r4|DZo9ePGnZ<(PT*-=a`j$fx!^*c zgYd6qii@AkIr(U4*0O86hHput!a~NAhH|@ewnlkv?QU6J(&QiEDcofdb9(+{zm;%PP%vz4yw!Byfq0Lvr#6BGB&xX_5NDxsIaYlA&K~$K!K5 zNrFmbo{uM{;uCB;(KJmSly!&-n@){CAB&)s#-?HQ+Y7Z;w-q}t=o3d861&R&+&kCf z`lm`f@h5wI?T%O{=X`Eso3ugmWrR;dk}c}l1r2X6|DS^z)1jBoK5EvV&AYB!l&-J( zh(ag(It@a#K;&O+w?)&kM#nFWu=sD?g5yyqgh}zIMl8B}Vzwrhy?MR7KB~pI#laC% zX7`l{I@Hij?FtrN)#)vA2h0D>-{+%zzxigB^Nu;0HnIh060l+0q{z|x<&E`Q;$50F z>*BY|$(>WT1p7Q_YCq&m3}VOF$?!Z`pse)P6@rzqi@`_wPlZ3rlbbieWsKasG6Uv=%$A zzBye23U1<4kHTg9}Ll-o?k(~H3AZc2P)tW361~~33mSRi}TikUo}9* zofHz%PV1`)z~c+*S~>u+%O+1OuF>4Z{uOrB>p%oL#R;dGa+@w0=`4T}t@!QOr)}wt zfTX0vg#sFl1bSv>Gv72y7s*m%!BD^W$At{REdKUKd|dHn5Yef5%HZ}~G+dD2R)aYB zV*pIfIJPmc%@mr84~T|`FifA|*TAu6=CG`%{K-5M?P8*#b2)11047~+iGNK%;LrMJ zxOVF_A^F@V&*BMbkH!O;xXlE^cslIT*#;l_M%*!g76kDjy@4%wFA22;D$N_ zh{x|yDlU5X_&W}h5&}sk`h&p!L7~(YK1)~di7`%r9I!XOO@~A(J>0{;+)jno^WGk6 zXi|;Bo+=ZGu!!PP#b$MBF(Ziik5`ZBUQgmhG8@AEB`A*GFylN#*aVkzmoWoDmzXJ@}c295C)TQm)WIk&FGg3vEA4Skl5SoZ*irORWnJb~>_5 zvg3j;_idalsC!G?_JR8-a?rK1A?o`yeC2Kq4fX}{0qlU+dhCAkLcsk{NzeQEKUkyV%BiyI+hEWDmx_rjEe3~bv`#2xOn-&(E++n z+$6+YVVk?O(y7s@L8B!!4)qC6w~VyCjVDDse7Z5g2ftCn_m_zuuSusm0m?7P0Ql8N z-FNLrx{^+K?*yMmng0!-EI5^O_h5y6%HiMo?w|^rY*gTB@76OtJE*w8I@LsXAR!0g zf6fbzxik8aHl2wqV5-Os0?5I2BW#N^VzO|^dufuwTb`$VLYy1g*QVh-wyvix@wAJ< z9*-0-3N|}9^T*kX{PlCy{yVE^mf<37TC9Y9h&&vxerke5-dVmLtQ=vy4}rV-%OQrA zUs~dx@ZLcyu%W1-nP;ADX!?bZXi?c0hChocCATQL?b7WY7uk`<9>;KCHrsA(KJwZd z_Gw_-KG#qxIEh5I{^5ld)x5x&0OF#aSF`#Q2LI#`_XNy)i}Fv>`@O#}pQ>`gK=D8D zb3X!Us}p#~pjVdHIkkGSQkJT!3!YABX(#HynV(*Htx$ukpKU4`?L39r09uQ#kX$-} zqR}eR^D!_y$~IH9cXA!^8U6;x@-D3y*y*aP#({?u_^TU#?i)_d^E9p(S=N> za-~UI?7JO)M$p|vHYy-F=LVGqALl?AT-3IdF$cg#rB*$q}|c(*p+6vzY1z<@@l*aM}T#xFtA!3JQ)_J_zy%keCsPI z@M_T~W;O3<-_iQs48v(J3kz6i=f(%3{+XLTKdG2W^4uU#*q@V7U1YrFcEyWVK)9Fa zom59Mx`LXKM7Huq?p?HtV{i2bD8O%8RI{UjzXylyRs1k%^0Mpl|BfVr^&V_Z^^zbO z@L5V^fTrmz_0$%~p~o0oIcxsW)vOzX zhh;{ZSZY*gLeH+6n1A40KMWYaTcjKfit=?iqyzo?tB!jvAa=Xx79}r^qttVer~XtC z;sZu+mNy;HE}77}Gzlb=M>-t(m3jUkvuWj{jR)WUYsb$z__2TB1tw36jo~f^$QEpP zYvp>m1bpy@op4J>vL5e3gDc5rL<)6S!P;SOiyII5PiA<-Afh%pTacQl@u22HN=~A$1fI6t-kVL`i6Ptm}E%aTg=wSk54zPuaPu1jJ zEHskNUtVzljQ6n8Y^v%==Pw&6?uZD8b`T#clr(CXBfp>Q7x7DNF-iJAHGCTr??S%y zGC}%dgEaPG>-(mu=h_<^Y9H>i=e_*h&NC~<#m~ij9jqzP_=jD;=sS6A)9~(Nnsq+j zvE4r`s+`QK%F1m1&|h@tq9FC2(+b(K`)zO7y)1L-|NHRAPnPC`Uw-^xXqc7r*M-=a z9IgmHuMf2;v5N`|56^0tvV5p~amu+a-+0~re;Eso8;*u<-Dg-jxws?py3YK{HH(Ya z?3|KT{rvd2zejlBKT#woFNDp9-U6%ym#dgY+pyGDp@e;@3I0x2IiZ96?z+J|RI=~c z*R8Y3zI1;CLL`A%*Q3jZ(}6zapE|x1-?2q@W%7(-U}DkW+#lmCF!Ojt-A#j*cgp^9 z(DdqLRHu28Zu^UA>-!F*&5t-!?fub(u*A&96oa1O=0Lp)X)52bNlC>_7GjrOMQZ%EyKc$u0pNvKec9KW%*~He6@Uj;y=QwK6m8r zj4-5$a~}Xv*5m{kUT|o+gQ9s+b4N#uf$#Lw`;&(|?2e3`{=v%YSEGsU^lbG9Ne(AE zcAhsp`nTw!hn>phXvONmpGBYhttzgQmMMbbtOqTZ{(Ul8I)9t-i-DS?uOOb!@$Yb# z?abPXR{g{wmx^_8z<(h=$=vc+qdYUGtAoN+&0bH-miOxpI-l=}tExXT-Zg&r`t{;o zuZgmyhM)4TmE~ruV;(iPO&EkZhced`mNz%2m3}wyrmt~Cw;}t~{ZcGZQ+?LIuFBWG z*6`1R9>Ysg(z0L9X;)~g&YY08=ucdeJwGYW7KQ9RiIwr>ZP=3Ue|M#8Ft~Q@&(GPzTWLS>bLG4@;0;-UVSEf-J!fEhosgA8=JjaPWake zL>*`@UY%r^smBGK*5 zC#PU|7JmCcZL@fu;LXynJ09#V2I_B<%tDJa&@*I9Z>|E7$o6B*i6Hb07aj(XxHW(BS=0 zykzI`2ZrjGhLX?C&MAk~@(%c_;TR zpGw}CxI*R}a*%T>!gHmB&i;&=Nf3|w`cG0RgVh}HCh*ODQS%n4d{H$J8DnD8_cuP6 zH-?824yBB;jTq~(W|ROpf3bC(>KgCC5Dwv}q7}?{+~Qj*D17qAmitOFX+*nhv^rx(ut502w+pM<>|Sr`*)HCoYvi;cubp#JOAQU{$B z30Feq{9BtC@5)-UtdhZjc$@h?& zBwa$!fj%BP%KEXJ0h=JW|N3Qz;$Q8JaBUkIblb%4iSLjsAUd82LbxZ?ZdCC~3}(~X}ax#%1M_aL2vw$BN}f!p1X9D9=Vuwm#> z;ku)yJy~n)uZS^1uYutZ9wBrFd5``jvC4UUAe#E9H=Ib zwux^6_~vRuacY7-+NEV~lhci9Q#EQNA*3m9pGNlN3ETj-+@^WMb>Vd{>-<}KbUSta zjnh`x;9bD2jsxZk5#C;WXJ|9oml2L*+HCB!U6ZUC_X*jRHD#8f0e1aISu8RF;HqWt z$ut?Wd7{5?UVVVSmw|s$4M0smu5#1wij@S&!_hN<)3#&588M`Y?{18Q;e zIIH-4Jkr&kB_O4KV#RJfAQV|gac*#9EAd)dC*tfQ1(7r(;Njq2V-jf+A^yZ=k&QcJ zy)e*3^#H0$fH|W#64ha2a)g!$QR(!C*V0#<5C7EM@izsHHd_}wh+K#y|yett# zh%cn0N?rQQ5qR|783x>~h8JQ2W;F4fs9FFsgTSe+xrWBch<%v6Hm3Q*qFXRNlC-p= zvSm3qAG-1~(v|mCm+>d%=m)&G*(7k)n+A(E;Z;@VCY%;CJExpsT<&f0rxF0fyJ+-T z*yYM~ZUBOBL4KmcyCO~xOx&rt>0`(JX!QmGI}25CUh8-8lk_e`gcVt(kw=+)5QUch zfz{-(2HfHfrNTS9sFaK z!T07+PZRH)ng~&Hs-%aSRjG;Wv{zRw*Uapb$!e{&lUnyLsIyLqP5!TS{Cb| zL*9Kj$-#?F6(BP_WKfulTUoWx2-1uF=V;8KAP3Py)dfip{MTs5iNt?k@Av@*S={e# z$p=|c;^w0+=(Qt#spye?F)rT(D^vw zexV2pp91zwgFBh#`t&rww(wZ)CJP4o;RK z#+{jwU#}kj!fFR6VW+iqmWsEZ-n%b{dLLC2rllftwnctOW^a&E!+!3XXt#jsL#VKf zHG@nJ?{|Nq4S}jac644lfQuzsHxF`3D-bD6a8Q4t+HEHZ|aZ+@KlmZu$M#rv;2mZ^0PcMr`f6 zC2eUR1CzXg&s?yQwh}BAeA(l9BA}X7zR1WL_9Cy)_aZql6ZWM7el-*i6i!lYydPZq zATMixak8UB0g_NG@tDq{>pP(fMX3p`RdiBut(lhJn|5!w09*b25S*9dPBoLbJ;>Az z6C(*FL!FIwQF0`GU`t6}$(n<04Y0-kuCCnIK^4xqRe35@ijvvW6V3H6f6*7_ojw@@zEiH!h3;GRbECVphr9pk|y=j88 z^TPuB8d49m2j@^-fulpmh9Dn(a7>J6Q>%Al$B}FDmqJZ~AGDC)q6=N~SJk6XgBe7aCwztQdXE`nixH*CxwSMRyXoZIqXTwv!K!_%|x;~)p@s8#P zqDyGCjLjDpz$d-X!}XUW++gU*or=UJ!gBbfoV7@>47|Vkv;#7V8yhy@T?#Y^aS$w4 zMZ1sJ?6JFE1mA%1umUH$g(&J^6+5SqaiKps-6sZpS%@e|gCYtQWXMdCwm=6YyV8ca zUFBhl2ss?37Fyp`Rf(^nVVAwf7VIO~sLF$U|I}SM2)izmjW`%$sEo`)@0U z=XWFK@1D!|SDI4>ao7WAAV;K-deoOy#Ae{^sjR}*3@chugi8xqV!iVI^&X*{=Xg6z z$6tv`d$VHZPjrl{4bhqX>+vP^jLt3U$ocyRyj$?hf-M!fF6_g?UF46m%e(j#!^7VC zU5%rDn+_Eps6W%5*%lmER%|#OpCo<~QIwsu{mDEZNFf`ul<;*EoQxYUO&)^CeP@<{ zA9(%R>ar_8^K%n2e+K6BgK?c&<%S%Ls*a&8h4u|iWwoJ0E=jgY{ViLP%$FLD`}srk z(0_MTyDnINs`LNQ_2vOl)@%DX>1%W%rBa|UnRD7S5y+EDqA;n4JY|M2J{vn?#f-kMV7 zw*+S+`T5leo!J^^zl!Fx$D;0JzT!H6eBr95K9{90lwTD51>N!Xb2_W0I0s8ltz&~S z**N0KRjY~0jQsZ-4>`tmkEx!CF;EzbCcc0!nxjEf{UHa7h(c+#x;_tf2Oq7 zM#Vnr==u&d^@|02tUX>WD7l#n;!RZdV0A^RX{6YH{J@80Jzw1{a(L|~@5bRTf_{x# z8Q6UORxF+>;o-B>>l4kre@2x*?)h8YaCsg7P_xtMNJY_EL*78zTSaA! z7oz32?)jl3!WIo=ICXgu3bnUseP^v`ddB4XYpMdvN3MGHeBl*I<@ta23a>aoITzpulDor*vKegRDcNmTcokb1_&*kK}jL8(Ql)p4~*rm)= z)x0J;ujkRFJL2c3`?eoW4r%LKa&(XJ3sLV?p75)(voF}dHSP*R^ZI%dzbwzFu7`?m z)o}$E+ji{|${IggyK>dLqAAk?hi@7~PyLtH^2peNJe&EIDXH$lgwl_qdwgDSTRr)B z=^F8XuQ)5P>7cOR-~L?M^2FjVrRCVoH%PEOz1S)I)p&Td zpUax{!b>~2Kh^UKC$}VrOmXY2O7_HG9C|+a^cMbbET~C);;Q%C1T==u?`Uo>P{qc` zuJK3z?XJ72=;~L#)_#gLXAOe)McMU$ZJ&RGwlq94VRHCuon9{JOKBPMMF_h*zVgm# zb#5(pkE>cz`Om0hSVFfARg1f(r*Y!gL9L}ecd#nQu{RgwHu=Jr|NYGnui%(S#sBL1 zWQz;`b`W2pwQCp-@+}{M_CwrTXV>*0AiY?${sU?uMm3tsp6U3{X48Y$o&xUw^>3#5 z#q6OLp*I@7 z!^p4_3*PmyDE0V(g-V4JFp;Sw1zVzg=!C-jcs3>QKdz98eQzeoY1R$QZiKM5!FB|L z&nsE@iy%RdPSLXsJsz}2BZx+2pTU-=ARdXFd~+aiyn=oQxuE@}(7Xn)MhuS;JJS)& zBZ#zFZ99KAE^bFe17Jy;a#$?zDnc{i1^ZS$#gN~Ucu@eeI zJrxgxS^y|tf)&L}+5`j60x9SrIVr4Iv ztbcl_plvogMMKN*B8goYpP*VR!%ZmDn)IAKyd851-cabW&+gD2%h`qwIgN*~q0o>! z6uK27ipV#~XAo1#R^PkUm?nlLp+~3eokPRj|O6@BM zp3~cQAT(#1dZhJ6?VyY=um6o6PwIGEaD6^NU#TboGYN0ZLbH<~X%}F{S|B~??gY&- z7^fp)0ctdouH|k<+wN3+@1;$5JgBZP2bjeSKiL%PuT(M}7+t50W;> z4ix{qjLi=g7{W+?7N0SAiPQp#Bj719?gsk*+eT1$2I1U=%^lJMa~8t7iXhR^`a$TkArSYUvVfZUx7 zP3z0;4cvd|u_F!?6mXt+?M4!OBW?mdnkH>WUl6h&&i#5AVnC$%RPa%ir-2aZBjXkE z+}XW^r1A%Qn8E<2|9PUa$E+C0{aHfvlS1#rK@Yv{SdMedZwC5N!qmhDf+F^L4wA5g zkhlh#jsGba+mqq)2eWC9MN5K^^1|g~#?ZyHoSpt|lI{}OojqvyP~j1v?_tsxi`rlX zCK=ENgbHv~;5W?5yn~1V`pqH(>h}2WZRPr&3&s**xkYw)q%X-_rm&wNE5wEj@iUXv zgK!@CM^`uEpvGoH8LK&!9lc0D(zJk51!XBH*47UiQJh3G2Ql%XMTB2{Z~8vfQ1jDf z_&slviBAu@#uYw0@C45A-I0n_eKm^X?5pZ+IJhtovy3+x6;J$bpgp494hC5mx>AUEw#x z$z2>|s(b$!UgNW;7$`3ooauQ$SSUk+_Cq{^CWa~@EqeHz@?z@Ny`++FaMKRym+eiv3N zmcV>WI<3(LK)3BSCPfEI@1!jNEl5G26wjF*HnbAFsgmEA$m)$Bm_Lo_<>$#I5dHr+ zcZgDd(o4@FKx*wtHRmv2fn5NN6A&xJ?tl}GWFcm|0wPt5*SQ;@=tw^V3$FX8QW7zO-z6UyE@v`0r`_;wcP_$Wsb<(ef^+F}=A?cWW2 zudNclEckabx1MC<>|wI5IG{2I0804!&_@~5C$#KF&7;S~YQO1w0z|;I>|GhqAlhbMaao}e)i!OwH|9O5G z-vxyaXdR(>8@C71c2Zz4yn+dppnH$9MB&cK`a1OvO$4Ob5sTa`$cA>7P88v+b?kPc zUgIKIk*T@7{dP%A*I^hi-Y%3Gm4CVQRkhAdIqYTDV$E`JG3HhxMwF)Sg}~ zY6-HapHQTn$=VP&4MH1-kA$Whsr-z^gyS(6G8Xt*?WiT#0 zB^=mcRr8fu3MFL0+k_l26lFSbuujf4vctUnIhls1- z3SBI{A5b7QXOH|i#QSGqF1oklWR?3li^ic%E@c{?HY%}iz5lWM{YZ9Qcc0YQyB6f2 ztHWM6*JVY&$G;-Xo!XDWF75K@ZWmmGz5*|^A#YNp854kldJvakFitw{CpoYaQa}#m7hui|8OE6X$hUdr0z`<|;!!^HqiJ#64DJLYKCOm0O zFU9mZh~^)&Gk$S8RpX0gE9{-%90pe9`rYYzY4xU2&F0#w3{`09M>!b4=?fe-UK^9< zI5}tO^!$Li=$mWb1^e4OKG(IVJohUIT0Qu=q(1b}8+lHd9lFyB(+5Pgmsbo`FP1M7 z-su;J6J?DN2eJnv&Ej>hgUHXf+;q!gu_mMp76p-o=T@4je;DA6y@AKayP`|GzAAlJ zz!bmVSvFX7zT(SHZWMxz0?WLsqjnyF{|ZXAI$)@?+mlv2e&dXtV>To!f|_~zNWQ&g zP;=K~hYl}<`ao-(mV zyD3_r@*u94GVw}8ddT?W?&=Tu>HC39OMUWoPFFi=OhhVCt8}0yzP{q~A6mQyrTy%!__f9sc}n|rCyw3T>o>PDtx_e) z4>Y^2;nK7HZlCgLQF~R6m6HTr!B5(SkBkckzHbWMH*hEvc6~uyJolZa?0VYo29ayY z;WDVDEKjO&ui#$vnRbK~XV|x{XZG5PgBu+xV!e)DfQ}s4H0beYJ-70GKi?5<`Bob) zos&6h?Qyh8k)9fHS&yLLWr5A`8sYfF<}brzt$18n|Jv)0M3gFWopTKlU%nvqZ>6It zRfVdTRH8z`*^K&d;dF}bCKvb9ctMy%}6g2+!b*@<9hh z{e;*N#iPkLh)bxa&n<5111hJITHaS-M~T-4t}R2i21N^~ZsKsCblyN6WH2C+`D8sq z6+e0pVebt!1Qm&R4x29+7E#>ZP)TSOPX&Jnml3S<2bjFY=k@5K=D!cTlF&EAX3O*Mh@&Jm#t`&EymW$mnwblNwvg#{=@FMAo+~ z2i>@TP9YQzHruA|AgVsRzs}8>Z9tYpe}Lf=ou=yCkQ%{FO(BVI<1-_CP>4_QLnTX1 zsLe^^suk*U5r9}lEy9m+qy5miL!%|05Omsbo+43!$#A|OQ7x*dP{6}8$I}CSmekTR zbG)||&0)^hBs7k?Gj!=O6ETaJ#vSJ{1ThB!iH!B#b614-)S$mX{RLda@jt9;j(h`o zjm<%EI5VlV6=-IXx&`PO8LDP%UxupdXx*^6ISU9~sZHYCoElbOXD&;0pWdjaY-b*| zpzo6<*(|VrsvWSMWz_U)?@#Ny`qIn?^FZz9BT>;#`xGt6*{m7p#FVKs*dEly3HoW) zvwRuyTqa)p-tO7bJfIm#xGmL|Zkv~Ak8L^-KVjZ{)FspFxy!8p+yxXaSfYx7bBA_W zi?BWb=@zuU0-j?u2qnVx;+aeFIpgBtk8izrI@i6eERr`Nf z@N9C@$m>&t_zHbVoGyw+Vz!fkk%Xwx9*GUi68)H)2}5Re@i3R+PKc~13J;g;Fgc;q z09r~d93AL0z){bF%LIC9Sl$IBhs3_8^aGp~NKdik3VSV;h!AeW6SxBet2%D+nwX2? zyq!Ksgr~>M>w*EhNrg6~4uOhcXdaKU0v~0Bgnu6C;6JuUgyvZ0?%n)(_u}`WJ%yml zSKw6We@WCWzpgu-*r&DMeQ4i-Sg&Kb{+=_K>l*Y`SYm~Y=!$FKL+$8s6(%l8b0_&( zJdx{`7bsQK_UM;c@*G>J$QW@)(XNw%-v#P9;*Pz~4((!9>RxP#G_ou|YW_e*10t=; zMkUqt+=z!sKawKm4``xO(HsdM#QC4@*fvLSRnvBAT(9{y?}oD;hA{PvrX+}~c@|G23)8xzs z*1z!VkRPPda7XZf>@}$&VF!kfNCyt!-uc;ITk(T)%BHFWV#Tv_U@w|L;##qH!2_IW z0=Qi($gHm+T|FXRLsT7QEdpr$C>wuXWCxZu2=_GP7Vgp9!i~&^Q@N^_xx+=d& zaB~!PO$--PeIBvyt>+LMqCPuSe}y2|-!<*CZES6!o{3yz<3KJ|`f+c0v!BYn zfYi5$Ba}S?jQn2oeCxrJYUVRlj=rcsGAOI&~+tZEX>5JJ3Jr z2h_*p0b}JNL@7-!{4t12B|!o3K|o|tSF-+f&_+j(*9H)8lF+($+(S5mFLR(i0f@?H>Kt z&mJE^|ICK_PvtL*UTZzIbTIc#ksV|NlJb=>Bp6!|AyPt3Jn6lI{O+xOD%dV%TP&(; zlYbVk@z-q*e8DE;e$*S>o17Wa1q6n2Lcew`H!ZDySYA+eUE?T^X{j&${E<-co?o;#sU=%Z9Y#Q}1LH6+KhUtVm zkI^ z?%MHqbnJ}y24tz{T~OuzbozB+-O&KdR+(IF2CqZNTh^?EqL>q7NY&&~SHU>fv$ zoM}0~Ng-}g&zH4i*YbDg&({^~(Cp8*9ovJA!J4m9RK_L@_ng;>ZWWoKI_h2J!Kvb+ zD{qLidN-f*(j_Q-#(Gf6%$l4`_$~4i2>{uFqYTr5Uk>P+2N^j0uL%WxvxxQo`8T-( z>tjmPsv+Kl{DJGaxm4Ll0UM=jBIP1JBo!sX6N|PZAd@p(Mvp^Zf=%quxdsfW+A74hw!57CW|2nO}5|8`}pE0keQR)xZs&&J49^FrN> zzT^M?gF2E;6%~g?z^(=KCVkCo*}Ei_z40lsK1_Vjr(Rb;Z={8OjFf&8E$0=NwD`=# zV7vG+6JbFrvI}9Loo{zTjZ6gfGDwn1&G6C&{h=iZ{bqd0CEj$}Wo`jXH$KtJ*1aoWy&!Ib=1m6-1P zjL;o<4+`dCokaZi!$2P@rZ4u?U=AJAP$9*ntwO8pay_1=Z^!E}PITpr9KkSqv|~Df z5yA-+%o#Z7s|hTHK|qATIeZN@>mmZ)JPYQmop*T)FW@*aI3(KZ507QEbdAYBpjfam z21P(YZCCI$B>zb@CSerreSq2KgFTg@{ zndst^%r@*rf4FX^8cffx$f;V|iw-?)h2NOVJjPH?nWF`2m1t0G5BWfYPYq z3mNFcy&wZaC2k6jq44uO`tfDtLJP=^XI#o2Z%;jP6#6GXwGonaKr^}05$*K-0!$)Q zbg9>oWF4r$2X-~?CXhvBP|Sl8Hz7+l-b^OmtF?|O=LptN(<7XZTmYI)0h8GfQPGxN zF+G|rl^7jX$Svsk{?x+(LB(>MZBC1J)kToQWL=fMa&2Szotw=JF_OyfPy>{y;F&%wMa7NrBpowstm5_X%de9 zM!Q5EWsV4{f<*CnyqPPBdZ1?iG47b+3b@OgM=Zb3QP|f`G$W zFY#pbkC$L=!{}^*sCG7ohuKK05L#SJu#WAOn0d~W;`l$?4U0`lRFPS|VXH$f!WZ0y zKlU#ZQq?2zJZm&ERxUb#%$)6xS?xR+K;l{9`QdM*L6AKm%Lw$3;N{|>I^tU(O$t5} z-tD8BVWb{nJY-G!DAIAwkKqzg+3U(47FGOlx-?JvO&^nUFK;#6Cb`NRFk2Im;bOHi znu|v~#am=m^X;0x|8Rdn1-oADDjw|WkI8qRG+qqg<~MzQTP;_iia`hB;?^!OYtx4! zB+B>qC7Grosi%4a+E4z6P#hH89G;XsLO8Ts^tDev6y`ttp|)t{6P6jEdbi;Y?3X5> zm77XKvZf+&25xA&zRJ2Xxi?p;)w)96uKDvwOo&rpnDmbD2UK6~m)YQ<@|7d*CLjUl zY>Tk^Rc%A-dD9SyBji{1c1Ty_1BVZFHparqC7To#erP)UUKcW7=nKGYUyMLbc6r%_ z$06kvR4l$}mrt|d`s$!{L+^2?cTOFQ?7X&Q$j@OsQ4;OU`oGzFpi~|RL&(IS@Q#I3 zgbt}i)Judc4KPm!j_Z3;n9Yrsfd0R!psgY8+rs3AdbgLGRCSg0VNQcn)?Kd2O-VIItVUIN6X*eq4!GpP{>cU8<`#nyh3Qd`ZCL%)q4y+Bo##U)ShfFHi_!)t zSn{ZpW3dX8(eBO?rt3^AF+2dH&HC*M%^YILs8Qy6mPcb6p{Jc2zW#uO^x4}i9gY_F zKT1LsRS(Uj+?L15Z^N@w_9Hyyv0D&gMv4W3h1gR}BQo|z1<#b_XgqbDhJ z_NP!mZa}2%8i;d4tmWBeFq~nkR*gGHy?^r^uv zO-KcBL*=lRmlws~aeuvYt{|*hS4+i`D7K*0zNr4z;}JSst!*84d10QNn6{cx2iNTq2DI`HU9*YKs!vl03r zeV~>+8JU-!uQr*4AOWwR8rWmY{1j;yr0U4lTo89lCN&V|iysI>F zx&msgO{mI}cWqnrjda@;dtH20&BH$9W1hm~5t0e*QQe&XdX4%nT2HEqh2Nx=Cq!mN z+PB8ru1i%kYRdU4P1=jN1{bJ(?BsDx*fW!C` zGoLmcbola>3&}t0b_d7u6M~gqyr0|b6^dlJSXH7t?esFv|J}O88riUFvX5IaS7W%D znggtYVk^xnFPu7YK=5OPW@(`6N;}W^n~L>2a<(^?zZnqo;ZG{djxeMw_U8FiseIab z0nID&b91GjRC`M4eqiodi~S_=#1_n7muk*_K(}XD>5x6SBi2Idml;*Cadc-Bzjl7v z#r^!DrYl#wyz>>;f4JX=S(h3c*f$va)1|LgOkbUVJJ-FG=&}wl5-jt_xLje@<+1|)4ZLcT z=jN{+RjV}SZt>?=R7_enZ+|Fy*{y%d)znNn(dStPVbEZ%H3Ec*sGJ37K7C~vbRwhz zIz-^2+dnH;7+=Y`hzZ3gce!8$;?JrGh-Gw}(YXIKFoV`do>unn7bX=K)0( zfE+QT0FSO~V#xc0(|j1Cvj8xED=rlo@;abRW>Zc-Vf79@Gl80Uq1Z*;;G{DLBmo;V zDu+-e0+|uM(tQ@ruju~+P|&lcdM-@^)`mS>1J!Hjv1kCV#Ruc}gjgAz)^n?oFZmO~ zXyi@wC<$2vR$inZ5+pJFMc)UFmfOOfbg0{7-vkmzSfX$0Ctfu5%M-rBe=JVEHnd;= z?Xym2lyP^u4RKr|JY0fYqZFfzf-?FDmX}@P{tF?@{>&PdCvj*6PY|u!K`a}Bj54m6 zmDv}g8CV}Ev&lxcqQYzk;UiJvWwZ;Kj9C$Y_vh&ewg|HTiDrwDbQ?tmM0WSQTmWT3 z$Trovp-rRG6*5Z0k*Mk>E14sF1~BJo$XY|75AHe+M2V)jpsOKvhH{N_bPXR?pohD!*RG z5V1Y1&zX%GY33VhOK@dVi%oKGb7qz;nhOq1_94 zE6gbsu_4eDfnDGgMXaB`Ifj;wN31z@-y+t=UWeR9HkEmZi_OI4t0?`$AE=nbZtDYu zfbkz`M;CR`a&6iA%cWM+}0k+PMxxW->c3)~A z&Tiuedhrx#-r~Gg!D*f8cxhaN?ypVV)vHTdRPWxj2>spQ&%#B5k`hvIDcC}ezv93? zZ74M3sqypCP&M!6z2S1lko^GV{e@ztxm5y1dL4?1!~JltQ{m=yU#hZE&}fv^Cqfc` zWDK--3ltR(H3>TUw9@BW#*AUorQZF;W4WzQ|BfZk;Y{J3l7=jT?b`r0&~1JAJ20DwEIhuhdRnGuckU7MXAVidN?rBnj_;(=}n zvdnp3g}+|b0hUw-I2B*H6~Oqqyb%eUUZew<{`%BOl0PL8R647kEm~oGr4`Gq8)p+z zh=_Zt0eZj|jmBO@#0z{BdKHlWX@4IHC?*F{&6^&*2fg+l=muU}mF}mp7*ce7uG#8O z$uX8?M1Qvc5)kx$RI?FG0NlIC(3t1Il0GPUvLgo^ z4??;hG(FmkT{Q;js`zja8^?RyO>dytdbc0bZgZ(jH{7V_fIvzka!>H|RnnFRo5WPC zQUENdUxy#s>SBoRww#7jPM6EET=2MqxT9|BtpTt&PXy)K50{1NzokRsf;(;?wd+I( zDfgL(p_D$cLo+`mII4ZK#v=7n3p)JF%=lw7sx5aR<$ji_dAsX}*u@@X`n<41D_i#3 zf~N!x4a~em8IFrhB?Iofc3z)JRm_$!*j^)DC&+^evv|Ojw-08JSaZg#rD{UAZrG%z zh=>BmswQ;LCP^@9ucVQ>UH)J#(y@IwsafTL*_>(Knrge8W6;0;C1xr|B{9MAVj(JV z7hE?$?>{;sZDMTmTJnFE&%|LvBOT5r`%7AZu%Uc;WtXZKN^xeb*=Ob(E6Mx~D0pls zwu3=LN;A6E8IKe^WYGRG{637(X&xnH6K(qA#QZ&Ig&vh($HYFOe z>4V5{ls7>TDL)jc59s_mUCSl1!pZrce~0yflp#9GL2vMxQ9&;WKVQ;cd$UwRniqOb z0hCEwuq&2KD9;VUsZQ8n=M0veLkesOu^~gHRlm72!5hrQn+3#{SN2f-9_bM>zkDJu zOF;u0dZ$5#cw-Y|Mi8GPXYU&LhO1cy>4Wksca=c?h9CK6w+UIlNCD0v#B&=T%m;MXeR9ykW?wLN9~C6Wef-h>W&`DCt0(zUaB-y;|~mMqo2{@%3j z)UBAvwv|}dZ(R7ouCUkejKnIskcgwtr(A(n-zkl-U6oJB-L@fUFx8)j(!5+^tNpbPH$Lg!= zz*&_md6Mnjyy+9u9-1)DYe-yw=V{hfFmZacDwCZg|Ye!o`Rad3Z zrwqOhWjSO87UjOLN@N>TyE?2Xn{V*xwCqtqU@;ne$Z>yfEhaEQ^Yf>lP|ht`Pix z5Ea{fDKST}Psvxvl`;LB@AgFpRR+fy*EnV@u^TdCUft>G^1$@D$&ri) zQJ>;rHY?ihiFDD8NzzxMc$*Q(d_$e`I0YJ{S21SShpM%qN3uM+yt{-ydTfj4xvCrN zzkC8=MvfmU?HRo3bF_19-THUl2r|DndcCOO*xLMe#^%m{;}eJFO~Rgs?!ruW*Ol;3 z7`fW~*_~BibbsOaZSwJU)ye5%zn<8E^L3)OT{lW(twG;R$Et>`KBF_%KL=my6J$!<4g~hzjMSq9n+`769H|!Bpn%%s1v$mR4QIX3UNl=G$YZwy@yGIwHlk|(r~{BK|1Jh5B%gF_Jx#I~FP{@DBV$jlCw?EmS(*nzEY z#>j><9zs5~?Lib`ZfQ?V8KTM{{-aYOoYv0_sMhc&`c!EWpGYW0g!AXfSMuGf-i5w; zuKcvespU|FD)Lz3?uKGm*5r|2m{k;tJj^_sHL*Ue`H;;SL*Nu$lDHYGnvwweB61U> zm(9L};Yw85pTOKkwi3BWX~ijFz5qk zfO?riIttY|n|z3DV0IE=%#H1py}5b^w|6`D3=vJ9RVCm@MhZh4h1HcT#4|fdB|D-U zKgGYjizq{AM>z%3pJHhS@Khg|#@O(|AF6Pn#q{44T))ZO<^iGLui&7Z{7Z5IA<`;I_b2W@!Lh4-*+;7-*Uc zI#9lvUKn-{N^pccQruA0La573zDW&g2RMm!%I?Wc(gzt%}V-Sec?r0+ro^BO>f z3aQjXj>77oULJr=b^@=nPx%8w^zY#+EPNmY19@mNhQoBgts~TB!8lMa#s)=Ja1GWW zL`I7%8#2Ij8JTXgt`8lAy-}mtX*a5=rh>*pM*9DxYm=nXF{I2JA;pDePs4 z&jbkI!LUYLJo-9iK@{Q%b%4Q5&HP1_N20_LV>qyLuNsQW=-Sq|4=-bqW2iY3Ak-B= zhZmW$l%9JYEx-`sW)s3y-++$U!RPrB+x${(N{_6coAQO&ubwI-JmZbT8FT_LmBA-K_Up(X0cYZ- zgB^me!32X6H|rvw`#gJNc3m_T{}}?JzGY~xk_#%3ISmF3amM6lS(OCefWxyEq zCWP3*La3wZjh#ea&&Jo2wF7Irf7)11jD)8`ZA3DF3^=Y$&zdmgz6J>bdVTG8`wU9; zxh7YZ&qOuRX#afT5@4um4>MGxuZF|(f+0F7+g+BIlcpO|JVQ7FA0V&7a=vV^Fi=Ef zpadapbI{wT4YUm7!F~p>YSUY|)}pzh=?=7cKWsnl(BQLfz&5pOTpcu+>|TSw6zC&u z#>WMq4Z6-Z=wz~g!TK@(PW!qgehQOn?MoP&#LVt06i|G0?@`xs^zQ}<`FsiYhDW0R zmFsq_vVOsp&lZf%OO_N|-WqWL3FJvx2Z-i`0e>aUV+c9^wW)K;#E+K0ywD$W@PeVc z5z?6+i9I+#dO%*j@9ymrUWWJz>Fh-B$R$fC3^%@Ajq5o93CI?++ zh#3N)4DDJXYlC|a?aK$iza7E8q;FWo`HXhnEPB)hLmQt!s+!a-;?K`7Mt&`eWq;9L zTVI3(uSpH^sZdTw6B*bY{_aQ)F9M|nB_@)FR3OC$uu7weUFpe97`y{jGz~#Mkhw)b zq8+Rt#`n7T3_`u08b~Gi^xua{Ii%=a-!~a7yVRal1|D3`Ai!&0s$ z^cc7$mN2v(&%9>Euf+ye-VSJwXya01pWP=+vmKvIN7cb_a@f?D|3O)|L>V5CM?2$; zeD8I*TmkmSXPB-Sl(on1^l=g51X=Q2Z7CDyE5rr|5RQMRh`DsoH7r~k{S>LqDb%^n zx+7TQQ{sZJ!g*tTL7^&2PSuy+Qkl!Im6n%2G?#EkJ^Rj~3=Ijo^SX7sfucsS3iNY{ zKPv5@bR&%vZowmNxhIzbnZkRMP?JXl_W9RyH$FkjHn zIMTpI>eNaEjb45ypUy_Y7DY0)Y&2B+% zGL#8H6mb@J^9-l6Z%w(N<J3mD}T355w zMYb+e_m_Ol5|{{wMR`s4Q3k-GzO zGy=!2c4?0~$!>mAl}sB4KD?g5^t55k;}q?avnf|ZI=QJ5W zNJAmM9exz7%?)XQ|5nfM2!^kH(pEpta~{SV(nmuUqQWjg)(amUMd;Vkic7r+u;lvrUKs1$x~}4bU5jVUgD?HD+?;@eF+M0z8;ZZa z-H~RVhk5L7H9s(IAu%#nf`xXFj?#`Oh$&$7hu-AU#GzSS-{; zw&mI6Nc(DTsFlY51q7q^!5}{F{+DF2iBkNskGJp=5Z?JF;n&ih-$>Ri*j*SRlqEk& z*tZY+qj!W)<*9>f4vIrH5k(wBOO#HTNG#z$OK>(|`aG}!Nc-YACzHIf(Y#3b733=u{dcPL=Jc4hT+!sc>~RoPTa+&!#*Rb^vw*%ggrrCZ~qNRj+m z*WX}~c?j-=TPziC$PP|j;)H9@DSb2dZB{8G+Sr5nHlyk3DShAOj+FWz zTA!3HpH}9@cQ25{Ms_D1iL&!PaL`hGvp`a&v7dCf^x@+9Tov{^lR{zPgJ)vGRNrw4 zdp&0rj5D9qXcSm<2!ubPdb=dZe|MC>q!e$hl;A8*>sa!B@aivXa&|Te^fnKeWk&n& zq&uW!Cv7>s|HBX~0IK1s+{IhgjVP0bcry_I&Bnrug0od6$(9;pjSa%fe7j)}-nwr( zUQs)JHMfNr7>%-D3Nh8yf zc}FFEzV8<8g=Y$Sp3izC@!J?W!OP$JxVq?c#}>w@vMq zW@t9|Zhh@W&U+=UYZrsDaVpj9cLPd{r1d@-SrNK=wLJEb)R-C1sB`V&)zss3|Ws5<92EN^2!X1XKI&v=G^W76m04? z{<=y6ql8~@S-rUy7C52J3aclgBfO8mVqrR^wrH({KYX$31bSBw2t4gCeM9~baUG&- z&0al8DfJE(o&il>5icEax_&tIA?=lrXKCZ$m;dbbJ#5jScRJM(M+@DK% zV>WdT9o~;L_Wh`txths1ui&BnFB&B?Bc|QnX3xbO?Nak#f5IR4Gx09}DLBrwqvJM2 z)BcPY4C*{*(vF&t(An##>1H=kfme@y$#|ITu6u!XVB7VS63&R^WbU=y@m<846mJ&) z%qo`@q)xKa5PI7xT|j1bn>XIt>xPI7iFXQGfgF%^q~Kv&5l|B1A7;qCiPH8a-apjf zLP#HtpbVB458g?zNm?{3Vhk-I!rhop&S+Npne~$5`-zE}aTNvN1fL+fPF=wQ!XCc`WR4ekP+gP# zY8O6$O2%|^7%WJGLksJ*N94q`ia;ZJ1rsPBrsOQaFF3;0>?eG4lE|%1L!al~ChB35 z9bsIEH=)&i1)gVW+cO~y_FpI(fbeo}ZF)h^&O4Wi3ZcZ~%thP(}5uv<1 zv>%c+#Y_hW_z~dyqE|SGnJ4jus3Yp@HB_snZ2qkHJCyoYVBt~Nig?%&9gm1ebEWF} zs9EVD2dN})3rnD>;EfN$?1#HMa%Ci%zAQ_FHUN}SL8WKedxN4C?CP;t7-@lW7vKa? z5(6WKMFD-0RNU}f4u)v5#1(^O8Hk4FyFD$Hd(%y#20I6^TY!d0eY`;=(Uw}w62=Xo z>olNh7)aCH>CNhhzBYaXG5!G*F$=T0Cv;|9TmS$T35}Bp0qE#YgL=gA6j&pI0xLjY zsF0~m=#?OmVps?hGGc^g$y6cLdcabL?gBx`GTd`dn&r2wIaS<%c}i8TDbD9ITe5g5 zM1~%VKnT&}Zai|J49sq}Y%o~ZGbpyFe^Da}7Buu7fJU$-V33#^nm)lY+Z{3v_B}b4 zV}&3*$B1r$xfJT`L^)aD%zhHID+>~7(_rnh;b+MVOi;jUeVPhi1=(@5TL2rAxgYIX z6f?5_Ic(XA@5e4e_4}KG1SOk`Pr)!)qQppG7mK|Sm&jU9P^30k12B>9svmIebJ!t) zq)f3!>y)H70jz{h4^~RT>cn1x4aD=InHcg$EG8v=i$ZVIq@hgYY(lzp;2IN+0{K5& zf%V7=0dWVg53ve8dh4OOuh)fR8|-XT5ArzQ5;FNi860c?q>x~LiGO$;c9p{&MR<&p zbtehJxVY?2M9ToaDp6f!U%aqZ+K2sb31m8r31Q^Qv>G(~9g`2o{T0H&G4n2IzvMcA zX1p7#ql!2O1$R>Z`C=rA2d2S4N6+=x*0M-OXN6=?^|}$ome?L7O-@1x9hzWE{019q zqmDr^v=hahmxQwaAom@!;Eri(YV(=7V0B}AO}}Kinp$Q*K{~hxl3&4odeErd3He+C z%P^yc0XFthN{iw7(HUS_N836ozo2!fF@w!9iR@nTMsGIQ$h9awBZtZIlIZ6cK3K$4 zPJHmHlFh+q5mdVk@suMwk?{Ki^!7tTI%g?}%dwzIJ>xxu&cyBTA`<{Cj*oagq_d9+ z!Lfw|z(d(sX9Y3WC^E8fmP9_14ht+5Ki57hpenUh04|C@t&?N}jbH}`)rn0G7l%s1 zVb1894np{iOdi9pI+xWRmd3-Du|u1OHsZiVm$t{-d!9LXxX&}+7t^9;UBrD>TT zEwBtyAR%fT>A0E}OV%A*9JaARXUwh7zl_wEOuaAJwIK6~{flTn16>CrpNO%AIfZL! zmR6V5r6}s!0}VAO)APr4kYMu1w?9ETj>!k2){irL-HVjTBj71%a}l6w%i*&0r$FlR z+M60%&+H_x;_bSr{`L))tHaERR)@A&rRbJr1=V)%;%~$D3-PBS-^v%Se!$Y( z)?WK}st-WB{XA^{&KssPT53&dy zlERPW8nbr3&RK%ONQUy<$c9MnG!8Hva*QHD+ptd&m`O@|(rQ{Q+8Zs1j&QeF0rIWY zz6m@zKwy3?w&7lN4y8S##%4zlD8D%plm{C`Uy!}AVJ-cwm_#2U+0g)z^uzBx8Nrr> zxFg4_e`-=po@G)c0uoov?kS+ph5(Ys5K7Riu?i<(10RAR5`pM}m+<{8?uN{J85Gw03LJ9LeSE=e69#O}h>r?-yqEU#~WKQ-Bco`;z$1dUR;3@zm z@ik;~|s!5;|BYqgEReu~=*46KHiAa)dOhj> z#;9X&^4V0?P}`vO$SQlk;ol8fqN z9&VDW?32&NAIJ;x8L&0&8dxPad-+1*!LsY`O$uK=5Gsh}M4q~?_I$Js?d(Ln)>g-* zM@OiT7*gjZzv86RXZwNzo!i`*9G+Wlue64jlT zB|rA7A4@nf8m#kvdbEZ2r>HkRZn81cy|JxoUznTs!Y1Tuh5^NXH%JcN{B`a}9=w@n z+xFC?t(8Q5BRJsp^Jx0_rM6~GKI01KD)ftckx}sZ4q?&r8;Ugul%Yg8 z!MznD5W#!uvzES9t4{MJgW`Vle0j|SDcaHjHk>g&XqR@e9-h1#1gQKZKzIr2{ysc( z=JWd3#F-UQQo+kb_;q;fko~irRn?q#=&~We+QO2ShmIKN3GRpo_vglu;A@NLM)7;) zX+K}XDgLpBC|sg6hZWw)IMLe<4J7G?OvBVgEus8|lEzpjjUm5@WmtX-bIhb++UfRO z&GS8#=8aRWx!6x?;>+5fw_;Wob!?y&RdqI4 z2gUgp9jaC+;wpTcPdyU-5;gT#sb(!gn*0E5Bei{n9j1C7I)SO0D)GoYOu^Ze_irQm zAt>>`{DI*XaPY=Cz{#^A9duj_vb3!ogy3q6ST89mhg9g|1@nV649M--1+jBz)1;!Kp6Ck5y9jzL&|t#PU>=5o zwbI~ek@hKO{eS)KuZM@M%=Q%R{$GF7!He6{Lrxlkr+H1vx8tq)Iw(akn7bf1qEdx9 z1dS_P5hizU=eJV+lp|!61No+6h|R?!4#NXjKEU^x>vA@9qE_R|Dfa6 zhq}gCh>nRCj-N43*gWj$YldYn-}L|_kKOVK~D5PJ_^On;~SQAI}NeNAwJ zJA+pNkd*=(AeiVkAZ92ITTvS>q=De>bB@Wo|BpS@{_!zPcdXTpWP&3$4W;Yjw zaa?wyL7o_zB0wf!3ji|cuLnX~1r#MZIJ*~D70pMXoIz_AqMr=&k6^ueBr04-KDE3+ z91>~sYcWP>E=2zzs-eu<1v6>ZUW>4hb1XQ_%u)#zV1C?5dcJg!VoSJ0tng9x?CF_9 z6BT0dm~i@li|Q{J4J1MEEMfdZ-P#P_AV4ExfGj<~9W^HfA}Y*bMkc_+qHGSpJBmj< zw3xSYNe!Z0UcXtm@FJioz~ZbY8NJOxi3e|bDgu&N0Fpxa0LvaQLntFNqQrqh4{$zX zPDNw5p|hHWkB+Yh2anhrZkuTY05i}TYz;5hbO;U+T^#zyC*dE#9d*tdp~CTs?Hq`D zczX`KzY1#S#UdrCiKI|i&YcY-;MJ^KgJ*f|vHGyF?j_&3CBG%)6?)2|)G&!-pa9 zm~fy~7|aR{=sOzpBd1?-hLkUQI)+expUqWmMn505zhg^5{oSNL8x^@2enTLxv-blN zs+?dJ>8v&M6l zF~j^!0*eCiIZ}4h8hZU4s`Kfy2`q(1@`5M<&ziy9rTRT}U7ve~b_V^d0gs&FYev-` z;iOqauoso+F*WPHN3aH)baZ{r<@my0^{gg4A#-%SQ+XU8AB$n-a;kdL2;5yZhlvV$b>eOFia!U9MZ*1sLg^^)?y74r&m$6duVAUp zNMR26O8oi*UI}+Pc-)I976Ui_)XeW*Y}fB$QM1_3q6TDbabY-}z{|_?0YfYv$7h6m zftr;MEG$Ww3(iNJ15EVfrRRn|=0b7s=2-@->^%FSsbzrUS?P-UCJ>WajZ~{ZC#`s7H0TR ziU&mFDV^n+pfCjH$Y=4h%3L*}2;_AUzVW$`?m)UGlUaSaf@lm>0HA}%4s&ybI9Q*$ z#40A4CRB{m9mU5+^22W?4OxyTW8@khJ_m5eILHj5Pn9CAwhmFY!`i&CGE>U&WaAj5 zJi@FA0hmmni^GWNilF33?p|cW>EwMJP&L5zK*8Ad2De`sH=I$lQD^mpfBm5e>YG#vwdl<)Nvk%Bl)1;UoN3Q8OBBgC|YVU^of7&3lEZM~3W<&y0B z?B6Q(t+{SxV^e17pr;SiYRqU5ovUcX_o6pDUpaD3cQyx}n&>Mx^~OjB3h$iY?1vnf zW*THXoyj>4aH*Cyxt%KM_=rCB5HN=!i&INe;q z6JSzxcJs!Tp$}#4q4`!&?Hd*hqK^=L$x9lMt8KxC{%Hi9uf)NHd3;4#PCmrAH*1l> zYF>Nq>jx4l?k{vw{$<}z?saUFZMDc#?e6d!Y&ytCnoBgIOf-Ew?Z;=k9NGpXI8jAy za=POW;dhk+lG|nSPg}wx%@bntY{oFw#f&M@DP~Gu;aS7$m-du}ZAQdvC2EBe5%pof zHy$qH{T@ay9a{k<>)`cGg0A)(lnuivNDTA4>Iu%L-|pR#d7b#lr>lL)Xabh%x>McR zx+L@$WG(nNamB)!T2fE9wozytfI2*HOov@ZRahI9FuVp+!ATp7vQ&-GwX|I0Ty``i zbclPcNozMx{7tw=xqeuTTvR@*B#lL;Yg-b^Yr-KZ2^C8`ey7rR%jOn?lj6g0C== z)qUzEycQ^fC<>uc;Y;3Vb~f5W!s^~#MeBnyPp z{IJh}nc3OQ^ zuIbRG2q8dyQ%11$sv#$bCk>rfkx5XzijJOhy>35tGXJcUQb2aTJy=W;{_`5>+z>wZan^Fy_~ZjcG4bVgptKbFF7l$jJ0pO@_g}7F(bq` zO9t2TAC)bTd)c<#n|iX&ecPeDr7vTqx2X87!ZF=6)V4{a{|b<%AhSgDwS&!7^E3E$ zRm-g*Zzq*pk(95vYwT~7_agO%WB5>rcS%S_8;&whIyT|PJvZ7r33I2dqPBgCoBtnK z?;a3kowkpou2Pb+Z3--B4H9yDlZSGH%Ph0>*^#K)YtZnf)BQcp4Pk9p=*?&2ggG+r z^bPOl-)OoS)%{{cS^Y>@Rb3d;T6*c<#}=l|UPl?jF0~v&R?#1n7$&_7e{9m%G-YH{ zyH!`nv*M{D>j~VCkuPr;szlWVd^XbeW`@2N027P7O?I<(*fs#+W_(Q(^NfmM9|IEg z2s6;^;R3XgT6eelQm63cpA~v}J{VTtGcSWcz74GwjW6na70WY6YcLtpCj5?Dh7TNy zKDW<2XnINSwJuR>eynr9V0H@@kW9nVMdvTp<*l7w6mvq?{a2tMjrci2a1=xWCKIXv z%s}GsSq>;X1`O;^yB~ftry zNbW%dj0quwWF-nwJToOT0KFjfUFMC%I*P}?jW=^aoz^pEkZ6(!ZJQ7RC;EyY^401c+s#LW!$SXFF?=x)g z2Pbyb!`dNg8|zAW;TE7NmLovK3i&RP&fGA$%@kG4+X-94%~#O5e2Z<^Zta zsA$|vZb2be36N39tKaF=A{w|oFp)DQ86Vkd0ehXwy}W%0LEgE`)AWjq>*tXzWLT24 zu0u48#z-hXb!EzZ6Tq-E)(M*XR7BfoGkSpfIaRTwd^WZ`OppQU;99>BFCh!04U$Dr zUhEH~fI|XT04}bM@Pe1M49&e~W01t$E=DNOq(;#;oa+d)^9ThfF7+~w63;ziycUEr znL55p;zn9HoO}Qi918wru5X9VbAc^w670?*_rJ%g-h;ouQ0l{s3k*S6m5?(aTTf1V zd*AaDvvT|d@-CcIhQ#Y^nk{JE%(;Q${m=E6drT@RA5~YtdN$oA%N*DGgj9yDKp>CXgtzCDWvdCs+d6tWztSm%}9`Aew1O5fufX$crDlpPeSDFH=hjT8^cM zpFh4r5YSj>uz+H~C_=MX)YxA1(&Rz_tSq@5v9OEu`2T(qx*&<5gw6b+Ur5Vi(4+q{7?&_J(&N2-y$NAmhS_&04Dt>zpy=Q1v+gvPz^i8yNkX|Ee_ z(eWas{@|o!HfGNvMO&2u8hqH=!JYsHYQr`Dnoeju!EaecARPxeQCMMjgF$9YMjCfy zwHyP=YNoqFcE@0Sl<~a0caxPp)%hZShG17fUPPVpGC#2Rvv;=-)9da!@qyS-()ELPqIs2t3ca zy;?+VBUQ~{XL-df$1b(SHv^eE)g9V)_rwQ@i7;O$+l4?3w|#4g;70rk8T$1_;$y9C zzL~lI&Wm-;$gDl^Fdp+XK7mGPXqIcWVxe8Rj?YAB^1EWVZVm@ywG4fHsbL{4>`N7Rv_5qFKH%QR+Q> z1`c!weV-V-*;|zt(ew1I^M#?FKia4LaMq#N;`IxvpAVY;b5DDFkD0$N8c#T&^DtC- zU?svn%C-De&%Ls5yUk4pMN0qiKVoyaGcggU1{|;M0P4!3G z^H%x~Dp8hT{Bqmg!+Bhup)p_K(1!ND-SE0gHu@frCl-s;sBO2glI`!6YouktQB!;j z(WRZ{yk|~)a1{Ny^ASQG09c`>W6LM+!p4bw9Q=i6yJ7aupZ? z0m~)5T`N?FODFF|-$(95$K?u$l|`W8R?((=&Y|ktSeHZ$_3qx-@i@$d;ErRl#H#Wi zxPXN&^6tc|?mA?ArU+ftt{^p0Uaq&Ft!&y@SlgNSDH3pzhXirHraEjAOHvbc&U=C0 zGr?MmYAORU?FaoaL3Zf3EbuC_9#I1oBOeQu&A{%u*&|+9SvN5?*YH2UyPvAOjNbGS zuRAFUqW`y5Z|+>B-+~YXdgic&>G@TOS%``%OL(0YP76jWDl*|J<>{3LR>rV~xG9&O$d7_gxdvmsW2!>9qxw#1qwq?QQYXp#cgyz)%;Nx9L& zZa^DZmGUpUYbU?XgV^BZiR)VHS1+$$Z^;X$)GprLG4)jYP@MN$Ldgv*DHmK(qNxZD zzk74v|K9tg$@q$uFBU@3{`GP}qp>Rvw+;V!+sHl(c!fN7847wy1Cu4gk^J`Jw zA_}9K+how*V;-W>0H?Z4(Eifd%GQ0~lD$c}$ec_^;Td{$v{UlR*8vV|CxiK=WZ_X$ z-5jg9-t&a8aKCP4+U&Jf@gP66#bE@^DocdhMpwZ70@4PLk~6&}PUgw4+2{OH+ouXZ z{{#mX2-In*zRjcI!q}US9sw6dJkM|*0z(S`Ne{>S|0anThUEJY(8V?jzF<@!hv}%v z%YW0%bJQ{^t|@E-PDbEpf{Hs5W|3|Z_zCli7scBtLtsTRM93W(o?(CkxD9k8Xo>`` zTNdI$1W#8GQUE+(|EvvQfUf)MH`ilOh<9``q@hq=a`YtK6Dg}X`7En32K$3BzmTJR zn3S5v8i5^dg9X%&si^<+CyImF{S$EguZJWoh=Y%VkPOyUHBd;%JcH%{Hw$Q&nXqau z`bJ2vBbatM>G!3?tT4NxZ2-*FS3uQMAUoY^QKPK!yCx)&tj~9k$I_yguEG^;4tS?? z{$H5l6*igA^}b0y-*`x*UKbL*hbc-Y%7t^Aa>(s0kjksM?%<%^c@tyW4yJGhIm{v} zBA4b;|L0Xm599pp7m^qp^Gxqnxx!HAB^f*QkoMQ zSo7{dT-=dZ($ppi5D;(_s*cpO@n6FIG5#&A&~P!=KxIH?sCZ@ttR{Pik^fIQFX?&- z_^`5UwB9Anp;|lALbT4~kLuZn3Q5#s`#xd*B?9$3AltZ+f|WPSJF_@2OW`=|Y0=q3 z$sPepF0QC9YZ%xGGfFybd`M2?IFgZqU>Wjv){&d^fJ#0wfx6j0xDleql+^(KW0PD{ z%<;|RBg4cHgGPo_a$c0ynm_OW)Hn1wbrU%nocXNiGpx{fn#K<_mxcXmuJUtkbS%+O zyXHx33Y;LFXxa7K@XumU$ABjLhS%= z2tW?Pdk}o6tE&i~%tb8f#CoD6e8cr}=8yY3%#~;|GzX%k4CP2 zYX6A|mrl5RDpc!`@j-+GxI=nq_;Qa;ZL3A_MSXaRfhLgUA>_w(j~sgj(Q@dA#Oo|_>^?(}_dgSu!|?C(q=uHK&Hq6+b8m;Fbo8z3Qs&33eRY9FG)-4X?&4v#Lj?kXmV^u-T*j~ zl=DnPK}OTGiHlO$95QigMFyZ4uZI>n@^2V+?E3U_k!plYKnjODDsJ(>KN--+2j;dc zAkzwB^@7nnO2+vEEu&f!6Q*ka{P1NNZh#a-erQZ4BGClIDM?a@vXF2=FSAk6XI^|Q z|MQY1<}lIewz+Fus|0KwDG;F1CERfKMQ!HYdYyV=QGs$nLiL$=@FoPM^>0nKRAbCQ zybY!Y3w9Kdkn*6{D1Lmr5516%^$R38)ZG=GPVK?q>YBw-Kz4|HdH1Z(w$%Z0O+gur zVZrc#*}^8Xg`%}J8e~*3BtVr%{{)KhkrM<3q#!YYE1_*g@ky0A-<|z#BnRV_$at@Q_iZ1$+{{{*{ zvdVvuYIp0A)=O}G3(vnZDag;R zvd@7jcXY#B;NM3uJ&~og=x1q6`o0c&ll2jH$#`HlCDh2Mc^H@vD>C19M0yIHc|S`? zEeS>MHb@hw)l}Iq_Eu?BpmR`k?48@=!QxhiAW83so&YsjHEZ#Z9-a~!Cu&lAWGJ0M?|W(^Gx6QqZ5k-25dc#$plUF75X`c}W807o zGJRdbE5KHKppdt-0D~=!n}cu^B8jpXeqnw%C~TZ4b8;~fLE4f~NNBlJojRXk17g1S z;V3;1D7GDTsx7r0f$;c_uDtGzB{Eb6`wj=x(G?nNagZ{lc%S3^WJf#OiD2yzGD^5` zX2)as0&<@kJ(9S#Zpk2s^y0+_os!E z#!PzOKvf5fwkW~24W^>GgRc{PN-#oe2u=?TPgR@G%Kapo*#1Yxt~_6#e~B)%y2307 zLZaj4vltP_3X9v*?;(Rd=o_Phz?xs$B}`PU!-6sk|CYHA+1X;TJrK(XMD<@G)!0u@k<;?nn45Yn1gaNXj&WU?^DKS|M98gZ)g7D!_bKqWW^Ah{!)?db z;WzJOD8{WCpT3496>9*a056K6cMLFRoF6<__Ww zQYR+77nf3qF3gO4G$;~lyqJzb49jGI7b5W+RcUyb?qH4*}$vREk zFm96?%L^(nI0YAWj2)Xym$rokr!^ND&^Rg`_E&ckD{q2%81MJJeX6hPqjA72m1CIh zZOEi`D?ikxrMAI!3eS-7@aK~^YHNH(yVJr7%*>}}PGzZ69e*9tM)R@iIK_KbRb>XKYiV{X>L>$(5Aq@+IVM8it)?DO3gMfDMeaA$CX z$L@MSj{j4Rt9FxtY2PDD>U8f1`C`fpq9xk6Gx9Rw`R;Pj^qYC2vZ@{} zO5$q(h>#x9^1<+GdNjeYOT#lpr%u>MCQA$3ML#-Kz@Km-e5>^a>sEu+$AN+igzLvf zc08zix^I{9=H&)EwXu2Iz@+8Pp3oM%_^voKP*!4({|PsIqaoYt51C?_+K zWjNVtSa0+T^8TjApz59VjCN{11e9v&9+^3+g<9G7c6potPJ{hawks@Ep#JtdEWEcG z(PIMAnXZu7P0tS86ooX0+UFq}Yj_kK+vC@=+5DH}kwrmTi`Mmy$K#vp@;B*udfqO| zbygyzV?}Jc1z&eAFBcocLM;$rk30Xf-LqHDPKKEc20y6Ta<@s9`p^lp{mZOARn$G} zlinxZZmWZ>$Alp~3|;$Fq7r`pKsF|!A{$-oFCz?#REs<8lwey=@gDrKss-(iCDYo1 zzTT1Js^UY&v;4+2dCKCu55L#;VS$FGB6lkYtX8KSwiB;-`m?o@VKUJT*RbAy;;0kt zrve+jN2fHbAQb9D36;IW=;+Q;VGV#i|A1bI55BQOTa$xL`c2emL)_nKwQ%xqBu@bO zN#&$^YF3D)eLV&%Ar3O2y3#{&dv&ESW+Vf8i-)VEy-rl0jN7k1jLy)S(^PW5_U0^vuwhE8py>kB+S^rbD{bcz#U|l7eZHPOGWe zS-6i{RUUbD?5lzSxN$Erl)Z~^M|yExIvlZMmsBOb-*Qdz%lZVQT!irDj>7yut2L9( zL`yH!d^phObZIWfVV=W5 zZOm-@Vlaoe;_6jd==g-VdJb=M;1T@STn=CXVRwYXi+}fvmHs^*9_;2cWNfVHF9><* z?(qc%E*X9|S%pvPz(tu%79)>ZUn0JVFFk6`%!a5+5Rg(>52>-UImKPi+TFlCuudps zodvgw#kh=kCs<|9g>1jp3SPBT2V4u@lJGPh3v~s6-(XX>yn0lWK?JWW1Dh0a!T=0$ zJh5*Stosfac*giugxM*81*K1q5>ln5fS!V$w{W}62MJPY4B(unIn_)%)KY=CjH?ME zw9a~AAIPy7;o zf+A*DX9P==9UT~Y&}Ymk8He8P)5H`$?pw|qEXJszZ?J&+#z?pFcnu6^ps*UqRQjA| zv4-9gCvrx`+By_$(mzL_k*@ABT=m#s+6K1Z2#7p4RIUJD0N5{QIhWsLO z8b8+V#3W&gSq#i_=$Ws;ufeH(KY29WE8TdLAt*K%u{3pm`3gQ;o$8)^g!Eh4^v7># z9loe+sjWJQo24}6P+|8`78MM@(4FuzFcbiL<=z4izj%QaXW;S;XReY4&?UnlexcA{ zmH`Z7qH2|hegaFm%U~l;0elvIgk@qt#(!mj?SLGd@%J-FVJiTKg5fXb*j4x(P$s|~ zbN#0y_48bGB1~+qvWj(9!L{f^lJNgv^3gIJJK?a=S8fGV#Utp7%5XR6n=#>&Wb`ik z>LB{<8HD@*rS^H0BLf-j(ahRMfq})Fp{yM`MMJ1f9o1-+QHwfi4oAfXuwcfbhAH8y zCG%-hU`5qg318CEV}99K@wA^{^$?;4jS7&92^F|?A$kbUt$&hb)Tkj0&|uQjERv|; zv(!Ou!p`uKm1uf(^-Ocll5WTcN_(cK35=6(!Byea$qXDVCE%q7(2^_`cleIpLuY|T zLMXcCEcXx(MHaFg)$72w_~eQ|vedw8Y;3vRd!P-#!E}jwa9=ZAoy;o#fhcuXv^?5^ zTp0ZwkB{kyT5Yb+UzrJEc-TcTe)|U`dAO!vl=;LC5`(!9VK#>M5pnSLt5Cy?osaUD z(B|=H4slJ>a0Hlr$mLW@#A9YIvay#F4haAnu?*ycJ+N8$^vFnG9AAdo!)r!JwT2#4 zCOilgYq!LyUNGJtzB~KI1;nhvD<0e1v>$dZSHtTn(&e)d)ou#}Nm#^C0NTMuYJg92 zXK9SLRY^LlG$uS|`fcf2khdU(_3xu0g(_@a6~FJn$6rq!7OK8O+C}U?b9grR4dXX! zwFgkwobeQznxZM0TP(I!F#V2COJY2yQZr)+dBQS-m85!6!jC{`x&98kLuEMZ<*H2` zs2bq7T9rel=adN2*ZR;TdV9`$^YnKHpxwsWhd>2TAFmucH_`>DG^az)EId-?gS<9$ z1yqZHqPUabzhNFY_d%FB7g@osELaSs+C_I^SNb9)6GY&eES(+vQC!{h|V$V-BDbg3vp&2(|mNSym>xlW|xYqHAkHUyycGW^%) zkShV+U5;#8)7~m+LRI)5U84fK?*MFKLZI*TRP2_i4kOM7>y}B0w^bReZuV<=nzN8V z8&kK{e?Fq&m56Fu4%`Lgn2kX}E|dXU$@J62B2%}%oXJd@`!MxVV7nm~V$7{ev)4pT z&FGcIjdt?e*0;xtH0NrDH~G)Zpng_nOr`V*8koTp!yFP)r*Q_R52QiYwA-)|K1vaB zYMz_kytAsonCO!n&dbZdE{s(wKJu#x?tPPAgYPZ@HCX0#ye6RbTUb%pHZP#-Ku7(eWC&Q7f*|3Zj_LvdzVDTU|o$T$o`C?H_Gdw z@ORY(17CF-XcE<>>pJ(q&}s>KO;J&7ZY=DQhBMGQEbZxX8I`M0+a$^BZkcph$+18~ zGRk^$;6O7i;JITz8lv`BM>W2`n*8|BQ8ac~?+-mOtoL3j&)gp<2#nMYNprDK;F~Yj zjLDCL8EI7gw6bgYkit zA`$}oWN-?n5w>=?2fJ(jzc6Yenk2>OMX~U~E>sdul@{CpZT7l{|6p0q%q!dOes(K$ z*7F0WpAHEz#Y1G*@>9Hx&T(S&pb^4iTC{Kh@+NJK!F=Op05`DpWKFl#>hW=1gmUus zYV}_IO2b$9C@WOf$6%kmXZDm8fT~+%Y~m-jBYD{HY7DQa-5%<8&&G+Gst;ot7*6v0 z&@n8b_UDw8y}@Uq{KpIj4^ljl2i5$K;l?fUfktUZ^oP+SYB)WY!d>~Gq{Hc_NT2F% zj651ZwY@jCWT0*4asg5o>QuokzZX$b#RUrHfkTgO!`*(pKnpTh`u9lcVl~0*e-EOrOHUK zH$ZLO{ils6=D%aU4)>r!nH0a&7sVlHHsA)!c428_-y1N-cIk9#+Pd;sc7IC0v@O2N zZ8FR>_^cwyZr1Jn0-sLr4s5fRW({sAtTE3H^6l^sw#3^ z!m&D^OOF^fJt%YGH=jH|@!+pT^_Lpfc`ptAqFK~8Jhlwe7+N=H)6C;HUkn`Ta4L^M zPV`USykZc@>4%N0#uAT*O#Xcx-}Fmk%6?;-c;`d%8K}BxJcR>5h7^%_#qfckfm`|i zbQGgQ^e$uPGV#iX}bo6 z$Y(SBA1NS1!*B|h))=e6+92bb@~*Jx-78hRi5ZvKkJZQ=tc1ZMBwTsu26q|T9bcUi zTOn43$qpBM`pzqVg2uE6N5iG0w$YITbr=XFTUNKB7GPE`TCH?RIbj=Jv{yDdaO@G% zG+*MbBI}R5GTPIoS%KQ$0&94lDJ0A6RHBExiyjR;)6J&Bn8+~VmP(|7X2+fEPrSy< z`K4~}bX@OM-jZoQG=*K0Q;mnmIRni$Ga`+gJj9=09S~n)bBuL227Y@SX3?LlYcB#E z&{Vn0p4-O|8KE-n7wb4mCdN=SA)X>D8X4=AV(|+#t|6=nH65c@y^o5= zH5RM=_c#D7{J-DKc0^D<2>O?q910Ac3l2dqHvn-^DJ(z?3Tc=cZRe^B;u2BaUBoLN zQj$HZyyX{!%vs|VPouPqo(-vXZTZg@y)n+02z$YKv&~kCbGE-5<3UU?Yww8-J{48g zXsf|O(N(0XqR@H+Zo=*dmdmC(V*;G0?!|k2&d;j>KXjV7q+jN+ww#2qJGd(l+NKwQ z5DV}?iGT25@=P*hXp0!+M&TRdjto|&=I+3%1Qzz2i=a`ktLWeQHovAyJHjJN!h87E(@)5UWF34jyyO^L0457N3q;VIQJOiXlr zM$(0f((W)*Ix*3%Xaj;zdl&DY0#fvj;ac=Qey#s*8F0Lh+e?$^qByV>X?OCN3^=jC z?aFd6*;G)Nqw#~0gBfqe9->&R8=u?{7gGrS_`dS+zl%xZ>TJcB(T`Yq3K9WK9~wTA z+fvgKj0(DOAyL785hPxQp9DI@=db=ZOO^oI(q5FjduHq|7+1=Ufq+E;gj`cwg2WX1 zNMQ_-Sf|@kjdJh_n6g_D4J#*-6X>Q5}6-cPxQ(W?orp}xuxW) ztrpY`xT1*&u-M7d}u z_a>ODbO2cNVP{4H{z2D3hhVDd_V5R8|5!f{=)_Vqh|V~mrTq=Br>PT+=l4@qY_T1+ z07Fl#Y7Z*IX1-ylD~5n$0){C$1DMHJkee=o*NaWu);}5(}VE|8aRU zh-}Nlk$#Ls=WPBk{*SO6&>@)1TQXQh_!?!(OxT016U!B)d`HbuiA$_DI$N>gc5=W& zyiGhV)ymub&ukZ#f_!E~e94RwbEdbSr z9DoE6unSUFn=Ulw`9LH3)?GMYaB8W>-ez==L;f0;k9q(NcAT6_xrlCgxtMuYlbW+_9A2 ze9ETgs#%|b;SA}727Hc{;9l&9f2M4VBPpGhJdm9pbASN6Fa@$)HNn(&{ z`VQ~CoT*fF<3d!}#eR&WGX6+FWoSOEW|P|sNH7A0T8?BIfI1^umP`}9NGSt>vAdW% zzfs<*e;=QbPYbyyI8T%&N zQ($1Uu|u%uhDLbxgr3#LuFDswJluZztWD{?lH}Gyl|e$O`V16Eo|8X2;DErVH4H;M zkLFB5uvcb4SBRdcPpKev&KTY{rvpZFCGg#to9hsW_jM-`8uh;?JgI=9XUyjpcxGS1 zL6$0`;)U|K-P1bUC9kNZ+H@m`AzZ^smg{Nwd;+~x!#3txm3?j-q89zzR11sM^Pw+1 z^Sz==In|oxd!YKFSncc5R=lakJ8pP2D#@Y3G85F_40uPRiYg^!hrDdd@%$8?(I?hj z)P{RUjc|r0{xVB$b|UnS@bdd$7-^un0s`~Aui5H8pnzuLq%AC{8^=FBV`q!yn~U!0 z28$wcjcRp;m3fWVQlEETt`;Yj#`Xgsg0dC+DzsY+{d-5>1cNfF$(RQo;|FI|_+pNX%`gD^Tjh%7L*p#c%TO!NN|wn_EgMiEln zgB=OqD#7^XVUxa7Z08^@^=fel>v=2Er)pA^fJ1mxoYrx6T=OT#FiDS-IXh zBpMCJWBoFN!rSXyIZ#?3-30gzHIaCo6iodDM<<6TR{T>#$2tfP>g+m{piV844d|A zd&UN^IUC!v@AGDKd2y(Y|JCG{SPf;icYeZ>qw;D+L6?80h+p;R~ClIQ!dcG$?z72NFDGTCqXxBCM) zyy_oSH0<9GZ>2+Cmhme>x-TU$dv!7b_3>TakYMTUJB!>j=QgPY?`AB4NgE#NKw|byBJfJS4=XyBDo2OM;>k4X=?$-jayk$UyPe0U?pRKOEr+4Bx|?d>RH=-)%Pkb_s8nJ_=RP55dav zk4mL$@^LbLRC$BRz;t<6pNfCJ`s-8G0V%_l%D2rEe18BNQi%cz}&*Cs*DS_lHoKD4EvJ1Y<4is-i{)j~3D4mz=Q z!$KKh#Z@OSagxj)$#SiLUWc#Bx%``?I` z;`rZx1>JQqQ1G-y(8`F|JgWDkvOS9Ji3ddxjK0JOX(p+rS= z3dR`IwF19w)%!xfwTxW_{Cw!+8r5rap966@SU|9pbSKPini)c~y5-gQ`Lxxj=4AZ`1A?$JL1F{1%Q zG%(qO_AFzvNmvXaHA9_sUy@RrDB}fvDA&z`$Ev=;D2Z|bp?R^bweI}UP|SrpG+sNU zb52)9TJ*oj|1BwjGn1bW$K_PA$|Wuhh;tdlKV@{)$4>+_vs66DdKX($K)Ery00MERR;6yW_{ARyrM6F%%+DmLZfNEc&j02R7s zETGadx(7e>|K!akBFJen{=}4keUc^ya8MqLh{{q^2tU|kG!j}v~qgI)byeyltmCVh5p|{H9=KznC_F}{& z`47Ql1)M6ASJGUeEeC{|5dEYb$atT1HryX1oy7 zonAjCC6WBgq1dNpn+#Gj?v(Zci$oQAh@cLFa2I{&U@?H5Xlz&u64)~8^ooqDcq;jA zngW9tR7ViMOzHS7Y3it+_cIpXW4esSafE(fQPKSYh*szXo~%!SJsZJPl?XA-J6Vj& zi3B!NzKeH{_XuL9;6n)NSMeqc|H6!J0Mi@~Z96SvQsd*+EKkz_V2i`Mg5@|y28E-n z-XEki970D{c^^H91>uP9J)~A?aQ-XPf3W32)`yC0R1Uyso%Z2no}^FIASp1#%Jpgf z>@FsjaO?7!q6(?~wCoqu3i%2vaH>5DzGrVE8LJW_5LuIOXdfo0#XY;)&({ppO&yLD z!@1QC(UOm$4bq}W6dLYZnB5-;M`%!}o-8FW z;Xx%FRRGesAPhhlZZqhJ9w_p{N%;)!Dy?EJABKQZ(FICxe)84V3~A_G>d|N~dT4rN zl}RIhw*NM%qg=lFgYkQ9?I5R-LkEAxHwq{ALGU!tCa(_*=7V;kWYTJM z9HNL26~ItWY~e4)HBdf5D2nOZ#{myJC!sGzZN1&kEQ|_*d|4!ONrXF%bUGcQm~3$- z!gbC-m|hoba>;9yL=+uR_6ml<5gMo%0+}Kh%)^a~@)_w|*-fCkJ;1YACLvl20SM?r z(vae3!J#}jYPOqsgoCwuoiXoJLfiD+eEI89)B?8+qOePz{^)vP6xvv$qZAzMP;AM= zwd(Goc%1xJa=YEp1|o2`CYH=(F>}lBwp?cbm#i1DRi#N#Y+>SwCWM$c5Sg#ass%o+ zAk2RVH0J@QpM+X3r*;5_ZEYz~xMF+M`x~54xQdmF&*!z2*M3SKP8TCsq63zMD6BPe zBn3P8O$?}gakxZd!z*e41joo?cB5>`Ny(aavBQG3T9~f@VL&SqEHZr+`r}(!)#=zo z(l|3Eh8DLed(R(BntK#hW5Y!0L8vh&3ehde+~p~pQsL>lJ|u(QO$Di_+Uf>7`N@b{ z$8WD|Ly7A8uW+}Wj{(lB+b zVx_|iv(qOBV8_O)O)|e7=C!rRC&oNyyL~-2wDgnJRS8x1dTJ%iqY8xvBX6Qq)T!Cq zYIo=z#qvY{kExH}!3;u;ITe8!&*!QhHT3VWc@Gx*SlZV7l7y6PK z>XHXP3Y)cmXu*YB>m_5)KP;zI7p@*H{Djy>^4by^c0A-9h#+b=1h( z83~H%rwPk?%*GlQ!4Fz3Xo0}N!1vg&a-mem+X?2pRHPe{cbc|~h9BmU-&$+F>DLFh zQ!_+}FycgMXqD*~vvl!VrVELtysGpxelfj%SfUnwA09SQdgct2k zDQ1ou^F+B4U2L0g?)M*?oIa~OvL!|sXiOkQ=~AB^gH&9dYZ?C{T)*UPdBI|1TOl6z z-xc^+wC@_z?`|JX4~?#Qs=#Ax$uHMfGv-toZ0B)LD)a!Nn$`gVmXIw)<%=_Ku7D&G zb*PBDyRVu*Iqjslbq1E|a4?thi^uFn*IU5l$IIgUi+^e9F`&cD_g@A4wf14+XA$5B zcZp@elKo}iU@tk=VMuoJdtcHAE)$XsyXj6_{OJ=SN0oS2dygmHZ3<_Y^(I<$Yj-aF zG_V5#{FIRB7a&${-dG^g2i&NwfXy7QOTmdi=#r?e*<u6q z@iDrPUz3X8aRy9nW@vaim{>H7z3(ql)XmPc@~8wnH+i^2wGM5@+{+#9vQvMs+=4E+ z#+#!(x+N%(1&+A6xVaXN3#zEYIH44kA>a~$buHZa+9^!|@Wy@hX=;D>n zrlc%EPXHUTXFA9n_nzpIc1lGJ>LJCa7`Da&`x5C)OpdM!tLz!_d8(;IXCg-+Tjrau zo-EV79oP}Q*;C06y=>8=gyr^xK493exRj3$#>Sg*0^eZ4lAXVrJf(@V*_4|Seha7i z)bjcR@;^D<=GW)J!^y|cH8AU1#tT4U`4zM{=;~fkS7bnMgjbV?dORJ?OZOUb)jgr* zWjUzbpTg=Q#dQrTeY)dXZlBVQT(0N2$)E%I+i-Tse!fy@!vjc%$~piYu-;UKJd(u; zNudz2LA0VfIPEO4>DpLn@9w4k(ehhb-An0bOom;m#n1W;TTOCR$MV@Yj|;P_E*DOR z_lBvnAb6JdKRSDV?S`~t*Oi`QeqzIy`?WUD8^s4byBCl4bZ*{lD8pH7Z57n#@UQls z=u_|3p*kMMSMXsL6{sO=l|_4;S8Rg@K8mX+qB(J_LD<^5AS=OC{YOL*I^jJb`>R&;y7Fn-gB z<@5!0E1D>WBVkq>eJW#gGW{gv(LlUY`R0k|SHMaCl%Mf_)(FGK(e{tjU}f;VKsSv4HBO_JeC zsP%&*@;T?_|8cgu+uywG2jcR4f{^8;@=>@ttgf_(@Yo1LBA^ivd#k*^VP;NvH1m9v z6@f!$5IAi$MeI;XHUY)J)tD(35w#aFyJW=01;+{jYPZ!AWc&RubO-Hw1Ob`_06$Ei z5D7DwMp5zHFWlts{>A=Ip*t8hR6bJhPQf;dnmMTZd(h2pDviiq2=*1 zU&Haw4_C@Az^t%!J9sVnZ+L|;{2`?8BBV>7!JwZ%y4$0F4%uU&T#j~4dYEw7UCrSV zxQh@YOdy9Ng4s(G8+Z&5!|4n7iW?j1*bMAC-cN}FR{l#yjNn)C(PXiWaUyzv5(^@k zD?l2WhS=g$V>wVo*n{ajGZs_V7(E68hxHma0bp&${505SHSOm23(b@*)s! zHI3--22HE=5LoAttFp#?8Ww;?r{RIRt+K6{6~VF6$E-C+-hs#>SR=~l7K6z=`V7iB zj^vnA$c2bd9Grw|*L5albutNDIEtF%C~y~`Al6)Cu8Ya|13S_nEhOPK*vbSnLUUh7 z&OOpA4gn!KB^q@q(Q-)v>Rf4S1lz;H|oHrWb zTK3jG{(?2fNs*VglwwHy;=71fwH9e6&Pu6_>kBhNg_sRArA?SF7J3dQOMrj^?W7sm z2x6IPtr@e4cL_&PL#}GW#j^QwT~xcMDcqeIk_`Ol8tCh_msB6|PSrwpEYE_iw0e{5 zSD6&y#Y2C~y*=t@jb+x=ZxLZ4hZiU^T(pc4w6hEq*W=8-*w4f745oz*d0t&(i)#UL z4A^F9t62pAJSX$!{)weYF9572_3cfIUBwLziihXJx)@(!IO6w^nFHD2Ud?B(`Nx+q zs1Cqh52rcIf20CPK0f|0k3w9`ytjv^07+dAMfB>6NCI0rQB_%K=^4vV`Z#y-++`^kMN zDt$%{c%k_&dfY}Nz0Z@6^7t!uV$CCHQ5|%-uUAykA?QnjG!xfg|7$?#Y4(ISDV-jX0YM=MEOa;XY zYbsO90a1#@u7;~QUxZ-Ie=v(c#W+C4_O-NQ7n%<`HrPNcB~Po23R z9a&ZSb^yuGb!Z!_QWkvZea_nYYW=KBjwrxSob7?`k=m=D1Ex_~RT^%iZ@T{QU#j|*Ntp=p=st*%xu40 zu!AtAo3aScXtBDwe_+^bEl8j-iAT$>u$BN)4s_6Aak@o=A5*!9?ocUo(Z`YUxUjLKBJa@pnn^Q z5loaQ)M$!jB1M&ms`QncJ$oyuJFM2-YoNo%37fK_>$&!>JzkMze5)}TTWuh6Nr~-; zn8}~Vvn#CL0ol2zk6Kb)!D@3HFmm0sLI9yJ^}^rzn%)woKj6qa2Gst8r(sM(lJsd9 zNisjR`pEp0l$Y~d17TzR=;Jf%&f-1OQL6*W&Np+M_@V9^xhTUr2abdvpiOs?Mgm_s z2(*-Td@8Gi27nrb>}|dR_v3k)VirQcpbj@Pgir{Fe#R2gP!|;Y4zKSS>qG7oiKM=G#^t)Fk0tXqNtU@w{8p&FX3m91?Q%H#vh zLP!x|aUR@VzF>MclE()I;Lm+!)X;m4KL~m-a@$WQZlyj_p@2cIX$l80?d0K^0zHe`lwUMqW0cfxAm#G0zEn7w_5*HqZIdp^L(DBr~%iNxRjXAGD1 z9cgNk?oT!&AK|>nuq1gmBOmAbAjj=;tS?53-2v4)yX&e*cvC}{YnXpWVfc$|GH#RF zYUsgjjJP5{wz968zcAWz+rH73+YcW0sU(9@QKE?nKi3DmKP?eJdR>rEchmPrj64yA z!LVXe9Wh)r{G~{fZmpXzIBF~A1!gL7i>+m?eC^=2hR>89`ju0&q6efY+q?eEddb9u z4^A4VF0JHpy_w+3=y@lYdo}nvlUGzz-f-HevqWd#DGkohubk`;KfmUAWsoSi=T{RX zHv)pNJjkr%(*w8bjBmhn%Y9vnHO!~gE$)2G$}&eP&~;oY;JY`dj_fcd#AS9Ka1HY`-${}7GhI? z6)gCqu~xsg@sT}5#^PXLsJdTGBG*DhexXa4;Lzp;jq_%tCLUZ7;Rp)x^${kk`IA$= zE{S;~c^V#qK}eVDy5`>+N=f$StKxPS8m_9BD~7sX)PL|jKGR9#JN?BuZ?VjM%SD-x zV50jvJH_#vD-%B*fe}<}w?f@q@7p0P6zyyGZx(&3+t2`&c}@$OMdx-6^90Xg@+V8C zXA4SXcYifmdrWK*%nPW{FG&eUJ%b+k*@8U3V4k2v{{+7AEEV%hhXS_=Xlo83^^TOc zv#wFpH-J{+-%X2pud^mlz=Xx~VyMY5)Vw(-Z}vV!{V-)3>Y5K>jGYwPWHhQywyI7{ zPy@kA;agQb>zOAC1{Lq#mQYu7s+ynoH+dpb1f`o@wf9kg51Xp1x6<6F3wSpUAAVgF zD7(CT$offmT?ya0kL^sMh>aFjr#lEPmX%+gj_f@#StmP}td~q{EYT0j=j}_K*LI-z zx5y z7FG2e*yr9?i+J*pvrE5uvpVd=2X`79d#*da&To+Sji}v-&+bchB9vZE|TN;GE2bj7@jY&hl-c;c%E?#nGUKhrCpK1cLaf z_WK_Et_`mvOeDZE$!vDj^zo(}aet4&GX=DS`|*omgWqn3yF1V-H$XWduTPkpq{`5f z(KSeE%t33g-VF$igmcbRKz;zbT)81`8;9#{J~<|1^$^yDgDv|o%hQkoP%X*a=gbAx zi2AqRUqFSuF%)SLvy6Ae0mZ+fs?HT9dvpVMzITx_8}L4s4%JxK@>(Rqai1vTor{#` zs5CAq0IW)dj0_|~4B&l?OgmWbCJ=vAK$cEDt6+p;a6$^fB#YS%vFcyCz5m!|v3MP> zKV<*QLMj>qx)l<95<|$1<{Ve(N82%YHu@Ig=STs~SXN{|2nR9qD$X_xwZL;VhJYze zd5?0?k_)i8PWKYQH&pdT%>70mz&>%_n2<=-e;~$#WbUQ(BQr`6sb-ZWYBBaBoF!2T7YJadjF^CM!+8Lr++b z_qIkpW~hD+8@3RHu&<~oj3m(w^o>*l4RC^P2!Ca){{X_rg1Kxe!wP&Z>10m&{QqntOl?u2 zp$wPL(Z|~fr=_^-C|#5?;4TMCI724N0?yqghjozL9oS3EJ}e?v=}5oYJgON7^fl*1nLarRF1^4NCeMoj`SQ3!G)6d^vUZs5n5GM0 z&&H$&fP4$_hN(I@0bwsrz6ut)g(!Uh13K^q&f^JK3_mIbT=w|-s>HW^qFBUJ%6 z0Q>-1KO04Fn|Poy1@)d-z}S_c7!VIL${)4|fIi$a&XpG9dZbjQbP5rA2P(;s8kBid zDlAw#2nQ>PvnM9nrrP*?bg-WH?v;m?S>>V=6@2aO1Z^Ld-(xn}_&4e$+I~o(tVBbCK@ws@&rozp#R zLooD%)gaK?!c03>p@rnE2#`4mWa!|ut9DI%moWv)uRr{#Z@0hjVs|M7w++|>>VqZu zW-u9dI4CgKVE@#=r+=jld(Ny-c-82)x;h=av>ku=_9@C(ejkFMUR+_mBg8^7_Yxax z;acT0@FtJ+RU~=c*vOi!zfPn%k~%&p7YSfgex42zPUgN1V`uE$B%Q%}EU67mpm=f_ zsZ8c1{x?ug>Yw|@ji@t@PPu5Tny#b9XsbsX!_;~0)ji~ zia1_6=+8Z|d{0~K|9io_zX6#yUdZYN!=~GM)z&(+t7&)qo= z@&T4OfYDTbvf~_WJ;#-o!rQ~?QC($fc)D3YW%?_$8D+j}aYi^2yuJ3Q_ZfiCJa}rj z+d19T`h|!ZS#Fb=EhFV%w@x%8S`KUIy~)6`b=0VtsN22;8O25QLsj{K59ITPz>`++ z(3lF0?86$P)2cDc6Jgi(1#4FxD5nCN&7OJD9(%bKNALk3mqE|TD-nLmofnE!#K9v{ z(@rdMSeS9?@)neLUqyW%05H_l@M!}-Sf}C5MCEg0XJmfzd<|D&IXUJt@DJFi#dO@s z^YZKw=Bk^aw3*P^ym8Z*R@RfcU>$ocRJ)57U61u^;^{B(Z*>Zq;Q>TeP3GtUnoCie z`+m0J>i>tA8A}@2CPxnRPpATuuysHAlKA)|d6})eug~Z+EW!GEqhip@BI*pkqZGZj z`FP188UMZ&5+u5@qac_!wz)+LW1uQ9vPk@8LPQ#2y?*SKW05JQF2IiVZ|EHqa0KSV zCAmaA6O$1gmDv{kPOYwv6r8~s#bd`(;CK}*fnr%x<w61Z-b`fi12-R;{W|Ib<8Q`oyBFl;iRYfgc8?H9yUB}^ zD5%qcR+DBUN>{TXaG)Syd>vv(GrUk`iN7jMK0z@A^*FOQTRGKHb#rS-q!rwr#M-!# z!j@y%=~I*#h{I}HxBqofh{fbRU!~U3DNV~OtdF_!`OXanZ{rpS>D0?5XdZUg{Xt%I z_XHAy`!45NKU0KQ-dc%0>=XaCvs<#Or#{C^@6KpGvd3870vAAWC0j1&4EKQ;DTbZCe08TiIdG+yiZdnML5ga^|FH+K~2eY0Py zz10hs{Hn2+zc;3Pkquj!g=ZeB7gS*yD@8Lz=Glfa^}Q$mY+T=BYYSBD0@5#b^V7ye zV>$+Mxp@BvVQXN)?7Ik^v0{t~@4p=y(c1QiL=}O?N;OtA)EMqsQxwEehq3(16zHxd z3^wZ96=xKZ$@j}MA3%D(DK~C~4O+FGK}m@4Qm1qSNn`la1|*X1-Zk4rrUQkBipmwe z|7eenec0!Phw#@r1=k+5RfUmuc;LZRtaz2k%z$EOr}lpy-yHJOt3kD*+d6E7&5F)o z3FTm#gQ73P9j|!v)4lRi=s>*gue0c0PhiXy{d&<*}*t({dj z-%5w!0F~ZbKaY;*Yp@U(wDWYcP-h`8xcOzL_Hga+YKu!wHiAMTqT~Dv+eM1XNZwPk z3x$14Vtdf@$r{C<{XX{RcOLK!Ften%u$2C{uR*=s? zl8v%aHuFMM6!0BZ*rnQtLH3G0d<>`2GD32<}-6WG&$kq zV|2)s&jId$`6T`nd?q>NkbP`0+6pgo-3wxcg6UPL945syOhZ5}A=jHtMY1qAW#`_8 zIoB$Dm!E44j@k&A!4{$TXCDzGsG>!TlGb4hB^q2K&iOvEfUB587MQUc40Mv1`Z3)6 z9jwKdI1K*(a?1aSz&W@~8?5Acj(a3!BQV-yoVkk5NP=}xmXfd1?fDe1!QfSyh?mPeyoZ{)q*Y`C z-35?x+7)};E>{qxu}FaeV8qd&>_Nt7qR}UDoi^l%MF@@|Q9w-WiSv`# z_Y-Dt8@i513I@^pF_dPQ*FAkGz;1{=1TQhC;mi;+4_gL1;A{fpAjK%yCmwoiifRby z6B&SJ|L1muk)U`)4zI!g=CN=UoEs=s`YyZ#Q_g3n7K2E*3c!&Y7y1<7L+%$z)EO;? z_NKsu#?3LnI@5AWlVBl1fkyo@FZO_b;7yN7C0k7BtJ1>ecw6-WJkr~G&?hZIs)VH) z=f_NPA-}$=j(f9GV!AdIm7tFi<;X9)G9zKq;xdZ(0>x7%9`NeHxu!QCbkUj4M(=^; zvw4Y|5YW-3&Dd^CI4g?*3d27B)*w1I;SP@cY61l%qw`8J(U&27`~-=TIjC!b5$-JU z=*v%Q&;L%pGNm7N2DuG@HLcGw*2#llEaio$o)a`37oEdWo&8Nfn6n>mgWdr^$vXBH z$w-OcWNdYYCU)MOB0x*3<25*I;Bh=*^i^7Zjd1C0!j?4Pcmc6sxPrrUhXEx0k!@mX zefCbD#YPCB6+j?W=Hg&IJ?CsrA_n%s&;lLd*u*?G(P>MNV{k+bnB#5%xL3m8xyxMl z$KZni*l3#3^86IDoH~GP$`xyd0IHEGiM#q*3_3@+W6&`fXjOqc=PdFVJYzc>>tw!|KP>G7WkP+tB|0*>tow9}!B8m1eXy1@3CA|@1aZ;bm&*n)5_|rAxOL^$bIG##xp*PLwvUV*4E%ugSg8m=pm03U}5DGpHc{&?;6%{#Kutl$*_(Dfs5 zY*zIidc`~jF&WhKuxkcHE)dn{rr~BJ&oyoN8%(T$y8b~mw97qlJib+|k1{UWvj}Ch z7J#hv3sJGhFaCg;%4a3Nhw|9hjO{cTB~epVwOTnfq{Rd8Y1jybhszFb#h|Y6=pyC_kwtc1|wEJ0Xr_hf2A;Z zJEWMXI(33-ET2%zfYFyyLrCHVb`iHe=i|Fj_%gy&H^E#vVr)H#73^Gbc!4-=tnOox z!}Se9AU=PfHJ=g3&d{#5T*pyE+$8RHrz{YVxku|Yu&~^p_CsFxASOCT;bAO4@iyRG6s8SmQCqCLp3h zWKIGKN(|x!#WAsJBQbMYqqG`=VvJTm9Jxj0k_3TPk6a|&1tJ!?izo~@!%Tkf=d&@V z{hq#FzdwjkU}n$WpZ(d-de(Z@s;?)tb#Ob%pjz+hNHxb}Vt1&#wGH>!Pa8OMhoE;k zx1ZW3^^*f-SWI6xW{$kW7J$8WM@`eW*v9#@0m19?u8kC5XoXA6^D>Q3ySxa<@*lmqtNr zg!jf=z12Cm)xHM^?5>Ir;jYTXcqQi=o7;=` zjG2RvH?4}1`1-{CIlDrCyAZ?cNHLq;d;G`mdyTjizqfZeJ5)iiA;MyO7-9Pb+uf3r z6jO%|#oFiA)cUW~HR+x*CPeQid0QOEVkRDInP*Jwh;S=k+~aFbi6(@?QOve9_vLmv z7$Oj#P{NIOErmwEtmjEn=q=SIaRtTb*wV3~oOMl4}t7Dwd7 z3mn42^bViF$uuq>Z`f<;sCVufKmPR%k~q}cz4q~*%@dAJ@7CW)j2FkE-yQbhAWh-D z{>a%&I(G(IAM)2Z=3Dwv9ovq*nVV@ySYu~Gb>7{!pMGin7d}Z4mb7$)dcf)ph zVOH@M`=uyjnWMn290OW=8#99vpi~Uua3j+MuGY8E!^fTgo~D)RpB5^iyCbiUNo;urt$?clP|^)ZP%J)X0~ z#E3!Pu9-_?CVDigEU;Vm8u#4wObhIdi8H=nEtWc~kHjHi~J50v<@ zl7UY`16!+HZo+3+H+GGBmOT|el(WQ7ZMaGaYPy@b0(|f8O z{EiA40YXZE3Kh~oP5DyYTo@9jO0wlzHG5QaL{MdDq`6qbC14k*0LxXULK%=Op%E&_PYq zqQSdGPMJb|d<0v{a1iz~LS<6QiUFx)*)LX6In-4Vs-hd>SRY|dg1d~L-Jsn5m!vB; zPE2X`Gi5%anS-jr^LV=28Okfd!o$WI(K& zh*wGOQ0hxf59#`iYkc2rWts!R%tuZrJ3o(9rpo*j+SEu+k-73P6r8VyJ|*##mI8D! zqmM$T9Ko`atX+)&6E=*roZv-bWCIRdMBy<^W@twrCJPvZ%5f>elY>*O0eRlka$LgE zEy93DEBAps3|YDtV7QGfE+zJhwqMAaK#^>HhX<^ryI`3tCt4{wNl$v5D>~#X9qaDt zaZ)b1XJF6m>*GH%o~eCA)oQo{&*mw^^C>HGRF^@$50{Xw9J-^Y{glgzyJ8JbLYcwTjP&bcDT(mSf^rr5Fc2vF{D z9NT?e*5x>vt_-@Z3*p2@lSW**Wf*;kU;aviB-8|fuwq?K0_C}A=n7cR+Oaq>;wy>) z`Vo(1IzNnJCF@dI2+B7ZHF13y@I*w!{My>3Zy+g8xa7pxJNwXQsxV4Y=fapl3%=w4 zJg0c|qQ!1yAetG@k~)292wQe+lo&!eapt6-uNIT;Xx{w?i-qqiO;sd}(8|mR%L3{W z<0Q$NMP`rWT3yniHbpaFx>oME7iDz`Bm3a*h>htUAzF8qcvbnETtf+I5(PYJYm?c+ z2ixE@YH*-t0`w+o(`~qg(gYLJfLuDCj_*-~W*pzbx zn7>kE1-AAM3#9CAt8+mTf1hJ%2TS>Z2Bobbcc|){Xm`*M2qj@lh6%<+1V{2Cil#)U z`MiV81fctG=)%vtHpsI4MMdTtDTAouRDeOBcS{f=!ft{V#RFrSbFkZP2 zA2coWYtVg1A-dnKbG|&eY0J7ESMP0?(;U_5w#av^>2ZC3PI<8>Z-3q`=bpvZ@m8;# z2*=b^mh56HgCdr6|_u zU&Q$$m3}~%MilI0H}RM}!JSMba6^rZ8R`6a_7jIW&_id!P$NSg9J>TTsIJwB1vlSGJ?a|@Fd)3V3W-+{qsCH>{x|`+u6(o%- z)7{*B9~&3QWc3n)rVJUr2!-#`13D%O&QyDg{~Ra zNk-H;|F(* zsdMLkr-7PeUwtLEaYN>v#DN30cOKaE`ukn;v!<@6oi|WMP?#mKw;kLa?9(->ZW>4^ zujT%`_BW;!SywWweNHN8`^6$hVEEbDGgQrmoaDxXh1eGweWNis&SxX|0~d?cb9zfl z$KvWF9=Z4WJ~@yST6x<)*EZ9y=ZO!e_A#A{D^Qs!QG9+%O6*yX^zP(?m3L7j6txWQ zpsnBQTDbpku;nsq?0iD|y;|dm8sn)-Ke=l=*)0y z&-vZ=>4D5RUXrmTnM|%KPid88fHFIw0tmApBXy4G34{}yKg*5#~e-x=hG`Q&??|@{YaVMJ8e|m^xUtelrF1o zpEo(!a&5Ko;r7olTFAusYIN4v$zS$2OuU!F1>vJ!$#Zz+wY7BlT(e)8^qT$gl|Q@v ztyA`e8^#+XOZm=Q1vhFjc7{$oZnE}D^N8Qz;8BY_k3}-)5cP_Zu zo8|WIy?*R_?;N>)D6>}Ia`$f1_Qt{qcg!1^>gY~gUFlWd{GrKW%&3T2HsP9IdS=n! zPImsDja%71rOnr6Xs;ofz$Mb{XA^z_D;{AvMu5ASy~s+vC91MJX@d*fq+K z6#dCywf#6kxWsE+-t{{oEfYGY&0A+T2@5Yn2RlBGi5qpr^GB__znA5>GQP4gyo+aa z#hw1$&(6jj4hIK`Rmu$D<9#ZmkuH#Fx5_&MVKXZ~D`Zf;PQDkNkn@w>(>lDTRrJP9 zC@idP$bSW~&d;mA4t!Lm&eL7VkHUw-oLJ=>T4nFfaAd%6}M8eGC~NR9EE zwOib#^*nX2;Xrso&FJPqul1JeEA6w2>~4UT!gD?pm{>8D6L+vy8 zay=IjPQ}NJNqM+4al-Qbo#EakhGml@j{o>zMpCP{n>=zVJlahA1DwtG>QgK&gV`(N z2cH}Wmlx<7AMKt0rsHs+;pi>XmF=l>t(DXNiZmp;je2Hf+OAEv1WF#zh?PfV+5h=d z?qP*$JU+fnh)1i`P7?Mrw99DKt%gCRzRT2xG-cvwCf+N;1b}>GqK{Va9_q3$V17}u zM(jIdHwu22$RX3q5^Z6mLrO0`&$d-DW)z1Iq^hVyX(r;-={}qTmMgSB-xWZL%Em5L9NM^nV4)!v$??b~0ik#ryp) zYNjlf-h?qArF=NdTF>!JB|kRi&+*Y=%a#M;dt?i$;{<9!%TQCh0(kd0@g% z3@_#vMJcTCIY3scphfV_!griWuDoR2fpv!n__-y_h|!hpw+8%>*H^ld`88 z{l`Zd<8h=qj}gub8?uAqb_UTU)D(rpS=G;w@{rDk^nk^W3k87e#pfgqRP0~-O81!Q zgm6?0;L5aZ!I0QgzsVjVEXoM`v$s3-tx|G>J&NiTUPrmiT8It2(1>U#S+!GSEj(yB zF@76jBXq>yI)z6v3>rY;HX+sWB_DmpN8m9kjoa<&(#x;oA|rvO8ti4mpt4b5*(o>& zrw&2imX~q540kjd}Y{al6KVGn>Gh z+{SkPWSAejFZ?8;U+ONGzw)~81_rA^QvMx1oC-JAS-l^jS%IXKzkNY6{fxzt@x)pY z$Y0V1;07ND_SVI75>9iz^;aE5a3_){&H#xq82#GncTFf)Zjgg5pca&cgUU$UGOs*7X4u=%RzSaIG9yyq>I6~ zXEGcZaW&30?KwDZg{)$ys+$mKbz$qgb|hrI_s?_7dh;&W57^x<6{nj6eQjgoYa9hW zcl%7%hvG}3(I50M%xLuqu0Syx;x_RO+0%TH^JW;& zOh_RLN18_`YQ&HFijctbcdSOm;!|ug?1*$6pWl}J!A@pz*Ah(NrwMV(6Ag}eUC*1( zS&lL`K8)!F|A6QZ+a+nyXTKx^=+roi{f?j_ajmMQa z(lW`l@Xnpvj<_kZSgWeTjb-Pj+=z_9l*#e)%&J-ay52T0Xge#vtFet@31TPyS%+n5Ft&|LNURiC*XZ#*NWu`17LuHHn5MoyE<3+cR^!VS%GK z(>5su2k=q)Z95!CEbY}YiD0~4SX<}F@z6`;M}zmZ%@+3-EG=e+_v(AwTO3uSoygRHJVC{$g zP|tGOqL?%_>|o=cS(!0=nO~{O4~<;z09xtOQ@7o_U1*Ux!QWB9^my`Gv026Z2Mguq z)qnb-CRz0OOW4R5nbrSVk$^ejIMz3J8O!IaJiF_sI6KudjMP1tUG=j&cb@-a(V;$* zOMtF(ZXJHpAobG`s@rF|wRZ6p&kHpjjgnw%ne#xVlGoY3f8yD~eGP9 z*7&fj^Ts9YrEQMOrtfXvWPR{@ruky7ZCTM^E)SoNU;1*}cF$?XmOF{|i?+q-oHH{K zP*;w*nj+VDCpRsN@T-_E*+TYP&OlqV*KZD#uSD!bV2_C|8vQqTKbs?3-wsW}~D zWEF7o*7C8{HTZ$~&deWJ?pWBk&hBxxC-_PXQr=Ge4+5j(2@Uh^_)FN1>b*LKw!Lf4 ztsYp})!1su>v_0_Ey@eohj*O;%dO9|XphPWwS8`F`Aq+37Q#rINV-UlS%zmoS5?~j zIg2eh6*))Zj+8KW>Bh7-F;cS(%>P?;b$K`bJ-Jwygi&7z_sq#&g2qd=4#$zKd7ZD^ z`n0AopgmqD&DS0v#yyG4&S|-1Tl!@%?I&Gc@cF%8Z8g?mSot7g+98v<(p80V*0d0Z{W0>CX_H$^1gUYyN5=!EQMb5+i=w3VjDHMw99xS ze0qlge>?-OQf0<6>DMg;AXbNc>vV5=Vr@v%gV$^rRDNhos!sV-xw0GG`<(ZCkC=U= z1Z+A!GjLtXr`|u_uD7{W+1{}fIkG!yuDxCK<>t4x^{sufl-p~on{%nLt?8h5wRe@p zN@<*X$U}FiX>I-0z95LWr<4j&*Jk-hD~F^c`lm0K;_rYHwb5oO)<-p+`cwOLErQf> zT!g6NtRgT*UPUn?IPS>*6P!>f1R{>L7Z9Ps=@kS1Qi9h$K1NESl?f$keos`$uzZw* zit17kxj+(|FqmWSY&4dh)$Cpgf01HHp-8l~GLi^dRMtxWAm05)A+P#>$+tcDNJFcM zE}sw#sv*-V`*Wu`m`7E%ZRw<@LNB?Mk69W)<@_3A`k zfqlk_TMeb&b%xA{|oU1{HpjnYJ7m%nc5WJ zsjW_+K7`LN?W_p|C-5xZ1S<$b3Ta4N-mP~8N zp-VX?ETqat0o+mAFH(d8mj&!4ML6qFQG`lZh4Zsr(Tc9Ogkm>=-kBb2ut=9dxNQ*x zlX1&_=croq)|j_X+2=au?XIUBre%p6oZx@)} z^6>%mYxCL~O!g;{dc!DhYp4iffUq~=uBgcv+Ey?bZe0}B+%D62j-kkQe2=q$LOneo zDf+`JYmL`RYV~b*|53p1*j=TdgdkLfLPx@e=$N+^R6WHuo{iuBx&WxdPbI`+CsulP z7*?!;CkZ^lo^qV=%qnWiL*q0IB^v|eu3Vze?(i47;*k4BS*ZNGG=472(3!OIfqjB{ zT~$^O8Vp-_o!-6$i#paVR3+cORyAmhP~*7^hCrfZ?y8A=LA7{eXGEJE^GsH)10>a{ z9Af!;Bxt)SKRN^m%J+QW>&oL;p1f$QRoYm@+|oFc?uo+o44ev0-?ny#RofU@FWjRg z_Ia2}qmUjezQ!vM7n9h5Vde&Lo^q1LO`>chG(;qrcd9HO77jsBajglm%kb73o69aD zHs(XpIO!n?HdYu>ngNSvZF?P#muX2srtGA;o@Oq{3~6ww6sPa*egc(-p~JS%@DY=C z@uk+9SePp_VxV!Iphg9`hXnuUdcWM{mmnzgCjypATN zd(f2m&f=mJh%}}oUG5f|4PuOsT0=TN?tB(5znI*soO7`5Yr}IeB;$a6im}I0XvRmu zc8wrg?ILE*a*{j?#b`^IV0KWp6&3;71*jA2t+uv<@(7043y)v@XZFZTX+Pg6zT!nfPbh+i38d6a7f|+HHCIcmy zr7mKmWWUzszg0uzDgGMXe(o<}F7f`tmbydY1^YjSgq4A+Io0f2282aBMBp61)4k*# z#Fl5a?wqk;x#JIKs^)LozWW+JNl7@RGL|OM4uhqS6~)Y(Z3uS6B{UjOvDaOXdq)!a z(of#%{!w5^|4KKP%A^ei@^ZsAz0=s1_rT`6CC;tz!Rw;?tTo^5?53b86}#xtiZmlZ z!@q;XhzkQ@5{f!Y9J5kist=}L zX+<>~9d)u8{%zX-P-9$8142MXdw=UR;fddkThDM^!J+>~|0YA)0VM)UE4&-uUY94)yBGuUBBz*bK*gbJ~@c?_<6Vd>kN zx4kjL&sK48Z0V}9;6=m^5#2Y^@5Yv$9amG9DDaUsY0SD!MT2?Kxtaw73)k%<9NDUP znx&sypA@*N*8ZL*-JkThI1mW(Rc@Q1;Cl5iHLv1!Uz?k6ebslO0YfP{!Kxpe+9?UW zQM+A=w~0~My3{5p=uXUVSM|QS^%=a^grD6|c-W_JL(7~W%435xHcXq+gZ7>zM_S)J zP$m-T)yEstRCl9|4F{*Tg$18Hi>WNYqO>cyov3TyXg}C>OpvngrC6NnlEU+v)-k>R zQ2@riJe762)|rN`_|}}~nP@KCKUE89x}gYQb~d#$%`eTIC9)zv+nXl)PG7IM5W04;%`VN~O z4f?WuQcIO|ZFBoskD8Np{G74kMJazs z-m&_ppC9$R+%x&gM#ESAZliK?**%PdR$nie*(I{Ozxd-%Ck9H2wA(b8bdvW;(my)xZDXm7vYmm3tl49w+XUyqt`GWSg#a z&a~i$0AoqlsCY-Op&l*s(bO&sI&hrwSa1W;yZ@*IdF7jO^N%T?aOSa$&LQfIyds7I z9h)tl52H@px$;;d)6_Q4&RMgo^daX1%DZX{0B~jNWf#l&&|~3}GHZ3y-d(f)T74s2 zf4lKu>9%aWFe83V|~-zs~r@$kRs-bu~iW?lf6M`YH=t#L!L^Y`TO*S2j!OSSQazPUICu-_1o92lJd zf9tVuWwsceg(aSRrua(Gi_w?;D!lVaNpcjW+eXF??pjOKRY~VoC}lz2MPHWex3b{pdaj5mMG2qL1*20Oh;9#a;pX;1S_Ga!Gktlx> z{O9QKr`S(u!>?%BC&d&`45~RbnkItPG2{MKpsZFA5n%_hHKU(QXRUY%o-K}YOeopMi21IDe z2Qx(6n#jYb&aL^@sa(YmYCnG!4`l_n7?%kmr9SRF!c zUl5csRW2~^;Z_dCpEc&EmW~D|RMe`|uH}S_b?3EM$sW>Mx1FLhhDWlpb~@e7W45ui zmyZYyIHal8@_s119#NI1l2p@$vPG5Nq&O}nQ?Z2M9W~D-3(I(dJ-5GlcL_K2mH^JM zXWGhNo`6FVE0g5bgrg@E0eHVzf#J}%Ns5)(DIWaFhzD$?giBlwbmUV(J%J&z#F6r{ z4n$zCfUQivqA8i5jUvQFun^+eGyKy&>0p41*&sI*xWC*AfVFDZLe?QnqTE|EyEx>j+SOe?Lp6f-(9V5yR+xVe6BP`)>S#9=2`cs zVZWmNA%dow?{M95Cihf2Rc-Am9WF&7N3V;wFy}1C0N85zE3C)2PN}$Fv(O&t$&-9= z2-OU!d=BbMy{i;x6O$)7=2@H|2@{baUgQHbP*jKMTprrv^dMy#;56l&C6-Miv)cHw z8URR#-cUvIX=72pUVN-%H{uk4d0up(S6pbZ-f2P$z;cBmZ@&622Rf^l0{XKpA_wLz zD6MS94OU&erG7;M~-Qj;!*ScfT&R0ca}j;JkEQ;ZCM zl$j*9dtw*d)nIzyc%PuALKNX(DhN=-7+hs(dH&5ZT80f+KGMdTa$-ntq8Cg7xr!mj=hXUE>r@MgTX6Gnys)MJMtgpA|3?9moYOl9m&yZ2I&8?9PSB-fmPvV3tTo{gJYZJsQqr_TUXmlUrmY_4> zxEy^X70C~?XGRALWzu`FL`;&d)`vY6w&b=oJ+@AN`%0BwIZ(M&Z>Tg@_Xn=K9sycg z8c~zK;hlbCPajYEnL&sA(WFZC-6x6CWWp&kOHcQr1G)9Vtcbie*S zkP$zPK7a*eQMIWid;UgV*Dd}*AK@Sn$LER;4eDgzDFnX+}yJ-r8@GI zWn)kk3ih+gWk74~gs}5)ZCdX38aF7>YnH|pQ!kr3erDMq^UmFiQE=h6m7630lQq@0 zcI4^izW=s8K}%jQr1^-|f9_v}lyB0??Bp?5ktjXYH#A;bw!^2$I=-fL#EHwP`kKfR z`v>-vuD+o1E3s8sfh5{ohQ#U=1iNDur_-#R;()uwMU|glFDt2)6x(FyUh%_G&0yIO zr;S>GSp<(aU*Us|s^uaNBGu1XeWble&j*hWMHq`x)0KybJU@;ETq;DQY-Ii?;l}-< z8y~Qp&*6E`CGtqozpTtBe5Z^se6@Aq736|^&e-sWJ4H2UMrK`?bT_x2TufhmhK!Y> zB@# z1gyH7p|2cA7*fE%=?%Erzu5FQhJ+M~g5}OL8tUsBgB@NQ3X_wJxGQH_iaIL-@ls-7 zhqyk1skBiV>5UhoS>?V+74#30I$EU=sv;9V`F!BcyexuWe#fP zyf*1n)cCS;z!SHxUMfy{&Y7@O-8!1huUwa6|8Ao2M`zrf=Le0ZhQ+u&ua*Z}zQa3o z0h)O`Q2BhsSyGMpNCZzE3Aj(j)llLU-D*GuLshPK_Awxv;;%2A&fZP+GUtcJzvq&2 zYpZr($$Z(dXA4Ehs0A}-9BZHF^`^mcJgS+c-zM|B(UwG`dFAs#;U}v%)OsH{Rlhmf zcixky238i1_b(b;J}vp~jFN}86n)QFOpY2FS9D8{simLA>cNkWJ zSQnk&5;!ox$Gh5$NVd~=R&dwXlO8WkFJZ=qu0mbPG*Xc1+Q!zkn}M-&nFToi96^pj zxwgQ%vswIDxx3nX`=bgD{khHY2mC~z8}zbFoO}6TX&Uy_i|BZn`x1Y;s%2_EZtq*D3{6NCC{~?kf6~Ho$rZf|ECCNtwWVzi6=xg{=%uRqP+U`@9e=RcezlqMWhT=hmB!ZR&+nz5b|DCiZ+`DUTO7!}5Q&Cr=#tFaNGB}rLpYx$-Q%~go zaQwDEr_-ZU(*osl==Ra_n6xkhE3s0l-bzL6g=b*Dw2OPe;e(^$p|YDk;jJwrdX z_U1(Fh}Rg~rWtBCz*N{DzWUpo>!u%e3@=&AmUG@{pDRnNJtf4LF+FSL{7+jQy_5X? z#3UMM*s*X)w%n@aFY%sk8J1Z-@#n-ptkrkiPxlx7ei!_DsQo}N9&Dt|>dpRj_85n^ zd(nIIM()y=r-WV@*!Qi|k9H=;bYF-*4#Da*<==yzaAPyB>YR+>nKr=7b}r3HjDRoU zw8d1nV_pi=Cbl>GnhC~o-4%dU7Lw@1zl2R7ORTFogp%wE<8OGwch35lqv;D!&^zc` zr@v(d|0I6Bos4*= zolodqoY%9jKF2he-<7!VioQBZU(*?uWQ0&I`AO4%qC0uGdFI#a=jg4ItsC9kZg?P| zN<101z53&2SNjYvmpSSpq06QJ(v=tL$S*Vg{bvyYAO6JcmmV%}zLR%gP#4{!E1YHV zeXFJOi|Av;H*!zhW6WGXtaX;fHPG^q%QR}YOZ9;B(#+gh`krUA&hJ&MRkZ8GAM9Ry zWk*$I++NBy3X+^jrlV4X{nMAi9P$KFaO|JZ+z><)x2K^R>mUE52~43V(%`)NeU9Ut z65xWU+orbK!owf-33{aH7Y`JQbiv9yjMw^59>-io=bN-bacx4fDQ8nQZyzBIVN}t; z9+XBP$Vv*Qf1;HU1pkY_iyN=1Xr-$C=dXx;Y%dgz^8QS({XgF&lY<{ukm8?Tn381+ z>dVvGzalm^7IhEitw0#8V6(^_Z^%Sc`Snd1!=`W}y7M2?%U{v}a0$p%5&T46&6(7YA}~qK(aO^XyH$Bv!hXO+l#yQ`vF;0EhK@7E zX;GCSWaT5Nu`pRu=c%*mq=8^H1~}yMEF6boVP%>VN(hIWMSx8qUrDToXUf5VNE;`m za!|0$kD^)Rl2Saj41Vim{PgAu$wVX` zE4HI{YAY~9taPsRknVbUcL*n3Z|&zZ{1|7=9H;TSjMq%HA>RO6icEvMDh}||QJjzI zjlDfDj;soQH~>Jkn`ozM1e^bG$!a>NbwYy|RO@dO8$J)e$U@lG1R=kQ>NnFWD3h01 zXXpAdLM~INat8?l(k3x^emjXqjE5P;*H1N8mQ$hQ$quw-2~c_QskrMzp11rfxz8|B z#y$ufDIF3%!E-{z3&v9Gi2o?K#o=)jtyZ%d)0ATFBMrW?<)gO?3_&x@!Aa|1Vtp0r z6tFGSDKGH14>*kJud{C(m7nwukteqF3Rw)B;-m|s8gDD9H1&2f(cUyP4LL%t2UXj< zvIJ-eeB-4AuTnBeaFhHa%_?;}MQJzNd_MNXay+ku13ssQc14%&h|dFis(Tt&6ku49 z)yQ>MX_E%^P05TivdK{UBw!}%m)Ifne0?wzc4 z1<#;-IV+#}_rYm*%Ka+$sI=l}`iMAK%#yWd85YEjBUSNQe}F{qgL2Pu*MZYz!cj+E zDT$NRw>B|IlkM<{(?LZ1#^C&6v%9_9sv3S89(&y>&p_yrsEXB2>!j{xCE}-tG17=i zeH;fYVrWHjx_tywTcXUFIqULBJ>9Gg49ailzvAa`Gf$|C%Rn`Xa)cUr3C9be> z(MaPx!<}MhRh$+`j+d0mPZGAan+J%d_(4Xz<}5Fu)PwV{JITEww}N>G0na(E@vYm| zGW!jl;_*QDL%H6mv?iT|2%nFnf&Kyq$4oqT}!|l%<94 z^W?T*KN{nGN9cGRdQO~gNX$ul&XBa;h(P*2hPpqAx0_K@?$qHw+PJ?ndx;pg=0b}q zDnU+A;HHa%Q${QV(|+spr%BD&V>W()Q&W<%Z@t@In0yMq<*I@(|9~L2rado2PxdRR z>D;y4VcJYnIr8NCQ}$H*V@cZ&NE)P6mtfmh#y5u8ra4LBcCwxx6qKzk+)`wPyYN~G zc+s> z2k>{+u2nwah-O}WOm(=hkP`M-oxhUS6SRG?xhj%Lh7`2X8rI4Q2|#sg@ts+StrbcY zs_13fNP*yAq4z7Iq+1>^Gw}G_b52lry#d+L+*ll7b;Y4y7Asp~Uwu!9&s|4;W0!ZY zh&9CReT8tawl_yu{|49ewJe-u=T0;>mhKCxqYJ0%A~J&{;TyS&nk^!8;2klXFe7>2 z=f{rnpX?qJQq*^ul{JIjMb6U*O~1rq9I#}3o{{k0(L2%H5WYwk9&6>cMqa$au~)Q$ zo-_PeDp$(wkFl|cyjiulf_DZF2|vg<3bHNE`dr`hARLr@s99Fc%7kJ8-$%%A!o%9S zol9aAH0(PgzEjWsUAig8H43ei5OT9gFS>gvf|St^Q4K%&;#E)&i#rzIjP*OB z*(;w;)8CqyD}8~S(pCv4i*4g1#{)8V+}789GF_S*|{ zY_n@UK6bt1l|t`te(;(+3wFbK9kprGX2x_}ed2dp#>d7#ur0QHkt%}_!FF-1*Hv4a z@5b6My=f=D(Ms911W-KzQGNfBK>M8g<&IC>b1-MRi^b7`@~*0nUmYgX>{(;H+{ihO z?>&}+P&N8|#`MmTn%z7c4{sS(V*k*3pCNgK=OVOm2`DFjHQ6a_*RLx*vuCG7Uq*=A zV%d<=Ge%}u&#ks++TGHo=}WtI*Z-IPc4cZ`_`t`Zjhlq_Ujo~nu9lvU!9J(t<#YWX zAz&;AHztRm<_N7&X4@?G^+;FKO9r#ols?8iiGdcy>?@o@le-qr+)9Uqt7F+4w;^BV zBwyyC#*93h?4~{w0_9D4ZA}ogTZkRgs;0zBtj5BbCmj`VbnMf`h4?c~S>4!`MOLo_srmU&hOrBY9t+O>4 zePOnb+%ow4)#S^u)bSOtfdL+u5y!gAXGqi9ZbK=lc!>Uu)X@GrBi(+UMyp!D=^mc5r3y zS9Ly*#l&Zs^VYmnSoHhw$38S(==C>$V;W_@n2N8Lp@U2V{lLP(X=1r`>HP0@S)O?^ z!n@P2)sh~+4aExBh{Crb_;`NPve)ps_v*NCuYZ4MSjpgIx5+F667M#&zJG^7UEf8A zCd>U2zBJPo4(Ge_!tN5|Xld}c6s8*1n+x>FG5cm`731GQ&>k!rvPm2@D)fa4lKR3?nPF5 zc6qNx(6R-|?VKa!K1w&r&Z`<85CO8_XKn5888ZjWL3{cCf$snE?<$A~UdV+~nqyR7 z-;LGslJc^Ak(0sya3x-LlK*DOTNA>oA!LoshWYx0RxS{D6mbf?l9bHqJ+-6N5^=PL zR-kwdEjaN#YJimiWZZK9Yb^?ptbRPudQYT2cTH`Sy~UiX$|r6g40ZuVTrV z{jL4OqJ}?eWg{JcjqD3^IKWtUAyHM!8)$8W#xm(&mfz+POJS%Gwy=$+EdJ32(UVKS zh1KsG?0T%+#FBRV3SU?sgzl{`Dyl_831q-ftx^@j;) zz`pS5WsI@c$w!$x4cTD}M#jvUG#Q}1(k6<1GKD7PH+-}-9V$$NvDX1iRJtVE@!^F` zM}8~HRw-uaXS}L!Zp6k)o#5$UG#fn;Cvz|wfpJ4^dZz)SEaq|1A8>0`6@afzOy1OD zF*O?Rf(D*nW=Fgw3%DzBr5wuDL>|g#t_q+?q)a{Hr@5*@GEx+5#kbG$W9`cQ{cBfl z78z`8eSNL&;pU48`*zZoM}d#6wpc__JpXkhCr5J+5~{Sh@dkGj*IR)+-T``wOKs%* zzrwB|qqtlXjxjc6z;TD`uCS<_NPsg4gA*k<8Y$D?7LAaBQ_hAA0I1nqp%`T;4VSV@ z`eGt8I5E7+?u-JThK$pSU~TL-8bHLV9G)km>OuEE47-asCsIOt$pn9dlReX6BG1s@F6kQIt4(jq+`yjEiohMWNnIWk z$)Kzh>qh*gfk^VPfj+51rdZPVC!NK#!+qF$GbCeYLogq85&fqR%_=(}l z^p=$)F3*5o_nbHZKS9Nb@PJ4ioX%OcU}dbT?kJ*o$0)I{#CUw6F64_Wr_r?Sw0KMB zSCL6eO_k%6@{U}_Wgs`T6%d)J_#%o*LXwd^s0I*R4BPUVT2bk`7~~U3w?Or%V5T8$ zRx*TNhMc9Rjb9NmUg@AEG+J6grK0uc1?5X!2bPaxpET+g%eJEse98n4hAX#-=GP*p z$4X`gw!SU`1sy6()!VCn5V_f)blJ3`TyH6hJ535R8hHG`(p@QON2s`gjm)yr?t)#V znSoeY#F{`(p1=1wG0Ia4FS};VPA_w{Xi8Qlj^7q<5ORO?8(b&;kThKsKpG8fUgQ*9 z=BvlN7;c#1Ocw)|%;$0tS?$l9I+;gzI8h%xS*MZNmMZXiU4 z>~@lMlPH!2BcoFM#d2)uE|#okjrr3S&#-t6NfpE;%WcM_-n#*b4ZizO(fpvFe?Z8B z{NOX;fuk#TC!?hc^RFbLm~YiS@w@GLzej%R^;%a&{EmQ=w~b@P{L*^0FuMT;fCRZ? zH+wOuQv7Cc12=W+C4TQAm(eQ1`XAG-JKiv0zG z{_H*G1P|mWy541Ia3>ihBkOZBR{?Dk{mK*>oZ_Mv zSzR>qV*tC?={uK#*@|Df)7bU8e?VZA1QJ)bE8RK6DoApS%R;tqH(tA!WaYWRVgJ5# z!o=}WMRn8jZbPvKr1sQW3e%>xx;>Qwvpb3!y*yFerTZ_QkBGcv-gb+)oXk}ss;+KL ztDM<5(36vE6ApB_Y$0&w@zRMhz3lO^QdG<-GtlzMqT6AsR71>B(>$43d9AC`NBMT= z1c`R@l^lLWX+IKVwskqJg5Tn=O@P9x((t}L3ioeN4FaN(mRAD4&5hCLZabpcz)U5* z&Su{ONNI#63@W!d+dLimyc<)TlRjp1DdS`ukwyXIF~MSz{Yz$5I78=SmT_cl77|WR z*`5*kkC_IE$)OLKz521kO-nmqnM%E%t)fe9y@N@Wp?gvDh}Fk1*X|XOyS}Y3W_jkA z@J#j~wwI;4@*qiiHS)6w%i>Ig%)IHZGgNtBGZDQl-0@fC zwkISThrz=sBT-`Vz`X1_``8(zaK+n_mKa1MLthcgyGGJ3Km$?=*=)ac~WTjce^$7MuO)EYpaAF^(UtjcQ!T9gnW{l&{Uzbna%z*1)zI z+z2`*teqw4@=m)NMb`wTanS>Q`Q{saik*i&gZhfY65XL`dLZ zgbnGJ{PKQ#mh%q>Hdr3a&qT%HyuHwtzco`AWG;`?rPbgvHNC{)74oe0Z-KtqHl4q; zbZ%qBsYm?}IqcY#a(1>PTt_F+HYIn78V-vl$}N&y-qMztUOc-jSBhs^3<(#yDt?b%gD-!o#B!!6b9_&W1CEezT}U4Z7bW8RoU4qB0PSn~=mo1C|Y-})==$l1%A zQc`dFSLtqbcX}BL{f+y=cb7+4j$i%0^$YT@qAJjq*VMA>6ol>ayQdrcne}wehXHzQt%9#j&xV_e&Kz|lYYsj`tIyE#Cm*gQq{U!-D4X=C!DFyYi>xyp?F*E_4=HB zM@+RImGdTl!zP__0q=sRGyNi(pPPF1%THS^v$p27zQm5s(=Tm$+Rl#auXP3v{N$y> zr|1m$*q(^t*0wbEJlwW+^408R*T+uI-&RGuaHvlD6-#Ynk8Hu)d!6AMyQkyHjpH2) z2&#R^UY54&*Y}LNdmjzNJG@$72>5v`=F-&zKhD?PJF-3AakI+5f!y z-}z;Kqsy*0!%x=e24d@sMR8<-M{nOa=y5Bm-}yl9h*HPI5Pc8k?F(A4!tQAO_03;B zjQ>&aw(4r8PIO~iU9E00221$GiYPJgUKBkkX?Ik8{a3)k77U^kSGx$ME)}e7*!~Us%Dz|GYS@>-R#F50|9sKCeRO2*uDCS-teJ%y!HU0GpslPP93~ThLDWI?n3=;oPtdSmafV3M#0*opW3?&u> zQdA0gft?l%t;a~Pr8aHb`XYtpM^XkBQ0)o>MBoG|G7cGuiMcvd&#Q7(fgE&)G6g5c z7>x3~LXhgd3$Cj0>}To@I>yPxpqd_m+UT$LibEs2oJQ$k2SY{eTqARWmwXN?R4p(? zO$$L>Ba?-`P52XM2LUx7CyV?i86jli=!S-T)xRf_>{q%c1v`=;j8C|RK_W&K(vV^R zmbTQ`Q7cETK%UdMRTWXpP)Y^Qr=;g#=Hv<=3@V`FK2vuVh`6hCRKCPnY{*$<`$d24 zUu4R2u&U$}7$%0$#>^D@Nr44bc9)5X5yiC(H%hqFh8$i4npi<^@4vk`%5`g+;{2J% z#qv1cijErSF=@;jbT2UWQb{JNk@3hCy4K@^`mk{#X^z(7V4OwnnGo9<+xX_E3WS1E zhh9s=+3XO)q=B4a7l7w{^uO&YQ})L&UIF+lmpP96qTsFI&aDX8GX`kqS^2sT1Kf&@ z)JCcGEse8KYJyX>j)`y*%EDP@eW2UCXpagZDfa^DWe<1H2+$4m=em0L`UpCu6V1l( z8@XYCoPsoTAsUE=uQqX_;aEo4_4a41W!j1!iEbVNwitD5`aXwYp(0SltCSB15 zJfLb{hJn<*HME5Y`L0wC{G2AS#yCs8uh5me{t+|@oAKss9OzFdS95!dzRz_S$0Z6x ziYGN9Yms3z*F95xqnjaHW!l+N>|MXeFB_aE9V`K!FOJf(K#&oj?SL{OfMk(?wa97j zO2igxr8u$FYXrBxtZ~&?V2Wd$|Lv~8)-A6Nqox`%6rb|^HJ5-+yUG5Tn8TaZF|yZI z%3do162_j<@TVq73j|QRn+&=~d#l_+azf(RHHFU{PNxgT*fvN@>AQxXr^mcIl zCfTRsq;g2c1f_J73g;CW0n!bTf?yOUiLcz^YL8QFau;y%ew}>!90^5A z6gf_X5g0aL=gc3ZF{y4ZtgT3vGKx`M9!RDvs;4*6nTy@40c>9yy&l4i#9URopXjD@ zltI|Rlm$$-y3oU$E}6f4n9!49BcmRC>-778i8i-HOMCA@GWBpDnX|TlxXG{XZjWvh zdt177<5P4m&kLu92!CCQyD@bD(NtsA>BwZ$I@jr)wi;g}$AWn1b`kbe%M5nU&Ka`8 z{R;xqQNI1ary!8P_1m!8Si z$8&971rsv&^j&xBKv(%7+z+T7hVbtFc|PCRQZiQ=uM{>Ae&m28F57mh2~|#LP`6*{ z;9`0RhK&NhnhPx$WijF$8N^629bT6|(Sbr29AVA0ju(RSluuOq%xGQL5ffg^k{~M( z8ZNlrU+qaER5ftpyDA=4hZvF+z-C+BAQh zQ>KSq$)I4Z5YMk2RV|!_n-3Go5A709zqczsjCv45tyX#OMXL8Z;U=l9-QrudfhFyk zZou7t+u9I7MvjW#QS;0Yjz7qVJEgn1z;4Nb@eRurPdF zkLj(`hVczHv(-Aj+rmo30mAzt9SB2oX1b7bt?8;XP;#C*-WSygSz`agj&A>1V!6zd zh3kI8zW=@RLPV}PC+{M+461yHrywoS@W*|J-2|DZ^CVMw%XL%3Jw9%==6e7DPm6QT zXImJ6j5J97F{<3J#mBt~vg;wh|Uf*2*+vq}GZ`RPfxjeUH?m48!4VFv!9WNQhbwi5@Ci~iy%vn-* zJ@me6(M@fgvpdZ%9k|@~Nd3cna+#xshXsCT_nt~k2Yn}>bxt~EQvDU|fk)A<-+9^F z@;XYRQHv{=9rC}QFMPU(+g%%u?MI1UGs$HDM|OUZ3IW|t`JVU ztna@Q0kgAXeZO_u2+)&bp8k<<;U9o@*DM|%w=FUa3=;VOPY*7&-N2K|`v_Nr5QcR# zi`nDx?B_f=j`+|jf!Ajl5{ZOF+aJGj;|Yp1t#K*ky)TctUlCI7)1Ffgjx8nBdG2+Z zWDfU@5uFy6PZCDmj$OIe{rZjT63XKrZkFA_wfV!nc6yQar`t2o-1@uysOP}GFuZu* zJZVq3?bv;{LvNqkwel%t8Exp~slUEZU)D}v+O~GzCxrIA7>TQUNd`~05NVTrT&cHs z^}oLVf0Q%ZWQXJSK*xMPH=S;}Uea9^FPD zQsX-O14qo5&e160a)tIVzcIte4O6{de{XNAb!lQEpIgYN+aJx=Og_|qL)nZlucBsMN-~~k&3h}83hP={$-s_p<`Sz}wPITm+u~mngqi){d1cO zqszeqJM*LNm?Jmu3R(Th=D=Isd%DvPy*8(!-@bco)%>RGLAEQIz7Cgr1^qQ?wm_Em zpW#buAUrZNDPvLwkE8^T-KU@1G_Sk8@W5Bb z{TA;L<QBHqg`PlkYRLcd)#5BK zC|ov>Ti&0|L=Lb^Dp$TV21h311OP~-DpS>B_%-zNLnz`Ansyf=?UA9C#CkCCdrf|o zFQ3k~cY_oSxH20`)>Uw|z^qNE0R0d{|7nyKT;t==9)s%9HvB7;&X14-yox&&eg*n) zRpb|oKQnDg{_ex*d-*+#D+9=kpTZ+UKnjs~60ob#igMgIB3VW+P?0Cbu zmr)^3Kw`O*8fcO!Tzqhh3B9)w1qvP2MQfe;Nz{{4@(K);k3uc1&I@P0==2Z*m$_P^ z9svq2SShROp9-1RHRwWZ%0XB@wos5VMCr!|roQGQLmus6gc(ihmVXGSgt_UAggv<*llGRx}x*X#2H%@#q|1c)}jrRn09UvSKHN|Adr>aFsi01VEXa5_**9eaeLAv(x3= zs*&*`HnZ}xoK4XLJi2db$_U9G>n^k{vRk@7>gJN0!%4a(g+@A=%?*1-tEPgQxJ5o4orl+UuRBc~*GRR*Utp{eUlJd6 z?}(bcq%Kc}q!VS7=vRYhS=jjT`9WJUZ!H=$l7h6E^i~STA?e1Wo4a&a$Zn!WeWa4{ zeo49#Do7QBtvZU6NR3{WZG^Z`p^<~2++6qmB}}p6*$M9+7`p2Cygwy*(BsB#QuQ=W z@2AZH2f4|kOW$#6FR0?>7^v}Swc!DaSd;ssUc6BixORevCvVHd*}iEf0L1 z2L~FCd3;QdrxlSgn{6f+VQMROXCJKsYpVwU>tNa!5ZZpgXwip%HtbQoE3FKVGTD1F z$13lygiDnzFozu1mS~e>*FKT2V6T(>;oJO=Oq~>rBj1-@jXRUmD*Fe2NmOTQWR$-2 zToGg=2$5CGCaA_{lZ@Z6#gFa{c>Vt)?9Bt3I@7js)E9Y2N116=I;dniomNLwx`MP8+tuM0AxVnnJ~*^Gb)2&r03E2CkNT|i}!C4`+2 zvK)Wc{T!T`@BQcZhpm#3ob#M#xv%TK?&~&j>g2j*NQKAX8L!H*Mj1kA8OCuKUJJ#z zE+m3r6oA`C{e%F+YFznvS z?$L!?kUA6gG~|~X)?xhR-QET092Cf1XhZXjwxj2i;48g7vJOT{_jm9(BFJ#D>|=u1@ZzMwz7fpfqSPtK7K7AA)gx@fB-Fc*PhoFTpliJ0 z0M=^>*5`jbC3fngGtsXN>Kq1@Srz0*3ne?S2WQfUiH65?*4G+55ZN}wEm4)$IAr(h z0oJ_%`l8ceTZ2yMXL%V4=5Q2gfkXdnsH%xv<4NLiQaGEYLC*sYOSA(D?LZ-0)heIoD(Oh0)V&)%jaCqEvb%_FER%xnnHnQV(*$0oKd+fR*_~#EZyyn3s z%DOS^+MwI>MBrF$`!4+WauH1G5;BkX?Uy<@!>`pXihsRMvwq;G<0B9#6BC5EaqCXh z0W0#w5yog7Jier?LYc$55(*%e0LL4a*~Fv-ARd4Z+;jw@yK%%Eeqpf$c3OjN#o7Mc zPp9vFQGQ+2?HSNNqCK6Gp0xwI7U^+Czwpq{{tXivYf}bITkgdg=ugc5h2|aJ`H<)q zqZU~owzGak*@m5E^d8s+q(^wyZ@3db-}q^}VGr4jd)2$n##T1Z^)=(RjJ-IfK)(5{a_9=IvTa!y5^87tyK6RH1FMNLQ z$WPD0ZM_Xa=xv4X-gjd~o|XBl29J=8XARhe{mw66^lr)0gK<8*s)oNyepIi{=ymK)MRxW>ZsX>%Rva;=5$)`hz9rYogG zNAp4IUNgOe+PP@SPp)s5-$H}1p1F;O-_=K&WtX{#gXdM|Kb-M~ne?LG3J!@{6mpy1 z^ldArxz($6Z%>Dmj#o2E|_)O^q*gWCEedG}YlP2UID6feCE``#ZCl><) zxPOPBgnDmQPpbOhI)#0MY+GznSWE%*0ZfbN0xJm>i6C3TpXtb<7AaIL($2O?F|+_e zSajlMo_xDMs4VLX-Zn8DLjc<|6a;2}K?O8O-gguv%`{233Lq|!u-Oofa!#T$47 zmjd$eEDN2T0WFzZ2(G^zM+zmp5rh-$nTj|N*J9W+Pfkf6=r11@?xtWB!gScWo>J;G zH1_m&%mPN}GBgCdjLVj>*0%k@SkN-bsC_Tw(TLEM5OPvBiPii;8a+wS0~{24a;UM! zBaoE{>H%`ya;aCJ$Xo>Z8=2O^hQm1Z4q9VHtiG#L~#djs-wr z1xTm#DP?ECrOzK21=bPQKobwFZAC3T?*kx=!p3`-8%$JbHg>cf`wH8e1_ye9lp?Mf z69aKlnzC79s6wwkvhX z7;*2vVv!-z#Me}l-4%Ej z);f%BSlAzCevTY<-O28a)ZrET8#>9+KniMf@x;3)EyT6K0QRLsgWv_qtdfI6H%591 zbmZg?k4DrF&POP#vd+yE8UBFxsZgP~ho(XLU$|~2gaLk#_W|FEdyK4wGY*&akEdbH zV9o*aQ85ImW7a}if=C?;A8p29AxjvhL_@X#Q}-fhZ1|{vG+{#Wh{PtHEVgEoV%Z(| zO$j2=04V-W5_2Sit`pqL2!a7?)#dZV?vOcFXwi{;MbmY1V8&8lw&h8oU#h zJi(YeLT(#tAG;+U5tDP1jtBP=*Rq7Ap!Y+TLk(F|P%#p)y~xRr%ALFYrgS0Xh+iPR zp)p3eo90`(U1mSHcctBVcUXa6Ytt+5K68eKE>V*x~@K!??moe>4q! z>ek*E`ToV4kv=YH1=5%>#jQ40n-%bhA8oPC69h8BrTKhgm-f;8ixsFajzn9}Pj#Ux zK&$Y6WlvjHv%luNb=^C0{I{769=ghfyIZvNXi&12QZVJXmR^7pZnz^hYy)}eFrYX^ z76P?&1Ju%_9L$;Ji?Y1>_CCbcP)=SkeyUA_zjBphvAP*}IePa1>}4+m_ypj-c1X|U z^G!J8VSe(Yka@zPw9$<6k8mH2Xt-nN$AN8dVM4MyULEOnS*Gi`qX6)#2$sG!t`w{!0 zpfKjSb|XUBVeEU^dwCkx$(ODwe9yMBhGkcVtWAHo5i2omidI7cyZ|++sr&|c2pTGH zMA(ahW+)7CRFg0qjXp2X;HZp%KoNkNn{i<(^K}9P*OfpYI19}D*XRh%h2yC%0CTI8&jOdR;0Z>UFB{C<7$?TB?O zk?y+#p!L6ym)K*qzHxcqKtUKXJ$Xc9LVpspw*L9i!FIFQZ)Y8xeFv^TD>jAQfK&iB zXlrl;+c_0IS{QOh6j-iCO{$|0Z$5rC&_aIcCJUtB5>3&O--?*(4e`bvCYRdXVv||+qW%?ON_}jy5Bp+j^oOke! z3OeLBmDuG`PmY$vrefRAepPzb?)7|hT5V{IZ-sgZm&hNJ>%Z}#Zd?elId!X#6Ipt# zzcBD;hs9{CehVREb5%ig9+YV?){ve$YW@^w1BEkRe(P7wtIbJLU07_M+tiz(&G(8A zsn8c;7?rxC@c);>gcOzPKF^OB?oT?Nvo%>X4_?uTPO4C@A6BGChfXMGq6#fihu?w>e)N5KXnoHMfcdfYP9UkGaYR47DM=K zzTRK9;m-N_Rij*f`@wTa$W|R}I`vj(GDOQDVaz&Pb!ol021N*7Ya-D|>lJ@#qMO3# zD#vb`BLBV{r=4UMs(mZZ+HOrXDZTeTb+MzReByZ_|25tdEx8jv4Lb;GY`T|49g3+) zj=15SFsna=C&}&14=_==g&e1>qG{$6@DV ze`Ap)ib@f)lj82B7LWcIPxa5%Nev zaKZvdfl6l~H{v18x6R1GkRETX<-GkA_Dv#oY&+lm8<;`>Tua+AqEQ*ff@g zVoGY2!C zqe9nu*k2d2V+ydh6XQVubuGlsh$sn=d*^<>6z-F~73kH=Tp01NN`7Mns22+eISt>W zBAFsA+mQn87JZU2nTVW}57MV$^JjO0{1reep=vmB;L9%o?+E#8x<495@Aoa|5VQp6 z;n_`D3%W~`EK`uIon!GA{0}A_?iAIs$Or&%Ng3hb1U5M6VT3KX2Xx?UU^Iq&=OX$C zYe&oksgZH9XvT|qt+3ql0Bx+;JW0yZ1njjWA{=F8It}4%abA4V)TUhDk#0U^%Aw#$%#nUJ4~MWt*w#4<($b;XLay zWU(WOl2UaNKHjw=kpCJ`Q^Uva0bXU_6oPME1^9iy7^7J`os6xIo`xa9G!zKcnpy57ihEo!ou{XsON=6C~m=)e5Kd@ zOp~BwvgpIJ8|B6r`cV{SS`n239Xt+?9=B=M2B|UHMm{`%^XkGs>pNNBPV{>r8`%3S z^}TQl0(4k?vmQ-YS{tw;%B+45auBXwEE0FqsA@=k(Ltk1?(z8VW+HR)8Susj0Uk5* zEqc6<+oW(OP!mI~k?9Ox#y^#M>(UpaUV!VY7aWY%lb(~Vm0B}XJw;aYXM-51 zLIzaVKSt{HVz!50PI5J6aVN#oi5S;dmde4GAq9-i+X^9j4r?K0qmfg=PKzZVQ(NHt z%37q)V&k&i;WLtbWV zIza%Ai4S&;h0a(n12{L~oe{PVnfk1TRqSKw`7cHtGD~BlY-E8rnV;X-=4&o>Gj1*W zd^}d%*bmd1$M&g1LC5+cty;2w5CHLcpfF&XQ$|rY2b<`}{@vs7fcajsqjd-y!Yy?^ z2DMq?Ij|HGKG?z}ZMpxkPicM@h{G{hRueskf|9C&`lZpb(OIGQTKMwbvUZqwVwh?R z{00gz;|VteLXvsNdh{gPfJ!sQQ9N+kNj|D|dxD^XvM}qb8a3#l1-q<`26PCDud<9| zZLD`wRkDnh9d?*Xlfd<6w_ zl3hbCGILq87iTyi!;=00hrU)mWL)n?D?Rr*r=R^M~fb{q}OdU zT&mC|@yY&fnI~4bd~8niO^?`@qjsVM=qB7xBLIzKk8RPhzA!sDkme>R9-R_xs)u~} zSR9;EboZe&KFD<#N+2z6#x=pj2=}+MqgdUlkA%SBZd^rUaA7syEUqfAg3+z3Du)z4dQC*mIhJtl6Gxj zdd#6fv{o{S(lxe{mID8M&CiQbZ-^|ipDoT)Y9+b*A?+hL$~!3Ih9*{fbM6ukl+6X2 zzd~BJ`g@Iy!4Y(i2cL%YV!BUN#4*q)qrTGPYP0j``w`o&X*qgc@E=no{xIy#2CKHF;O779j2b?a()$6Kvamg)y^<-QUT!x*Pt}V zdW;38@7~M+b*6Pag^R8DL^jLPzYnvuONNhrtF>07n{Ydoa)L;z`m9zsGMbOym;FZn z?ara&KcKMJK8!Qa5kiK6@?MpD{o%^rq1e1C62!7#;&uhalsR5QERYoFjQOx zcCz}-Xid~sSQX+dF7p{2I9|~iO*NpOhV7N==F3O@#0Ag>yrXr!n;|cv5vT~%!#DrA zqQ+>pL$BY|@l;QF5YhX;R#5X;dH1TEH_0fF(C>6y?SlKQr>@mCnbYTh=@* zdNfn>dd$H{M=Tc6C?qB55*0s9v8JtFoOsJ=*QXWOVK?T?^1J8M?2>2N6uw8P=|3jd zEM2qXnLAp^eSvYU4HIco8&V~JqGzgV!UxJ0JHOp{_B1=!pb!8DfJ`&3HAQRH{>Lo9 zD_B7d>UeQvmYm0avndDaGU?a9?g$TMrjercVaXYCSPpke>~&Wy4+~ zD#OuP%4ORZHE7&!o_^o(yu|uf-Q#>=;BChc56i+PCVYWxT>_2Z=8px?X=7_K1R*+?{IWI}h?ZPy@m&lIxCu7b zYtf{aQgz6~Fp$siE}_Bypav+)#SbXKDSQZO^pJ=~m<|X-!S??Hxv6Tg*aH^R9t?mX zr#dORhRlyrCK&|bAhd}`QV`vGO~qc)v|umI$53`&qwp1|hKgn@I)zW&QH`7qMuqn6 zi)gRp{5Y4&q`e%(J@*NOQK=nbb#%iVCzu6D^u^<@rQkJkQA+HJX^&}u0$>z0FJl5} zfehhkCu9l%v?>Zj?1%iXkQyeCI<*YV7OP*{SCY*z6%Jh;oy1~f&p9zX=^@0dhlJQ2 zwZmvYNXawYH1<$(4kI#R$}UR&pJcl)OU&XnAvvH2P5O8r^3T4A%HlAKm+Dsbe}Ikt(*Am~<&V4O%2h>S}}I{@2i#BU9dSwdP2bHzP`G6+zT z`kqv|lPF$#y|;~0=yY=w{p0Pd3J8=>IvY{K*1$%Y0~0Z*%B9Q}(x=UoK+fH|F2WiB z1KIY`nI(B2cKcC_nK?VEH$qs_1DnPOAw{?tF@aUf`DGxg9fRBX( z^n$tk%;Ra)LK^d&o;5r#D2Gp;)VXdj>=i;ZJe2A~{$U^6({&;Tf_h;>Q=+2!`bR(F z&fsi5$79Rdna*^-7)-mApDhoMUoXFS@=r)=G9h`+(kUbu&kS8mx1k?;CNp4j-k7^~ z=6xC(a7U3?aPX?bE-U)~M7J+#5} zi^aQ6IV{7mi?V+hXOQzT#OfAgSw$FqQct~v>D!4YfG&gXlv=#ik%nWg#B7B$#sg+v zH3x3m7GQn6Eae6VlM&MybkPk?{WU!cJQtaoB&?8bkgObG1;Q?5sqT-0t-*8z#0I$_ zUr^0b@pn#InUH5xLrM>>O6aBg(aiPmo?>cHd|1iswKR!XN*alO;TuYrT7a|&SaKxf zoyr9CY$71POBVu{2FxL7hKYluIfg7c0C9`BI(9F1l-Y&Xgw2O#o=4?Ig5s4l_i2f&tPrL4aGhAGcWf#M?Th=TIWaqNG^xaeuz($qwvCL9y zGN+Ag8%>!`D(DGHP$AxGV4PE7fUp{b#!<+#GfHSm04%&~gL6NhMT8j^6s=Dp#p%J} z0e7~bTHOaHJmiM+gkJ%ZB5!uBpci#$P~dk2ebk^|Cf9};qj0c*f(Hx$A`fT~Kt!RjRTZ!iMVp|%4aD2fNH=hhj#uIy08%m`d zdF=&tu~Q;M23JO;h#|C-jlQrbBtvEcWbYv37Ecx2U-0bR>4K1g7pPr}K>e}X=q3z_ zopo05c(Y?&Y=SrI1r-*RX?)yL$zjGb3wr+q66kKsVK&`0X#M9Z~{g=jp zVpTH&PIZqFayeWUQbGRa<0F@3O}XGI@cn^Q)+>~0GDU4XWk6vL0n_o2L%X>S-M!`; zu%0!*q1apOwaj7hG^CAcrXW)w%)ka;&Yuc5?IZ+ z2bpgHuEPkdFK-EG0Atm3DOxrOrpH_Zr8iB=9bOekuzJyDKMeLe3h6N84D(_zpz4$Z zq$nOSA_uE;ZVk5*eXqZ(va{~;y~bA;8Riy$c73P$ne0*GcH4gSdN;r@_ZQ0ikgqC;uI3;w|$Nh1lRM`dQh~Mr`p2{1LZ|KBwny>L(&w*jjqeGsvOo>p;~biSeeShfede5$jd&*nf`eG zeOrlc;XqR^4uzh6LV%Up1EL{#1f{mjmlxn0DTrQ^mv!#!Iq-jPY%n@d>s!dn>1Hcs zjht_vSb+_*>s|pW=jK<+ueF)a9Lv%y`WEP<@f7d~=GrIdadk7a-HLCjNEs`v2y8Xk zmt!xooh{4Gf56jbPLST;t^L@*Ehii!PT~YC*mbwCUgWV>W5=V$|+my97hRL*p8`+n&soyzMDEOxsgE8;Ul8E4qomKfo*Oc*^|uv;D{fubVZN?0|F>5R<&_^~>bgt^F&nFF z0OhXt!o+Q`O+|f5<96sy$9AUG*%r!A7M_YMtdtw~Lw5a(;@SLG>!)RCYnqo+IR9ja z2~JQ84&Lq4iF;P+DCZZl(e(N_piqx8BnSOayJO_tsDgZoZot7$`}z6!a;F=PVv~KG z2r(08;vFYA?G7t}YGPU=gWoQ$>=Dlb6NX(PGHdt1Z9`{Tkzr3XId}8-QP7-aH-Mu0 zXO5L;X4U^uWfV0|7-t1TNPEB;{T3sN_he!b5Z@8n10=@U_*Cteemb|`np)4#?ik7iwFb~5IppA2{0d8GoD}Bcs!^!az zKvvad*rN?bO8@O65aNUzga;12hJu5M_e9}Fm*f-~Kh^tv{@Md;ZPG#!x*OS% zjCnv8nirzes3-QK)3`{~Gp_vXMiCx=C4!F-T#-Zju3SWWR@GQo&!0c*RlqXt{;^CA zGQ5F!S#-m($~V|xcMT{mFiLl_Q|sF5XX~zep8Ue%_ma^t)U#)!4*>s5v~Fg5zF(b` z4DB_2D>zIB<)}8)sGVCCvNC>d^~6diCnq$no|9zjeiu=$N()N|oF)A?Tvbmv%oFv) zjJ-bNsQ~{kEZ(UM!&T$!#SL{2_nSXyth;pxjNaRRqI{S{W*@vdC{5Hh&@3HA;OgMX zLJ1#a^ihr75_!cshh#3M30v3zDsRNV9-+K`dHHR0EsO16a%f7Vr|9_oN(L{H{YSJ)w!LQ*o(dDm!jK=o6!;cPS_ch>A14`%IfMz zT!nRpG{YF?Z`^2h_Vwg`WBL%xn$hZb_@0~`O)9UtmXGQXp~~c#;$7hw#4MBU8%}bu zGxDvPEAYU-x2Joh;g^pOV-x!_qQ1@Ot3-&23z0dC6<{k*E?cuJU1Q?~S`On`R9aF+ z2aeIjN?f1w2t(^#cVhJye**T#Bc4pq|1}mR> zB|cd#(N-cK5h@z)!ecCa1maSi68BoTqfGtY(gB;3y9BrHxjiD6o0M2aI7G=#mZo9< zpY*Io{QsYlbV07o?1l3MA*q19apoWoC%|%aXhu30ie~`$$v+gvGCw2ayaf-UlBFo< z<;7qUnSd4?1R1qbypQ6$SUm9lV1RX0>t#nEwz&`AWR@%r+DPq~J%D%!_*tVEoZhW3Y`B>=gA5lM-S+|86A%HSYa#UJ7y=sO5pGj0Yvjc4p^ zxcdU3uv{X-OBha*P+Bk*L!O9o*4>m_o3|7lqoDns$r3~$VX>Q6p^v4Ip91hFseCi;1}{;L8wi^yX7~in zq~8S!697JTQnZGdl%I>Hl0vrk zE-n)OO8XJl14bjc-boiUMUw&*F)k{y2xG{!)&lXPx}X`LJ0Ly$rfwpNqv7ER0}PDQ*H)p{e;1OLX6)g*?NB zm?~^u0iGi=y-qLIOasH`s-`RN z-@*2RDdSKuB`|5VFXlXw)L;>@Rxor*Sk^JAKC2)gy$Nj$5EIZ2f#PcHBAr6nF7yED za5f{%PfQQCQ|zn4HZtkIg|&bv35iu~tmv{ZgRzE45ur~cwojHN7Ai!@(cY@wTLn1cNBp$pk+exPvp*sV5 z{OuWlX$>;zbC_5OTOnK`{2_X0)`gbT?|LJ-xe=X11IP{&*wCSmb2zF`+^<0b1SJsW zX1Y%uPO5+B>R~p;h%(mS=L(8Qpjt&oEL9V+x)xXgS4XElhK*H|kD=(evg_x1hW7uI z^YBZxxbBuCLcY_E>Q+P#1m7{uV#ksyVab%sc@1)LZy?PMk6x_z3({|7*PYC(J(ImR zHmq66b*PLVv}N@;SDaEr>qL4T3o?S%TVpIPA?(Ip(+{xLcyRztrJfy2Utn&NX+2d! zD%X1*IdA2GcX2OR&>81$7k0)4bppwS4-VMb4Rs|TTmRd@s6Bh}dUR!LH(!U8>3_G5 z8^pS0W7GW3rz28^%sqkI<0~3OSX^s9ByL&r*Nibr1tqzkutmN&{>-fE9#La^U>8DT z>`mty9KJVwi0&u=hvg6myaE^_Ex*v^Osbf|-C50HwhO5edVsgY{!Y<52*V2pcCqKn zwp<@3yh^yaBLzlPOOef{+_ldw=7d+|XZd$IsXvyRoOBPz(uHmB#~yGs9BA$bv7l<+z!~pThpb;>_r7t;eg!5iVLsAtcr6_;6^%kP1MQse_{4R5$KH@ z$e&vdHxJZsyWT}7iP;5drve6^z%{537gXN5C(;v{QQ*@0X*Vw zh0;>vjOR!4aa($?@%ABeoDLQAgYmeAz|-}0iWq#%T=+8mJo`Cv#dn;ZWr-&;ot%&z z-Q0a32xW?pco0wju=2rDY`NyR&IuBm8PFMVbC_IQkl2@(frZp` z@M@O_Ces$@70zDsZE;|>*v(a&5|`y%t!V8^h_BQQjD{tFvd+EB+iuX|%4GA~5f40B z2n##yD&xC2L}%V9X~Sm)oQ{4l_HBmI={3+Of22m+9XQ4Rs1g0-y2iM$uHsZu`s2rs z{Ww)ull`?97RNKq=lHrvXrS}_jBY`#ri13cM=mdK%{jcPUYjfKsqHI}ej?7_n`j*Z zpL6xZ(Oyrqu=?gFK!Qw(^ekTvo9hB|y7-eMhl#IE9F|DL0i`L$-^#RH-FRaJbD||+ zE|k?~X%l?P3^(5H(2Ae!jb^3kVR8dHKgOeuA`DmZ<%n)|z4)Iqhir_t9sK#e-XE)Z zD~*k$Te4t{6FxUG<@K@fUC*)AV?M7}WP^~Y@Rm~^z-~-C?MH6LSW>R*`v693wlLo>be15%HFuU?FNXH`^@Pih-{3q_|S^kXx?q{2)6%W)vu#scm zR%lnm8N`YKCI3oUpqGBp&|g0QgdMnh_H3-GRPQo$XM_)_P@7A3Am5H9fV}U*u4adU zXtiO)=nud7=Nj{j(j@mLNypp))6dPR=Cb;P%s%%afO+%G4aG*cDP!NNwM#*Xez8QU zt4K}*Z3H)pvOxYdaYq95-P^JwU>VnEHDn9z|KcI)HBgz|h6V)K?ZkCj(coRlD=vGK z&dbCf5ByJiXh&beyise-#-vt_C=h#0xKU!dX}(3(b+JX+^d|IJJU#ybKc3du{-?j(d)M z%-4cR9&A6FjpkglqO?w^Z}!2WQsfu*1l&GkbdhP@^=V;e-Yna1_o{x;U<@xg>=}he zx$X4n{Zb!sf!eAja%m*;UHey`J=u+mJ^aB?qG2{xKub$Qjjj6;es;G`+3hQ8<}|n& zw4JZalIcHKA=c|?U*7DR9mVZ=uanbz8J?qOg!QX9#Z=~aXnQxhXd2gS!1xqtZTlnQ ztE0eQhCUsOOtfhhH;Zd?vd;FZ(vSb6e=p1`1GBN+jTAh)SAx&(3x_%&5}luoB3Q1G z+NQ$WIl02#PNw_&UECXN>7*s4WXwI7^E_p5?!g)())1I$nRryKH(W(y$Yea}zs_?K z>&ADMwLw%A9jR*vcHb_}&aYa||EaNCC*L&~kho_Lot1a>i8z~gVZptcr8#@DYEXW> z9G&ZCH1dsAqjBJfN6$>t{90Ww*sgO;eGu5&zUvlaEx45j-x^x^V{6}fjYsOjo%|2ax%bNC0)#|1_ zd__7okk;@J@eGr9$28r1IFZEV1=qw|2V-N6IWdr<)YgDK(JZw_w z-+1Dh_*9t>tJ8M-D{zX)U#VM(`ZQlr_>dQ=Ab#)Ar{U1CU^6sN&0N>BXW#gq6E*Ki z(awUc;!&=;+ipJJSOS8yymqg@Woz0g8FX^eJ&_2vAfcQW|2D6}7oLRl3r9<4Nw?nu zx$g&w5LqFNZd+t@EGvU3FLq&PI|MEfm|R)(&H9uf+H;2`P1ODzJJh$_OwpcM!_ zPx26qcVOCj;u{#M7TTLpqMUILLhuVZg39kH24)74VE@qoRj~g7i9n!8pgQP_8H6AR zL<(J87S#Og;__i5zt|XwLb?3}&DkH)vcA-32$-Hs%ZdIVh%#i7}HJp&?gKl??P{lP$YY$RGq`!stw8T+-xq;bLCW2pCED_OTqNa1rQ| zgB(%I6*5t#5@}Ejlr9x=qh$1sAa^}NgXYZ^q}BxhVG{76p0)E3o8xqc;FuiicU~>? zNk^it$OcvK$)wP9HUfrYcm)&?x+_f!|WUv+&r$C*WKt zz~4xBJqmV8D9UyPCoG06kQvJ{{R}`6m>c9pL1)q5{-r&ciDC)VnRnATtEhqk_E+Q( z0hmPMa3iQ0oIT{#BhV?JVx}^%k%yB4)D83@)EtlI5z1VcsldzeFU90n-ea^2YZ2v! zAw-L?q34q`%nLUUqGb4D@sIlWlZ*2fj6%TRhR)qk>%zGQ!7bhm_ceduBePtvpPp<& zN8UYXheLFdYZKJ3ZtVu-`JnG76$6#Agy;r>^D3B7avgB}uM3ss3*39HTMY1qE zJD5fKj1VBDAxq2-2hQRCd2g`=R`s_1Na5qEWvlbd1u!8Zrl?xCx}6SBT>LhpcZ-6S zqJU&Ae7VcmFJXl))CgJe|K;Ju#@r}>9BG7W0OTSb;PahK4P zdbu`9gQ$*-UjZHlrqt-5g-1pSXq@jPA57go!(p*_NW0gIDFQ0XW=5T31u;KDHV(b+ zdQdMyb1_uF)LP#3V3L?>Anut!XZQ!bC#xzdrJqr6Sx^wfwABe~?IyKyf`cgtm~wcB z(}rT

@DxZSjg4hXQkZqeLUmyFg z|9-x=AosavD=XLC`DoU))&$p~fDHaa)A#NeolB@n{8bil&RwDoOnfg#UzwU)id|$; zze!1c+~$#IEFae0!ko+tIXrwU^ZKFVVF@j6eH|M?^Y>%9y7q7Px7*|A@twcgDKV{~ zIa_%xSedoR-n-IB#ktzmp}U;5ESS}kKhyEJXUog>u7UYx{1@X7GT8Gc4tex7j!7n? zIh;Cy)td?TC9j6}?C09}Kal5#7slp=nCuX%wq}?2D6b&x!#+%ZiHS!0ZXtiJp9x>~ z`7qLq<=h_>!_Uk27WT?^<>baDY%^Hre9Zo+m1U3>e{NQ)ALppg-=3|))jKHGgyn_y zMdZEo7zj6AV#Gx+HtSYhl*%O?x_*2PeN7@}Yk7 z@t>nv(8cd=YRC!zO@gLJA9-D1KGiSaiwlLd$DLZgh`4#1jWeYEw74^B+ zg+-ecUx;3`)%>2yw!6fcrCD&{Q@hE-&S7%T33u;bB6W4%wblOW1EsxpCe3~}x& zpt{<>&A2S@g2<(@PxgEc$IC%K+PyvhUnRBQsvYkFixVfTybU1|T1{t8o*l(Eg9-3>Z}<9i7S$4YlU zkm>hth{YN+s`DP=NYv1cC8xCI!n;z}3i)w|JWM(v8+<%idsvv$aN(N=rk4#{E|ymW zno0+fFcj0VxZ%5qgonbB5g(rCS-1?SD+g#+LV@iV?Bpg=o_l#VJZ^Vt5fuG*<)-A8 zrp8NA?P>+*`mx2sks2#iXR@|Xws-vU>%*cm%FpJ4*VJ@qKm7^|Sk3g`{PHj|XtWonZnGJaF6`tgfwPMuQla?~16#B`(* z*l6FAdyRDOBM_IpA9=X|+(>^Klho`#Bw=UZDI35hv;grz&`EG*)*Dv)qj_JbDOXhG zRq>1#f_`YWV0X$gw1R=~y>7gd8Lb#V0n(N{I%o@jMgqC3Yxz%Yo!N~;<()9fw~y2c z!c-=(rUqX?X7Mra?p!k@3ftlE6zLOnB{tgAe)Y!@)5qG}?;0cfLf$ZW!7 zq}ML}5P?ooR+9}x4^}9E$sff8MM+nobX@>P2auI)j^_<#EiB+H1SP3U4pY$+Q=-n! zp~;A*-HsTYvk3W0KvR3D7M z1{a)FNM|uSauY~4UaVrHU+#*!%nMDfMtD4B^p%hFPt3(Pg6lK-+Ilac9kyKQg`=Al zfG9p5W05CSq$kpy*k@{L({?~}D3b>b@;(elj&-iGV{8fq9@fX!4L-;=;T5d0c&I#pjW8q=yw|EPo3#pGY~LX-oDkTr%vP` zsXzT;>_w{I-PEXS3UjtBt5B6ItR}`&d(8SUx>tm%l{&(qz9wOIUj7xGXep1w!ve)n z;t(6GF$6l=eeikM;5=n}pZGMckxZlz$u};-wxV7)RI9GziS>sQfgyrnbpWFZH>ggB z68dn?wRh^t5K00R8^V#HO*#PrAm=xr!#MGVGa}o@QB31$JCQnLG^gKZbB^Y%yVA** zQ>q9Y1Olda&5Eif=nt+p^(QXP`3${PYU+shfi{Ozf*f_Hsg^922oU9Pct3?y)lg7l zWi4BZ|H$Safr_1o(m@YFC{4;yl=;W!O362?rOoVWE*oX)>L>cPVkc}Yo$aW;uE`4; zVh5}}hY0199h>~dH|D!HtJVzpfBp4VBcW0Y{WEPjlO#7NOUVFw3E>$~l@U$?o>qps zCZ8dglnhPAG}e&C0rZY>bB<=oH{|XtgO_5&ea)uYLGys}&b}%vkzR=sFeCskXmuNq z>}8l{PBR*l5bh&J-tH#6J;SOaqUCTD4)g7`wO5AH(c9JH4-$61oTZ~NoXgsX&zmPK zz)G~m%GBRh+HHtd+0ShUZg+r`h-=$DX@9Lh2`;W-q8#UnmM$oEKwONDsD^4!++|We zP&AtU;ECBB;23RDQT@*cLn8fxD@gQcTZgMEcHn4XS-EEWLEPs@EPif;jic8VoK(@n z`q}4nakWLK&CooZ=@I49y=rAj&&rhUTq=nTv*k=i>1G$sUvayZn+nAO#>oqV!9M(? zZrH=c4yvGR!WUi;L*9Q>F*I4ssWVO-1&kKH|8 zN&CT=P&eK0gMkARvgVzl%zI3pS6^k?{jomYGN`g4~q`w z_aKz0A(REKCd3HL^aw5YnXfFXAO-o9+aPLb_!44v8v-M!{y#GULvc%qc?SA>D6RoV zNqAKP}GjMP{p{^BO&Bve=gq;WWC?1Zom314CNOl$V2vP}actje;B&u^|C8 zfpDFm2r9k)v(9JAdBUd<%7Gy6f_^$5K1V&LiHSv+A|U`mnX`;|QlkIIIE3*J6cN$Q zN7Xj@cQx|%$fN-A)(8A=0K%^3cUH=+y~y=iiigiOECd;K03>7_Hp!t{0)9vY1{%qr zFg-H~m|3tJn8B+K0&Ir*A(yVh8Qp3^K9JddYEns#lRzQN60}e*U^4m10zD$tpcifx zw%<#PE{Sk}#H)b3DcaqhB$W>0`^g9b-!bCAOx?TI0R7hSsRBfT*_^JbzIbF%QNi0m z5RWuaK!A-H3~jITauVKyi-^5#e4>V+PBJi0^bBN{WvxD;Aa<@4;1>Uk@^j?(fd0Y+2mKWA{dt8vm2>ea=tPE73{3XO;7;lV zua_^G)Chc!6-thQX;>4piZlk^5eYxMmLfru^bp;P;pcqm>9f#rVE#)Xn$UH_djOUW zSfx?K?=TSy8jJ!N`f=kooW?3bZpd6!C`mJgfBP@ z|IyPyE*6d><-4!dy)MX~Aduf!cU>F3<~!3?u9VIg@0$9tJmHvzGy$ zw0i<3gv6=f&O~n?at=ufVI%NFvuCjN5K;3p*18l=(>E@`BS(OwMsiSOy#U2i(EtvD z45JXfj7%_$g0D!!@nLG9+xBnQ$V-x?Mq)J=@O#t@KF9Z`Tqfx6K(Yi z8O{T(1oBm~(5@cWXTOY~AC*}Cl2L`nQvw6>O4Db7ya(MN$R=UvJxcsPfV*+`Pu^ z;^%)|N@yrp$KJ8gu0cxZ{Nu#Rfu3r5v}D?bIkVIJ#Q*#6WfepYV(kgpA08i@2doRb z{?J>9ieWO6h*C>#DSbLHREfu|i6p1QCMB&LHQ_VP{qB9jTfYlOmu})6acFvRlRLBB z@W!#XA=mya`*M|3o?<>zcH3^Bs(JfALR41wOS}&WhoVdm1v)ew2@v>qU^E*0KwDM5 zMHuf;e*>{Z#-&B|aSycn z1~O(q8MupmyIT`DG-%099oC8(vvf@=+0cWaV=A+Gp@UJgliT0mh8^DT>(aV%Py3QB ztreMLb3(QTW(?ld1ai2kK{3sj>jo?Ymmj>aFl$_Cps?RI=CSHZXi&2D@0?lT?K{OS zpQrR5TW-;n9=vMPWc0;7oTf$I!+j28tFrX+SUm|GdG3`^cI_kXyL|(0`*?z9&ZpCx z%X84##66P?|_FK`x zQfJBemSZ+G4l|l}?8t3sUH`jQlvQ!W%fQav&b7U1a%X7%#NCB6ih^ZkKZgsAN`eEU z!VGc}_HE-OFNw?!e$Z+l9&SZ2C)@_N`JPoox|PkHI6V+;B;;~Kf_3NOI|A)&V9tRF zSiZFLXqi!-vD@44@3qD7?!(8r(XnqiX+@j96)(Q?<;mmovL2aOVx zt1Uo_-|gGW-17OE5lyjO4L^wk0*?RE?;;zqiHd8K@AQx3jJ%y|xJUkIXos$WeYZpB zS-e}B&~s?lgCoT`Grfx8XWwLwM~rMJrDphYrp3<2fPoC-?KM^B+U?h?yM=ikzjJn8 zgLcFHp7g9i)cs#o%jcO}#M|mDtj(e;#of|h7G^tW{#Ac$-N>h{x+)DBZZLf2=`X8K z%zADxF=3>r+#Vx&Xd-Afm$RR4%9`<@(D7i=60Z~Hn@*UE{mT!kZt30^*8~Upj@UN6 z9Md>$-y=!cd2(`Ta7t~2v1wvlJL?}yd|q^|Vfw? z1`p{zt2alPE!H)=$7DJ*-FhD45%@ePDp?;#=@TgOOB?C%Zn(;j06;8f+oEcwe)m zuHuw6uwwH}xmR$O!6|3{@Q)t*+FUtS?Tgq0{G0~M4=yL|J83o%_mCf;SyzDvV*tWV z`W;({Q$uN6V@#dy65MMe0+wI#BO#AC)TTE6buZ!*hwH6=3_WyFVbwJu?WMO|Qg*`B zmhoxdf(S3w7CZ{ly16p3YK{IqY|{~$#8v99+UfL1E-G}nILVT|x77I}tymX}oyrol zj!uX<8u1|5TV&2bgV1jae6_tKKa2FL5-}>i=!uo?@>1LRrbtKV*TtA^a2{?$@&q3E zR)*B?(8-I5TAUx741h`Uj;o28fI5t}mF9mvHFx0%WVb#QQ?QBD6w` zovKd+NeI#3UqQT{rBWzvL7?H|o|$Zza^}>r+rxZRDWZ1Pz=|-H+(bQt4;_3YY=RDF}YPC?cY02csd79k@uctxZ22^jR_3AJnFhSjtAC{9m*ByZzp z7$4;gfS(}70Q@xP6D`7K3$6nk7IuaF2u7sW$wzBD+IPlWeh?-Cu!k5Uojc5XGe&Z> ziW!Jb|@g%uE4^qbou#K}#o$J9uYcl?WL_dGETa8Dln>*l|9tnWJ(lSHo?^ zw_(fk4*jk7B31F2NmzoHN?FQw3q1uJ_Gmru? zN8n!XHmg6up=TRbp8SB+?uWOFO2j7fDtw#j;}Yy_e6I0?1DrNL4QH~non>{mPP;I@d8k`6BgY0&p7k?o=IEa920MB z==qX)ZYbYrQ}hqL@A^uqdk67b5QD+QDxWfm#BdBf3c>`$i}6+nx?rLUscSy%^cpR_ z3BWeAc^co7>Yl)iDR@H^449NA9e&VW)1GeBzMet}!btehy!t6DL4JG(H(|fgNh{xhky;)*^EuR6KLoOk&yvpGKz8S8~8OiH;#0I zs$^&t{5V>{g}br)hDK|PPdLeB1zeh)ZvE!_*z04+bw{6?y@A ze939DIAK9q%qxb`1|dy|@`OC~>Y|pW8r+~(hncQ*{Fgitf*1&aT0^c2dgeH@Q~&~% z|8$?9Jnk7Z3#%W-spFs?32z%o7R3b5*X0{w{1z*Afl!m}CM6z|C#lFhCC{~Kgp1WnrwOoN$qfMG z4(gmhhjwsg=t5=)D_KGs#x9O>AU8wXNqT@M|aExyhRsYO-lE zc2kFo?(LZa<=#vByXGHTb;oYQW{rU=tAe9($t}E>g*4PfYck@ymBB9~vZS@c#qa8} zGJLj48#9utt@lYxPUb%tS6)+XMLPQJF4O?#hpAD4WvFS=TOVB!+-mhQc#Bu`SfXn& zyE&}w3My-MS7k(cTVv);`sC6&-*$Ho1Z(ZvP*bw|x|d~gaC@N1L-#krYcnTA4^ECY zwjFF}8|~F4`*G|RweJpl@o|>LN`8S4u9BjhU-Pfpjc2c}wX(1cE6fUBcu$*nV?(51V&b#x zgf;HVR|u?&laeK?Y@D0!c*o@VZmzcyaZe6!?-VoznFsHL5%Hc>e-#V#!Fu+=@vPy- zb6=MkesFcAb)e~ERYQvW*vE(1FH8z@AF42+COnm?|N4gIpQQ{$ZXC@uVmgIlc4dE=vTpLnO=&3_BodHp_JZDxs0mooa2j~1EBBV{F( z6-oSFuOQK;5yRBF+`@qq-CkQD9NwAlAIa`DQa&jhJdnf@pRpWr?uKVqt7oEtY$(urO?30n#4sR4kUqALwq>P=uXKX!S?D?KP z<{y1Dy89*NKf_<3Xv7}reK>m(x3-a_9KLVn6J6|uW1MSorGjsNJ=AY_dR=g|Py3>T zgb{3;%F4c!?SqNg`x4>lmHRJ(2y8z#J^7a1^5V3<^G{>H%*`#{9lFB((OK-ztCS7B z;cLH|;5fh4+F*C=-<`$(Y_C<-cdYWx56p`;TeCC2b@AL(t6AOaTxF%{Ds&hGZI4Ue ztl;*qIp=<1yIZq7Dfxy$)b?QKoYyz_aijm6zSBQ2Fg2Ief3q;>jtx7YIA=_nZ8kgT z`OMc=_jdv4dZ!E!5xcIrbpg zcs!p|F0(PV5%wlK=doREliq4S-{$ck{QQvl6STc}BlUOQDJjmYKa=y~iIJ>5I>x~I zmzIbjt+Bk}>u#NJ8Q)-{OPPDuYR9R-FcE_Ol+<+eq8^D-r54z};;n@f4zv%^k zVtcOpWp>B2hULDQ+{^D3X@b8g?;e9v;AZX!t5H#zU}k3WQu58FX7gqDUl@L?wHy|_ zxsI0^R=C(XCL_`(QsfS0?ve4R3T0R6VcjJ+Q&V)5*-wW=Ww0`-- zxfbN1H&~3UaK2raEXUKw>D<*IuMns4q3Ujn}LELsnYQWH>F zYn;w$%X-iQZLk*ZGs#Pb0eGV4pj2zDv*V_|rbc2O z@)0ASxC|@dbAh!vZ}y{e?K=nf2%^`f>n!X}0jN_IeLW9?LK4eSzx2wIU=N7LYc_oN3Y=o-(084W&V%2fiC+pc zHqti+7t55q?qvgIpsP)U;?6fB}Ft0Gk#fkZof*)UVIcp{vQ+VmFKc zoh2?zM7eO2O7gNI%;L<7&%tafdvKkp{Wz}q&}1}*PqYFMRNM58FKUH|@{n(YK_NS* zQH$AcTDDNMvHCH|QosmEGoWQC(DxN(=8&T8r8fKc%HX8{HycRcdbfTKhByjLM3Kmf zD}|bxmbA3-iV0+XD(`#vjv>&~ouAIHyu|9U5sr<)4{)Xq#)9KI5C2k_egaR{!F(fH zm{S+k^T}Lm4^y~hz@^i3G)D=&10=IfedWY8!W(dx;eK+>9MJbmL_?9BqpR_=Qt`%+ zaeB;on~5bC>0)tySb3x|<%>Fg*DJI^=*<*K6g>FSf?E$QEhsav2Y_7wsIPL#DS>ne zyBQIrQr+x~3ghq}c1Nx??189&H1!bXg^>yvLgT-17K@SjD({L3UK@35t#)naZ9}&x)smj6~d0PA!V?K zsP=!cY<;A9gnj2iA#U7-lV(^}8T9v;tW&VoP+H&|OBl4Jhx6M$} zYu6iYWc>`5QuaaY-iA%7zdaQGGv*hJfxh$1T8e&*#u~{k_8?hC0j0a7I`h-zy`E=D z`JM9o<@1f5P{82-!D@93W^(@B__LU4tzvl~vGF_9xj-FQj->BBb3MApKv^48iKD>w zAew;iEPqA?+Sq&k6y9enSiGNk4V1~`?9+)_Xb@mjxqw$Nz}FRF8k&wZ6Icrniw<`? zWuGuS4e@gZwy&p&@InJ{Wv)9)d3Lt~?J>u6HA`E91{t~8TiWwz_oAI zIH4o+E{b&ujc{WVoC=x~iHi`z>r(W@zEdBKS@Fnggw5%w*Gk~lDYAegAEeB{Jzj3z|$nGkXh#cLW} z5eTgZPms}myoepn7Xx9{hSXAxj8Dgl2>@>-$`}H4u!LX&fGH?CzE9R8!2H~CFQhh3 z$=Eq?bn)t$W-I7Jh>`iA3I6{v_T~Xm)@%DX=Bu1Cq*BmanA55W$gxBem2o-fNmJ@A z_ANm}0VPFbbC`C~zw7=y*g5C> zzTe;P_lLQ}Jo7xC?YXb}y6)>Xo>1qp5#{%E%~Sn3kS5qYs9s-&Xmpcyy#dl|DR?N* z=LY2N08Ab3sAQGuxB*t_0rW^kNO%<%a%Xdmy)#-9ey|nU!#=f0$HQ2VhXEd}+y3yh zPf(n++%CthPWgbjE3*ERAhg>P+3?sRqmSc+-oTbt{Ku~**U0s)5S8HFNpIMzpV@s6 zCH-U}?(S;wbn-?#-^>;?`eDFbQuup0TOg#GNA&|W+L@xnX^u7+YD`G-5W}&@sAxhz z0Dwx?MSh%mmr<A@m+o^^O_D==(;? zXY}Vtx1a9*qeOa$N|AIo=L?_33r||CHm|jNvq_hAN2q?{<=w%DUEjA=vbxow zx?jfxujx15`R`V@XWn_QTG%BYzM11ap`J|s-K@;&^d(+bu%=q%%%E z!h1>b`zPW)4|URvwu>_jBOdCEErg1bcWQ$`SAhSD-42f-DuUcrTz*?kK_-?0OPl_K z_+R5IcR`3>8CbBVczm+RF%rHRDZ1BPn&5&C6unt_j_;2|)*lA%URLcCcj!|i_XVp> zi$+#77aG0i(ywVW-KzVmXUuOaPx~q&#vAcgtTnGm36!dTYtVkavGWs`-HC@Ic8WTZ z=LD!%@$!F)dSh>?(}d8ZSDj=x?cC5k)4S>iU3|0;I2t^B`uvHaSB#IR+!-CrclUNv ze}hPsZ+ox8WNd8iSV6bf^R-58xj}=H2Y6|>H>8N3pqT?Nt6{uUu;>qiJ+oRWjI(6# zhedDIn%z(jyiyq~aR0k3+2y;|GskO3bCiMmLdClu=v&o8s)!EN#fqU!U4BD3tkBw= z1~&gvwks_{nOb?Hsbfw5TOr%GTL^O1qozptjyo3p&Av6ousJ(rFe`X}bPMu@irAWvPpwW@zE)S^y=niJ;Z!iQ zYj<92sEXScslE~!?t#Lpl(C^F6M5P_p$*qO-9?$giekGPS7HkP(Nw!KcH#P+kps35 zhL7CXm$Wa?(!t%WbIkk2wx<5zMv7avn$_PjQ`a^d!&0oQ>~PA?b9Nye&bHtFwRZ2k zbKd*%U%pnb>ss?8Mf7^VK3mD;_IFYRm)dpjB);32TV0(~GJS`nLD>}M*LD|Pmv5UQ06dR&CvzMRDez{WG;pq~vDD1nola1NM#W@A)jBp9e)L#0mYPv)CfCpi<#bzz~ zsDKmM4HhA%XK4?O|NViaR@fy|hKLir0`e|6H|AVyS4^xoKR>Wkb*t%MBS;P?GLtve z{wg@vuCvV-yg9nL20r_H1nLWIj}aXC$DZMTy4U4myo2$cBWv&3`t_J~~#BFNYfQLeM4YH^PFqjL12mFM_sfdlsFju$Acq|TQX7vvXbzic*UQ<^@@ zYa-!&t$Dzf*-2)+Nsm*I(-yw1v3wJz0Bt$RF*Q7NoDUVhuyG=RVRK~K#J zy=cEjad>`o^7AgW=qi$l@CrQ)8-O8|=W@L26tC%KqBEmilC)8AI5J3Ol`a1<D6)!yBiOdnrD6)h%B6{w{&v*W*TT?7IQq4c25SS z2ov35V$rbv#wn|GewVuNwENbKG&|Jq_@@O+o=4d_m&*4;gu^?$nD4j}2NFyeKrAvo zuS*@fwqi(g1R-`ZI*4a9SPxD7)H};W(JyNrDyerKgwuc`O;VknBxx)`nAuNqWqlJk zyoT&$&>@q{8qDEeybVu*SuH5lQOhPqczy864X}mRUckbO7e%ODg1hR)AtU&YsRS1~ zqp$e97kS$jeG>Gsf5tLXWsPQ^ftP1QUMh^5anHij1`ID=B@ra)u0e|2N3snt zHiG?u6P~id0g7j&lG2J0<@3N-*93@hbK*q}Cs0rYr#%sf&H?w1&rn1^_}F!HkCVAw z@e8vM8(7J6;K>bT@mN=%>o_?iR*^t5kmnDKbJ~S;cF&YC`jVzBgkkV;-K+5I3H1YS zgd+Mkbn)n5zt^=PDiztJ5KuJ0;H~k9u^R`WebDSlcQgIXugx(6P6tv>#eexsFoR-b zZ;l=g_E4k=>q^Y|5Mr#MP;kfL%jS4pUJpqcgVj<)h!5v;@MhsT>BUp#=)aNZB-k5v zysEt7qslmxCGFM<-uW)w59~ghkmOb@jYI&@aJ=wV!ooj;Iu4*mxQNcZ+YBBZJa&mDy-76u~T z>&A15R4o<$AeNSYw5aj)-73WYhv&~g*EFi@L$~%d){y?XT;2#ruc>}{sGk&1N(9HV zl#mX=Q;`r3V-PT?_LQ=im(JTZ`!iDF>7K*0=rV!Ai31_+rCz)baQG^+Aa_|X#Kp-7 z$Yw;|udWT$H`SK2j6&cPm%PSjNcT>vZ-0G8va8^XTxXl1wBmQ}(|^gYL0$byOFk41 zTlEXt~Y?-IS{SitBCqa*7valLS9h?g<6{bV!VQXpF zsFK{ekbxSIi>PZ)D}7fR+9a$4TQMfq@tt>FYpM>uU@BB;J!ixpLIyD;N3U ze_)!NKR^YqG*zk#cV^ytVPSa5pClOkepHp|C$is=M8s<%q|u3A+QL@^eObtF5%yH& zTJ+k5zL@Vwvxpk+=VzI~Vjn@sSc3pL8Ne3ZVfcrwlXPn!8E~?+Yu)q^&_Rg!HKgxj z3+QAW`=y+b!!Ofl?!d|rC?@Plj(HeQpecsCEv?>4y~BZ0;9js^1WC<*3WTpetE>$T ze~;KCY!iYk2hTBtfI|~^Wx(ttNEK@K9bJ$l;DeBP5(H&@BuRc9!6ekh8|+0PH-no1 zq=-I{^dMm39=RaHb<(xD1}OWjDDcgWsRW3~%@46SCw51nGNF5N0K-OhkQ|`f!lK_; z&HkM(ioDS*UO+KUgk}`kB;*c_FGo&Y5`Hq&?2%w}p5(*vCCbIG55z1`PFb-0N+vu=(cI z&t^jkp8$hRJS#Y36|fXNJH1d2yJGTqnpb16ED0TI7mvjQz15v>!N-g#A&4gxO&Zd7 zaZ`^$#VyeQPN^CS?odAnYUqG<1FsQKt<2>OsZmmE(CxF2A}@{yAq&AGW;RC)N%Ukw zYeJQ6EU|+&6HGc!vU=ntiII+<2KN;-YNWOfSgreDZ0AhfFaMLrovLHn^zR6ka4iG4 z{~PGVzk!oxDL4FgY9j#gW=xscV|VvuUFXYNVXflYl8!nt9QF4~Wi=ZR5M)O&vN ztxxt_zmqm=C|i)M_cX7z3~{g6vC{f`vki*_K3f#fen4lQSDTg|s7zH1Yd^;#JpQF@ z;oHu(LC4yx7C4;zEmJyZ{mR!nK7ggva|+4y*(Ft?l^>@qL+GmpiKgcYf+M-~{}?Q^>Oi#mlzO_Z+6 zuUr}ZKJf3CR$TYnx53Tb!LQH7?Ah#++wvS^)sQMOpvoLYq=oC#g818Dkrl4q@!wt4 z_&Z#Ep=bS)$ah+bEc3K~6P!t%C7Zyy^$RoSH+^w`q;UUI*JgKA0=hicYd2?ZbI_)$ z^ZODygF{3SzpZorMA-3W{;JF2^I?6G*7>MTc|;tTHPBoT^kSb&LFI3iEB2%{-hSA* zAIC#&vT%E*me&kvux5wp(f)6n%yKexUsH&_&+O5XmwbhHSLayX`8%Nq@6)xwON8yzuynd^$mZYM8lNC}c;D;22lZrE}& zFm-vbS^)>Y+uIj<&v9=uesUx$$?zYUSufvSc_w8A;%IE00o%T8@e8Tl*XHF4j6Pkp z+YNoV)HzjW_k>?}P*=1SALzeTdEDH{G-TfSBhE*fZuj+hXC`q8Jw93AZsh>SUIdcRxXdv9KZCX{Spq+6Ed7isO=0!|JG#Rdk2@NyTMwmpofIiFFzmH}#aSScm15%Z0W95y9)3}E2jLJGFH+yol{hgG zvO0Fbc3+jfzBhJ3PLIaXC7a)yH?JMV%1Ao__|5i*VU%Ap%0O!^2c{*Qs86?&;r|vAJFpfd|I5hxT&!?f5AY-3X1vuX z15Rb~A00)gaxz`zgz{!MUF(mgAk}ft8~PAEe}sFy}Xi1S8jEAOF$s6D_ zYZ36vb6TI+9hxfINR1do+hn)1QenpqyGO^vKXy=Hnykx%8$4`W;jSf)o-|Jppscu| zLC#YF*8!SP7@dIH?ZW~bywE16Xbe!KFX@AJLeO6v&#r19)_RS$WiD5sy61B@ijtQV zAD-qNyVfmT-Hr*-z}3zoa4=QhLWvVl&U-^I+?aeJA*sPHBsFUGO z&>n#92O2>}`o=|%POu1522r-3#)!LaYj;TMnM1(PJU2@#fajrQ#qa!Bji1STK$v{!eU0??mrKNFVm=1 zhj&o&gQLeFf`vvF9>$RIT7DA}azw<>;=?9a)TLww4jz|80 z{vgQHuOFR6Itr}7va3ajiYq0+8C)VbN|4}-H6@1x;Mkeg|U2)U_O z9P*rw8NLN(S#Kw*B}Q6WUwp<~^jNG3+D5iY0D4W4=2gi?d#dBOupDj~W}pzv`zcPe=ivxt<5&G*E&C8-2i zm^CogRBH`j4z`*lY`8*JGyx`6G!}dv1<_S8v)`5?OsD7>A_3v3h68fcShPNZqwHod z#5aXYXCM;^NY9bN7Kiog93s_77G^TyZ=*E9Ji__@=sAq`ZG5c(~ z=+34PBL`Or3NlYd!g2(HK~Ll}k{JsAxsH+W$;jrYK7y$q5Ce)nWmBWX@(R@Ojuie+ z8!A*mj{safU9rliT`wVZ0o?Ee8=Wv1y^A_H3RUO{CfnNHXkqX?Ze zZh091wTKV9E6|-xg;Rwx0B)txfXpSwG^7dXfna|o`qTc5ocYl$fC;K86=A?}&IEhm zD0iRWE&>o}mzkU!-;DcRS_JkJU|K{;3c3Swk7FqoCN~Gr&#Cbv@3on*Q=s~2>M}>5 z9CGeQBmGEk-o=$KPbZ5xX2QmF=5xIOk}0)9(Ai)fL;_@1M(7+18GJFfSn2!Vw}wQ{ zGq^Bv5Ei}M7+{d_hvkZ1n4Jsq|j;ffZg>$-=9k9*QBO7?85!Y2M7D z_Xd2GtN&Z--LbvGykfPTT%NOcpKaBt&h2Z4viAomUvL*z)L%*S{dr!r)=*iW(Xrjq zxb)-3-dvtv9uAIe6CPW7YJ5H{5OoQ7$_t*zxXMwvbHlaOSu&9eTqu7z-Jo95>5wG6 z9uwfJ$UoTYHF4~kE~{fmUHLUerRag#tlF)yhsBxJ<3rl|fz}U#j?NKuWbPmQYr)LU zg7o9NT|ItBN39DHJp1FCo_belvrsB;NlDIsJl1_YJwR==*U2t6oGW!7n}cESlyl0P)$c=VOf(4_fR z{O(g}r&BG`q$Ub6OkXa}71`ac8A_Y9Rxhk*SADU@`dbv84=Eio7AVt?#a$g1y_OSx ztj??X=-=~ef&~2P0r8w__2YKsTbF7&+%vcGs&3C(n4uPD$c;O+Md%W9rXofWII2FY z-kG0zPFntG-&+ey<`w1Gywl`yDcIG{0&K=sX^v}B$*XuJo;ng_?C%$Qe235Ap*1C1 ztCnDA#pEWd1A3|7uVy*k+g@|dek2_?iIQz!up_r+#YSbt`J0QE*y}5``+w_d$g`_a z2MBYTFZ`-~e-&U((bWfDZtlT5)-KA7KKJNIv9CIQiy#2W-5zzsqXR>BW%}sY3j;aZ zuN+?T%t*mvx2m(f9j52re4PDk*XdP}Eg#0l27eat)8nBN_Sa85qdR#x-ELxYr}5mD zfIQ8cc@fL^{|vR_?VNO%d?X0&UwV?hg!jqf(}(+fFRlo_`)fzZTYGIT_zxDz?J9%X z3e06MwoM!wFb!?rhjl70RJBxwdF}f|-CxzVPg$tDUUZ@}C{~f8EGdh;+M>Jq=vJBg zmELZTiMV|)>&^5{^+$IcOyenEMi;per{cADqfAUMi(m3`|6x$r-CWdlxAa~^Zq~p8 z`=O@MDx;8D0mFZBoy_a(i~vI3yrOhqx8k(b3xe3K52TGL0hMF>{XB-3e3fG~yXyX< z<~BJ>ne*z4j5}j1k|VXPhnl+-t(FHp^dqm5vd`}6dW&JW4AD%#P)I3w}OP;JhLca z?TJK(u6W~8@7YJ+wB%>5h=kAn@3V7m-@R=%Po4H+r#>xjT?~NSoQvyswkyT`=Fk)m z52w8dcdely^LveB`vpc=U6-sLmxwxoJA_7pis%;>mF4j6U^9M%I z5B*B3;L8g=_bztak~nVN=w!L>UEhQYWTb5wz}NAWAw8mg>8_3;Ghv|dExFaO?Wpox z_CEfu8bq{_9;l+X^i}3&FPiYM(eCQj+$?TK$fwa_l#)uqB!21WPZikr=)+P~!(J zPySK$w{k(ZbE@&ei38+?oxD(agumxtG#jh^z+yt_PVYf#IEL5deI~#;-z=+7M@=pI<*^!B@j(CV zXudMnMm*dN-KL$iEY3MQ8&~;98C;;zjwYIAJ*tA^kDo;&I@TljQ%LsJrvo6FZ+xuJ z5QHbf<2+j2f1P70Wc838X*fVM^4V&1I?KT9xilO{A^;ZhPN-*@z`cD9i))~c063B9 zVo)f$E}2giy`nSbm#KEwK0nPJArHKy-kLHhQt0Xz-TI-=WBax-e7Q9LM7@h5a8S8IVmy@$IoR<7;dodMss>0)`;nO5PHVz7{NsxufYdUSx@K_l;>6*1|(u) z{-ayEr)V@vq%1`)$x_ndUu=*I^%}bxWQAZ|XGw06N*q&q1DP`+x~wrLGXpn-?<^kX zLh-`cYtJn!M%<6$3)Lj6)x_PAq+QQ1AdhH9eU79EFAJr|SjXs)>S@R_BO=1GbBOb) z9Rb@xaWsK+PFn~fbi<4Wy9)gYqKt)9-j__q(=`<2;z1|0$#V%Ld!*pDO)_G63SKUF zqcXn+L4#er{)dfKdpWr`kO5C2(S%`KNvaMO-_yY4iYpqDG=w^$>|kTn5-QVx0@F^? zRi34nDm4wq7_7M`z=?rQ0(rF9EBs+;e+9fWLBTd+1y*+jTk|-ruYCDUQR6IMN4@RFb76$-`3+;k2Wpji5LWull zfOX0)3=uYBX%v@E>9Z+n1g<1b<{4-Ync9_w#nf*tc}max)RyYczml3BvH(^mO{5N zWmn@wnD7-7fSZK$Bg7lY4Jmcj0vchaQr%1-0^G;wYC~juSf_JnV_XQ%A&kyK-G(g- zq=~%)i&v*uiNKfobu*OD)Y&swL`~=kBtBM9IY0j2@H9B^{-E)@PBfKemdgx+^-uM&y6)Bv10 zVIy0(Ul9tzNMZ+0f7%zEpcFA_3WAsuEm* zsKo!|3pbb|%Bzc?N{Ml^a@qJM2W|-CQKq618WJn~lW3a(H@r|{a9;*UgaTmTF=R0? zmYsypP)rP=3+&ZQ4fCiC*$7_ArS#Tbn#DD} zLsbCaAwk}0;r5(77?&W%ouD2v4Qv(Qhv^RxVH193#VN$T<~2rDqFfzc%7zX`rpnOQoPIEtY^Pp!a;3yvT7_~{Q#B~dse`xI;c|M-2DDZF{S_?>gaa|2AR6?i}X zuD3xX$uBe3gg#IVl>I78>5z;k@P3{r^v%t$9)d6U?bB~Ug_s(yOGJ`0`4*JQo!AV# z`p4g&mfgO0&4mgIw+)n{98SGHqB!B6%@Oa%Cq5akExOq0nBEj}r!9MzEZ~IMbtuLn z9(wx}?4QnSk@O;-b+-u>_vW0wJgdcF&K=_jzt#@D>E8Z^&{CbPT9cu=BKRa^I(&1E zrW&~+B!%(3{;Y=tU1`$&O;;{n?d!`AEY!|8KdE;bk6sQpt0Q{#h>X|!L&`$$C&n!e zCt9o|wM!dau3by1ipC2+)Cn@Cp%6FCj4QVbv?h(ZvX8@r3ftlad-Z;0@{Q`{l}WnX z;gXsmPxIiivcL$#(#pT~&3gyBZyj&3`A_vFWqtvbhxBvmrA<*So>no+(+wwsRz_}A ze|6BSWZp+n;W2eD{}kN`5B|x3e~#^U5#E*M7Z(E?Ib=1-uU(Yo3a*xqiv351PQE$* zk70hxkWp}_!R8bQaM?)7+dRp!Xv zKao|CwLkRi*>{ujT2|J6S-;qAR1x776jYGrSak44b4%=cGp%xdU-S;2kah3bEpUlH zZRs|oi12GnDXh8dFaMQaC+c!-C@hO_pNu-l*R6lRb*=YO5Me(9$p zrTX<2qaU_hl<{@*5nwE@|6H92_ODcVd|WXZnqvE7!+7A~1(&rhKf_0+c1CwYkolE2 z4k*J@Dns=f%k#_@H(d+1Gp$jmBBQf(Tm9woqi)Mncl;pHt~1r`_}FXVkfI{CE;19S-qfiMx>=b_yWL=kpKthX(l=HrrKx*k`+As9E0)OQErlpK6QXVFk{ zR&DWFDVBcQf}FZBSspqFMp%6oH9xGSGw77%*eqdIpC*jgdoFLg{@eHa#Iwi5i#*($ znh}d72-0p?%pE*l{bHKS7&nNhYA}> zuX;?i=FA&{_a=5OtXLt$h`1Jrsq5)--pG*ac&He$_}+}n)YD7uU6@}xh-NfF+0MMs za3LPu`;!vm7c;ao7jz_hL6(@odaPYU)gE89U^HXNRdU0<(c!}C&~mQ$p}^8N%>2h5 zbG$`ceJ8R?y#uQcSYdz~a!I5pC@x9TzXk@y>rp|~kIR8t<>Edk7EUGUUqSAoRwL(M z8<;20=7k(v8@n08vIP*l0@ma?TcH&=qRhRwhu;{BPuG|Pi~5_t*IR&;Yi|rpDo*Uy zi8UcSpAxNi%gy>MHRNBjV_o3Q{jMOz(}~ZOL?t*CY2Rq|)7=uJD^+G^=5tE1<_sEy ze!H@b#b-+2rFu_yIN$5zJc_SpsMjw~aSQuKniw~(u@Anq_HMl4$q*6wo=Nc!2T?kA z7EZbR0KU`kl9uI?zBV%1J-q8y8S0Ly*(p^6iWx`+{%sksiz?lqPdOcXAADo-ywKyL ziZStq7=}%WPoPxm^RRNybrh0`#TE7vc1G{tapA6{@u-CvMTOeXmb9oECw}$vflakC z#gXhqkAx?M)hdaq8MZ(!tiN8`3RIH=m}`s-H$He;Itc9`(^@+YVSX(9^U>lWsG2`Y597f zea>%VXsbFcE`O1(+pnQK)z1Uyjvp1*mJ0zN^PC3k%obX3bIu7DX0y2qiQat~&KT&0{W`a$<5I$!>;dSTbVu*oJ!v+WR~ z^Sn=ytY+Q5 zj~FeP^TwUNnbX3;%?pT@e`lU_$zWWH`6TqwezV!LOA%(x-mg$jfl!b+=7gjB`c}HnwA?&(`x_l6s6nfzs z;owd{-!2%5gETmKIUPj0dS=rhHaN4el zmX?_f6C5Pukt1omiS)C`JMRM+8rT{yA|IS&6Hk73=YN7m26vwFeJo^U>k}|lg@rPA znxFWIKIa-PmENCj->wU*hHM%kYli+9r+bENJn2P&7vc3zOMos&09JeqN5I639Nj`7 z1rk&i&rShna>Av!nK6){8(Fd=g9E}8={{BLGH+)%-m90<@^_4QE4AwFGTz82I+wF8 z1g!44{bLK)8b4BmM|{W$1%y|qZW`{+FkNH%X)K|ghlz7mV*?|u7i4PjC0L8YeE{qP zv*F4|Bt)q1iKw}mSUL2}*sR$WG zy(beUVFG}A#Pqa}j1jz5qW~p+GHe)G7ax2%IX*WlqUt?<AkU!7o`o`%~t6 zPMZjJ;C|p@A&2IuT9`=?>f}HemNJtSk?Awg9tPCmJ3KZLI#U{ECjT`>0L1^pz!L~* z2eu4S<%IQ$=qo6XWDlqC_orI3p%Vrb@zmkqWx8Nfq$lwQPA@>3XHM!xXqj81@ZLj4 zuEvjY2;p1l@W&IOMO{%RalnM0Nxs6`tp8y!4TDBdm(?y;qf{6o;}5XXs0!X0wb1le zh=G>EXdBPi3;0P}jS-=oj+w@RXlvzEl7jmPl;TJTaY&P3Fw!Hs=l%`NqJ!lB25;HF zEraR=a=bnQiJ1aBv?+k_bv7 zyc-QS&~1V!rrEMZFEcV+n?mFXpy^o};6E?VJppC_`B>V%Pct!3W!>1>q!&S^j%#^8 zGta_}GH~I1B~N5Dk>DUfdWfIxeA88aQ7=CN=Lf{?Pt<=HY)N0c;tzvhI}to$fA>E$ z9(s zi3&EUh*;&RLzN*s-;4$n{}rHz@p!`#$$?74o%8VutE?{`LzsT0)gxFh!kEMnrPXVl zajkmYmPchbV3Hfle-fT1zGGVIn5{1hf{)T2PkdWFs-|x}9uv1%JLCxp3ytTja5%sD z#yrX6GyTC$ZOEYSc6JEuyj`*0cGicx(M>D$&HL)MXAbs!{&VVBu6C7J>aY4#{@}@? z?Xfw(TE01|40WC-kGns)NjFL}-tTz%BI+bu%wdwJo)+wNbO+t}Z3_`@J~ z&?3hD=Z`Jz-e1~&=;o@pntPK=hZo0n!?n!ev0-dfMNgCWq89>s#9wNpXlQm2&INJG zHiHI>vHb^Nc6}^(SLpghL(dh2Z2fRgCtn$`J#ygwFOLUBS9-^t(KJ|YGqZUp(Ydf# z|IY`57Se+kiZB1@?%{=irm|)wZ|&Ibv%mWPVen#3!cQ_EizwN_7NukQ4sg7j5;Cjyci_t~B;$}av*&Tl|-=cb>2OFMVtmiWas)O4Rv zHZ+XS;}7?!3;S+7)*Z{!pma4Dq}!H!-O|{Kvz@*P`bUkzD-EI(`q$8TtngA<&>sd- zwlQ{@M-MD{zNNbVW&2w{r?f`=?c9^`#QJsK`~3FnPkmzdMF!&amt9XUL|oFf*#59C zVAH3KfggUef6>-^3tJZ|=R~LIwnR8XKM)w^eD6Y{v?VC*fMvh5+$xVmtCi!E+i}!Z z)n+Lz6oFK+>vdqs+$iz6UHhZg?tiB4XQx%|yE2MKmS{#jWU(txKQFvDdZQ@_k-(SV zNEMO;+E8n8a)7Db>(vKl#Erwf&8Fc%$gW$J?}&#br>u_Jgs6GXyXdZV7K)x7SkQe5 z*5AI4B?~4^GbduZ7Dj3%bK=+aor|9laq_X*32blaxdL@;On-0iCu`2V^oN17(UPs1 zlr;j#c@h!=7g8ZnoH%?R`uLcyHNO~(4FFVLuDS2IQ@Aa=1ggTHGfUFi zJJe^7TJBfu_&OiSn~F?-$qBm|6Y+yl^_|ggW`oFq|cU@kIez6l@2bkuWdLJlAH+!I8ntpPw*ITP5 z(h}~LSb8q2n~8=j?_k5YZLO03^NoOuW;Vm8drkm;T&9E>_%H~?wCH=FGfw89 zZ%eFb7x2-^!?4fR72YpW?0j84@~cgK=xTv`>5)}Vk^`A3n(F#=bJVRS_ZVm1jho2I zAzzQOZofYMfzG1QYmFR#Mh&CM9tNBn0v6Wu;d$*2U!9d=WmuMegEOyprO>yrsbAb8 zK9B!|loRIz1f8i9m_g zW-?5^rZ&X%!uqkE$*@M<_1I!FExK^!t~|Pac=4FHX(;8`VKa;2o@2ty!If)yDq)WE z)_T#}7f+R0xOVByv)~vdMAS`3*wZz&p{k0q)8QVsK{ zZh2#X@7+-GRBGkJtCu76#eBaIG8tLTg7skp9)C&WI2vB5+ zba6!aeQ5ebp2%dDC|GK?@Oc-!b>Nu#D0^iriZ^Z*XHds5eThGyy@28c!}>GP9Dz}k zq(1)xfW{8s@^dyn&hhA3N-qxtU`Y}(j=}p1rKa66sp{LjoR5gyLVEf$i?9gU<|6hn zvsa3$M^C?mMq_!d!%k$E_4tG?Q?6|9trvywNi)uBcRF@Gi;&XJ`YYGkTw1|vS06S} zf+CBkq0&A88u(+Q&0Q^9oMS$ljj=@808{r+$|zCFaJifY8umnV^W7ovc@(>Lp;E8@K9DjZ^_ zpaafTdM;{0wXbNhz)8?f2FaFH!MuC?U1?E;r`-y)_f-wjRj;Ep)Q78|gBFH)iRBFt zOC#FG82VvKyoiPzmFNbL;ST$1bPZqqE8Yk!mEt(aT*1m(<9%Uow!WJeAj!gmZAxaA zuQi^AGha^^=lY8ekn(P-7S#=c;pi&!Hq;I;tN`KbNQ9_-W0mP)XqxNvUKCcOhi`4< zdR$-bF(xyisd#+KF$Tb_sXk)%r)WlwNKZ+0h)Ff7ep#~};)K|`*aO5|Z70X|Sr>5^ z?R>2j*_=x&3H2hwl1JhMs~q(I=s`uscLt7;aR2=dF)mgTWnKEt6$kK@=WODxu!KvJ zYEg$D8iY-8U9^mwxhCnD9xcQa$S_=fPLm^Ulv#U?Wb->-N8`6fdq47NWpcYI&db0A z?r~U1AYDL!6P^HZ1BflgT407?DCh+dDr}-zXnzugKCBoF*^@MsLg3~w2Pz$Fp+1*0 z(4NsgIq@hagJmFYXO*S5+FnTrmM-ojE zND^L#xm*r(#Hg4xM0$yRfEF{+OQLNQ&ocrErfO4~P{v;s0p4=GsfoU0R2!>9{eHj?@zmV2z;`c2uU@PrKq%1~c++5%M~G=Z0H9%*v2SQNbJ$ zs;N^KR&dt^PI!$0pUOH=^%Cbi1~E?)AQAu;H^x-BoG{h?3!-LDluo!e2KH>sJ8TN> z;ycj|5U3yf8nxtTk%ZD=9@=Da((PuC>7eB z6sBRHK|l_tT;Bu-JVm4WQ;y>p4t5DZrm18z@!r@d+?!$$jnx@>ZVsIQ1N`-n;o375 z0axM;+)J62r~i+g!(dFG;wt>!!=ZXy6w{1Hfk@(Vukn)*+_@PLJmF@t4(HyJnBVl! z*cCYCtbK9HkUk>?0vP)5VN=xJLk)9+nHypRuYQ4L$)t@p*|5?;f?mMS--#eAF-i&b&bp65-h?9rzmat{zdD%|-8frF*Ng#V z+3UURho6bT+88J&G)yx>=?+fV@&RjKN4R+k^U?}u1H!tIsc>s0l-5G(m2y0^2WTyS zN137j@r!z9v6gyBXJLXdsZWd9$$*`u(4g3U`ZAfahhvE%ngLWOP$qwho@WURE_c8k zk!8Trknc6J3?2Wp>jnNc-hJ7l@emm zDn(Kq^wYZi*Mz(M&uLoEPlkDnm2JFs(M(u()An%LYn#iy&#FGKCM~+-JmAyT>*%Am zs=i|4xWdb=rOB&4D0TOq82cJ+es*$}=lJT|J9rA=-Vzv?i0`X}v)ruX@68Aj9&9c!HiiXC ze17)w*pJ^|%xmx!Dm6@DB>!^|}7ViGatI4PRfuq3T z-VwLlYb!^hWJ6AtsaukR^Bd+JmC6KeH4n-sPIa`%7NJ|$OD)a)g9WkccZY3y)c*Gd zixL?>(D|nS_W|2uH}Ja}olhS1ZQ$3Ilv3esu5ft#!;|a3%AH75*qUiO{xDeN={0ev ztxBfQ?=+u&k3V~GQZ%t+Wx$HSMpz3b9^x5>1l0XD@JU0V{Kj6p)!Vlp?5NnNJtdu) zQ8O5ppYr-Gk?x0(x3}K&FK3EW>Ib6P3B)vGaXNq1-}sR(yM z0jWz{*h`(7UmEx)gAQ+(8EzEpiPWD3PU0OYxcn&ROjl`}mFWe~pKGEj)v@A;-gPK^ zhrz9v{6y=-U5#vezP^mVFCQm;06)|DW|)IZNoIClMlrw7q;FTo644KPl7p*pLjC?0 z@)gL!`lJe$jc%^BZdaK*_3{*69}o|Z0bI9Ol~kw~;n=hC9@bUv3a-^8 zW%QaJmu@9HCztwkGHQA})&u$U=6g2ZnF;F^ReUGB=5|9!;|a@@aeCQ0pNp99`5Z&7 z%gP{6UY=Btg$hQ^UP6egLcMp<1L1t$ga&T1CkIo>Ttiq_=6w0*1DLXk)UwmTnjF|P zBgMDbskp3O+L)d!z76C1|eUz{-S+VJNi`t#csa)3|_kf<8 zM3sxs@=W7s@*tOl?%)Ryn%&`d59ppFSb8%nExPjFMGj$+o3vjucyz>$=aa z-!7z+OkR|OtgcK6`p+2snQ3;nG37b40fuxqE&BO)k#Ry&JySyvWHw%p&XBpKydO9m zpg)Gs04gzqX}2-Z-dGe^5Dz|JWH6_|Cn~Djne>J%=K?|+V9yv`O!CnRTW#|2VGpE-LWw=9D8fIlXZ1?yUTL`d?Q2fZ=yyeU!b<%BdXNN z(pnTjBmOyWn|07F-FY-yV`@K3evfwUz9otylF`wFl)IrB(7{g@{GhNxG+E^H86jT8 zjd1JX;wi@%c5AZ_NtnxTF?9(7>4;wmu@!Fg|C&*iNBRfA0%YemECgO29(Gg!qpP#w zZCi7|+>VQZpvMR5g*674+~j6js8?Z%JxAG2l7M->I&-BQt<8Mubnhr%#`@0lI_PBA zq+XkZLik|4UYe*3rxcudbiUplmU` zc=90uY5kIMlIowouie}chZv~zVkGZLSYi6N`Z=iYS>OMS`+ zFGowCyTz|AkvaCJeYz4|&Pg?l3?-N*kZ3g&no_JOA&H336v=}am}L0yfaMH&7?BCY zK&6b(W21I61S$_A;P!zSUz(&w#E#gNxGHQx4ixd1n#am@yjZ5qzU*~IoU zehTP*Dl3DutH)2NcOVAq(JCHz5p$i4Ey(1bO#V-m)*i{9(z0SSEVqSMKs(isLTX>O z2d3aOi9bGse&iBavnAZn6@P%vSWjEg0W6SVHH)|j{XiR_kDX#~*$SlsyiiYRkhJ;erfZT3g-VvC7!rNV%8pmG503iQ;l_!+Kj}sJrN?{OpZp=0C zY?~?wr3-=Tr9`bLM<~moFp{#s*vl}Aq?MtJR){|a7Ym`o5O5T?9hW9h(YJvwBGpiUj`P) z?m(Vi&D1ImcRvIBP@%WukSthh`eCLi;to4IJ)SsmNzCNJOq2E5hkV@WK2pe z(#@n7XWbBKV@@fjsl1txIDu!5=Y{$S+4HzIsg0pfkD)aLUkwZaEd>sYvTPL7959HAYXP^iX_-1USY8M_ z4)a528o^rRmAHJsRBbo)0%LCr8cd8hSa$C3^agqQi<*sr@h9B{C7rNNraa1s#%6vZ z5Z+KeWa>4?6zEo5p?^9G${4i{k2R*px9ByP+9AxRU^+De*It0f1p#nKelP&26Pt@g zO6h~AP@I0pfyrPvr8Eeceg z9SYT-NP>e@f$#;_cA>xX#^Ey+T3#7XHSDU#3OXM-R9j+WQlL%yXefG%09{+Dsw`=& zhqv|EJER1mwip?n?lFz^*}{o9)JF;YgED*1MZJ{QP;wWgwdq;%HN86jhBdqwh0=7` zr$^SqCi1!pjX1T7&1`BEIU>z?!d>C;>RNQl!O{7^?2Gzmb*a1lF!+nrvc{{R0pg+6c&qY6{0YOlt zLy1khO6h6%g_T7RGLMp)zIFRMj5GIrUz5JSccxrh&=_{J->~%cuusNyd@73`8_d2P zVD*ylm(7YFV@8dioshU^YuBIB+GuPv>qDn2P=>fRd%)9X-FrJpz+bRbDR~hkbMG9vFMDs* zs?dh^y=kxMU)nwdbaNdQW5XvjmWCH~7h~_mOHUs4j$huTaN_SvHy$=T#qU`Apr<&q zdT_sTaO-5T{J_yI@=GsBGG#+jw^7jxZQeLARHH#&NB*ih^w6r_wb#rrt0e1jx!8wl zFdN4Pgc;r*s7MbHcdVsIgq*g`VZJ&FnQ!m>WBLZa{^mOk%Y}6TJpJp!;Vv~VyCG-j z^V*(|#2&c;KrwxF*1z;QC*1St)c@Mqr5Bi2KIpsKQP;7CVHFiBRMbMTt~k?|8cv(6U%x;~!si5(xSmehxc z<{VvHZ&<4I-Mm299Mw1Td}-SodMT29BHojubQCt|F0XUCw5)dJ&>a7}jXQ0h`+Yx5 zUe9YoCyU-MJLWJ4^*VbW_8ax6%YV`psCik=8`0=fDlzF+pTE!@FW0S=f6$@yMSH9E z1L)J*aTZNS%i=DbN7QU0>u-BqFRj(Ldi-JVg|%qn-IbEO0KCuq@>4}8U9uhrX-rNZ zhFT%O=T|KVO%EgXqV~{#>qpkhU;;2 zq;d2&;dhsRrb$2h5Dm2Ap=3$7lLx5#NB%H)c7Uf=dd_H%RYX;&d3nxjdVJolgVZle zhS#E3j0}F#_iQ1)K;t3Z)z2-K*`I!MRO` zqmlxo9}b3W@cG-L6Pg)`8-cc^>A$^2iF!S%N!!wW!=g$(dY*aMG3ow^#bDao=I3FB z%@{sBviAW%WTUu)5L|!bqDxk+`iSRyt|Qxg;<#)c3Z*l^U26~KXhT94lw_#yqlcSJ zBVY{k%j0o`YZs?`XQOh}8Tw!OW0R@~!;N@KtCrR9^moyEruy~S(2H2q;i`Y7zSnYB z&Wb_Kq)QE@8S@~9>{yuSyit*Q8pvGb#*yP7i1ZX4TR5*awKSfT2&I(>4pP8aM(Xjk zdSS+fg`|1p)h46cFqNvgT!n5q5^=>8$ALK3wR&b*VmvPZr)M)1+A#HkczUh#nkp9- zWPwjX{DC?!z-%>NBh)Pgd)EgpJbUJGDR4=oEwV6Oa@{PvezUdvLOrS zJ*qgocv|N|bR7-4)h6eKL+2af2CCnx#jeU+G!K$k+6-CFQ461+f#s!dETnp0uOYnz zt-OxDi&M7kCw0GS0~#TlnITE8-LkiS42v0xB3EHT3%VdZQ`{=nIPTqNjStzCw7LO6 z^=41=cFF|k48!SDqnCObVu))ZhE-{i7znC~4x5Pm+RbaBT@YmVSz6E5KsQ0O(oxJI z>BE*)U#;rlhqFdm0V1Sd8>P)9rIDUnPZYg-Wg=*Meg+KC4w$C~2K(5oYgdG}k7(V% zD~%!8oJc+Geoq{Lg)4fFcy_Vk!v&2pa?l+?h;-ThR$Lk_!RUG* zCQvpx2oOutgI?3fTSG`rF4Z6m5W0grranq32(e9YGxFn#ARG#=-j1`j^Rdk;M9u!X z8!h?@#|hdr;tRd&viJy{z58-kQfVx+ipw7Q6@%&`u#bvT&Q=tlNLs2EHU+D=9J4UL*r)e+1a##^# zGoXjK0eFNdP{2h+_9#J?qaOmg&{8Oe!Xr@qx_&Fft2Iu7)*p%re<``#v&z!SA`q6|JCN15?g<}b z&3c@ZsQb*OXb90c#YSr?40W@VCRk*`3@5uryML-7hvjGVi+2@kt#b@FAoX}fmCV~; zQj|k6vc4iO<0>;4o*nFa_eiMx{73%ny%9GW{iP?cyE9n3b{@Ys#H+4F?{#y{UG}rp zwZUpUyew4&53m}|`-bR6wJO3igh=yZ69!)T!M!>=qsAgTpp3V zKVg%g%lar!I}f(64U1e`h4P%EL)!Hdfx^3cZXjKb!A%taRA@fc?Ok{DUC?$E#v+-M zr(0Vc-pwL@5>Hd)Pw)VaN;t2R(Tr}dutGLj{DCXC5HUZ^y=3pjc{clVt_2Xe+Wix< zSr{j|U`aT9M^rRLcz~c(g3B>Mj47nh6$pCGU4qiUo6Laf14b${6S9i9dH6K!ftXB| z7^f*IIkd8b-#G*XaE720fhYGxr2PakU<5iv#%oMF@W~L%FR)PfJ?(nCoBt1AUji28 zd3H^-hE}7bRWLziz9e6xp9&<6xB?^bBQ_c}CSetr2sI$6Xb@#_culC1KyaD`X;lP8 zgJBy-_GOqv#AzX{3d5qHP-Iw!odIT;$$#$W#iZ?j{r`2t5nT>FFasXt*F*VJFUAwv-HrouDcoSd%aG3&#)$Ba(2#0}yY10+Rw^(R1VV8XS?5Ab0& z$*7S?V5P1&8F$^nNAr<5Mu|NWzG@!=Wd_&i>XC0a&DI9^P=GcP7httu$AP5wDZJ(W zSr&*A766;RRqVcqqQkR53HnLJm>!RYF{H$`^P@{5TOOE@dY2_1_yr2=hL7F>ltN{< zLl71B6P`d~!bP2f7EUnS`q>9R7D^Ck-l4XIiB_+QBQx)>fxnHSgNN}2UH#b5ZCs!c z_o!zT07zM2btC=<Jaz;6 zK}>-k?758@#pV*R#?SnV)LI$@cz=a@6EeiOz+6Pw5L30|`qFQB?p9-QoZ5Cp@!$p< z+i^-CmDRHsE%BC2N9n~7k<%#wQUE|}t_D2&?nXxZYUUbG9fJRxA0voeBKO<`CPxFP z=oH2Qv#bk!4~Hy$T+N$9U)$ z@h-t+ct-#;OJ}%aPxVm9$qBpLjNSoIteDZxLf@y_R7ZTQ>BjeP<{(x2G8#^zjDqSC zm~@*i0M*^M8HDE@zh`T>Syq5r4PuV2B8@j(FlZM10*0T0V*HIINx41t=jl_QL%9O} z4@(VU5NIISAIt&h9G-4k6V#54brV=@^kK`3GHA5WSo#Csh*zU3fEmD{goW2RFnJD@ zYSTKUkHhp3W}#@?7kMxwad0-xM~0hHhiu(ay5pBWSie7C@zX1ZPa6D7|LtW(-_O46 znIq=D_Xeojlf)87>+ux_n@6B1Pb}vEJRbzr8e0sqlr}@YJXI^HH?}-#i^LF1&4w?_ zFf)W&P4%hEenxGS{I+{?)&Zy4^trs-@}_=8BcHeP*`%{rf`s18Dfd`%rw|CyYG%ko zXSlBY`n$OQ;_Bl~cPG9H?k7bNW;+*Tor`{3cqp$R zThX;NCV1JKpN`(G`m(h!`8PPjx)lxbh{3_zL*)YBrpT7D_hak_?oJ)@l>O6c@yLJa z-I{~$jXx6}$n~D72sed7b|6xEv)%A*31=2|ZqoStP(h1X?y?|kS?T42!JoT*p3}KZ z2o-IKrm^;6djL1~Y*`UHXQ56P!P}oDwO(`cmtR5lQom3Wg2S4j zKjYWh*d8=Y3cuzp{#2_gt<4l{KU=jYV0il%Yi&`h7IXt-^NMikZHbv!`oC5y^})Jb zh5g=c+S<0r`D9>#F)+m1Dp1$$Knpv+PdRp832C-8|v6E>@);EVCKe zuI(<1ZF0Xi7`8@X{x{-u!&cEB9usA9pO6QtY(YT=9hKxp$#w@OWIuoCgoW zUh!DpF*f3lU3hn{;>HHz#3{Hwa$msa>w>-H&xqH zXB_0id6yF%Cb~5Rt=1(?wec9qV~@GtoFqo6OAo!9DGkL+>8;^#%mfcCt?S3aQ=yAPiL8KbdY^T#exD z^J+<}uxB*7e$Td;#?o*!MgolpOruOJNLJeo7Bq=|^Q&j_4qG&VVH>#g;7gH%kH^I= z(&_1x4(mIHgSqu}2UmJE*=VXErY}QyinWMUDYj zWcVvlkVU?3g&>8_y)<_I#q~#jGO@6=%^ZZt!iNhAH%Q=gg}JjK z;r^*NM}DupNp+5e`i!b&KLm>pI8BBhS!Vc)v~z{1X|f+-DTKhlaiw4z(>sD>dTZ0X zv*ioGa3r`RMKM!oX%~cQX>5{;yG6Cz`v!#f${Mvp1ySa#@TH&GU%80g5cfUfbPtRq zfI3QIksCtpDSgs^fK#tqF%Po_M$regW_f$*D21C1l#=N9F=3II*f9p;&fGU=01na) zxC%&p&57pB10e`bCjxODuLUj54yo4t1AF!c#fY9s((HWVN-F0wrjiH!-%#Wx@17M$ zlFEqUoGKBB$|@LU>O+&wYmOy3r!`}@)Il|%56cv<)N~xc3{xQ{irdy@OI$fv z4N3I74?s*(smQrcIV#2xSfD}?2>;apQYoz2f7h}6#nDpxp!68`Xy>ZpHKs2D0l_>w;jJwEpHocJ^WN=9?^uN5 z>DbkU%u9mimerDe3?r%Eqj;TZosmZ$dhGl#i5$0xw525ZQ&H&2aPZgIA(L+!6MR1T zC>Z@W>b15wYIPY%=fgS&;&F9O%4p@}H{g2Hz!-^H{W)5Hvsi>6Ea)V)d?U#@$FnaP zXgtWn4uFEF2occ)@aoRMIcRI{13`!Q79x@g^n9(!MB|<|az?}NXW1KQa;J?f29o)| zt*sggjK=#LVSo$I9^N2%X#%ki2$eS7&SdnnGZ{dk<#{XtPu) zoQtk67CQxI>~g?kRT`KZ1t1xZUiM3bd2=- z`&^;M?&)LPr<)xnl1~rX{eaOE_S!vQ55cRa)_2$Z)%?7{yyF_8XO+Nco6TOa)q+hU zuy~Wy8J5@m!d9932RWj4l*!iJP;y=aPVTE|w$K^cC3uVewS=X~RSVeyMDP?hYOv$t zKk;2Z2Y;Y#P!UZ!QkxHmc1F+$>^#emIMtXVdsGi?E}=eF>W3@Y!9Fab zg?E$5?VthUTJh5l<`MO?` zy7InnY>>+8*&f3B`w*!NC}x4A5eBV~@G#`I)WQkqo?N*C34?>i6?PH(fX=^!(_;A- zz?itJ0fZkheRW8}uqNEh4hqOWbFaTkKkU{(<{IX+cfqHmZ6Y1#kL?mpty7yY1k7L& z(!Q8R%#w~%PB0;i(Yvf%2Vnw;NC|GQ_AfZ4@zy^f3kHwJt+?2W5mJ`&qVDh&N%SVl zkfBjDOx%C9*Liu%LyRwy=n40dvuvrVAs;vadls7O(mWv~h5V`che%GtV~bo14K!5= zG(vTcK>;6&BYuK*zgCnTkS2!R0s5a&(vzkrQp`O{I}uA_m~Qo9^dMUTW-{)Puz_D| zE{L3whQK^PqK7;VSKkIUM2;sChTlF3zC!~R{)Y9@mvmUe-R-ffD6M;|%M?rN|&NYRg!2C7%kkK-^233iYr3o-!=g5%?&j3`~GV`>BP z^d|rF;g@j$vOGN)WHx12K~2oxz$$p1(L$u^bKIq2J#832GtZc}utY>RxThI1GKx1` zrz)MbYE;N$^*XE~1wP}^k+{G`u{{IX1loKS?5rg0i5bXF0ovwhEO7a_(fhbts;>T5 z#W$^;e;c5G9V}<~84!GiA)K05$1wdjL)WB=!pSiY%q((2L6-=E&&NOE!!XX;wp~RO zM7s^|8lIbz5`^xBwA(>v(569NJ3B;vh(?>r6WB(4STJpSLFUqrWzAMIW3lN2W_fYG z-23V9{fku^lbQo*{|Ekl3aUxNj!}8T_)>t@>c59qRK%akc6^vRH8vm|8xwuV{b&o4 zb#zR$X68u!Q0a!QHIPT@?Fa>Z{Re+r zI9e8FIJvOMHv_^=!(#Md#j)--Qm7Z5fl$A-sOEZ+lB`x;z`Ovr7WnnL>zunM5*o?I z66UyC*JW=i<7eLX%k z)OL8d?CupmZ^dc5AjALc(mXaOXXF<;o(uWNJ?fg^`CKl)_r&CV5Wsh9Z%@vt%$spO z7{7agVESZHw&;fws^Y9_Wx3o+*V_JwgF5p1e1E%!+kq#n%p2D2LIc~K;s11f@`>No zusv^%T&YL3MRmouN4fr{BE_QK&8k0#mc%a+iN6gw>8V(fm-KS&|HkdfZz`+iZnUEK zyS4T5uH*YN^X%>S=s)-hiN!@nqketMsMI?aMT!rH7-O`7ee04xx3rR`i({+2^xIo9 zYBj>niw)2*n*)kUB5L&qw31NmW3Qh~+;bdi(+vj&TfL{gQtE97+|COF3Z&~t%rD1f zhAnfK-;x!L8CBlGGZS$;AHLeIck9>go``-7aee4HVHYHb=U?%wE#H6H!qZC76d5X?8>B%<93$BnQ0W`M)W_n>)PO7;rRUho?A^YCzA)&txNLN zjj9amxNC07GjAG!-~OCz*kE*Gbvu&c9*0bsr`sk{e~$U+LleMefZr_5D=fQ}vGfGz zmyVvmzB0eZjxvKJ^w^blm10IXvSRS-6eOCPI{YU}q$UlVrp;K^e=l>qr2J8sjWqZt zlVe9wTJU5nbMu{?zcy~yDq`&x1{)oW_reje{s8C-4ANu7h~^|R&fN~%5A{fLJrJP0qXs-ZmYTb1p} z)c(ZcNo}_pY3KaxXkjxba%46n=-jqo#k}i}O2OsrDueCMm|@Mpdbh?NxO_KH?39cT z<+e4!G1(XgBWE**$GapJaQV_e(_N+)8XQ!6hpWoDLygu$c^PP5KE_#i#QUl=T9>-8 zNacm@%Zb6-dtN>}RE>Uy|n zw69Y)X*|_mJ7h1&4lDoQ-i*st!-e+texuOZMs4AJR+s;Ti(AL#d!it`2x#i-Rzr*i z@h_fFLZ6j&J<7A42iG}R+%4shbW3YB%cwfJ&~sF2Je97PDG`@UTt$^JU*Bp8B-Su0Lp%G+*mgoLpLivN|0m*b9y66+5oj&E2Ymu_l}>%T6gIg3~UC;wh(07QN0V+9JN6=Cs zugsU8)M~8y39XMf{V;dOE?yKI)+5uDo$II)YdyO)wzl%s zm>1eg)TiqGW;%=XCCJ0bAaW;{sI9=&Iy3D7aC%rfp9P~2w9hAobB>K#+nbG&_@*fH zoVvp`v0MWTpw2ES?_ZhRDvln=1F1f0IKaqlAn8P&ypzv83VHYnl)d#jqv#th3LS_8 zDQqeybj89k87^&4u8Al4kX1hQr2%*R1_`4kt~QK8JaQlGng7FfUnqbZ3O1FyCYsDslPO)`p#OWCp3bl9@qV@CKp(m9Y z$a)^DcE^Uat=C1f7JVPH5t}D1*&Bo&ai`b8?!gB=3eZpeI&iS;T8nCb0Eq1)!k*5e zNTKdysA!B1HjcT=3*&#p{C$Snh0G>vV0MQs5C7$VQB?5md(G=EZs|%FmuR5mx;i+C zjlMMqKHEBfU1)D=#Ra9##q>qLx8Zux7@-11Z7|!}>%%>Y+kzeSo;)JvQ(RiU{oXV+ zw_or4>%wui|5G5 zGnNz}ly1td3z0PTLNaY6L=;8-*x(to6h+fMwHjW*80BIR!aQyUS*Ag_jXv-iT_Ztr z5LS!+Kwyj9e`{1mIj)vo#2C_F5(0d8WxS6HVz{X>@N?PU;AM7&OcssO4Ij7nnisIW zkuoKs0s^^!4}YEauxI8J)|5!w5hD5NU(^hY14qBM4A!BDHVf#X z8JgB(*Fua>72kSRE9!^mi3@%QVrRX#r1b{+IdN({0Sw4$0(AIDi1TBK^V@;PR|8un z8K#Dd53sQoCFU^u5A}iMnSLr#oPoIZ=QdAFeUC+2v=}im=Q?zK!=xz8^z}0YUfk*{c3C*vzNYvC5g~ki#vicSIl5~ zcqX#YHe(dN1cSs}*U(7Amf(nu@=unx*$XZRj)ZK#C_2@l1VNQ;2x_(%_~79&od zObR8+r{OzDOYF}a=mCD&M1Y|o5#Vqo$1A$qXDMpPxUpI>f^qjGS$2h{FuG*dcl|T< zamJ^UdOznHd_QptU5)(^6m&S@S9EqjZVdUG0M@+<#nPy}XU-k`&W+uhpfg+GNR$G( zs?CUZo^27g*B|*?Ow+0aG{cU*gw}BaSG{eS-(_m0az?%|gg zVorbXA3&Q*Ntk6Iwz(xJ)1D(|E*;L@^x!ZZ? zn9Y4H+{6Eou_Yx!T&9D7PCf z+~>s(lF#i;Z@71GwaKl4$Ap71_5XBFPVd?~CTt!7mmRA&CS@cyD(ZAFiAA-i*w!cy zEQpU!)c(ox;a{+-FHCvuI4O*}z2#b^E#%C6+>RX#_k)%2 z>bZYdhXruYrkK0tsak8lDdN-~t-f|VQh#LPeqYCt;SOD>=Gf$hljWV&6`G-$$OkRl zmAddkv1BW%Om)^MP~D488kzh2jJ_84IpyW(W8$|{0y|pP=XCfW>Gy5GM!T|@zlP;( zFB`G>?wHngYAP=?a!-Nbe6hsL$AdfAx3su+=FZT@%wvzM^t*OPE_?km>lodNdaS4m zX5ZaDFU+^iJ!~sDlKOb^v!UH*q7cw^b!ZZ*9=3<2x#iWw`DtO56`~jE>_qjNQ0Z@` zJh|rrwZ03`n&X6X_x|thP5kNMxow?G+oYj8DZ&U0-RiX-)zT>ogP;s-dcR!ooqb9F?F?xfaRsr~)OD|VVqc<;7C;$hAzl%O?- zrMl=B_JD-#l#O4k(pU|)>y3^94u2ls!aW~$qP$#z?f;O_U;p-#T)PAE(&6f|IBlEc z6%7|q?N&5mk=*s=5u3X<*6|*>^@>inV}9D9DQ(J#Y(v9i>|?_55wsaz7q@KBi(V@N zGtncs_JabVq~U9fE_`+m*$_8yt;kf|2P^e1)Bjp({l zvB%uU^FaXDmaLvAhu|4ygOddvy-PsNdk&ZUZ8GXL995RW$M$6qP(+m*zjSAw1?Dwk zQRA2WR#qdg#)MfL+dM7HB!1!Ge19j(aZAK-D6^`VR{t5;gkIGDT(zm}ChWMRm@;`F0}tgv4lN!Ec+nRL@yHUP z?pTJ!`Oq4Y{h^@{sWf{C?0y@HdULfn0G~L7{P`@*1NxGU;~5~JxgOesm}~A0cv>SG z@Z#yN=B3yf$HqXX<(8J!4`vNzN)-)j1aLSaD*4m}a%nf;u4Apz~ z15Q`a78_pLZ|3y-{jgcswod-0Qq$pI2)T$vnAeWVZwPZ>mY>(~A)_bv+=lI$vBi1G z@+ZvR8j1!F6!U)uIy+&)9o5;tWNTbgvh%rw5?!bfBsmc2NL%p>> zb^6WcQW{ZXiuaQz?U1oF0K>kSE<@OA%^-^WIGz#(r z{RQF$$|3f;Fn5?-k4CX#ya%z!-KW&dbiNK*}tqXc!n^=VU5x-UotmP6{i}q0o2!Y-$`&o2B}>G`C1%gJ z)4q(vx&^$|A(wga0ln-*amflhmLIFu_CZ; zei3)z{N5L5nT_;SknsXc=uwAV_sqE_*L4)hQ@uh8Fb?n?1?;QlRM5XEhQx!LZL5_& z?(#DNr^d9b5|;8Uj3okj{fjJ;w=f0@;{U-wIr~ULkxc&$P)=bsDXi7Vihx-|u_pwo zDgSJG{3aMgD8bjvo0T%u(Ij57{1ay;C3ZH$#xiaKLW~3#0SE-s7>;0ga%vQp-^UOq zi*fnw6%)HE9TyQ-NjEkQ@SOvAz%ms8;IFqW?8&L?n^d?1NQ+nLel`6q z-~SB-=`cZr=?K_6W=}uKo3wkP$Q&!1m}n&PNHYrxy=#zZ%7KIssgt2Qc5zZ8V7g<# zmWU-paD&VK7Z`5y0RTP2fA9vJFzm#MB2v5TGni^bV`I!A0@NU7T(Ib{t6)KNZM8sN z=5EFy?z56sF&wM(u{S}Fv@>USOn1I<_|J%cmPnkB0|#@_+YNQ zZ-q{H?AX?MNT0xSn&Bv>nIL&T48cIIw%OcZ4|2^VTz)qvGqxoQI|H%bnK zqIVUVJYai~x%A09jB}Z{J={5rq8ns8PB{|@$rl}E*MJIXdUWKIjh`3u@9gR%2@UIw z4nCI^fFqN*yhjBL7WuSVc!>!->XAU;Nnxh{V8GINhNK-p?@Ku4(2N4G#tAEpd(eZ(*|LIj2JyIY%rG|qv3`BL;SHss9gKGQz1eyO!{>Zt`d3AvUE}sM9tt zrh&c00ec190|5}$s`_JKW$<}uOo4b zLyMQ`6NOAu?B{F$H(mEHa2Y-9{u3_~R-TWY5kH5eU`V{rW0;#c-Or!T9COfN4la%J zR(^B}B)ES$*Q~o<5g&hS%Gf*{+#J*_4A>B_yG_~uBxX)WRe+f{@k3WtU*+-o@ zks0Fn609^WJ~-t#@#QcbFldq6Av${Lv^7$9YZQUpli!oDNlG6J+2cJB8 zCN3^{YNljG#&?Yzh|F6!F?3_uqB}x07a&U+KdQ71rxqq;zvEACyT_89&q6*g!5Qu? zyqzQ(Nc@$@a`9i$UDb77V^YH3KD}Ersad*oB`R^yv?yt^H5*NzG6}dSL>)qwKBhH06 zD}pzb*wut|9_<+T=*+`ptpj|pZ|WT?tgPyX%BqfuQ90cxF>etC8n+b61!L0{mT!ST zPWK8(hWtVjRJrJYu@8PuH}!kkG8H{F#S2iN+1eN>_XWckC{J#a3hzk+>}R&WS(fE* z_=2mOHL?=c9m?by*1+AW3{MC|WP1EcH+fDSC;RnGfWvb(cZrYf8Fz^XGV==BMvYid zUvH@#LC+>lz``FvPR@YS3q59hscCI!oD@FuncnRpu6Dw&F2eoLYkb{|?BGYjF7T%g z>dof>+y zD|e%^QFyar5{<&}b!~!Pbe}QCglfC+zHwt_%*5ec3lyI(Rh?ISh|+@89ZimHL0-lT z{8n)q?ooG=n5{=Kawa-3qHxw5_2p+ktO2owI(rd)FCK@k_Jtj|Z zh}IJO<}N(D)-#COGD2CNzcxyhT;L|8Lx?;P#?#H5_s8n;FzKKwZh|#e6W2Oa*63`y zW8G{H6+d=V;)*GnV+>Wx)Pzu{60iG~Fbo1sz} z8rnz?5jq3ItTw#jxD^>MKyfjg%`sv*03&n=B%@t$fe)AF((X1?ebO_a7?}pn>a1a7C!0XUszwmkF&@bS{bEuGP>&UeeUf3peRGUDhyNB&P=trg z41xMn7s~2x{=F_2&dG_%ewq56C;?|!@AR6f7(`7Wcx-;?hK*rx&gKy1y;Iz&D?a)? zBBpC~&zXnaC^$@P3KbbPPa1{65biS9FQ1yY$~B-P^R?)!HkknKq;=72rC4Oi zts6`8Qib?ZCuAHrKZweJj*j7ko-H({L(diNWMnv|UDJ<;jX_!SIk4 z0CbLZB+^cDb6*NVieQJPCHyL;@D_?`1l!=JNH{=FT$*oYfenS`M2;6BfHKF)fTC+C z%AsXYHbh_i-E38KEIGB`>dUD65NYw2#&c^m0VWX9zJr+rH*(B`C)$kGE{#YX3g32P zvtHPC2nE+HWWo7ugEm$3OgDmgQnh8=#QV0PR*Iv&aYt5MN!8%_51s*}jR`syNedG@ ziSl6pGrYl*o-)whptk?5KguJ(&Eo+$WtZP}75Nnx;IT(6dLwb_VK^qgvphZ_RPbEp zz;P}|W!AZ@RwMaLHAg=8kN7%ldIz>zTIO}5Sj1BI%AW}**I>FrJpU2CCKpLAdI zld>BPhXxM8Nd;NZk+;q)0VQ&os$ChTA;N+j?g_rpbRa7(F(Naz*?<$=YKfSUDhXXQ zw!#L$1D-(j2W-P6FqTx5WBnWz)y08coCxadW%VRczurw|(# z@hMe!E})njIT3m<7#}i5!&JmDk^M5z2(T4@4Y3TG%z`F_tk@jV_L4alumnDyasU_) zY|=qTiQn`Q#lwgfqU=>&OOWR7|Kv&B8| zUSSBk(<{0`1CL>H=Q17^a>jYF){f4TOLM|D}W?uqTjg zn3U<5uo{7y>Y(?s>;&n4We;(iEC&IB@9B&slqkcjhxfV{wo?P7H;;A|*!jT9fVvJK z4dsB*L!U#=T>#@(QI!nIIDK0ys9Uzzxig70^8+JIz%MFq0=Gz*uyo6}=-~hg z80#88#iK{PIVGr5>73nt1q&Ew47Jl@NRAn36J(}~U@Ch`sLzU)c=vL;v;Ws0+8wZ& zuwG@qph7k#|Aub)FCJQ}&qa+?-rlEb3QSIa zmT*OzXqu+%kcD#|+g(n>22#|F!KIxFhZoJA?9+>aZPF_cJ0!Lu`NBhL5boHNXW3-R z0scb&5*Yr^KW?Pe#x}0reEtX8C7naQz-C5Ysh*>_V{NH)uRCHZ-98iP_IGN~8XK&< zg!$hn{!;sTc#zg|I=QR%tWg-L|e#Bn&L-|zKcKv}YrTM!sR z^6r_`sSan{Bs86wHY@w?Qgs(aN-hKaz60DCT;2W!R6}#TJecsV}_^pgg z52^00M*=u2~6!X^N-`Zxc z`!RB7WgP{H&IO8F82Jb#{?B$gO6of0_IP|MCS{+x302~UqUv{sWc=ZdVXyb)5G2*B zq`H%^m8u~NWxuIU8}#LuR~F*^iE%-dvh=AT zAOce+xqwEzb`T514c&BAmN&xIsQ_KZQG=-eb#{nSS0z8^b5GM0k=7=)C&`n~7W7Wd)ZXR^XVJPP%Pv#kdYf70*e z(xI~|(JpISmOLIjtR+3YY}L+=?+g|J35jo5qDaN-u9>mLw~4&ihRZAR*%EJrG& zu#ZsC#7WCNZBobD{aQ=2Hx|p#L-ch6I*yJ;lB2Zrk0T$+5aRt>F4xAP6#T*%(Of8U z2C_GG^hBjAbqTGa>YTd4;wEvoX@0>Q&_*^M!<%d-SmzgCO6sx#3_38;K5sDS@_oFXw{C{AyD)R$0sQw8wBjW+0w){ zW69Pwnr;mVKNt+t%p<7-$z%WIH1MN=ybS&qykAoWyiFC@kSA15kFBkg?!eK;hv+Sh z4dh|SrKzMJBM{$~NInxT5I+9ZePe~TLnF{dX2W)Bsc3)76Zfx$10s6+-f&|Kt|B+E zS#JyXTNbfIUqjm-xO}VNmw{FwhFHjTDOR#ZT^ME&jY!6u_qHp@W(kKA6wXK{udc^+ z<8dWg)*>1)UO+lpWwZ}NM5abnh`|NO`j&S+i&xe`eY~zjB6Dv9JvyNe1@aWiG)G)|i(ZiX%v*o{(j5--c##GBHv?jWFRo=s0)8 zO>3s@)<{~d`(VGMbAxD9Fnn3$p(cI7PbM?rtq(-+hIga!@<1XikfuiU1fC?o{ERW7r_Nn?q)})sI6_c_|%gX zv*UU@$*jHYb(9n{+D!YqAfcO{(+Qh1wjow15__I0G{+q18X5t`6M=?J8=*X|X>vhB zD1~n1+wkT~TRl-uZqqf9iJm7PgN1}@o8oN1KzL!_YJfxpvPN*C2tz5APiM~ZZj|mF z^sk-a#20XlET@1d&gz@x`6f^%1kmn$i$SaOqtJ?FJdKQl4N|1C7$Iyv^CUa z)Wc+)=I3TizyYpXOM7E|5bAOY%8!Ik#aGdZieCp?-^de=c16NS?f9q7!mcW#SD*sb zx{RlVpw4t7q~cs{Zw9%{qKdh>1#Q^0#U`~_AJ_*m8_-~1jE`p{vyn!#8I<$(egLlYdQt$ga2~_E+;9+&E2fA-9GN~?9d81oHatje zrE+*^=9FZ5GCR5)9o*f(l1GvA;F1S!Dc%6AX^*`We;yc8<<6upYEX`fFhI{9xN!jkpX)3Z%2KCTj^9upkrBvxERK!xVt!K$APB*ry`y^cCdo~5ZbP6AfR#c8kv1T@K zCughV(e8H^FyCE(HhyqS(alPkfW(Q;ZFf|jh(^|02(ubMYq%J((aE81xq2p1l{#VL zL!g)(Xi|qSCVMJllD`&B6VQl<0S-!Z0=eUnvC`}B@gFd)yer|WK^XQ_klQoSYO-*i z*p;AuH3tsLmL^F5j}9@O3=ap$RI~J(r_FVlYZ~hDX$0BG(t49J2#>fNnLJ@LY9)Pf z$LgF3F7zW_^^^Tc{2V4p^YNt9B>J#ks0h@qa7o@3&^$_f2OAG@38JpOEa7fdw zQ0>6{Ma4NK6Y;9CkvQFId53{Av&;ePT@zcbt82qgCJ9TBn{PyTh-pHI4X7Ni!9Xyz zFk^*RV_b(u=n1kWU%=ck=#osett3N$6*cWNK)s@iRq4#Fn_T%wZw}1v3Nt6@7bwx- zR6|Yf8IYi^J+w~WJudKKIRS}GjHt7bM9Gxa-qBy%K-WNE0~?SdGH?+nKBf}xy-s+D z{q^`5v~o(;?x5L(#N7(%p#X-`N=mT|2#UhgiE>2Ddy22;C*)#6Rk>68E3|Xcr$4oi znuUOjzILNj9pgmvMVwO0HGr^$oZCAb^~Jiw;N169qb}Gk!{oaS2l{bfO>|Y_`yKIh z;Cz!Q)eILxa~`jORn%$xjr0s~DpH!1l1PH{GP^uHqpRrZ13P}gX}`gmz~9vMIDuC- zFi{V`M_@hg;0t2$IPf{hy*(oS6WRFtO%o`D{^RSe85jYZryO(`PHSlCQ zCLA5{)46SpL-9VMGQ9cpUUD%&^Dx8=FD&Z%mk&1m(tL5x^K*meWCU0WV^? z9c)TJ55v#M7CVu%2`*vp)~0TL!_vksx_%Sk#&bU-*ovfZg)(_zXTPy5qi&6`(?SAE zEo|-;;mt|4ey`jyy#KVyrgxcBidhVhL)y+%e(b z$-_fH4WSG*KmC_;eX9M}3?=&a2cZ85etSgp_` zb()D3?%3gzm)w5axx!aDP#n{Ci0gVXW>_TO_V!3v>2O8G`8>aB1g&xFB8T$Y;?~&W zqtUr%;ueWKy~VlisO0R) z3%N987(mMt^iaY%kt)7gj3RKsfMUuxv*sVLa+8D_ihN|l`U>s#_oMZHYWuZAfYr)( zUhwoVe9_j7{ww}?dKRz1|N6#=>Dlou)yG8vs*b{;gI@B7*vS4|*`di7s)2te4?1-| zmaR0)@PK9$&B=~1hh*v2kwxcr%a8e`iqrm{Ex?>?p^!Pc?`G{bfl-KmgEriZRD(pA z(`9%$Z|42sA&=pS#+aG^E=;UM1;;9tWK7D}rA00aE??!+Xr3A}{3ftA7eNw9CY)Q&j^m zD~%%Iz14TK!u{|pMdnvJt5N;jKd@OfbBe1Kl<(THU%unE&Q)aA$)E|+T zQrf2It<|6{Pn!6A=iR?^sVZBnKN&UDZmf%R_pX;1qa%Z~U8sIao>A;`1?k<4HrDRi z^C!@mX?((IK1n_ zgGPzb(Ahyn!Mc4zIS<6=WJ7RKhE3TGlaWQ(9IPfgi=b!H?-;aq(4y3Q z;1#0-+Dc=iJjgAYLlq;D%&B zdsp%GgbCaMbGGccle9g~{$zHLVAN8k-*t=;yzT_&oHLolQoyF0Q-0et=N-ohYoQT` zVX0KG18AY>?$Q${*2A%rFRlfGG^}P4LAdxl?$CVqt~25 z`7Omgn~p6zn+4wi&mJwU)k9$&D9=Mzs^FluzD1{A7Y&?nM0i%vbb<_sHHj`ha3@Zk zE$2NXn}mK70gmYY@+TE;oi&497nFMcX5?_FD6t^6XXEV}#u*{Tcf(c$P~(L{r9;1;oSQMNaxXEv)512w2p~>wfL7%I30`pG^87 z8K$bxeV-&}Kx61{$mu}us7k4K5a?G*0e~i7w*^mExE~hgh)0TRuZ|Y)+44LhhdsEB z`S7#`3ES%;!t(0rzy9nq+K!oD>*}4sVWD8v0?`(Nq4T0cjVQ;f-e=c z$)s7;423S}P#5O4QUM=S5PA8WQWM}4c0^aTv!G0lB-M7@==4an)k{sbd2##Dy+;cH zG4dbHgXwaVAK^R@t3~#vPXLN)c3CZOGP~HLI`DJR|R5}adr8@r|Qdh((j5uHSmqUZ;6={so zu7R4nQOcoE9a$t$AnK#m$teh!DD9xOI%ozdM zF%AxoM3;MsF5Gk*K|KS`RQx*&X;{EWZ$f!5iw=3w-&4mZmJr}|%p||1V}RfXnN2g; zf$y_Xs6SW{^dVy4jkkf57tA(i!l0R;@r*0MN3_s|pDXXE#x$F(^X#0bNSxWs@^}Tn z^p{YZK;ceY8*KZ(0nbCcC?CmUoSgKwZNQDtMIdlpbQ*ij7w!;vgO_|3eEWE7;3OH< zKGVlst}CcHu;dGFGbPyQ?n@l7UpmGOTLB#w{FSA?X9ggX#6Me6hQ-%yS2Hn!OGeqhVuEhJtbc73WzZk(B zAb+aX#@CT_6$&yiu~-(fb%&IyDV}h^Ps?GgrwcOZL*)iI^4WrEH-PILHhLXwD z&vn3WP+CFQ(`B*bT~ifpVGIgll!IYHh@Qxlb2(xNAi6Y%;cz^GH?hg$FX|p_vKTT| zEE!_`iR@)%yfAN~gm;Ej$9}z?2?1?9W@zfiFl$zWlf+TNO56#wCGh4;n66|9%@zYz zHin4J=C$h5?4xF^&H#cr?@j#Drh;B%MyPQz-?9QglT|<=(F=tq3-`-J&Camb3jf)c8jBy7 zMwV&RAnRX*xY;2o1pb&YN(Lfc%NqNv^Cn-Krp;ptgqu_zJ4yt~+L@f2b&If5~( zWQhbGNaEY5weaKs+xH$`?+ zP`NKXuh@@ehDPtyRMEU<-N0a&@UHEz#a{$kt^ob%_I2-i4l~aQ&os_c=hsyu(N;g( z;U!I+2rdZFO|~hzo|J{WUM|QRKJPb?`Jybt!*HvqP+ziTu{M6uB)V3050+3)C_y=O zwx-nU?a99#dFA~?kT-}P+4QDJ&C9guWg@6TRYu34H&xqZjT*=D2*+BBc(FP_Emis?Tnx-3U8db7utKYBf=g2r2%3!+fRT5kzBGSMM+1)PaMx&1L^{AP2~S}Uv4rRNJQt+0t&O@i@p2q{0IXpU%{Y_&!*rzSH{ zmE7C|Q=GABPeGf7qy4ZPMC;6{eeWL6su2vF>U#mB9G4ES+@DC(cEvSjX2z$zN&N6J z&V@uluIj}h7N7%6DMoEq=$5Jdo7pO>cji016r)<4qeB}L88%Hc8*JifLL@U%y_nIG? z<@`^{iiL;0Tb}f58mrK7tQRgRZ{2Afk)cK{vE$ceFLktreaQWN1CG&c&&s2>PpKa7 zSrHf6u~>c5RYxAjU4u>c43?!`cZ9tcChiu26cjAAL$@cxIhEnrPh~FzXQpn0%3YNx zIx<`lUto|oo&DM$Q&)j5CAp>0Bp{;&WUty(9rWVldGaVinKK&XN(B8x+Poo z#r3z=l~TpzH6-Lo-&9xy6fWyY5R&~p(U$$tT8$zq;8f#edf zINs*8eLEu>cM7K_3Up29viuQJH;F@%=D}-k95gmK?0D_6)wOzo&dmo+MQadzqy1{2 zg2#H;tkf0>K11*rQ>fiKMkYYsmgpDylZEmJ;(Q%hex^M4vnv#7RoB772ucYPbRBm5 zN4nd7u(=00wM&ooJi5Wa23)9~r~_Z-*XvaM8HCz%MpXmm$r&C^71u9=o^@`gQfuwd z%*B*PoA9{^ZIZ~uH|GTQEA3*SHYe1?qKO;=n5(b?Cez7N;kT%8c4&tZ3$3-T!A>GW zXDESsJt$@|kPjw%KW?OVGPEXqKi(`HA9A%B7!U`_zXchDY!9UKW&nNGv)TwFF&k zg?$7lW6{Wv%isX$eDnYUMr9w9*Zby!=X9ieNP|egW1=a4)5uGtnH8h4RWCp!693dr z7(zA_6IE$&a9r4bdAT2XV5V?zzIly;#kdGsYVVs9aM$yoZ0I1cYKeuFkLglG5q8r#-8;05p^1j|?x{?q}NsYOBYviJef4t$-tDXKJl6ZoLAITyPz zl;&@jmdT3?A?t+X$aL?uc6Y;9=+SoS0aG_y97h}h5*P?@Hr8U$8u$sGFIT)e%~6|_=RS~-!##ot!Ayz$h{45d4AwG%l-F0 z?yt@Fe1MXVtO(^m-ebjp*!EqOEoFnr`2hy>JB&E@gm&ZGKV8QSh)oC9Eq~!_UjXff z*dBZ8Haw0VDbJosYrD~gX%y_ou9{ycJJ;XRzY(bUUdcVe~tdbfEUK9i?Vd1*6)&3L}& z6^NJ6Nd=^H@;K1&DUruRm`OzNw!(RyN zMA2L4F|L#R%5fu@sJxz-dD$Rlqv>jb1(?lz z%;26*a0gQ9eW&5AACO9MxOap*y2pTJx>`ixw4Dj8(MpMO66@;WUpd}5NW2j|w9wYA zcM4x@1#0#~cRhL>UrPrclbW-Lpme3|uK9A|)Oh}6c@$;4*}_mTFT z_(HUHnQ(ld4$S6B_}by#Vr5$V<+wqZWP*&MUr7c?GY4?@n#A@--x_SyOdt;w!hZ+a ze*-=JyM6#TAsGj_CNJ+Mus#duC28#l7i?`DOM94~Pu$DonPDFXDY zTMkXHxm36bCL2%{vl>E$snR@33Nd_P4%sttmar{i=*q?gayJJUH!^G{>{7yVBSpsc zdy7yJsfl|Wzb9G~KZEv$9G?yA`D97TT+o~NRKaojKq~44CMScbr7)FfeM0yIEH2GZ zgmt(orr)I$%&O9-*GNQfJ_6NbPs%1B1pv)2x(U|~9QT~WPa`(w zz|2?tvWs5v%yykQz6_6;bPces;|0vKsFTGJ>}gZhhou7 zUPb*nDf0gV&*I!pz>;O3PR*uh?cybLIIfmr#^?Fb7BPREX4PIOJ7&{uu&jb=C9V?F z9Zd^URRKZ$gCi^RM6sDaUJa5t?Z8ne&nX!Mp#O45Q1E>Umt=@oZz=) z@EnW|&bWv?x!$cUQ<{A^z3gjvQ|+*^UoGsZ)vWWVM`OlB?GPO4Pux89=J3PwH-&XE zI_In;gcD}Y+H;``mLvC49RcbkzQ@AVPIB_1!}}V@tRJr0{91E#?aGCqbKOmU zU=nAp*8{h`omIBjE7Cx1Uv4kNSnIRjXeobVTe;Hc<$Q?q>ma9Kx5nhh!f8uuV~f}) zsWaaC)Jp%N2=Q0Z7hk_~tznZsKGJA+J z#Lz5AqY_YKw6ck80s?V?X|)PSScCvkimW5D4ahK)_j5mkNzb|d@Bg|=HHyq{d4A9D z{@&l^wz|}04QC$U?zC0kxT|F=n31u0$3OI*0h-%m<7;Q;>EF6ioIW&pyS4(ru5-=` zE#-zEZ6y;kY;eI_gPDXyo?dO;Z)7&%+|4)Joa4JkW~VWS)^J(3xs3Uc>jKRC!48o( zgKc~G;T1Rw5tSP({I4F(hiM-$u`{3Z$o5>BxLs>mBHUMPL0O2;{dAkrULh{c z7Qf)zg=@|x^g0(u!tUk2nq;^*=~ggvx~pcT-v7yJ_bB}4Zl(%cnAU*TY?mydnc0Ku z92YfXgzUEdc?wwVd=?{j4_pf#sJi&ui*uX4ci3~cq|>x_$xYXXuXkUv%?~k@omw!> zb@}f$ZvQP;GV@$oK5d9|DYtKQpKnImU{$Bv9x!QOZKH?Dr&xW{H7Ry##LnLugs9AHS>bYN<<2Mrmbx&8;ekxDL{WFxc z#~Z6bLx~Ao+vuB-mlRO-TZwfMVnHDv#_V289UQy$uj2=^qRr2k8zu3+C9_m|0^30B zC78AoTAt}>Ve6&ymgMbKX4?Y}iuXWeEec{}$=)KOLM_wSZp%N7M(F65H~G{JtE=0= zj=uYfb&dm`7bp0|TceV#Crz_Mo2jf}n+zxL?v88NaIF(%NYwk^HnT5E>=qz(^@MyA zq+scuVIe?b!p)5VbB7=UgC7Oly406SsGNYI5(>%`g8AfUBTU=DCGU$4 z<%*3x$_DCp1dLo4H|yN^{+RgH<6`3x2kTe$$9w~DeJVWFLv);{`)=&0{>ynj!Hx-E z53Jmi3YoIQ-Goo|m92*+WRZR?NsEo$?WSVkAgXE)a3@;|yBZft^Q6bJSP4WIJ8JhD z=#P6dpJ(^g$1robxoW%RsH8-?k=r9Pf6Rp)tq16;4LD`zr8n(gcV2`Nn|}9HP73jw4$)}dU4cTX8)4!T;zNNDH8Na zE7F7O#*V@@==lfscH0P4r$r98(><02hj1MF37!zb?Mqp!d4>~f``B1Hjp>@~6B z(?#J(-7mQxly*Y)lB$*{m6WsN;JBLtR!!47QF=5EW{qmDHW`}l0Aj_aOznNTX8A3Q z_QhGD!Ma{=r)ecp3YFgLBmO2)!wouhFD@QA?U%^3E{BA$m-~hO4Y0B5^nfWMKnPlU!A3C&{Dev9{oa91 zK*>9~6KP&j(F^_6-DzYS)lMuSuhY zt&yl5iS>$@i@KBQn+N}BuM|{<-l^00p~84Y2EpB|7;;oKG#Au7uXBU0+yqj<;uJVI z%5fntMik#3(;$P13zKjgn9~4t3XPg&2BpUMegpe~ds;nREim5L9Ki z#o76_hq#Kb4O6A+R2?4t9GA~jI58*qlA>@GSben`JUH6@9IF+uNe%me&5gC4FDPFs z5EBq7pF@!k;N!c+u%}=SWh1)BVfCZ^Rwzo#UR&cr5aID-&~3n@;ZdUdQ*G~rDf)a- z2`+z8feM)e=oPAZylGv4U5@T>?0PJ!oj&hF-%8|_&{K>b=C=`Z|LYx^8BNW+dK3as zFh{|DM@5fAcLj7d@(b2TwbFqTGXDud+Z3+(h3`*v#+2X`O-@rRz}kK)LKK zn$ym~<*1QGGl$q(29l}4GJL0SdfYBeJQfzt?gp2F-I7V~WjJdJ8;u<^x0qA!n-Ogi zJf<`O`z0T1m($Q+i`;4*t@CflQYBu26vFFJ<#v)#LT;q-`;tSN=$}CYLTSh1zDc9k zVf%m&Y&Zc{3YO-90aa{oWBYJ-lRx1n{#5pDd*94q6)*j!dYMix!i$At?dFoXsWfJE zV)&%9#560IaJljgYphe|j(Tjm_EHHtFFhmd#O*Au63*B@$?0V0{V;*W4;KS1c<=@u z$3AVinoRmH2-p3-#cjwUf_UNDNRdB60~t0?LaIcFuIZ+{*L+(R%!lKjQaFX85$|Wl-;i<3pTzYVa3d%&yKnLhrTm^Yc7S8$` zp@-qMm2#SqjzD4omUf{@lHI>c5R|Gc2D(%$gcbdRRKmf~JlIQPjX`BnbiDRp#u<;Q zyjj~(WsFgr_tsrBf?z}iPi}Ky`no_efxkSP0$j&96n6L74llL8my|k0=z7^Uq^+cF zAxZ-@ZKqD{@@2&)+IdaY7K02|7+8TlG|ztAAh!$-6*d=u(?3uasK`=#10~~$Tad?| zNZ%!C4nKz+g`(R}DNPYx`=%>7*VWwYCF2fesFM`3U-S~P;TPH|YdeY-ByInaZh?Bn zk`k%k52T)!QKl~Ks$oZe&r2bB<{enH&D38guA(&Y@Vk}`UtSr!E zUVBatEX;nq$F|vLmrXfu-`DN!8hHYHP5ZH)vzVl{P}r-@U}(E_zUOn!Qwt%Oh6fRz zg5#3*3MV2~V%fUr(1@lVqV{XT&}hhwphaE8TLc60RABJ%&iUi7?ky9(<=>CaJR`7u z+c3lj;SVWyyq?UqB=2GNF55t!=hGT&@&K;P%MDWk;&;WCA^!=SFyF1<{P}!>ABus%C^oP<)_RY!Fb?WjGpgDe%c47%G4QeeI7#3A9aXbw2loRVqvkI?+CJgr(9(^O*A}&| z`SdxGe$uMe2uZOuLKOal*aTcEc+z>B%f5=6Eza?4@NWNW4v(;|r{81d$0>fk4ED>s zwiSLBl@BceEGu=Jt?*&%H2kAIxx>{p9Xqb66E`T{uhcZeW&m>TS~8R@tgG zT~^!2K9R}XyaqJPElpqPRg;E+-0D5al?b3lUolhvk?kJc1jpm=e>-b~@qF?~(X!%l zj5L%NjadClBv5JT$)zsDK z_fFt*3O|%Q*=e)3;OroP0=;i-+59J7HUBfZ7Yq9_#(*+*wfW)VPl>8Aw~?$Cup`vm#+!REjA)B=W#zp0o22y% zc<@f=E7sK9HeL6x{~CA&UYE&Dcg$H*C{^n09B#6Tl4RiUmWiJemmKCfR#?tbKisi_ zzK$=BF9H8u?F3^Sok)FZ`-iXNO25P&nNjdWQxukWRelDNT5}1~OxM>*t!vw)^eO|NhODZ-+V?Wy>#f(lJwifqL~P2S5#Q*^j&CR2L_Bil;cr2V_&hy-M);LIug&=J>&#DT{OVRj%~wS&}xaP@7Vb8QqJV z?lirmzNXY63xq}e9hIsrZ^4Ek-fQK_o?q=~3a9ogpDPOSB&4?&dq}#}ZO9&Uswr%0 zpY=wkNCS%qktDGQ(E3%h-dvb1vmODbt?mI!yCzqgw3lq3iW-?LY87_mN1_(ph4Q#@ zZ-+uiiEG&Lm(b%Cen}f{TJR-P(L9DHp&q+jIiOXS>^O(Bj+Mi$Zq#`-ZlwFuZRT-KX`XPG00*ViuS+% z_7K@zoe_&M;u^~LYReb?vJaR>#`x@Te_dA>XJIogoal^owZs(TR}L}d6U=m@gi%gT zUezDD(ejnJXkThlRQ;)onqo}4@)^nPrQ}A5Kc&j;mPuoEH5KRo_SQk9SrHwRZ4Fa? zfBB0G_xhKl-K*an60lm5R79hZeivxaZj0*zUUo$R!OA#FOCUO6N-oM|sp9FiJ0ra8 z1(4A-;K4wY%a+K}67I^GI9Kr%4kt7T*F=Rf7N55m+bEUb3K&~lrbNfW*9zdGvXbO5 z#@Yam7$GUyPB#I%GXIB0-z$$O-yirw>6xKo2i;!QN`~wzdAwR1HgI2y_>6=036wx( z_qMO)-iF)7fh?7n0MyTow`DO&+!5zXMlwgvs}nmsCjqgH7hQb-0%aWEs)U?c)Ffmt zCyk+v0YBuw4i~ZRh=VDr>~MC3u&dDZ=hy0`; zmf^ntb)FQu2Cy#hy5f^`->+hhXB8__TAns8k8(HHDqbx?pR(tiPlljD1AGJoVYJqW%?SIY0@zSkCd3xYw7{R-$!UN39_z4X__G5v^$ z#WLihgc9W*qF{IZbamWAws*`g@!ZKkp|;huQ7Vi>JNpuy_t|_RFL<@KKrUb*0t7w}Ji3zL1^NFb^>2gdc5)#@ z1yQp(C^}a7NQHIG@Uq7{-oOzybC_dP&wy~H{4_arXw}vX8M+IS5mY6*dFHp@BGj0Q z{p8$cX6!ByeUE7y?*M^d<6;O|7St{s8!XK-vl%R^x;Sjw^4a!WH|f&IoFDZ;9KF1W z#xPZg^rZ*XC~*9LF^CFXOIbn~#Uo9E&YidMfAX6LU-Ac!O6LPm`2YRIDi(+MeC6HCp~;s`T&b+T`z}1So!* zuEwuDE5kfB_b2;p4%n-0ey?Zpo}%h|@TW)C3&S>)Q>8;f790ZFaH+!2LZK&v>V&2m z6^JIdxa2ji59fF2{vVfGB3FNq6OTpl^6kpokanxhCKRzZA%c?eU^a}ZTU~G6XLZ4s z`u4NJYx2;Ce{FVd)_vJ7Z~kb?xkO4o^K?RE&>f`>Ee^!483mNcP07BH6>FP*Z`St? zdqYC@TsAj!KbB{6zaIYbOHb#FekX9G|Kfyr^HaC@p_`Eh z>XbPXrNE2llAWTQ!)glN3DNH#ePjk=W@LrM+f#O5)^)X?MAs`c=@xS=ezF!9m-M+s zse_yPwtlg9@=;^!WEiwHme7sNYw){nSyD08h}!b1xr6k{ilmOvsMS%A``BW_ZpZ(m z&PV@AzfG%O94ZqB*`UrQPRQPi{7_rMbM=j5<<5_EOUo?vNEbXJr-dGPwXx&HJjT+b zW*B-kLx)os9x_+^w=DtTx*Ic6g`xcf;Fw46UNKixzf{9 zv$+5aZC9U^ZcBQ)dix40w)aA8Uu?6Gx11d18{kuRx3Dy|u*_opBqw6UnR>(JXj4OX z^J)n+yG8w40oxu@^&9?5H__IKUQr2RPEp9fAVdSJ!2T5YO*0D#!8Yhi3d0Cupo;EV! z^B*T#6=juu#c5&l-M;9w8pO~|eeB*92RyJt`nP6EYV28jU%QWP4giw`Tr|L+w`Tf# zAnaq4Iz%I0_BEG7)oHg!xmZj-24ntPXro09m*4o^v%7nBKi}^3N&&-_;evHV(fgd| z&au256|m^qzJD5O3R2#(P0C}&OS?ZUhP$Y)9@?FWACR2 zXt#!!YP?+QT5J`5XwOgLj|E|QeZ@-wd@!gc(*ttfH zL;9629iQWwn>{|w@p~_~DJLl6jB+X9(io*A6K-5|K`Fa$xRyzdl4dN9SV>8b7hu*t zxI3YQex9N^$QVN0v1|6s!-6+cOM3S=o8~Vy=TUMmGNW^(Ln7YrLr7y$Qrd31Gay$@ zFN)tV`o9Z^+GWg)52?EWJIwpjPPc0|<)%cELFb%%#_|xc)1RNFKv?hJ)jlwR2YNXl z8jzWC{0Pksn6NF90NgDpN_$Rp(weDTM~G46lfj3u;;%_wRqIn*<%S4NS7s;=nsc$8 zr|l*7&ke~UOJ2=1F4}I`!`eH3vr9D-g3hyYrvlr}b5ngC0;Rl&2@qgK9Z1b2a@#-V zw%YZE;2lvoGslrhv!i`pY>JvI$q}^Zyh?XmeJkHHPkfn``((0-(G*z!%J>8OiOHWb z=R>Nx+N&E<;Y!i%U@Pq~$Xm_bK&4fIXNgAL^Ta`_@pF`W0uE%*QcHiOh`5FDXjVWS z9je^zmN{2?eFHlRTVsAG%p6LnYqZ2NIEgAY>Q6j}&t{f_sYOo23AhtE4GlBgJ|V18 zfQ%%xdPszdXpL#)Q#(uE)7P1tx@G`&i*~jkQ^KiI?Ro^5*QyroqT6Po-O)#i-Xm_(y0ycPUMVT=1CKtFtftP&m? z+9UCU?m@ovHfop|HOIP2pYDF{jtrHejhK>obUlhwXG3q3H>Gi_*I6w2emA5kLH)z@ z*?e)$F$08HQ7({ZCE7Bkwb7W7G%AGNMnGcP;*-ogg+hIr`*wFXPaAwqq95mUAd?NZ z)jz~C$|1bu zrrGls*giLZSaoja6J<#od-KQuton4|CBy}Wx&8F0tkE*-Y?R}pPnM=9K5$&7r)<3r zT>rscDC%;V6Qp201&P!*f+Gcq$+}?u(AXdK7*ll7e+gqUQ?ZH=~~ zImb-^Z)CNu_0F!z$fovB*RcVZ?(7;lm&LGK)SX{N^OM;VVVj?(mqx3LA-eo+t<5)x zR9s|($f?Zj+8{yHhWq{3_K=E9nTA?kD!RFL38&EVc0?-I;2%3d5CZaQ-SSTXU9hw9 zrA*2~MxlLWJxi+r6j&w|s4m3pCWvK!g!OGy?%^-0w5wrMX1OuTPg7b zT|UO5h4WObS+7K86HnW$B$`4(#CbA#RB;!Q5+uB8+G0v^6I2$po8AtI?PaB4WCWf} zspLC{b5Vs;@;UWYHr2B0-!@7i{wpb_1y*yRWU!Bv%JMxbHOUUK;XG%Zb7X4EO5c~h z77(*v(Tg%Z6fX38hfk%_ppW*}bfLb58zg^fn}~ve5^p6F2QHke`W_GYRJe;Wo&oy1 zSCg;5Da{78tiH*bi%!5o4fBn(BCh;Z8Vp#d1$vrxcf`NWPoPGZ>x`u#tUi;*5M~rP z41ia3`{EPLR^2PxoW-rXQua{Eh~n4JxW(*-v8Q>yNw)(AtCVNEi(=UoE}+8hL#y<; zcr$B6*m~Tmvufmr01q8h*IQPdRtb$HRDs&Mj0pv~EI3Ibn@SF%t%Bo{3y0mG&*2t0 zaV>ztcB{K6P6bVj>sd8%`0 zpf=K-sN_g16lYYb#QrZ4X8BmbuAYlgSEQlkp!P&=f@W(iK zaa1vU!!URPQ%UbrGI^CPL9W^>+{R7?E0Jq*mny=UaD`_4=&SZiG3FtaKdsgq-3I>9 z@eyTV5>S(rr`HRkgp#GpkUp0(k=T z_!{wp>iWT-j$l~~BRDZrFvAB(3PAohR42ceJr+ebW?Bi^7U4ef66CwUNjk6M3{YFK zU0RKP-?^|gPP!-zEwavL)-wOR8_JSv)^LA|Tq+8e6!{n-^-(@^Xf|I1^X$YGo3s*j znIEm%k^H|B@Up?%_dj+E)#j2_L+|b1c+4LpyiNZLt4q@*-S)eI5ZljnBfs^#>C_vw zp)}F?q|GNJIIeBmnk_o_pn-JKqQwyt$;wp;vm=0umcCCx+ ze{cRub2}p4tdw^(ED`R$G>Hbs~Q}O%oTY`G!u2r?T7apEBMvhXsbHk zpU^s(S@MS)1yAbF7{0oxKUSZEeD%ETr&r(^C8xwy^LWcx*h^d2g`u`OaEc+Sgzwc$8feHTT03U8ZwYU&<~hGc?tlyY|nBJYAag z*hRC^R^RTz`!F`l0k{^l{XKs_soxujL)cNLKle)bb8GV(YLjH}X5txUuw>1g7{Sa~ zV8NC73#|KYujmzl_Hq9kAu|tue5~%K&*PiEeP!!yzmsm(G{mFk+BV`l8mbMMGqjgH zrfV-Hd0X>re&mbhTqE4*T|;?cb0u^A{ORkW;3*s$T-lX}@hTlUB8(E%k35v!RCkChI)v zwK|6_{cnf*FBQ?HJe0>RPx$%zINc9R(Y9v~4ZRY+YU?Rbq*<5YAC&#QLtqMWz{wN* z2RH2~+y@M<^VpkwdtQ6VKzBk<*Z57Fx7wa9!yDT7va8Eon4RE+uF+}Mv|yh?bJ3nA z$Mn6}!8%i64U+WfJb+LX2Y%OW?D&kL_N&l@b%4NRC#Ut!x|)OB?<(yzu>rpvDg#KZ zGYuD)nYB~q)XZ~orkix>?bvbD|Laqj?TqIaXpvVIcKi8Fn|0;*%G2)-E!(*~Kbl_a z);0I85AIVkD#FgbOHVP_{37cHNh6*&&$N~~QXabQX7iy@T|c=y9h5dNF*>W7F|Euo zbwjMptI6136Irr^<8S`n;i;-GcJ^0fx(?BG`eWv~f>ho46e6r+tKwU9o4N2Q5g;CT zc+W*0w{pi%$ZW12tcY4xG898;gtaoQt~7LP-x_j*L-wcg(^2|v&M&o42mapIb)hBxqnmJIYKk;`^ByRfU1B~m8@m*+Q=9^E&%NtIA3Zpij+?`zvp-`zE=>O;k-hrUgQ-K|wudo`8p>tNQt(>H zT)0{YVerMuFbq_E91Eb+K{9+@3cIY7;kFSK2CK%GxR$Oyb6d-F$68%^B3%vON^DKr z`~^F36}wC!NeW$ixOFd2RF^u?EYrg?;XBd06wT%boKpj6_QGRtONk~LaT?WN_Hf^MS+3i z6~C-eNzh)I|KbIO9TY-N6FMugbVq~tVmbcLI80K~^t2Jw^j6CFgb_#h*lH`)haT?w zibJXl5`%el8n7hF_tJL9DMD70KfX1&70fyZlxr$gpvs&pw51w5bS)Ktn6qR8M4$E7 z9~7|6QP;n+Wlvq4=XqlM8I>qptl02&kS+_wAr?&va4~U0DFk^Nj`qC{3jK zXOf-LtPIyo)UnC2_@iq|D^j(!#GONx3*|F7cH@RhLwDmk9`@kbZyzf*XUzO0WX7q7 zi%xuQyLv3Kt-r8mMB3Q3H?FJlx}nj%Z_`?%$-g>$vd9MOq;0Waii0!wn5mpvBb)NT zJvAbV%cAQxiO4^7#;FkjmnsP(vxE|>>=F4XWl6?#L-||?i;do1l?xK&N;2Cj!?Lcz z@t3WdpAKovwW-;FZ)1LD_`YXj-Ae|3yMz~R@S<$qyRlJ+S`busbv-p)lM-kn5jS3C z?lK+;e==3N41IhAcqu8M4R1YQZMCcx^Z zWgWu%;WmMtY84Tl;7)-NO5X@s%oz+;1_LKr7|F^kTV5^|VN8^d#AM;sEl!Xx&X z5JC1Y7FcG0^)b037}WU`goE(*oTW=FmvPL$$YxDzsa!<7qP{R!1FPdPoMDaU2Jy*(>f!ASv1vb5mC zg`$xvT;RmYqBwarAJd~(tG%!h)2meK6%>L{z3Cut8 znH(ZM_G9&LN!bgz)me|%U8SGi1RA83TzJ_d5iWnvqEsN;8LM#d=`S49r!zEZdw z_Wf#%jFIlp^HL`vLC%v`QIX@KwQTRbJm^j6CQxd~mSIn;$;+EwNMSaRJGNdj~$ZsqU1@8-G9==6o$4OJU*cYVkq+31pvw1W^?r zASNpx`=g%#O?BJVRaM)!on3u^m43=HfAISzmd^=CVMH-`VC)pjsxX=I=pkwuIakg| zHW$uH5T`=1@|)p@hT=eky>>J&dkzSkptZur2AzGW%W_X;tG+_B2+XNoqzdwy)P4C8 z>XMAiQwq(ibi@Lye?Y2fhAuBXr z(SZ&|+S^?iU!C91P;JT#=9ftqz&QPYsgO7?e^Y2beft6O@nZ6o z;|N?VvtNJML%Drov1jvu1ePs?CfF;p&WYbNH2GcoQ?7vd12~SMU82De$_U3EQmMKz zeh6d&hn~Mz-8C5ySc+PuAh1gqz1!<$#D`1&*uFigXZ0__dM00D0KsveGJWv1f4*6z zHSX4))kYK_4d#s>-`lsUVbrLYx)a8CLys7sO^qjS2B)zH^M-tx zUU9UjO*O?Ll&zTFnN(2DCcnV7Y0JF$tfuPtWY@8GlV@FB_IXFR8T7b@b%~1L=Vy=Qq)m)p{K+K8_xecA`@)vz^nSR`T)bNtWNqfZY`>_~ zoz<1P)@Xmx?(FjWOuUt#r9Yh5=d#9j#dctqCF4TGZ&?%Q;ddb786 z$$NcK6J1m9??AKqa}nbDCb!I~N#oi2NyUNYp?3_8x04LP4c|Mgqr46Fe?we#>-w5J zD`0c4QWRBN58J+NdEDF{xn5zTnnhit-A&(BL3@sU`FR_j@RlYW9)f>PlL|--| z3qUiFWILkI>uA`JvdeE3tEuGLHkT{aMk2Zc1-y}>1?b0oo4Y&kXuAe}x=Ls6_pv1( zZ=f(d>GjI^wZ$I?>XYhLSKrhO3^GEwW$;e4?VSj4{3@GE*RRS3ShxH8B0}~@KGs$F zuH)sziBZAPXAQ@EYvjWmwZG|~zk8S>R9sU~9usq$I~XMs=+m5##lTyU|$xzS(MMdGXFU z{bc-yHwRK0|8PDnKQm=((dF^|krOTJl4T6lpnJIE`);E?SE`BV2=7~r#KlXtg-xV) zDe)p)~ruzKz0l`2}2HWBA%zF z6m99pUj2sFcyhK2+ixIvL-&rdYfwa0F)Z)smw3rxxL-y8VA0!VS}%91I%fHs_L&T? zC(J~Oe5E9vTqySm*pI2mgS`FLRuKc~)a%4mg^10a#|; zbau@24U`$1{*qmiCcCqy_iOF&L>5f^<^_*7bkHX_$Gu@-iK)5PP#O~ayb6+H?C6Q6 zabcKU?M5Tdip(gdv$ua7ego~>icN4>c70bt)RfFK^H%Cb<@p!vv;JmsA z_Q8}C3=Or0vX)~#rb5aB(L0pVhEerm0gf+qng30mD>6u$`T}~H6 z!i7mhk3OoyU^@+~e|)LMsixpkgr0rj+O?ts%;q5fYomp|dx$YYWQD*7WerY>9g3jw zqf3LYmg8L;owe#_P2Xwa<6`|=o>#Kr*r`%Vj()4K8fLcFV^5r?igQ92>@ZDH34jHm zBb~pqb4X3R1kHq%yY~jOFip%$+dt0`QFD zfXVu=WB{)WQwnPrKq}WyUPJbW9jFQ-)M8;N5bI@V@-Ee?+b_V^bLzJ$TmWUYVv0eu zKDpmEkGw*L(yL(6~@~4B~cn4uvXyt4R6>_T{j|My9~SXv)N8`(S}XoKp_>*@4R!5yQJTgaY%D4s^s z!-XuC7#*Pz_h0?q;a|m5G+m|;7T^N-3;H9E{@qYcgW}kJ(WjO^vYU5%)fkm-E(@%; zMy`;$5a_HMZM(FkEQ8d6t~~uS+_#0+veo!hJ{!}Gn=4k5e5|L@l{Mo0V`Zd7*pA4J4# zB^_$dYdpV+&J_*mvIU&|7B11(2lCZ!hrHH)lTRg_2k5Pr%Ur}m(G`N#(LU# z`<+{9I`_a5;w}SnQ+8jNP=q&@Q0%V4y*9Q#7x(yE^T>z$Zgsr7A^E$K7Y9yf`TaU4 zyE3V;${M}D=WyQD_N)G9thsq3^BCbilHglE?M)wjUEBdIwnIa&tsOCaX;5Ijdo;$M zdS9Pa^$q&NCU@~u+2%(NVqvxBmh2A>xH)^ea79~Q*ENv|8jS4UmVY-`xa&dB+C8+T z^=FYzf4;BcW8*-B^%?6tx`Ov!Ok1-9T4zC!;dlj;Uk6Kw{LQiCJrfwlRcT!CGp+Fy zkJHhb+M91M^-+H-|4&L{TavZjunD~~1QXOxLh05C6S6OFxxFlC=JFu;7Vkz|^|Zbf z9rIkw+ni%#>tn8u|MZk)(?`+unJEP`+qJ(gA03sneqgqBMRohf>_yO*_4`Z4 z#a126W{}$Q%zVeVzCYhgJF)$e<+pyjP5Ue_UP2CYFh*D0(6M~&je5(4_^wW$J=K;; zse_c&Jn5Q)S!P{N`@+hJe~Jb&x+I2pPgpS&&LZi*+0wO+*bn!i$JOAU`l>}8$p=Q} z!NAw)fzG~(I*;=OmD)1BdBWZ4Gb1JiEVDRO2gh>W#WSY-Z1$74Z2#cgweD+0<(XL5 z*59zfGKw9PqI7rJw+K7gXJ#tV{9+9kGrnA2r%h&ifvS)U<~F8MJ*ScpR@yC8Brxy2 z!utjJ5nFNre5Y>w$a$$}>i0KwPPNzjqkUeU(9#!GIDS>Vw^h{H&Rz|EsDv$}whq3W zX7YU$`|XJy^Py&~p^e!!fljoE2)`l&smlXKy=tam;ESN_s@dOc-?zl_E0+GD>U&E* znD2VLWiAiLAx+I@9bW34FBa>Do;;|1k~p(`ZUv1c+>Xt)PP6>FyNwnPBFT(ajVVT? zt)Vw|UFuMDQYciPr7~S|cRUxF=S{40Gu1Qc8ztLw8fagGil( zf35y2?F+76C{qkf5#BDp>OEU9)UN9w?RCoBgT?#C9n6eo-RgRw{IQZ|>tkgUK^@id z2v_N2s5pt_$EFh#2Bn>{`$Hq*`p63N2UvQBLI$nK+$ld<8 z^&NjZ(gd!t3Q<&UzPQl6Y5njP^M*mXOg`P0qWz4;{Jf(2mf(Y-t%kAnRqO-j!{-CRo9gH z9e>z1-B*O_q$u&}&5DXX8vC_u4c<}_J>8eRL;OH?kK3fUF$!rc<>FuJqM_aZ^r@R=SIc$%(B)GM&b7;4T zS>1PY$b9&{%ab%Be|y0YlwEErY@#5&TCs8G^2(6&OT)I@TCM5bq^+_}Nbfy7kML4N zHZXCtf0($%?v(*+p!5i7bod~tr6Lq!1(3fY6Li98AWt zB_N<|dzkvqIn>_Y8bjR*ER`-3ydJFu7%Q`X?=(_q-!bR9*`WujfrlVxFtR{(ahXNv zjDWYhQ#7Ff+XvOA#1`AfoC+-)b=7-X3ovl5qBTL;fPpbjM7;^oNZapk8^?%FkUZ(% z%R}sDXE?U(0rYHY5m3vQ9VURa7LIKht|j4_)8%he$fsSwI9&YJN?Vc&Q>sk$H65(C zzx~G7RNHUa|4b@ZN@+W*$?-UELqK+Nt`LKOKs6>?rU?mxVFINgi}h2iqfy{4nZg^C zf7gDIMEodwi9qWr>m0-JGJEaLJ~lUcfITOAD)MOMptKIB7N@xzhX!s=aGR1rv3XLL zF6Pc!ng8bNna*Pk+gT#brYJz(XOfBIXK2OG$rYcPqY218S0HtBx4Km_qi85!UugQq zkYG);eKGyCQoYlnML{#X*($SN`)Xx}^z;06Da!}4T(P(Laf-PW-ctGB|0;bHxv$ZZ zsjCdP5VBK^gCR@`#JNP6*vQSI)go91Trc_0-Tg4E@OdWtL<)N?LP#X$ls%6 zN7fF20@Z+y6mWs$K=ZL=FO1i0B-BC!?D(KcxWrcgwR`)2IP@GuhRPp zS{CWW1640}>*0=&ZJ9m%m?1YZ&c;3jnx|(f50oT3iEVt&TA9bLU@i}lqR*S>3Z*Q% zhf!jmlD?8JOPic0!LnRVR6kZ(0mH;J4;=gGHni5vAW+a#^$lb*fd~H)y#9}d09!g4 zQ4gNI2ft2wXc^m;Sc=&uA2gxtzbfl4(bGqHTqh~~y9Ckx@VBX}fxK8X*@W-K$J8wy zDjrUc>7tYftTmGo0`hi4aX~C z9|A1xz7b0Az~5;qqYBU1_dWHoY9X)absMlIJj_INF1cKyDJpmB3R2|ZeFrjL8En~- z)V8HkuYbV4_VA@(Ha*vb_SN=#&e)tEGQwn=8~%d9U~3$90Ixnc-Hq{}&u-UsgrBTU zLKWGT|AM`-)fKq6W9N>9Z-YXweQ?Rg{NMll)5Y5*9}k5bZ_s3TYN*Hm&YSV6#A@xY zw`*-cGmM|pgRj~!^PC>|=haaG{TGq0;Y8!(6Tl$@;p96P> zE{m90+t3?6IzR_Qj3()mDigwW-rBA86RQd?2DcoTY+K~1*_1kY;4nO7sBI4q1I)bB{XJjgzoOS z^w@OUCDLxgOS;~OHqToffr=64Aq&kWBO@tF+eODI!E!HuBiqKacqA(d|fdrY9H$twKm<1w!-&1by?F5 z?fi%9NB`8-FDJoN@S8qZ_939)eS5~!|WAg2t1dWuzk~LZ8y03 zQefyi0lpYUs!im}@@NupQ*1Gz6D>&DTC;p?1Kssg_r3_3?ix!^FfTd!KY}&)KH3n! zIbXLvJ161D3v;9Y6K_^LkK*zaon`G{r!Bv|5;mAbBj;%Su?GD~ll#Azr?PWR!100N z#*2k(?(8;RS{r%xl80qLaa#KsQnTSIh=I1eOx6@1p@ zxD;%h88cyd(<)YlbM|;P!qsz2@`PzxI0=X2M2fYV+V>Ks~vuGIuGdGC|K zk`Bt^F0*&Z`L5sPSQEDdM6BDi&>2QU@9IL>&CxK?x<)UY$OuuSccCi5?iKlMNikevKPuF42o!f6#P1^w944aimYFT9t(<2g^2hZrEZHCBlEKpKz@J@3| zoG@W|+(uHv&gNK_2h?XVhvo)s`MU?At{9=-%)qLZi6f8;)iu{Xm!Gor!xGCIn`mYQ zxC^~2YbGVe4|~~{b=Gu_f77u}qMzrV0v^18-ID5cxCKBkRh_$?Vk%!Ek;S17o)za` zpT&&Nk#(bMP0kJ7Lg|MdO(gWoCsB*~O3+wrtA0 zC*8x_S5LB6;hOrfD7al@d-J9Wpx>HnZRWb%RrMLNtVAj_4OpS@g!64#qrXLpHfMR+N;$y-v5 ztrT&PU3iQ%)zdN#DX9BeDToq!w){>%F6O9to+hm?C&@y_Y|8Fy&oKOjSqSTG`$BI- z_m0H-bR}1LQ{y{^n;}SVeQ#2yH)&eUF@DH=5}HSO3wee;>Tg=3-n5Ji^B7pBatQ6Z zzNV_7?UnjfmNjK?Y@&TYKCACAyp-y0t1_^g9rh-=^3U?7fB=vInp*LY2{${3(tp^l zCN-llY;w!E=DqQl;`c0Lgxwn6Z&vKXSdD`GkRI7Xf)@T#m>&<(L1U6~jt{X+mnk`H z@5cC{K_TrVjnP}_xJWG`PfFF3NLg=&dehEi4=>$_HYmOQNHgq#R;g-1aEuC^MrSWQ zp6ya-J(wazE+A)RcupETt?0ENE7c$!U&o<8boAyKdNsZD0d#VV&R;xf4-}`l8NDHu z5lHj^j?!z~9%A!P4M`Td|AXBs&?WRMqhQIqq*(8?r+Dt4y2K41P~0m|Q@mFsPLBkLkO+~%)giOin) zHM7Y^Ub5}+mX28LRFZk#an^Q#*%H{cv6FX8wExp-MJm>1MaWrrWtryT4$sdGAv{ zS{VUtMoYkJ*`*!9)moQ5ox!Oe$0P~{!nP=6&K|jarF6|F4~454J;CrZYR06?t&9!W~d(MNfjXp zToj86D~_h*ke|dC<`i~TV64=X6m^QcoT_BU9(q;%aqJjE$Qx2855tL8Za7Cs?(Q%N%8`!1 z)A=BL(kpfjS^t61K>YO4%1v^ReZ?&sjL^(E!cx33;;>g@! zw5$4&Qppow_eVe9)s&3x`#6+v!H>+LQTS1AQeD#+u^w{IA?Tt9(-u1nBSA$s?^3iQ zTY%^Z#K~coxGDj4bS_Z03&RPLd~Tlu-}I8*FdyCPlqsrMeVG|pdU6A&p-_cNrJ}Vm zX5v46DW&PGxY>#AV@dAvB6?UQLzgi_XKa96!*EeA+GqPplfd0T(J5$+8wHz=LHCT% zWp;v#GveBzs#z<5TRJTJgWnuIx88b)OZk2ja!*I8y;!AssIYW^=*;t^BOg0a2YgWoA8g9Z;+lrc&$@ zYAS6cVGxoPC;KeJE)_~-Sr;|gg&HyfF373ZO{7zC!_W5j4)2Wd=7l#uh_z}2t~%(c zOUQy|vXEz~%XV4)c9pK%6kSW8hAm~u@s6;n&$yN3JxCCigz!p^t2fx&olIhh(I&|7sQ3;dxsQ|j|VyTIbT0ua&wW} zOYQ-SowgHAi%v55|80fK)2NW0z!-^p75g}V5~Zn(Q)sGJ_(q)Q3QgVfyUY5OK1g;$dK@ zzViVzROZq$^|zw<{ZRz0fFnO)+av1ws8y=ga|=okJQ!1p7AXMguA{e5P^#L*Q7bHO z)2{=|iZBEfgi^|~$l3lmVR`Jn-p}Z->$0ila{j|$Z-lsUmBLtfP)=OJsATU_U+~g0 zFwJg@Ck2kDhs`PYO?$V=b5UiM+JpWL@cr=L3Rki|xXL#eg+bSE?r6j9klh_Smj+wm zVBE&`=q|gBY6u!Up?_z6dB?MbEfu%FcbHf(kxHOwd(Oe}mKi_4ZcB_x>&ZXy-PsC~ zpm5Pa%w31O%a|21s`{q&SaD+Ov+*s3)nQOI!Tik7u4}8Tc_Tl_iN4qGWiwXj&U9`x zG`8hBJ>g|dUv{p5Br_j!PiIOoa=Dv|XL@kJrJAqL-!OIlZPIh^7QQ=wDakAKh_-w+ z|G85W5%kQu0I79)Ek2j8NpSSve6FtB7fZOACLvH{FVmCAnEeJF(^Jgz2kS9KH?`3`PPslWC*LEw- z-&NBgU&oin8}rUu##?|k(ff^-bssS=<*@lsDcZ88hN1Hn64auF5^YWpd}YSHn}!rh z4ZH~O3;A;3lVYJQi}Qpay;(!T&#DIfSM|Gpw)B~ut1EXtmDpA@tB>HIy~k~z{3LY_ zH5CIj+FhN}t5$Zcy$(C&Qx=jKE!+5tiwCaVOI!AOM%O#A8Kc6NTx$4vOXR#<+vR5~ z8$vpHUS|CZx`9e9VDX7~^Mb0{q$I-)*AH7S-^y4J;Bjx#N73e}t^jkqH8WM$l}~4W z+qlTzGkpf4E~M~=Yc!KDo7Fd20x=u6Zx4!P zj7iDOQwx3@aKYvGRWl<(&OrHoRi8NYeC4#AOHOPF*u?H+$vE8~KI>p~@Rs<2{?q2z zgL92F+M-2wo_b?Pp>vp__pWQgjg7YI?Wb+OX5wGv##0c%)qN|@RLwC>|L7tCI!q)_ zjl5T9miWo_I{ceVX)A}}}Buo=VH z)rned9U@08RakpZIDVNmbGo*{9Ky^|$Wp*9eVW@z`ZJ^cWSwcBU~~D4ap$V0W#|R~ zwnjo9t*3KI=dz^F|AJGmh+^bG>X}1`uIcEvC$BR9B6sHe$&I77UKot8%?R#H|CY31 zB6C1fqdRYpuCoV{6mQ+rP#sLrl||#77HaMwYk0MxJdQbt zWKOosoMF4L-B7;4xUZlt(90ZJD(SSPFg2C;*bozV3ckb}>Sg_eK+N`Y=TUYajBjb^ zYE%NpqwWVLL%T;BOm}&f8cS?CGC8FgHG2089Sq)hZFuWQyJdR9Z^J>Lom~FXUVZp? zLD|?@x0cMDihGR&ium#6>llsO5D)_-RWD`wQF$FukekD5zH3}_`;yOi(TbT_Fz79GF&L~+i#+lhz6jf}h5 zTu>LGS0=j4A6>8N@OZHq`r4Xm`&|sL6{r#XqvLGzgHLSUCT}I-8`cLdZVH{gYx+z> zmb<}O|87Y~-`X49X_@{5gF}m-Ir6ndGqf|%rgeQ`DGj`r$kH%u98T_OnRAVf9d}fh z#e5T_kkZYOTszPb-MZ97Wb%?Zn>a2;g1AJa<)r)hRZRe>h49za6S6-f>&%VB=h&3p zCDVwz=dC0)M5f@g#g1;3&tFgSy$Pv^5>hfd7M1jUs^tu(rr+&oUyGMH(It2EpS0bd zGD5K;Q2L15$amBlZP3v#lQ*k;ipfqB2R?m46PR{3EurUnWT$mBOBJlUqIaHOCh8Y^ zc8R#~X3A^`!VK1yUl^2_Ef}w9* zUiI}J3|3VHOGkwNKo^*WAc_&s)5xO3K5K1x$LyW2Y*oq8B3+NublpGgqb`~~UIx65R-BGQRbxj78q!X{! zw9M7UT5}?JGx~)_(~aN&mso$3w0*A%h&yR@c|c=^3@&FdoMmj4dHm@?PmI`)CK>)7 zXq^WMv^2fo{H$-hy$u01*4)H_mrH7`8f4|~&Y$(Y!%z32o;2K;-ZDo}=D>=34IPHA z`T5-DsMCE;cE&$Ft?Zfzm{TV-&)@jsuf*sn4mmqH03V6=OGLL|(07ft_vtgNIo+#@ zTacZdzEfBfz$&~~;!k;yg%blB2=^x)ZiSYFW0dB}=^{yE21hd7gwRmIpimR>t^6BN zqe@XJ(H+M-{8>DG{2hY1M?Q)-_emYav8__m=WG!8058K^P_Ejot0Moi%P*$S4zL$*11Gy0+HSdVSNW^IPOTvh~9oT~e|da6R_Ia8&f z$dlC{NPP|Qd6Vl7XoTZRLOuZMqnqyiSU z-{ybn;X+Ol+M(Sxzp$(YHlz@%Cwqo2d91x7=-^2!oos{nsb7d$4fmSE$}saptsUhV zS_U1a=z1E%fmQK_EK2k^GD-E(*+VnrSVPqV_X@<)acI zqksy#$5q%fN$G(=^*6;9_AmuUW<`pY4_;GOes|dE;E=}Pkv}m^R#`Pw$no@Da!FCb zT9#ApOAfG4;+sO#AiV}DrJP8+E(<{Q-@bZC1;arHD&8znbKmWdqNAj_!px;;6Y(y+ z`m=-dEM>$Loxk^J508=T-$$Yuoa1Sk)SpQY(cGo3K%fU5cIxSGgoQgj)OXgh{F=bs zVI8}+n)l*;3>|U+GKI^3;f`CVtG$VZdQ2p+zwC_ABb=qRwixBgu4#Nug`TOwc4(ir z@TbDZ>N3>2lNO8uOUhhezx%9sq~LP?I8HGYt4`$9Lfe0U!llv%6jCD7y1>dxSpQ_2l zOMtW~?LWu@7ey(LV6LXO&hH14xW52=t|1ue(6 z6{#OFytDk5K<~#s5fQ7>(A!T{wh9ShkS3AvtHU7Pj?4G}luL5v_CC;$T%8=2U4kV0 zm&8=?H2vFj8HtIrix&h*4ma0Wj*_aRe%Alv*Wtt^o;^7*nRW`K6bLG|72cxf&KJ>_ z5{&>O%>9&?`nWQM%h^oLZ>}zsqd-D&UfOT#%@d~PZ+!;?)iWn#E1Kb6x{cNan;eMS zwnzS-?T#zQ_X5vBwyZ+=zSkb$+yC*a7*>ZzXboQoj#bEXGrt6eswzsVO;wc6Dzc}>uk~P zbboKnxCFuYF_Xv{M=@mQ|w>SbB)Z$ zslHp-THlsP)@q9O+Lqu&bvdmruJ`H=+=&Ho`uSr#10_l4`TsKZ=5bNp=lVaI8hT6& zr>RP+hB-Y=n$(Tv#3gQwCdb%lqNWY2ELCbiL5TrG(P5HUV}+P$o2ZQ&E-@MwaRfwA zAQ6!ssj?UrAxM=X>xc*pIKxc7@B8y$PW$`)zOUEs4-$l7=6OEPXSuKYy6)?acm2Kd z8D&HjegyV-3MUMMA?sVb?&qmzCPe_9{^_V(i~5p_7N)=!B? zYL9GA{3ymPCA`TpmgUDz`FMJ+Db=44QH0QG5Dj^w0?InW_O&QSSBypjcdVM>b;BSE z!6Sa~315dXNM2Q?ZC)rv4Zr>t+*DFnKj~a0Bw>X6TR-|kZS$c28i?v9PR&uygNeP; zqQzGB77aw{n^S`Qvs)?~0g2uD_x>g|vo|6{)t_Q}Bi!(@?SAf+7yE*Amb2MqbrHHY zoOWxY!(#oMKfPz%9%)(E6`4_ReM4+mkW0=aK3MuR#(4a&dan1|#?ze{rxMDzw%skr zT+hVouU{H*IzIkj!<-5FOOy2Gj+(Lhl?cs=hU4jB*{OdrC@{nycdg9^ z$#zD(Q)+%TUP~y)PBHyz87)s7C#4L_DIFz5LY1}7Yr>|RJr}XxK0rS5SyJfz94mP* z!CbwYW{QLzoLG+2W89_@zkG7Uc10g99^YKP0h=(Tho*~bgOR=Z@>AtQTooh?} zuyv+0iP?7}ZOa?0><$n6Lh~`7?&kiwg0EQ{Ab_%lyk8Mr_o#o z-mPY4)K4sK_>96qH|OHzhsGGxd(Msmw}C^r7Dd1asPynCc5Fae6@mkZ#(MTpO?dsG#At;W5fK@VV}8 z^Se{lxe=oW>%LZMJe~85O*Ykn;`+UGX^fSf{PMSt+plCW!E`gebLr^{poK}WV6^ne z5wH%BnDB}&wyYa3P^GaHGnn!F|HVTL_8~o(>m=G*tmB34=m*x{QrkDRV^u{-k z*agaz%)m#(A+Tkc&E~@ZHg>I~Hm!B{oK=KZ+S;4X6jVcVS=+ixth)O<=*PmOt%kaRheP*U0p zk7y60FTx0;CCPq86=!HAw&ycBv!{3!EwYW_0$&=_rEj~UqBCWI!GVoEMIeP$|COdV z@hnA{Kp`VsJfeBy0tANA=p6nX{rB*3ky`3^DVo#d#LusIUt?z1%!m<_+_O@kYn_@< z78LQ@Nm#SG(To>!+oU<;%Pb>*3sI?F?RVZjO6n!S9#Z>TUK!b$Q0m-oux^RZkcpqa zYxlcTuDx5R3CxcwD=^m8L2@BxXq;tPhBJ0&NM!iYYq`)fPOW^|4}zVz2j_!Y$DW%d zq(xp92GfNJyJGADOw&eS=;<3AS@Ao*E*nLE=$W^SmQq~p^G+GR z_Mig1ramL0a*FLO9UE(py5V$F%BG zDYfKf;^(0}gwW{;e1ijrdlKlmN(nWkgf)4hi<0rBlBk}5hw?(A zR4OGBZU`D>QY%Jdh*Ks&VvC3PLH;{J!4hU|C|d?uZ9&e>quDY}kR5>Hd5(zvVSOI( z^a@x!1YVTO!(5k8>`_UENSj}Zp;a4E2nki>l!0fw!j(dXYJ3e(h!_kgG!#v84UHG& z*TQTfLm&`S3+TRuYDkgcj^<0k;kEgV_zMlgB1j9r#YTq6?fdxu4dke^WkrY}p73|1 zuv=P#Qw&W|M7fN{^!l89j=*Cj!jxRudn12Lm5>Cts&lUa7swNolCQgMOh( z7s)M(Sqd{l#pp5;uuVg?nRzs>DCv_3Sp8hCB$Id6XhP}O2vQ*6H7)iO&=FKnWP4K* zilbe~+X-8$CT$Sm2c4$^Q)s<*quw$mM8(Mv?I=T7CIU#6Jyfz|mRJ-Ii8UmSR-Z$# zRX!|ik#mIM5JR2no_xOxo=G)pL1PT|#)ZgOdM{NKAy$8-b!HKjB~_y>k6mE%8EKPQ zZ8AGD$^GIfOx~R5mCDs2SGVJYBwBqQ3mGZdngqJ;4klLwx6IWAhwRezbn~mXq(?13 zCAV}NGxi7IuBEtDA8$5UPYiCt;lbMCAcRRq{sOGC(a{s6f`u_0DM+h+kpX5o;EN`^NNFJ z;maJ7Lil$tBKlQ1B63Q33LDocq^Q9=wc^ywR|{_*qa-Lu`;yC)<*25tO#7jHj|RXL zv01J?6^y;Lj~~L)-{Vhhoy%JP*DnQ_WtV1=uz$RFOfzw=V2_o$1P#}zTnsZzYg|JT zuEkED>ECl%i9-v39;LFQG5pjTsTxsOR_3|#d6PXumeZ^3OWYCx)^&98YWtHF|GgTP zSv|D#P#7@`MeO_O7a0Jhe3i;swv2!Jg7D=BlDi}CCGO5@2Ulq-uEXO~6fiHc7{_(f zI-+nC3RcsD{@e`iD%E|@Mz3eCdld!N9++PC%H+Qta(`;8(W3n6!f$yBpA|s|CK@KX>2-``fR5=2$a<&mj6cJMFw7 znzKcB|AD{ljpkj4c5g5+yY#D1ZTl5{C2MMw|IQp&ctlpO*Un^_BIliF8+PvHMeD{vQXV@l?%My{tw(8uXrY$@3 zZFu}ulMRqwlRS|5>Ns44tP`L3XvQ;wH6pC`#s{&dh}O|X>elJn##YfK2Q))76ifdN zXO;J$@8=<#-o11C%6yMm@t}I%$RPXSr%p0}BPwYTq_YQ9LC2LQK;~+Hf=YY3s zHCazXInCSNA{9o*0)6H+?B2Dzu{PS8WU|Gii@*#;U9BV4QjOu`uwX*h(p?r%V}9n* zE=yzG0W?5o15kwyW8Xru5{hE-bLvZ}%I)v0N>25!`HA0_T50Cmn}RDSgI%ANk-F-B z>do+eoKPv`n&$DRa^uCn=XdY8(HXm6*P9ff&!m}of)rD-*L{q~e{exjyjR(VphbiK zvA6LC-A#JNwj?(=&I?>V5#K&7J@jJX4I^DxbL|U5+w1g$=Wb`BU>z7R7rVatGklYH zALJwYKN!nT6%_5OC2ek6A(bGd>AoV;#Gn81U-p|z8*Wr@PjBwOv9`G3U|3zS;p1&5 zqC@vzotW$JB?A#U<{NUmWxK--XG++i7F3^{+t}~3c^>U!XQx{^3j31BZhf+#qW8Sv zU7M}$_Dw?njvePK9ABoFEgJQg1No5wn?AWc!*pwl@sS%L#!H3FTP3{nyO+Gvlgqtt zpXtc2`Ss~l{>f|;B^iI8lzghlr`ofR_6&6`ed}~L|FZJ4=$s7UjFL;ozbL zR?mU{vcU%!g3v{`^p`!khlOr>e(w1LETh2(-bsQ>oO4r3tkfkp#Cmvi zKes)`;OcxXKxbQ6GnilQTfH<8aeAD0a2G{mdFN($*9P76yJcnKqD*wLy3{P&lc*^XiJv4ZF0aA5<+wTjE@nS%Z*f-O^O|DEMnF|5ei6NIYz!pL z#ll+s`ho)TQ#%H2@T*2?O3vHrrJL-6!!4IG^-`Dl`@xm;sYOP7q(0Ec;LDjs5OFpr zaJ*-h(#CpTeAwe*A&iFN6Hy@c&N-EZ2??8R@j>582)6~-bPDAFd`{oo;N7ad54t%5 z0x8!J2UAhTek5H7D>#-^^MaTOPyG>#ItP3CIHiPoe@c>jd@-#iD zVvPxKdyrr^NWd;(?i24`79*>wc$K9%PRq_ugT0f^&(7dFXtBn=jD2lrml*yjLO=!j zxED}W)I8c(vj*_`AA>c(92P)DDq+59Qb$DM6EAvj5`8Js-sMucb97?VS|xy1$s)wPSP)Mj(Ou5GlBp=GD%h1XIH z1yHA14SUMX5X}IzsgVD+ zs(1hvetf-k%`4tVI(v57K^=LTm1BahI}fC-W##nK&o=eT$mrdD>pU)(y_M3~R@h@l zR&3ZD;oKLo>iQz<7bCVTnbQ@}KKHp>=f{`zTGrYfI;U6EB(KV{e2Z_l+fL8yq*UX` z*0a+l+%f)P?rVmoQNMBi!qjW%-xzKzyOX+GgNl z+uI%dr@!ygb%t8{{1~^@4go*atFY-d!GV-}s!V>XIQvLJp`^ZtfC{p2D>({`cqoH}2#|Rx zzLtt~grqzH-!4WxGF`;nhnY0-`B|`SQe(q|Bb+6yn{)uerCH$F-ihYE!&lYMLSr@!`!Ov)`NVz5|HiM9vDI_}*eGo*<`rj}rf{#Vn zsF=Nxa#wXq2CI>dA`4U%X9DEpAB5V}3*JnV2w6ndvGI}Q(<5HZn+zxJmO-hF(;-qqbx?)Nyz5S_>YD;v;+OU)r512|(P1H*IFB3Gc~P`Mg_fR?%P5QJS* zloEx#RUdm*5$a6GD=8F;xMKbY;aD}yC%w01WZ|nAA5^p4dq_BYi`~;UHeCAKD_yJ_ zeF@z?_3Its#~ibJoPh!>dne)yCH--dMn85bDR@g&>5a(1Io_F2qXvdK zIaHh{!8;feb0ZN5dX=nNUx?+6u2-ow@hyh>0rL*+We>|hJ>>*Ld`Y{PmnsCOxENbO zOZD`aQEsscWXQ%YvR#ce^~BOBj+Ywg<|3^wUya0WVEjZ4;d2x$`!srUETy+Hn*r-+ zSmhFXx3GSR<+5RO1Y0<0;2^73zaFP7bgV-E zD6q*gS!NINJF;Hn_W38(6}sarKqfUtt1mTsTLl5gx_O=C4Wpbbb4P=(N*SsQrg&Bv zC_$Htt@=I-?UGcJoo4hhm=d|xOSX*~KP3mTVW!KS(P}kG|6mgv zSSVvp9(!qw0u|z-OD2@gy~QZt<1u&2WuZ0JdQ~kXQ-kSvh)f$P8L`q1cVY72cP^yC zE0D-yl!V*0s-?!X8S0xkO5YyQF(8Tnyk=~c(^9BK**lJLU?G_vUhKv&_YSo$B406o z#PMd8tdPknMg!m^l@9yqjMuV=C#)p96pSXb5rGVs zJ@R1*@)&^dkWrU*61QR=HBZc(h8`#_QPi#|Q-B#~%0tI1+u{=E`#ij9$rw#bD;7Df zRGF!)VqtmF$qM~C1ls^8fr2SmrVzIG{pas(op-n*Z|3q4e0pSNvc8 zFMH4kAfLovNX84Q;+fHspdGU5rsJ^eUsCHW3f_e}t-I3l7}(n}ArPO6_GN|HqoN+z zj^2OIU5uE&RH-wT^c$Qq%~xo{^G|5}pT2)+ICOgGHvZ`%xk+{_?dksO7%W1)j1;>t zX|C?bK+l46r}HT{oc+Lf$-}K+U_;aRq)A25)>U3tH?Iky^IBu?DPxoS#q&CkHF542 zOPoI+n=RF0UMF_Ecw&2evS)jqzV=i~evG?uojt_rL5TX($JUD1=+j&8KC3&ma)Yn$ z3xw{DA3H|oB5HfEVN5Gy27OgIschbb8PD`=$@j1qA@4b(6C?h;rHgB;vNoh`n$c7| z+tc<&evF5Qvj|U_&@Dhkf9pOb%(lCCCmZkRq61sbn4jBUr-M2==WY5!NcT&XmcF6@ z)daM%usqUO(O!^#ZLf6_y-g2WPoMjw41(ZT!-_`#nvj%Di9NSHx2-pxja*ky@kQP5 z9DhC|-RGJbU(PQ$%Y~fCo7xq1!8Ie$aJIxU{o0m^HH+L*4SCmUocR%YS`p?+pG}ey zRREIlUJ(bZ@6?Z-GQ!fF!HTY2*Ph>K!%=4)B3+K2@`%%wTW5yeUJ~-)iTV}w^~SP> zM=njsKQXwJ4k7R}U6CD7BoIQY*u?Y9ykS8?gEjHr7I|7rJ7&{zfy}Hfr?72|dKHY> z+nz6-k^DoWZFL5Myz5gEF6!y+eUo6Ggwy30U-8MbuSZkoci)TQ~ijn(6!L)&?_)&u3PdWXpw z9Z0XDty43%1pkRrxt-E{v@mJfmX3M9am=(#dpg8_mVH@{F$(Xwjl@;eQ#R|u(+oeS z?$I;NvVJm!X*aiByENvmv)F0ENe0<=Lw-0Xlj{5s{imqUyy91+Zw_o?Gr)Irp_&zlk^);q=0A$V_`y;27xA z@1`z?PNuQxA4k7@O&mDMx=?Ug+lFfn0_AJ}hLg)a_qz1X$W)y@)baDwhKAak0m;4X zE>O{}&N254)g~%sOCyqYEpr+s7JPE7HNLoJX4OBMiuOHQo0D(t-6#ge77+Gtxx zVb)q_Y`zZR@h+d6mf)ne7)Ji^O# zHbhDt%HQ3ty!_~`DGf%5I!VG_KogS(jKm@*6jGY|BxXylcaL*t8JbcMry^zC)C0q# zYVZi1Lks%!VYWAEOj?@qybC`-E`&{FR<*wJsG6X%k?Gtcx*p7mULw{ zrFPDFZ5Trq)|S;d8-qqjD-m?S51b@Jx}WXav)8#D#{a-WTad%#BpgT6{I9#vmnGNk zYg_Fne@yj0crF#6d`&rXAe0+{3XVD?*!6cu#Kl(dT z84{X30lX$DV_@(+hNyeu?+~f!vb8qbcd0gcvT@TVj^P-1dDJCQ(Eh4rUF-16c*Rz% zq;C6!Z8bV{T9J&6L6Gg9p<7H-P@HIQ)42;TSKGXN`n*xNQP?DAyS9W>YNlO(g0NPy zwdHGONMvVfLRo+WpR#ZFkd;?A{AJZB(~g31f>vWK@j_mw*p`|?!vaQo%=p=pV}#O7 zn%x3|QK8pE#j3G1XE6Uw)%AwwO)o7v)kp7SSmI-|}xoG7j}?H%mN+YE$9cj|;6GxR(qkyTaV1xg@qM65$}qWiG0>l^p#d zWm`3Wta?+-ng{F<8s5=E7TP4?t;o}T<#QfByHs~CNus`{nnD+RSe z3PT+;4g`^jW)u<%;lNHPrK1L=F~y;+p$m^p9Fb&|@kx)+sm2yg;wV^_{oC9WAknLp zp;7+P$DDKN_u9@-58)TNqs?NSLT=bO_eIfWa9Q67VJVMV671-$dIX8_p*`D>T zj=7bv(-Jx($C`Rl!d(hU|F;qtGz8in8xuo-nj-i4)=+{)O;#x8LYq^Qtxn9BQj%%> zIygbtQiiN><9yvMQ=`Gk8VP*DOVzIS4}Ve7`eCyPWE(IX67ho$caQ3`!>f6 zu_RW9hLEQZ-Ex$n7n^ix3HCzVhI(?cai|73ipr81CApS~B^>doyj0qmK>kV!f(jJ! zQ*`viG|5v+FaajRRuXL8RNaAKx)mj>vKe5L=K;L5?}4)h`;NIL`5Ry4to&C|jE_k2 z9Q3LZyDqAvHqCefN;qE^sf(~1l`cnLXvoXz;c3m=Y8w(Spy}yC z5@Ex&^-Nl!dCaP$E5Of0X{#pe2#;IRic-5GP;u+v7(V3 ztWkxppQt^R;gCFQmu*trjz*$B!b3Ucc*GLJRq$ybNU)QqHw?6exKj$@S&C9m+uV3gx#yH8Ti1j>W|M(^0@X9o4U6kxB zPaf$cG#hhyS7@+al$7`Wye3WumNF#;*ovI`DG9|})$f}^ZC{P^ z4d0}lMY`2RpS27YWfJHmb(CS5*_Zyp>K$khLpK?3y>yQZbc6AjuZO4GhkfFn z)Sq#1ZAtrI0+&@U-_-au5nzhb^cl{$AVTatnf{yV@pXjyy&Pu#0i_~ z__$qA@L6B_cY;eM7>`xYYmQES<-_<1#w3cx2tuwGN88d{>Zcf+@8l2fT4Z0HzQOs^ z3g_iDhCu7-+E2>5(4UWJh}^W}S7}`%*NoHsq3^|`!4VtljE;FhHN*OT@!6w2?jfz- zq0T2m%(TjQD+m#uVj~;(+NS%Wj|G2sQ@HCk@LE*Hl!#Y~3Z60BeMSlGYV*~{DFv`M z0k5@d!~BA@ID~BVy56oRKietOsc>uIdi#j0zLl}E0&0rmJbv80V|$uq>g2i0LZdgF zD3AWm<-A8o+RgO=`7lts53c|m?!{1fSusYv2HNi=_c z6AR4i#i2Jz^?$uOmc4QGgdOr+B=V!C&%TPbL6_6OQh`%qi@mdZ{!hMhaof7`6ot+uwe-y`xzw=AeraDn zQbD$H;YM7dT&!GrM7V^nv_?k5cN~VX2}>O^WNZUFp5Yu6kU)Ut{WB z+rmYZ+iF&>c{`!h+BLXeaf_CiVP)*^{&n1!pR=}5IO^)HK8D8njH!j?Cm(sm{@dUn zbhcPDJh-1m7Zk(0rAu(DbzZgW9SO$ID^%T4_222ln9$)2o>+VbktA_m9ZtQR{rkjx zPlA<9QCjZUk!2isZEAY&9M4SFWcy7w`r?kHn?s`lE&Fn0&QQxdrAN_YGjksE1gG!3+&Dq3-^(d>n!($FzC&>Lm()fWjgN2v)Ua$`wdsWr7*{l!k(Cl z$I4z`wVlL6BB}>bcsXn8;7p_QW3QrU=O=z>;@5OP7ijmiKBaH%{c+!i;aVS!3`eGw z+l_SOCc!c?^?_~iV z?kOF+ch8KtkhZha#GJKyYI@n=mKCU=(AA{*SE){&0n`#w;jx(*A4WQeee}|G$87Z0 zrrz!*fev}N#_ox0>}leSoqL64m|C?;oPFR3!sKTPfNR1~x`v=J2BIzNEzRhiIFVEwpL0D8 zG5D;DZ0b7OsId0B%cM1>3p6F{Y?f`<6>li8>+4bv-?M`BBIf_Z`NMlpag24KZsqxx z=9NR9>hHLQg$B6CeU?pBo@T&I7j&DE=z^E?>Rb7^M>q7bgGCx64M$5lr!(RItHV-L zxC^u{xh`5I#`sqyn)g4glZDT_rr?4}R#VT;RDAqkSQ|}#bC@x7QN}uwkp}mOK3GDw zY)U^wDlY{iu0L6IJ);>o4&By?%mdV&gg9&6L>U^cYy1KU^G`HXQPL?nwa5^K zL2tkqF&rWQ@yk^04g9DU(lpP?E5lp(P))&kLdwD^0kXsp5R8RU5T&m4>=X)!Hx%M- zxv|BD;mC;58eb|6`@~G_QwSFj44HO?VU-aPH#yO(1l{li<+cNWqXr9rIa>Z4IhKUQ zT)GXA1RxOVjg3cYtieQ5`MB&f{sfO|`Y@QTFH{U7Q+jwJe`?A@Q`ibtH9rj57KxMy zj$-)_c&LRv&CC5OeDs!*?_7pGKYKW}0%KFzyH8x9evG10k);(=U8ur7ric~>itXgE zRY|jl=uVfZWTrcu`jgtNs4z%;RSSFdnYU#dJWle#J#M#Vf2(WDTE_>>c5xltT3@Aj zSHO%gKqK|{L#7Ev*W7!QJaH8`$MY8nC81rfDS3hr67n&EmNXom)#MD3GRkSH22VUf zhL;1Z^?#>2SYU_9ZKi4COzw0I2>^f4=gKO(RB>kQ8~Jv}rxj z^b zCzc+@svJ)PC|n|xo@9iWji!!8gXax9_!H&!#7q;*IHYrOQM*^srwox}v$m5acC2uD zGosO%K!%iFqne)l*eeHKnnA%CFaIWh=y0n3?PJcM!nWJuU;7&&{C03*@h+N-I`AZe zUS{h@VtxK&9HcMUS3_j>%h-jaVE}DgCrVW{{w@Z>ZY8SyoL_~4xRv5~A~r_73RMl;yik<@`%LW;hEx`H#zH?M&Eek> zadG(&8pGd+RRoe{U*f~FLlxW;O>p4JRCW_MJzr^3fMVpx6R~lN)s{6HW-+%QZ$ZVm-1|_&%}mv! zm!fR7t7!WgvIypcTqkw>LMVgWbBD-4rIHZB6F&_WLIo>X8w9m~sWl9E>u=Mne%(lW z0AA&b6Mv|>^9ho63mB#SA`Rh-vQNePKU_vi8m;|Ge*J&{n%qi0Q904dIvWCg%UQy> za4P^PgD5I`*>%|0T#)10q@JgFzB~#Iq>HzbcuUcHnZNR{82Z!iF{IY*`JX`XBP+Ix z`*DEaPnB%^{{I}MC+hy`f+gvL`k1?)brEhFuPeDKzpAV)x+lZk$9ZJqaFgvFFJN>* z`N4w~J>R+fVXMw?FZHtTxTo2Qb{fhC6POc^F z)FPer!5W{r{#82L@uk&{w2kAy-4q> z#SEk4MLuqNC^k$3yvd0oU3VAGK2P}#6fNQ3D}bMW+@3MGnU!DVde;1$CnaNZ0y0v= ztuyx&wtaDS`Z~DU_>9`vjDh}Kp8uS@DYQ(SNc&k8kb#&OgoDSCW zuSte&>`QY?UE1u1v5Nx5NGl8aj^?@j-uVbuT14?=H3GU4U}xiJPvg0u^Sj4 zOoN$;jjg>uEU(&`2?ze1W2&^byyW#lLHD!ih4qm!4Rc`U$Qb`OYsr*Sd$0e)jL6>c zN$rn&oq5-mQ@a_sYtt>RJ)ddThu=0o$jm$ZZKu~y28u2gjIYw)&SDrOdgT@0V2pS}PJ75HZpK#n{_KbCB3?*47pOp?qEMq$&8o>U#P?3Fm zVf$F@LhAc^LDhz`I!rxpFG{>HGs3webRE3*XBF**G<}e60~v9f9cS>|%vP>E-1Od# ziE;*FVk#Dgw2OT(!jVU#A?I#Z>|Dq5v+k0|pE6?Cp)W~=6`aw9)_9egsx$i{=IF8w zA6m~R6?V?K>S<|Q&!a!3WYsYqan-M@1}Rx2Q6=;l^KO2#CvAQnyhX+e^MS)nk1Z*) zZAe8e-0R@{X}U|R$?;J+7^k>?G`*Czkhy7&6_@tbB)5Ho@^X$Y9>HyYJi+nxuE>O| zLk1?P?(zD1eUtRy`JEUF%@0%`#)gf@56x$Um6`Q7< zY-$_Bs(AnoHpM?-O#g|xwzc_@N*SBwTK#(bGIQe=@BO5!Ew-8kSNlE?z0D?i+>Lq~v^sY1Ng$(0HQzbv(cv=x5BcoeA5 z6ur)EUrcT27}qtp=|+g*P@YT1MoJ+M5};;fc5M;kP;}aKE8y~USzza~l88y;T7P+I zoX1afG|)f`(({jSddPZ}S+6hrL3$#6XKsEPbTeWv3c>a3y-GD0N+smk<^P3#jR*dE zn~z9MDpN{qV}^)LG^%sP&q8P>I`-sCt9DpY{u-q7W?9y!X(dh*{SBF}oGL5B;^}jf9$7mbXa>H3H0(EmgJx2BB z>XO>7f>LW|IVxSL;-uMR_2^6(4(d8y@trcUor`*=LiAc_+Kv(C%R|J1Sj z=2>w+>Zq@CDa+Kg7Pev8C$3A^BJl@3v?EOVz2X5A8&Z5|o|%w-r|v_hNz{UCOt4U< zcsm$qX_Zv=XBHonck1fh*9ui??maxp?cWBXEXntq7S%_x!<9$2(la=-(phA+m5Ean zn#%sT5{_(x_(-)VFjjOwy z8nu#-qLq&-yg^xQoq;>?nq$bgTb#j@UKx@R3T>M1<`&*xvk$``VRdQC5WnaHnk3=F z-cdiU=_UM#LSL(3QCa9jG*3eOf{Fz#BmAV;>pl**WJt!WO4N!hCE|itj5wg=$jT^G z^`S%gr{a5Mt>WU#xJ<;z2h)pHF!BV74ZzXeU_bEBP{REfzK_d?8=+;&c2)Tkma4FI zY;35=&MEGiSb_jD`x+*;>>vVLPWXFK>)`M* z_mDQxcz77Q!6*`v6H{VDj+*eYa!_ImSHYOV8A=Q~#0+5P!q^IBGFD*->f5x~55!7< z$G%gUE){Kc!q`uJOT=k15)2wwy;#wEQW_w!;Bg2DZVx~xatOjaalF<_%L!($K_H67 z&aCOucln<5{*@Yth0VjhRO&^sgn4uR2TD^81H63LW>~G!F49;KA0pvlVMW+wq@5m0 z3!?JLNOlz(7;X5-kzwgwPFAJvW||UYeHy%6tVNm`h6Po}-&dALJjBEQkl306L$Pu~ z?#T{wgb-b$RoKQ)T;WPpgv*_P$_RkFA6U281ki}q64)VL-LveTgLX*x;poW_+-j&} zW2CF?-^_~J)ZY)4zXcP@5kR&iV{lXXFs)&b@aVKhiJE$n2?Db#oR`2SHE73@v7Rf- zb{Q+yVm$pipf6MVMx)a!T&wxJ#+J{c!PKW=(M){F6rm4LxNtO43}O(WI9^N#@iw@~ z>`#<3LJ!Jo*02))uZE)+if_ep(NZm6j1Z4fRe&`Xx9AMDLU3|;f z%TAfh_Oui=YbrsxE2Z*sqajyDZ(A% z5K2$F@_8D_pW;fZr;E?~=3<1@EEer33&8WIj-MW(RWqvD_Z4L+=IG=skX1pIG9(6w zO9cv7=CKjIMQA5(dIi*{jsncnb4B^_(rH>~db>avrJvO%u`g}5E_>BjL7RR&d0n-J zRoxhxURK=1{0w3yGmTkT+0d+}Jme9Yo3*c)VyPN+S!JXPylDl?TPUe7fYAp>D1UOUJQeaadUuydoY_4@=+NCI z3R%&M;bPYO3ePQHDXwMzz)fBy7%Hruyw-M7@VrXUAyj6Xl+V_GY!=$bgNIY{$@R)z$+~suE;UdarT`mWMc0y_~BPRm?-6D;2f> zf+2;P90JLrL;3&n^*zEyV)EWdX`n#R;s(trR%ucpvY=eS)!dyVx|)=!RTH+^IrS@O}eLd#Wm=gVUYA50I?w^vh>)7=#O^XEn;d|9-) zu{v%p)u-{r{U4q>1v1{!mDScC(%oV$lrqrU?(b1}`kUEP`W>4aYF0KGT_x##!k^T% z$CKTjgU{?KyqB~oxwG0PfVSqNgqH5jqu3w^H|Kj77=BI{!DC{y>XV4ZCO`y2dL6_4ZRlUrI>l>*Ct%+3q(lsMfVLb~~)&M%+TLKG^Rc4@Z*M`_t5m@>I#p^7s@;D7T*4>2JCE|-E?<=Z4lp4d|DP< zgFdwX76DU1N?&j#WjSk$C5h#h^;<&PGzYBO6L>qp`eEMlpMow@=+^s^sFUxV{QKtEz`>L;fHRY6p z#=!Du$I9JkwM)D^R#pBjhU9^Xk{dEVi(CCd_jml)>;vSsY&^;q|x171({&VsXM; z=}1TU;OQ*2dU%xqCijFAZy)I+uP|UwcOxiVo8NHIRbTC|ZYQzazIxX|RkSg6nrrn> zxTDtmO;U8o#;DFl+^=iznpX$8&a8etsxzr9K17JJa_L0kaP6GVWm>O{%%u<9#9FNj% zG(=EKctAL#l)Rp?qNI~vwFP4fSDq<2+&rnF)matNxzVdH)Uvl`Z_rxUSx#%XF*FS( z$DB_6P)m97;%#W|Yp(LMyR!*82wP|(3nnrG!%TH9CGwbkyfmjC_uZt3I$?V$pB-Xt zxqQ&GtEhM<8%s!`G}4vsWgf(*6K<`swySzX6-OW4_wD1PlhhJN6Z?iT0FGQU4w1H0 zleXtWy&8`MyV*7g$$agdEIL@Sr%saq5J?Bhjxzwz%R6qB^>);cU+Xe0d05B3;+pzG z;%vIkjvR<`{Gicr^mb-n)RUW1FT01VXCm*6&G!X9KJ7ZId~HoKceK>*b|amg`^ja* z-qJT(udfV89%2eP%_1ta>292SMCa4}T>Uy@c?zDs1}jAR(K1NZZ2P~T`r!H^8YS0f zEVIpnNR17(BQok_z-e$BipF~+@?eugr+P#z8yLWfoXU%H_EFRqG#X3#EG!+WMF$QK z?{t`!*xwPmWcYuxePQ@xbL$uwoRX4j3!AK$TbvEgzw|ddIwlVieZX^KvC_NZ zeI!vnz*k)~?{4IjG_A<~=N&r)1|iM0L1-h?ZsCU>}aNkMpMuAFd_ zoMl*dOSq-{mITtmX>hCZMHe>i=PkZt>!T`Cw6Lli)vWzqW(-u6%h(;LXQzY67hG1k z265T(%o??x@tq2`g3H zF6fXt&CJfNhP*^`80b0wV7PZY1TAqiZfS<2(ym)Yg6^Bin3*QEzeH6KU3*De-tU4+ zhLcnB02$W?1vDw&(Na^-dE3x_|P(K*0>J-XP_w z#Ag7i@5yW0 z0#IvE05$@2H7h~5hY821W0P?hzNlnD(hu0_Oz{EoA1Wm_hO?hO=4++`M`SIPK0>!5sOW= zzzpfFZx<(LeQw6#x%XtqU~MUb*D9wm7|T~pfGVy#Z=U3P5@$*+as8gq-WjCcD@m!x zBheUM-0Z9J7LUOW7h>Y1G*3V-xXawHl(1V;Fl;4EL=+tl!JDeS1Oa2&Kze2(wiZmk zjsxi7Nhu487a;ycRXkY`(qF1UvgoBj@-i$4qU4Ay#Hm<`qd2Kq=m2)vFY2Uor=f~y zqU=wag}ZdbMN#pc%Q&-w)-57scxltJ>$k$FXW1 zX!|Mcsh;|NgdR$(<*5EAvUg&>>w;B@!lnwTj+ll4fNX}0kX(*do-7p%|HQZc%lE>y zbLSRHmdSAlGJWW?{XME>y-L$s7ur;Y-X6-+@N`H$C7PLF`gD3^U*qHg&{Bw@y65oZ z<%Uq}Xaz2E7cwpQ6LxP_8%A#O_p+6%D?6pxJ+-fpk-7LUSNP9Yj_{3VMWipe6s}8B z7s~!gJ_ujo5lKr*yA}fzKQg+0=W^?rY;Y7T?)`sf9+WP=i}yu7b#_sPL zRrhVU@qQP#rrTZC-ni1fY;aS}tkzrp|FGrkn3PPX9ogygKj>O?m2_o|XUc%y@*yC& z=xv|5$(OvIp=C>LLIWk1*1aDr7SB8=-bxr&W*Y;FQukw?2mn z&9-;@VoF;Sf5${T`{UGcWrs7;_K@bOSSxul+Mh58_~T_=IQ&Q15~!Yyq76-h54e(a z9H*As+P?Cjf%PSRl|4sm|J#J-R6VMC~Hmyl_>46Ycy{ zkn66lX?ohet{2{YgY8#lyKikmk>)!dq4&u=b+oHUM5nQ#H$`u(;>4ulw;3tWU#`&I2wj!I)N>`IXd9j+zor z`x4xELu_$}M>b-SYh0IdE!{f)N?BEwk#?K@$jlwNVcd3B!KpI+k8;ha^sbWzk{Rt~ zSr;r@e4RC+)_{#a_^ZBrYKMo|632`Qxiov;c1s35&SECie)Pu2M$Y8(nO&V{DQ0xx||Iu`Qbst=CSYq6TWltf9&;8NY-51nAumKpyj+o*A<(Q z7MtXuD5OL60D!jL96oJMY2b~BJ&jIdPCT|(N zKyihWp;;H=A0)*?1c_j#WsW(H@JDnSoc95D4HnWwd0zjG4GX_8S+E42-0`x!4=A;` zsst~8JMN~xS#hA*2y4Gv$y61Lr$#G!p9l*;SL^6V`IE}vCBI38#yfTQf@mqWXwtugz!(U_=tQbg`yh{x4hVj@G(oz zgS$9@n|t}_#|tLtM%7_cEfVn0eOCsCCGWhnXDHrXhz01IKH9NOB`&rkWFQE} zXVEoM#pB?bF{?f?J?PFmJVIqPB!r|ib*RAr8c3+2p|$}-0ScH6Ybinx^~4E}aF6-U zrKR^xL7{+1!k0AEDko^%4!@B|=`_TKvqY1EqtjElatdlyCLVsGd2EcJ6d;2D&Wbqa zUQw-s328lf)sO9e;`6$mrr(7tXz(JTo*p95v{q9`5>*n0Dwc7~DmX(Gmnm#c)B>x7 zhLEUrvaz(wQSj!Xde8imTp0|3KvMzmPrL(=NSYne+x4J3`|uB>_RhPrl>lsVD>_Gb z8N?@C-BA1|92{|sl9_5?lt7cULqiXusV(^F^HXM#n9$CHlT^kRUd7(6@`}+DRtk|N zq-=F_qQW?;-b+e66Q7um1k~7MCzOg{4sg#$uWd_@q{M)k(;Glctinz)S(Ty!Ft3i; zoqXMAY8iuHzBV758j{K#WkmMGz7OFiQLXcRF!{2ft3raL9@8*p6^q00XDD?b?FGkq zxnd?wVN;|2ih`^XJ|pdzSfXFc6>{D1#dLxq`yDmVCm|hb@1u$~Ijm|DjgBArOr=XM zKmI;`9io3Fcn(I0DAkwrspXIhS$`~vN9i%+8=-GR*)0U6U@K}LB~TPLM|l@(k6j+C z%uF&9R87dOFO~T8=zCAlW>yZ#af-)!{?4^_MIK8yD_vo;sopWt=-_#RSE)#K8~ouh z$k7TKnup}9(zsos;COEDZp175uX#qLQ{d-rY>f+9Puy!bCY>N*-uVb=&kO$3#Z38% z1~TYs!4s3PoR5d_CneC^m2KED{jV5_ku|+XL(!YxfL904zbVBOhYbbqO}dHKA}MB- zijXJlBJcW2$3jKV5+JK;=#qX$B5Wavv{i_7_FGm}H+6=*O}$O4AzW>2)bcBUqZqI> z22Z!=V8OMa`=g?RJI~wmm5k8BwvuY~i3$r|cdHUP8<`ZqXKZBcVuBOHk0tD4-jZYgd`y;cSpNL_W(x9+|OOO#=nIR8jcQ z($RFQf+A$t#1|Z?FPjd@16hy)1o>PnYmHx2XkcIXJ_W)agV^~Ux?s6e=2yIsv0_l> zqH(Sa1h7$4RTLl8>Sg6K6dU`=Q#dwB-8Ly1q|cb^QpgNy;@-3vk$cJ%M5gq_+}N`7 zsT45C0HM|O8pQ`Yx@r~6Ao4&lai?@c&$vrr2tz=H0Z5;6wXui^3|Xf z3x$jU-^)zb#&U>nRia!*6V->O_zf_F#2F8gUtrk3zE-{azw)#!aLC~2F;wn*((qCQFK#h@gS-AwLnC~YtV zB(-#){P=Q?n}~{8p?M_jgk%2xOX_(^P73wLZY&^G3C0^Q3UJp3L>oY*xMp+jQpUuo z<&V<3qV{fTnNyJAfC(dD%8CrD>dud(d4hHJ9h+HjUtdV4%Okm4@7mw#dv%Xv~%vwyVP8^o9}k^ z(Btp!^mRICV05&fE$2gLNo>+Yw=lPzFIe!a42y^gH5j6!qlhj(mB0#@)&BT-)?I8A zXQ%ns%&O@7930u2wPyC^+^~}R!j5Z{gxGS-S5ZG^9yaNbq;{UF)G=*65piYZlznKjr60bw0;Cwf0VMUu`rSOy9Xw z1)^7vF=8re8Ji#Kd(U9HK`T(}+FQ!6cvST>W#74M|MHY!Z|AtGU19ASgTnGiq!RB( zoMB^<1OFZd*qd!u1>kqDU8?&QZ<=D4aZ61O^nK$ejWfqFC?TR03@N(#3^|X8{S(X2 z>+8qnPdeQ5HxdwHuS)_uEd@wUDHuC@Oo5pHed zGi~%H0Z64bIP$WlcutO*c7YEMqxXpW30;8O9H`5NWXJic_;NYZ3NNP`KDO88{eUr$ zy?C1<(1LEjjd+#BIrlGUx&6ZP&(mo)a&6t^9Rw2p5%dI~(R4Qt{Iq}EGAok?ZH;}p zyRmj-$o(ekz7`9*|JW6}b>`|h#C>=BlEz>2D#Gfi^?YW3ES*57zuQ>I09lm-+!{TLx)w8BDyzql&Mmm^J#ZmA6<%BSolA-mbV*DEw_N9S zNhCakGtyCp1n*6zss~MUlQ7}b0IU!=l)iSjH^O){+{sPD6EIvIXm`7wA1=II*ztR7 z3(~C=Alu_?3&$RMrT_+Sl<&`N{Z4&kIMo~tYzilCEvoA*Y_Xmdqow&1aQ?DTjs`2= zM6D-D7dEBPJH1H8CO#4E(b*W`?lbr%kh%((xI$Myb~aAi*cv|>ve9gxH~rjCa30td zLh!IR9%O6AMfRC}EjPT1th!7eV{Jx;YklgMk&hZRLO1nMF??$~dUgtB|MI2@xyh$t z6n-HLx@(`WmM54$M1b}*9@X?UZab>M@93RKJre~dL<_ZfPc$ouOa5r=u`eoG%tDU<}l)El)m_s4I zNF;Q5W`B(EqRg&K&Vh1L^Ts7BbLIhL50V!(9`SR0(d0~MMcfirH;QhcnYXtc@G3EG zb|!Bz_|x&os)va^diTDq$MNVDaZg7oyJoXtb8LW^I*GP~g|fwZik6>Gi(=MmR>rRe zR^Myt^bM)sSLMsMY5qaR^$|Y$zLwxW1%^HP+>>xyt_Ma_f+~4s;ul(|z;laMH_KR7 zY>^i8L5e(+&=!pxCh{t+P6#B#NsvUSM3->BUfAN?A6z3Vb%~Vb@d^SZqK|cU2K5xd zM42S!2RvIl|=B}P@jFTQHe>3SToOE?nlj}E%g7+*dt0Q0&un|Y1%v)F1y zeiXASz5xKl7#Qgab$LWv5(uf>xvIWZHDTiSB<)UH?1Y^q2ic35ffdqx}F;tOZp=P)gaT z3Mb)t3Eo;=&<=pjlIsEl<10)_NKBE2W<1PJcf|i&0Wb=%NxUebWZ+5plHqa@GyoZM zCAHkxe7mGqOpe-uX4xtv1mloO?(3q74X+`%238_xWH1-HKE;3h>H%Rf!^E$%;*iz% z8}DREP>oni>E}Ea*=%DyBqHw-9Mavju(39L+_#cNs~|D~++x&-53NnlQrk?@e<_Fh zB8BaS6WBIMhk@Opjv1*H&iAd5%}lM3Zv^l0ENE94(yMs#HPaNMjut4`Fn zG+MHlA;;ZA?spHNTnZAUsizDt&_k6SYImscthz}t?Hwa$E>xSW0t8g=Cu3<~{6_;i zcCS{gu`8jl^!SqET+}_ZNLBCznoTR46@=<`<~J+e4;9)-5~JOYtD}8?T{~P;K3;bg zrv4H7u2r%T6lU}WXt6&MBR*svYfI2D`DFnTU5bp$2*de75*uSeEaZEZR;ky?`heaO z(Y$Rq0WCPUQcw|QdHkRs{gJs z0#cciDPrTHqfxR_g7v=2V3Nkl2Mf=DL(xF*VYjnN*UomqwRraKlNWmTx*p>BX%CK9 zE*3zvYg4I} zMG()W#K_QQX*Ki;yQ;(ySQI(!s?-|sz5c$!p85!l_(VBHm0YAK_*k;SdZP;hOwzG%ML=Q&uHu(LsL=7 zAwclSPDHaZts-DN@iaxK?J{P|k^$s(nrNHf^(eSf4v`s2MCWV)|K! zvH=eL2`fiAV1`S%E;oxY9{ap7ru1Ufr0DWrMf7S}dACa)MqV^b zuk!R#vt4!vQIX5;@ULJ0-{dJMa*wo|-6~8h7~eo~9Ia|P$_6+5 z-L88w0GO1GHm2s=ac50OHWj_@Eb5+Ref-*q zLFf2^2i#~*1k$>EDUeO~%R28Yao4rZoT=|^lN#Lf9beO-bwyWZAlAOVS=CV&7DZUc z*LOtGyJv7;#`yNLUUy$Qo>*H_bz$a_fg+3U-bWz@;t|7CuIrU!9s4Mm?X3BH?alGd zTBzT>bupGuXQ95SJ-3?>gtyj@V=;aku-gTB>g32tNy3JGQa=0n^nxd86^H zzMPS`o#*!N18#vte@oBkIZiSb)4DVxW7>IB@3)3=W@C;t1!oZ>#NA? zXdnWB(o=6~{KByh`xwW^iqpEj%k|iYqgE(y67)jEWjP(P`TH2HCAK$QUpt}#6C9na zn3~vikR%+*@8yO#W}Kb--0fvly?u}?((y_vAZebmW` zupWaQ8iWw+-eHJtjW5V-GyDeac%$)pP0NCczE2i`Q){Lrx)_8+?Ir znOIJrM`Kv*{34$UM@O=hBxfwA7`&r?#g?wq-?FFpJHE&@{%BLAf4OJc0i%`~T0w!N zZHkg_DjDa;CD=X`#FpxN$KxcHF8M#xoFDI;R2NK9;)hl^M9ZG+WJM=#+Hz@M%Qo@I z@?R0ieLePgwBvVxa?@h-{~_zmA2KhC#vj0 zatEXxJ-Yy;c+Z~Xe+}L86Ujbe4a!717cxCobi**ofdQ!N!2Ze~Q=e1^4UwU;lsajm zB{L?)?L5uge+kJ?IvRSd$+BJ}oVkBl-NvI5pvfiSThP3yz`G2uiM`mE(l1hGmfB7B z8KTY=7p0Mub$9)x%sAItw=2?|j>RZ@d{pCkAAlF%5NkYxC~*jTz|bkU*Q`cOFrSBc z<1in9!{^#?HEMV!aZL;7eZuA<(upOcZwYT zC?s$w?kvr{)zOT3(bNv~KbeAXwG8(*fb!H_YZK>(mze~Uu)%A^>e#_*yr%)^hchhR zx9E190(rKF4*rb?Q=bD0!{!983yuzhsRa9Uqh`f?8fjIM^?DWeQDAIcyJiZ0UzO=? zHwDJy_$s0(uzaBY56PbU0F9zRu)T#N{W;Aj(q0Z8nhZjnX8>%9M~2p2CXmUubg(Ss zZpzB~=iZ!Iu5L!BTgtE6=ZdyrwPGo^rvUDfF#T`=2?1~q??NGvyr)J;NV1$OS}A{0 z+UQ(>rV%g{4%!Rh6Qs|)GK@O4j-b=gZF!F89YlM$n(95}z31?{FI;mq%v^jF7hJ?_ zM{~Df#UUDjejQz{yGkx7*Q?BD?~N2Zc;Evhnt7A|Ay0?St*1wjn7)7S%Zl5PI>Dof zEE&Qv{~BrnFwu95`W0c|(VKPRv8PtLh1e8UC1X0Mrw?Q`-xIJ|Da^sqEQg`D*n_FahKFzBU`nHoFlabebOuoPG*MmyXLW<2loFB>N zW8a!!qsaVfNy2sqzHqh7y18ldVo5sksdcabE_r@wr=ocRl`ne?ZSNML(eH0UW7Zqz zC4espG0QXj)Zsz>YpM^fsIrfH1r-MscRLeA#@Ny8X`pEE+ZO32*-3ZL~w>`%&orVVF8RE=9|FEAKo}Ka9_w z2fW7POTSoyXNbuubzJ%2tP~GG(QiM0hAga8p_Nh#pP^O`&5=9g!Pst7LouBur|2mH zc3W0^Eq@NeXX^W=gDk%RLPmsFP7SeW0F6^pn4N1l!lto0*u2W@(7~7w&xGy2GmMX* zCmFKYvs|ICrlQ+uhhOR=$hch_4wLb#onq_e7HLsnlLy1qM7{&GIeY>YFvX{2+$rBY zo5@H;s4P1}(HfJtu|zs~QKPsvL5vH5*Ad~&jC;saTUxpbi;9I(RJ+DfwJ8)PGFdN^ z5D<<)YJ|GaK=xYfW4_Ww%|-C+SyLIQT!(SnBM;d;6Z9jX5L7A@L5B+0ML{1RJ99=W zuKolrE@h#m53n63@4?I$9a@55+F3k4_%?YwLs3g0Z5*tb0U%yLT9TY9o(qK?5=n&J ztel&NB>jl{70hQU(FZ3lK|%+rMpDHRbej*10{(gzL6GUlHL%)d73pP951>gmub5HW!^MQNXvFj*T0}2FA1;tPu7-a7Zc@NK*WK2;W4&3+*uF zi3!DK!bge4r~WrxE_{nH-5EmS5fp1LIJCIr5vDNE3XQ1;8`yU~xJJ|P#05$au?AyI ze7InR>s{1KK#~r`Ew1Mq@S0ft{C<)C1vgQ@?u(sKVIQP1QVRK5>;FMaPmTf@N3hU1 zVF6N2OhrN`bG8r$l!M#`R}LYb7o&rYI-71YoqEyIwJ^j>?Ncd5$E8Yyd=N7`#(}pj zOU0*ppxrQ1X`{_b?CCA+(3qFhmj_}}dMsa{5fIicrWktTV2lj=m@naHoFkjM-t05< zOP@SSTruW@rWB;>bhUI7$iG|VQqN)W=7JQn( z_tfAUFx{p?c`blThG+;WIWOy~3MdGkj+dFJH z+2XVV+NPWx*eXvARntf72FNmsCC3XOj)Cc*2G3BsO_n2g&sy(~Pw%_&i%uoyhD{1pxbxc36gx z?Z4PnOQHz;AOt1PRIdTUtox(~(L6(j^nW(JX^@7GBae@4E=w<48qG43U{HZ%V?fGW z?80CMktepHeO-_w6KH0z^8e-Jy3NX^Z9Ge{45 z8X?Tm0fpI^!FY(np-|(%(*Qkz$ij|DT2x)p!v=K=@_`5(P!Ht7fG^|lwg1Hs;K%qO z4ZH`KKx^iIu}PZi51Gi!T+uY38$u{WBv0WHb+K|mNNgW$P9Y_+vgi6Sb>|r7LuF@xG*RnpjpR`mr`O|slU%ZKOQ(WNja8K~)``4b zP^hV+xk&^(($Wr=|A^X4{ML89;q%W1{uC67%;f3-{hqr%MmJpopLhW4I_YYc?X7O} zT*63}w{nXr7<||3ki09+qcxtr>I%CCncVI;H(XL$5g?-q8J1CdD8Fn9A$+U%DA!@Wz85*qH7%{5$A3cHxwwH+skQ(>rSQ*28M_-Lk&GC$V{T0Qa8$EKfIF@_{1Gda8b-WX%-p} z>d#TQWBZ~{n|~W(r>m1P1ZBGi6}Z0icS?FJk}_{zd8;h-Uf+khP0@E_(D-&lw?8cc zJ6Y83`tQ*n%U^Q6w-|5XAMDMqqI2VJB`+?>&D(hZO+c=ki4BKy%ShfMuQz}!deo+^ z5f$}*3zCyb0-QKm;8Pd($`Oa^LF0?xj9%t##^Kpj;V;s7*Qds z_I_!ckzQ6u;7HT?MlK2zwb+h?ey^95Dh^;$kq zDKopmD#v7-#i?-2Yy!Zp^{6%;(Oqojo8^sY-IUp`nZ`bF+qmXIIed6aphQQy_jc%> z)=iTsA&qCwlzpfZ=R=bJ)ar47{TA{tT+iS7$&ib81cVvEaKK&ET9aQp6k%;Tvm!G! z$R|V$FQe;dXB>A|H+ZoIU7eNu$+wfz%!LCd7oAeK-K>k)K3tA(p(C;~Y8M%D@s~b+ zQyIC%s}AjVFlTxOheRJ{OSNa4iW-9X7xJ>$1XYJYbbi^K-i!uCx~x4NujwO^LQRsk z6xhJst=qjAfdPeL2wK;_18K9!5cN;Ms_zq{wVpD@)u7!8@{&t-4}g&g28INpY_fxd zxp1XPboHs#PkZ>|#j>z0sQQh%OEES1qwQzq*u8xFS~R7j4yQ1W9ejHV9$*0FwVKxq z_u+=#S-Qkw&^*__@F6z{qDQH#j=LK$yZy_uE+pWFMk*TWhaOrtdISQK_w*DQ?1Bf> z&m-hUz@7_zF|?;gQV#YOuGnZTF()A%O=ZI4BYH2L_{qN==kCDj3@&+)lo&I+J!Q(z z5uw{J)oo&^mru_9l80^i*^MGyf@tDW0yjNTnsfJVQ;F#N1i*E*Us#i!G`h2Vp2}?P zR$*GF68dGe{ugBKhnxA?v;WC^DLig%fl0F6ZOrP#2YXpGb(RL zb$7uVE}W@R!BGk`EX(^%TsLR*4B~%12vTzNMv-myOIULG6b@xe^;>e$t>xs%8IuC} z4NyMFpc>yJDxhgg*kiaQ`|t8Gr}J0%7I)L{V|xB&jC#%RqpF2z`o}uGR$vTZs62O} zMqH1q%XfHlQ>7}b$yJ*rA1QeyO6yrDd`P5$+ULm_eM8c1boxuIi$YCpdkkxxi6F-?$5h#Ct~h%IX2`oW_V<)nG%$@7Yh z&nLHJij51OT0sGhwSzH4gaogd&R531#iPD8RPkURcDQ$w9{Psr)^cM;iI=!*E4Dl@ ztmV|6Vr9PzRNGuV&flBAvTO-@qz9_e01TXLH8k8R_-gOL~ zD4TODe$?T{nXR}SC1$ZG0UW&-Cu@+~BJJ^k? zcVNy$Odrb63G${{^kH?vhvt-sfHM$2h{uh=@b(&7j?T1MXb(E;3^g?BBTA3e!VW+e zQ0PIZ%k}*kRiwAbMX&9iNmOZZ*tVOl?qs+3oF6nV{3hRA!&mW_AWE5xcqvq}x1xTt9k|+h7iuxk-t#H$ z!fDYS!QT{x|2QLH>@(5499<-5xelVkFGML*{4r^(6AMj!poWq8iSi%qjQp4G$|3+Y zV^lu^ZR)Z$l|2`pW?Xt8#6_?JzuknEDs#U?k0MD{yZ3?4r&iX@2qwpjzR&^%hj=-C z_W1kVTPS!$w&T)4)EBU3Ti8DFxE+zYna_lavl#F)xPr(t=VYdv$^gUxbqPe~V*N#) z;eUaKXGmaj|E_@CWs>=1!E}qdi;)2ggKF5x$k&3}3RNQ&N=5xj;smUOCM=$$ghqM~ zSr;Kj#zsXD0v6IP%P>wQkT!z|U?PEdE4&KGV_Myf*ot8g661+`0MHQvyMZXcP+*5;{!#kDQ8LeYa>24HiK@@U)$N~mp? zbKksbY)*qZ)$X z9>_|X2_I9uPT&?nYzgFWHR#~m&yX1@dZ^zKLR+Q-V1zV18j8p<5X?dW@ofmf7bwJU z;RrE1xgJP0mc2w51@%6Xu}q~&kOP!gcO9(jW9I!y34J;?RqO~9dJ}>Kb8TKdtBsE<>zDyO{Hn1#bvn^6= zJ>Re?2_?S~pZ0EST8*>5=42+QoMlmhKo^U{;uh~TKK9NOO}{S4 zZ%Z1#!7~DUac_pn7O8k4zSmhXd7Kf>l#J{eF5-e{{`zYgb2HzihK#;qn*@= zFf@gs)hq5d@&V>hEK%wX6@~eBBRBFgd;pfQPg2g?3(jAlNNynUOSlR$4CboO%Cu?S zfYFThwhN?MFH{$hY#EOlxzXhk)_EEH2lGA!O3j=fc@&L;Mf)~IaCLxF$k!X~`B`R5 zLo-yeBGk!#&8{n6NGDs2x`Y(`A&eGNq+34%#>9Y&WMjs_Z~Kxo1ZORk7Q`iV zD^c5mN^{U}m?5avV~7=a9NQUEndtWeLvczh!h3b;5tQxtcwr$Z@bFxCn6(rirP5HCjrBeFq;iZKPCB&2&I z#!Uj&LO+qAlWC#@o@H^V-Wf&75ctd_B7$T$o~Y-gYGfS2it;sB&Xk*`a{*z)L=4EB z7!8D6@W=-FqCkx^%i=z}Ml5Lp@N9d-Cd!1>WHeZhb|=^nac-LJEUirw696 zlpkBFKoBrwjd7-aUKWNzpaBFI863nJpNFO%Vr+I{aJ{pb_c0hQKyhkwN25azSD!QiNWUQ|N9w3ZJMW!9z_8CMVp#SGT~c zNwI%aX>fy2vbPGqZlC)58l*0L9yO(W`kl^IwW6B8{Yl?lu6ZhGzcIQcXIbgGMs6Ij z-=A~-_T(M}m5Q|ens$x8XRMQEI>c8G9~HA+^rFHw$ca!J$6Dvio2B}Q z9MZ*m`d>o2*DTKoUo!VkPu9aYCRPjhvo>J|=!LJm2`Jo0?QdL_6xNVj>#K3+b&Fh@ zOGnkZd&!j{t&-C)6Li{LkA6Nik-HBp6?bYy{-cs6!1$bfgLm@v>F8J`Idkvgk^+@^ z6P$+boykH{9_eu}6g;ninEIAC+=Wi>J@(e9=+xecOiARo(QpLwHpOu_IAu0={^lv5 z>iySHW|>!V$7pX@*BTDIxqPpd>q?hn&nVh9YjAlF-oWGamlnbl*7$LhsQIatjS1nn zgR5h4eU6iuf1z|yRnUZ!anEvR?JE3J6@zWSM(y9Yvf|rLsa}uG`2Nsp*%|v*C0^9^ z*_n4nr-sxz=e)sUB!G*|GsDb2PsY|KNkt81YsMCRi%`y`@l51_2!B5NC^!=S)YV6A zP0b4ndJff9k-EJqc6PMnN zn^RU-@hD0~I*seG?g_5HGA+TvceeA#R#bnzJtb1dbt6&%EBHhKeM6{Ybng?G7U7xld-^4%C@+)!pLwpnREYL|~i@t1rZep@@SL7b;Xs(W%;w`~|7-_1!eN%~8cz}?z) zMCYxq1Q+n9wTLnIf3aA-Yy1svD0bwjRl>CqcSY{zeEp`e?^JwPrOI61La=a0*Nr^T zP|j!8t($M%xI;uq@uuc5aXr^}zg)?`tms-JGo7wj?BB-Kw=XrViYrA3=%v=b3^+Q9 z>Oi|dF0}Rt+Q*jWWKehVO;d&sR*4#YZiaevc&iT}_typ-07ot?{Xk&NV zZN&5gFD}AT+YrQW%P{;P@p~zfcNptyTbVr+l|8SwwX2Wc$T#^x&t5aKg6jrAe+2}9 zQrdC>y~0|Z@22eU?FQVh@)eC@7U#)YbHrT@gIs9Z;BU>BK7+*Mi1(0UKUz)|-N6mH zr{EzRj;irJcGP@%L|mijJQ6C?O{JMH zmh3;vWd?iP#j;u#9E*K7r{TnC?iNh-cWWaQqbtj2!JX?k=4sE^-Tw!^XOBEAP46`d z8BV1yV>`u%FIFyGth?uc=WL@M;+|-IXLTw}v3N}spp9IjIS=clS%E#^8^mIxnUz7~ zJ;Ay?t6$GQ3o6HH(T1FS?z>Fm2gYb!Pl3W*;q4LqSiBgMNU2SH09!m!d_|S+g~zd% z;#b>!6*<2#S;d9d zG|Q_&*f`IJFn@n*mhUmqSU{g4x-C1$=~ij&q^dwtYP9Kv$XpU>{=Iol@0VGyM}Lo* zGVVvwoDES1-<2ewap&#sp>^VrG-EJ-dF8St|H9+`&N`U5w6|dYunq)53Y_Pf(L(Kx z{966DQ)rxE#GTIDuu2By#h?fh582o>e=hJGMylI1j5cG3(FUnLbnYq@X`g@#9ObA+ z?`U`w(4xWBYrNH=cQOn`pRL}s7}Fy2Yk#c6>$41*LkDzM=FxVPEXr>@m41?_p_OZD ze2;v7Z@eDe35L*?8Sa4GP1}sFpJHKLCM_q&an=;!i=85`Rp@BC+?=7*xycHcnmQ^F z4aLG=BQQYvWhiDdwYVIYzPYbqQIY->MI9Feh%L>+LhR7|_XH`g43DI<{|yROtnUfy*L+8;H9$_+c?`wYah*s`_skn_zYZ6dVuqb>_E_RvrDTBMNVUkHZ7Wb~_A7KEb zZQyRxj9#Z?=JlS;)_slymH`O<`D8p8F?sUo)*eGq zbxIuXC8Z5sc||hLLgyE`N(uEqVzc;2x4qkl8kk8>xTDx&+y(XTPRU5Zc&5#5hD0z)?4t zWa>bv9*kQcQbJ~mtECJZu>k~T1wjI{eZ_F278E}Z0lt3}gz%Qco9Gmdh?ijVY=M!1 zsM{?C&obGu&jrj3$V7M~qL~8wn59RMpj3sOxobO3;~>pLq^jBXWLH+qih1X-kd<^er7&hr&5Ta+8_IHE8uV-{b z(ryN{sq|f3uce@xA*C!2j5#|MATo>WDSgVC`~Y3!?*!k_vWP!t`QcST3}Osv#!d*> z+Q8JR2+kRbr_frUMJ%q@dHBs6RWPcrB`>u~mXr&Gmy%G2OoWfPyNVCGNf;HN5#MD^ zcj5DZ|6^o@d_9i+cwl%rj^b~9g(4ESq5uwpa>KesHoYlu=Ns6>G zzQpQpFfdu^S|DaMR9szV&4E)rFr(Vw>#A&ndAD$AVq6HLP!#}4OQ@jLnQkflImt!V zW)}noJcNDrGaZ%hQgG-HTR1w`WI+Cp=5wuL<9ng z$`ssQBU^BHD22u+_lx^lrBGmi6vHWrje`4tRT*34^Cjo`%QvmA4XetviHt78TZk|* z=mXMkXxi}JjDj)%tLGaM896FICRd>wz0E`bUTNu_Ue1oV=PHKKu@*337y(Bj{4sil zgXjrJ5cyYFRv?p@#t*B%|NdFH4O~0!`k0Onq9n;1wl&Z?2m21tIO^O9qnATT1L7b_nW z6@2yy5HI(XfL)?9VPMKByZ@f~bEDQ1s|95VG^&9l=Aa7#cSoMsjQA_q4Ydj+S70)& zNzp*0G81ajj)vCi{*SR(KpkT_IhI5Ifjz>~ysr->Ph1OR?t+@1-gp6E)Udb3c&~@G zH`N$Wu@Qswv>HKKQuPvvwhU;7ZXtl2kQPRZaP||NY1=ZkQGmH%pF|Ho%U@VwJSGwo zK;W!2s{bW|m`%U12=z%F!N`XHt|Y+D&UE(RCQxF7p%dy3ur;vXEW`FjQZ_6NxEV09 z7pT_|h#=rVF33p0Ff-ET+03B^d_+vJJR%)Y5Q7PF41oX-1rP?V#t4C&Jz{^cITGedg%GHe!Un>Gy@X z590-5X29aX48#-z`k1Rj7soKzO7G&AkJ_J#dnGJ-!8#AM#h45 zu?YT%x=ALhd~e+Q&zKahV=e^Pp&acGHRikpUWMg|&JYGTc>_-fJ6><~`!AOb&{ujeCIiKvTM5%>y)7>X}Sw zsGLkb3wLakj->cjA2-yJ;kzh4cfX=O>cKiJ@*;7bf0L-o`LvGY=I6ZME6fdTsJNST zz-}rPy zB7%D5f{VCST(ckg1HBjprV%c(UrXi|1j;rVwDm5}xh{Y3X|#*2e%Gw;F97Yf zc{i=zQ?U`o$JlKKI!#-;VX`JUb9ccSrCzO+HS0jugTT$t^rtzxPf z#~e6;hs?Lg@*8BP6LF4vg7h!zE_M%VoeL5&69!wUjLUU>s%F#7CY{-wA?~T_S_8sM zqS1kAcKD;wk7oac;&krulA~hiWCz7&n+s>$8>E}ZY_&fc{leG5i@fFfh*ToLZw3g1{}Ncmug?8 zl9b;BeHOR^lj8FM`BNYHLsi`}}6{3zdamM6F(F=EI*Bjpt7iq+!WgE9*C?b&bj;j;phnS9V z-;Fj7gJl}W9RHQ3`)$;+Cn3~{LUI)u2J|MH7m1~K>f5pSCkT+^WnDKw$(%^4>Nxhk z{%e{mpSHFu%&><2G)k&X$dc-(vJjFTOkIw**5A)7?Jo!FyHUW4^`CwpdFNxEAt;Q_ z+2f5ag8{QsQYL$v(6Ly5dRz&+Ydpa}8&_Drz34eyV}ENUoX>REv7BPdGLR$|Cd?w) z#2!pq#2a93$*KM z$mgAg#QU>FIGIX-JF|VjKh+_#o+;i6$qb|%`3Oy~)enh2PSwa+8AAqvqiOhrUx+SB z3f^jbp1P$4`*(iuV_89wyj&cHojh2cS5gFyA`Sh8qk>VX0y+m}AqjTVc_r5dstx09 zLu);E$FKa+AXg2vipzcKmP4A7FzB?f85i51A8@9@gW0)XM`teMQ>&$pvex&G^}cR1r` z-p;V*gJa<8OL-wt{6$w)gj=)3^RSClze{At6^Q?xapXMm>2~t4vF`3fnZYjk`+eJb zJ1`aGCo;IJ;9_feWwdUvb>&^-x;ZhY&yVGGt%D}wm}lgHfVa<&<#xS}du%ga95e1f zH@5JbqdGbE3+Edp^Y*^&cXAUpjP$d=1io%sk+TBsf-gRZq&{WaD+-ITSY0|NWVbrV z9nYL{$YEWB$D=Tp$|+NEP;I8UWW4Fp;KlBVOI0Jj1FL1f;i?jQtYwq$hjd7a;J`0V zz}$2i7+T4HdxWbG7kQDx$o^#y)dM$ItdUNek&avy;WKqOx^)w5KFv2WTf;@cy;It? z6SMz`Vz`|_w(wqz>At*zQoAXMz0l0NYk(fOSH;$Jf>iCtjiEplB%KJCb ztx`Pk1p5HyUoeb`{QC@F%gu8Byx{eKTS7tY=8BBOxusPj(|lHJRaIM6?)1W(eW+}t zEaA$v4<&j2{pK?eGOc;-_Nw6#SA;UJtF#Nj2>=NDl$Alb_h-x@^dw8h;k4TW`!=0J z^71*DSGS@_V4n}@ms~{kndaN~R4Can4mD0bEZkVd_zqa4eVo{5It4le64ZGtR^Z2UAR-Ux8JaV|Km z71?a+2jukGlZt!@0X&ESSdS#`!AwewdSH^EBg+H9Y8#=pCqpKXhb@KYe11r|b`sFr zi&{*U{XcrAQ~7284_>PL8-<$f!E9lML&=AV9+;KljbQzkNML-RBckjrTNO{-g%$2QX zHs!e8+pJrl6bdECNeX^aQfKI*P+`zOvSt=&Q!exhXQCa zk-P1|L?{TRg^_)=$c90u_<7fFiKdUpQ6;drBftYCMVKsvkp$r~9rL)2GXw9IETkPC z`m3))goNf`WN^p49$@nAnzWk)5;i)3R$Re25|cv7B9>~bxpe3u0)AV)u838vJQg>? z84%FO`gf0P(Hab?X9Y`>&buz(%|?*W;Ld{}wTy0RQw;i_k*~Y8fioR>HhFAbd%R$5M~?0?zp>@uZgECxyz|H~77{X^b9;@{95l5o$_q;hJ@OPv z3AiVQW#9l}fRWvN#N%UlI(Gh?vY`CHiC~fvStX2DxZzK&}Mf+6;CJxbd;S%z`qZl&OoPjySb+8-uLq zs|WwR5qyLH!gp6KeDGQ;+M$jT8mA-oq`^n4a?SavvqPrtrmz6TSjmiz6 zcOtq4_2Ez@G)PU>w6FncNzY*^l>q<`MdNQZ@=vlDw51E?&k>j7i)?1qq!p+PXkePm ztFB{c$x?zj$Y;-sSea2lbx!r>Roa*F1mxjrUZN3_=@RsOD5rW3a961Kfiwfn&zno*NPt75hhpPlUZ-+|f1IQM1Fu=eOgG)bi$4*uoGsbN7$QKg+6(EEx8VWiU1G}IJAZe>U zJ`3z+lTZK#tUVr2n=>&OLcc^dfHRpg1`-YIIU2(AU}&%=nQDWSCk#!)*dVIf(_Kt}V55_bP3jghHHf8Lhd4eW#&9}D>N#Duf&br>z9D|OrVJ7Z660XK z6LB?cAqX;b7oa0yBm<)bh#FveEnLzwTI0W14P50j`2!XXS%ENV1)dg`8UAD&54o>1 zz5`+!=3;|?f_M>B6VU3=Yvvh0U}gCLdtS>xeM+$np#L3)7Q5no%QYpquyLe+(QBxd zN>%{2=yZ(lzyf%7;SyZl22ANzUIjE-DI3=Y zobnNz#1Ad%l<%-PCWRqG&{1W5r+k3?-EILSXh zVsNK_+K$cD1&@k96W8baiH7_>Yg*SJ4S7CA_M$!ybU;>sq|{AbT)SV~Ux%3P*Fa*P zouz(~c~wKLiRA^7-;T|FEdD+MGVp%&_zT#g`-bbvDn`CfR*#uuul5%qE$icBSiJU+ zyLX5P15xsB-|d=7Ck~a&s_Rh>B7F-m{7?O6AgPEb^bb;4AKQwXd-3YmVNIfm{#{dX zwxij7F3#VFhf|$NWc}P7FCAN720RfTth8UNp6K^mZ~<0>O@8(JxxwM;J4VH6-8!%j zx(Fj+I7d?TJ&+nW$1C!3#QnqLCofOTRh){$ZY3R=*n>#)@2*r;@qCiGpuGGbPB(@P z`Pe6j$#)b2xt`|V2nBMljr+aih8p^;&{=m%q3wv+$8|-SFCD-x$_0bD!3fw4X}Nz} z`zuVZieAbZR@Co3)OHae(XYElKl*Vco!d*5wU8SGmWqw}cwQqT*PQ!Ow@ z7+ZRZ>s`~886FGPAulIz4T%@}1HhcM6w`JWxWSe=Jy+9He=x zw>Pt0|4+y$l25J3&XkoJqru5XR6M{7Z!F{=YaWD{t;H=M9mc2e&T?L(Y{%PVFN~<0 z_hTZAc$FEqA3`uZ7_7T=QQ43y?$@16g|I?Sah~c`r8e5WDc_=HRW;-;nVUIy{C`p8|BBgi@tT;wnA-E>_>r31a01M2;~rP5!U&tRC*idhV)*Z6R8 ztO|r<)zp*Jd2#6LV?VSOEtz}_+hr!+MuTiV8vfmZ4FHZ^i(Y^(D-U9i4rrOWo%6Sm zyAnfEg9%jGwu z5?xtfv-uk81k{pbyXG+0#V|44N3Iy&2D=%P=cW0$d1n-}Hmi4Q1Ay#4WOW8d;l^?D z47zwD?MAKM$g(Gu`|ej9D4?0hx6d;iH+-wo$57L^s}#Wm^%o)~eQaF?-qWLJ)VnT8 z<=EOE2Z%G5h<3S)qtlEqT#y=$=A0bzZsf*{?#(0o1Khy;EHR2>LOOP-B^f0r(*~0- zOHQM<2av0G;(radPziw=ou~vq>BNZf5FI-awH^G%@L;0@f*k+cj7oJI{0DURiuo+j zuJt7CLIa2il=h6almR9p5{wx^T|^};{1konN?fy?vtQO-@YZpmmD1Mmx6tsV;;!Gq z9mglDI+CD#I+6he1+wDYbU;kdUWVJ2za4pMZtH$EdV;)Z`I1 z+F?W{IY8{SaqrvFf8VIsiZiA!x)iZz2nArcl;WK3ZF(@WVzdbBEM2H!qNp?}$+vuR z?t=b9xTQ==673#|4fd~!lo|Iet~jqlR}6S_2E}?wpTV!D_uR$0e*xtF3DS4@q(sCY zJGGahECv*uGZ&%9;$;a?IgIq|SNjGAh0IZE*EEy+hs6>m((EJ7{|S7Tsvq8%Pk3?o zTO-h#iT0y?8d?l&7kQy{EVy02|K1bXEiW93DIH)SqIc!td!Y0$zE*!XrY8lu#e)0q zd}63oC8#EXrVkfom|&o0)Y?771mo%{qWrs9dqFW~4tNLs3zs^R2||Q>^N_PGdWqoO z{Y!hZ*tZt{|HBy*&wo*k_6Yp>k2`~=Tt*w18CQ}U_@o)0QowG)B z!$tY9bxBLkAM25#6ARvo+gw{DNMwSF+Z46n%EB6uYOtYb#Y{&TM-A5fE_Y|@Gp^E3 zhQc>fSA|zzRrw*B(tWFG@TdhMJY~_4M2O4_|5Fs7+&V8J+m5G@gJSk9U#=;U`Y4~g zO!;H~P^?h9-N*%Z&n&k%g;72PKSEh4Qn0{5lzEGvcTi!D!y~Sw44+*pwWUT_EySqA zb~xvwv8&uI#dpV_m#3_@0_erFGv2{#Vm<29t)!Z2^A3JjK#qM!mFrCVZf zU-sKKtI><;9Tybk3jHMtcH>fP;hXnx!tyC#e9BWHY{43Yp(>oO>X01=_!i<_%0N-1 zIwVna_nruWE~6Et>#(B+8}lwYS+_kPS1?O3!`!)9X1Hc{Typfwa^Z?8rehtlccNF) z6F`MP5bavp zqTm){l_FNL0SIe_ZHrhDVi<5mOw}!9*x3@MAaM<))8m)@8W_9C5Mi3$Ngm}%Y>Le( zazZb7B&Aaa{OpD_j9>{({kPK?+I3A6vTf`Fk;JLAOBsYgJLC*A?KJ6}*XEL%Ub z8gY3kurr0N6d?=D&;&3LK}5#?&>sS@MSSg+0;a;kkg$e-!Z6e50ohTD1E>XPJwUHO zMGlnNBr6qvP}P6JKp~rs=tt=A2jqo1p@o!T#ZW=m!s23xv}Xx*O>pQRM!5Nin#vK- zTZ0bb#ALz36s1!|WLA%h!C7FHfAlYa|Frs8Mv`y<&1#m&x2ik~|6NF50>*$Rtq{j6 z)!hOoMrhCKBq|bWzP7L?c)`_92=z(U1c)e%ud!nKO#%b_C9?HEDM4JjTYJycb*uU8 zd~Et6hyDx9HEO+zytwUzBWi(w$t^uw-*F+3g&gA4bvsrAVY~6Zc==agwno}hS@+BnAJ*DOs0G{&muBrGyYMVu# z>BQ%GL`ktWuhw9t8#C>>{a}!Qre^3)L9GOVM7YLwBQbg;L9ld^ggX^G6U)WFVU6Ap zXOMVD*%+vfPi$~WwIoDfiDM`Z*q2B!#LTWK-%OeZA_faEfiT#RE#M%<_&v#`@J&|x zgpS-e1VE8z^^Btyk$(<-8`KxTBL0S32mjm4aJb+}@2bJzY-7K;B{_Y3IVO{Lo&^=N zXNP%#92K@DD2Uc#W1yyVNPMFPVLA4dgi>^Yz~`C6!_0Ep`R9wN16?2O1E9VzYm!?v zK)l=0)(GS&c^Jap@DrwOLYigp-hUhknMQ+fD9sMCg&_ouv+W^nlu#*N4|KJb2}vJB zyTjTCJq;)~vJkm!rq9^TK0WU^IMcYh1`mN>BY@AgNvB%jrZC)KS|$Y1mB7r9HiBS2 zu_HqLjDYP0LkCP=jHW;hu#7)_A5IVKW(e$4zd3r%criZ+qC4>~)*KR+GU!M0HzJnc zk=O@rq7M7Zvo;0jVPHA5Kx{^y0&R#HWWAX!B3L1UXr)`1Q#(D>pID4ZK!h<|j|vnQ zdWNlYiU8UjE3MyarVBfP`l5vUC{zJ4!HN^9)`xo5;iUlHV}#!%wjnkG{3310U?h(S z8%`?5iDXr9%VeH^!ca5e0zOtSf^|3H;YnQAkl3*G&1#Ak5~y4225ced!&8cD4T6Cc z(Qb@!^e&_4sKlB|P~3w0CHE*(aN)&kX{J#3WGE^S`2kp^HHvGoEBSLM^iXtqSH7tm zxkdJa@KXp2Ao_Llg+Hbjk1HyrEwhvki;QlQNlAqT=}0hCsZC`~(sq+m4c~BJcU=9x z1K@|mQHI~@s^+%&2R1nasLQ^~jhBrR^Y>@Gqf5}RVB(FvBgNAw+gPWleQIU=yjbF= zZ55jol61aeP;2{X5I!4Uwl4etJo7Gj0N(f4m5C~^EJp0Nf1pe0D>tq)OTZYkO!(}r zvX0*vSU0ZpE>W3&SJ)()>_4j8gIbHMX?)tNw~bdj-5Lw|PGK{+c$ef7>QDO7h+tG5 zkcZA(OL4jv>%2ae%7I<@S8lLR6D;NI5X+?@4-nF>7gq-Lj>{(_tfBw^A$+Le-XV!k zb3^W-4Nj=$qa<2x>HVHz)7Bi|+N8t+Dpf#OS?5@Jm1RrMWwf>)H+&s9qHRh>k+rAR z^)ApIpBUk_CR}WTX9rcqiX1105%-r^Wx~GA5}z8#{*{J(laUbXx>h27@q#d;YtaU1 z;BKhBBq4Te?Byi>8M0x(iW(BR=v` zHAD&fa#o5bE+qOw%Ilv9w+YmL=Xe&^L{3Ft?1i7dcE2vZng0R1y{Iukfui3=ywZZ( zJE%#Z%$!KTYz^wPp+nZZn!jRoEX23Bntodw$pBuEsf-X1-_lWp@Ub-Gi4 zPFNF)Tl$qst;?w3foQ%+) zN9#e~6{liOk*mB|@z*Tl_T5U-^#8;86qnES4-ZQ-;TiG9Q>$MbbE;?*cOC8GCib~X z7lbbmw{WBJdS_m}axk?a%XFYyl`knoYLAGpuw4un!sGXThSI(9++A}51be~lEtjfg z{c8pnT(KSVl85VF2-bziXf|#djYAc>_|S$+3GuSbqt?3519ph&j#*34swc1+N-uW` z%^f_4MWY$~?_stv_J>h-gV@!f3e}VS0dDC-)}wuVux~GTu>CU4!-}kG*dY8eF1B3*Nzqzg zQLB>-&m%q;EuFTIK->IA9rvNC<)?P;h9=r2QEA@P;qCDiA?;Zc-1JPnLn}rY@0bs# zrJWFkmzYbBF{-$DY^htpE^pkj1X&(9|n#$TdCJ&1S zUtO-h#J?=Phg|O~dpN!I`*=m7yueP9EVCI1(p%$8EyhPE{e)6z@YUD!VxB5lAP$Nz zNrwI>+W5M@3wZaKQzVGvs3|wW<3dO8ZIO3~yP+zpbsEJq|7J5=f=P68t2a&|1hX{(hZ8p=%hIe9zz0!|5!A*fp^hw9%HbkcJgrK z?o~g1*lXDFt=LARo*46mSBe}PX?J6v*AHpO~4ryXnp75G(@rS?k8n zAWndOm(WG_+)|fZ0s#<&FGQso(9lFhgfi?7VSM`{dp`99Zx387zb{K&Cz)eYe zd|4DhV9mdCGEGEpFZ-mc!Uyp8S%&&mLSl`b%$DM@XU;OL(XXo#kZ7eRPB?9zPoGJ3 z%?~z{td@?6qhodamYsUm;M?5KNiyifRR_zxSr@^4royFG!*q7LA8$gR$TDP=?J3gi z6`KWv+fd}_j-7&QBHdaR+zHqJG$L2W!Y!Tri>ljEGV2q$og!st^s)B z3ZFB`*+J+Z?0t=VqMaDM;iBH5Y!=>k0Gae1N>tG&oe^={@BjA#tI% zJ;h)Ljj+X9@et;_a{SS(|iH?10ZA~<9Wh+xIjeY z1S{FSE1OF@n(Ln64Yf93h6otZTE3%#mr~jaIXxBtQp_crj$t8vKJ==j^QEi{Fpt@> zg9g>%3qVd>D6Ybq&JEG+$@6|^SW5@xbrfS{@3q*rA^4Otsj@I|!er?Yo&!o_s7?d0 z6vT^=DE4GI@d=n2Hirt&8=88I1Cxcb-qUO*z=97kBlQ1dCbt@ya}=Djpfh!~lliki zUy2okXT;9*Jb0jVRF`tTz`--FvYrbIDBid7W+(9h2HR57@3riC8yz}PQmpETKWZoD zoWS!@e`v-(NRCcHG_peXj@u$j2s>~@w{LtGPdP53jnL-VGLr;7*TFc|TP`1oCjv3Z z8$sVp&De?MLC%L!T0xBzS3UzW!72FS2pr8U8F$jthkkw^qm5b2|&8fh@9gty+zz6k#8 zkJ;6qh{m)Lmcx4vk&K_|y8#PmULRaCL-#tE4l))T0vFYqh=sC!{|u6PE1E~jGn8gA zS1$tqLT}Uen)~LGZp%e9V$DCzrAUWa|`V>4Xs0S-Ge^=8R1n8+Kh^_$}3{_Yy zID~PT|NsAwQ4Cb}Ljn~B7g>gurLz&O?)ZYS0#uFF;|6^^Q3Pz?v!T9@M7XkUj38eL zDVF1gW_U5Tk~ZQ#aXKi{g_9nISs9SH4JZy*`pe8}H18uyFy|A{21%eA@uwBhQS>&b zL4S854&X*X#{_}}kwAr8^KmrtD`8iG1#h9YaJYB`JHo1DT`5_n82NN&Bug~`mU05j zzJFH`mh@nD6wg5PGH`XnD)R#3>u3TqTLPrKSrVJl{Zy2NDye7A4W#cE=sKqFCnEz& zkfTe(Ou%FCrMP0cZahX0+8*#(Rx^N4TW$wG#?yJTMnJ!qtQbM;V<2fQ#A76{bK zb>qjwX$;sd&PWYTV_>Wejz(Qmkzyu$mTKrXNMl*k(jmuWB_iQltb4hyZgzZ5o8}_22{Qnu0C5PZDB80 zsOm_jfhGD-qzQUr-2BVsSZExT3qk&phq)TRKCdKo7EKm+p)JF;l6kMOri19tfQqsC z9#lH9?AnP_G{o|+IwWTYCM_kg{89-5%^)Q$WS-`0S-U#*-xZ(^Vq1<-M)}P6^SUx%Y|_R)=dv z{70M?bpvI&>wuGN(W9x|)~`MXwQMT$LWS?fU8A{=M?PDDJoRJAI>OuNIroJ7H4o=O z>nN#yyrLM01-^&9hEmrv&wrny`pKx6q4y?HYO>7q9BMvw5%MyugmQUtac)hbAOEZL z$FWGcISup;Bja~zsxgFrrKm})f7!jEMZ;x%x#|f&0<%k$S}b{hK;-VG!PA3ly~J4U zlhZ+|_S`M5a=VD%tZqef^=W^zdPq5jPbCNSSJ;ssc#(^1j?!H8_u*sC(W4@Ndx*%# z#@y83aEo`DuiZn&r>o*I)u8LhW7+o+(-M8;*M1PwBhLRIFRv?B z@i*>;l8)v!uej_{uF{KpGRx==r1Qw0`-LguGL!5-(&Xuvjx9t!^$Y^1{*fWrP*h8` zKgi}n;vgIIL{aRcMv3~k$ulJC-kG3d06c2Z^z#zD=&f)+r(;#79}MllPoWbPIn|st z|Cnt!E^aZE&W9duxcZgJTAxVX5Y6T*9TlVb=1ZmnNAD$!=DvsfR7~utspCFU#3!Ss zV6<-tTlM6$dqXp~FReRvN6TlF8~3e%Bl;m&blZ+%uLARLoY&h0$ELx-gdNS|t~EVP zZHmtMPp!T!=sE1*U_Neof5n09BN?q7kXoS!tD$tg{N>JZ&g7`D5lsxPuq6OJIqfBu zXT|LDkzDJl_#q!(vIU+I6{$mW(zcpEH`%7Onos}WhoNk`eh3Z`NKUqln$u5Cs%}ph z#pW24aBt8$ATwzkg19vZDA38Y9<}UHrXK(zO1cVb0Mn8@!2EOGsK}pkZ#WyFO~YZB zVMXr5peWzp+oN;jxY};xhtWFGY}5K;qHWNSc#qkt?$|o26Sj66&AEh@*4I-)0|rGJ z1%xXiul2YTX@Y4J_yV>5>#P8C>AfqC^@j#wE0~}A=TVvI#oW#ZFf}MpS$0h+o4jH8m#*;eT9j5? zZ4+ja#`EWxK0Gyg6(+p4m-QS71*zlmD}(A^LFNs7c}BU~5aKI@OS8}l(L39Qz1@rN zuYyRuU1spy>m^Y)yH;)XS7l6I^@RQUF!kRCV2g+OJ(C^njLznAwx(@p~9E z4yI-p5!#%*(I;jRDuJ7;)RzA8b2z;iYpgmP=`unwD}Q3iG=_&tfC7Q4Q=jAT3EKBO z`Fn9qIzuyyu_8}Mkywb_#n4D{>j1@4Ffl<7ORLDK!`xmWj1NyO&7=N@EMof41Ch)XhTcnZ1MT`_yOx(Jb|x|uhIZLGPEYN zA-4J%Cq6@UnB))y(pk1psL7)CSq^wvqH&)EJ_8G~(Ktib%`SJ2#PW?GY-N)k(AW+k z9g=OsPW=81DPDk{yX#ciB%#fUl`21v4RR<$Q$iQO|2pT`bd;|9oy|ZwbX6#@{V%|M z9If9GFA|!g>x20qgF|hTmIuPFdOrFzv#SNB&fFrjdhih1`=PN!UA3~2K}}SB3*uvW z7)(yl>gng<1{n$sXnjQ$)~Z7bfnT6PJUHlqeJ)v z_B*RwfW(1HyAT)!AG8YK|~P=*UIRiG0h8&}pg z#d$Q21istE|m1#+_yBh)onD|ETnBCPQxF zo^$;(KXIzXWI?fer4Ix8;a~4P7=ExEDq0i{yRG_qFGSe_%ojZVDT^x=>!IbrfjKw3 zABuq8aeAmzuc37$6=ehXIt#%31AG`dSYLaRf!5LzxdZ3AXHTb=F3efDMNjlaV_=v` zxW9R(fnXrnY8RH9u!c}bJ`2qo6ePM$$dV+R=biivNgXzg$h-{A!qdc0^y2Y@*oIHj zK&k#|k)>jUB@G-B8MJ_EVRxBo)6H+nreJb5cCB0?lTO$8A3Zt)>Fv~#3?Vd18wsz& z&5@EmXg5X&gBhLoOLn-pP{F?PMv)bpZ`5oKKxnKFgs&IEXLL*Sz|9uiFS5E;{TT)< zP3>`17^`JBpib|Z@4fI5OKHas)_sGl9_W^A!1=%5)OOAI6g#nTY>@l%>qg&#=$> z%^0Mbyaxp(nL`PB^@*6vtUy%f@8}h}NLS2GZpd|6mK93WyPe_2ZxvV9Pg`IVwn4jlXpZ5CbL1 zZD?kr-E%^z%5Z=xm|6Wj))-Qw)PbQqMbtla2xgmVRC*&H&!-#>zvNma6ZZA20v85_v;3@~8{yCq<~pYztlh5TpD;TEAMr$jabg;X|zCwGlt z7%mZHjgUzt-4H=QX$kVG5C*U%=GG&)*ug=P*e87B*?+VES(=>cF1T8E3?EB@dhv*OuRO-x=n7d85Ib#W2CollC&C%?E;vcUE+7W1b#oI;mK9b1L(&$L7#C}Dx z5)`@kKW4J)G(G?c2PEs~LK%QZvJo(lkR&!8Ks^Ak2}i2Dw_}^ovK%}ibW(-O;kxAa zQFf1_3iyw@rEz$&rwbA!Mvs zod}-BHspUX<`(P?2UU8wq+DSldYcKP@GXKWvS18^ZuTZ5?syh>;q6XPOjPGHlud>8 z7%aA8De=$nGW|iFXaJq*^E;92-b>pzYc#k;F&}p!;N}_Cu%XYdZfsQxVZ$NLep0M%fyOpuL_{bz8f zzGLx+=p6&rhTtkHS6IdzbV(U9>U}~<0NDl8OclUyBY|Hg1;EquaGQ||8ADlWH+T*_ z24=Xzw;1hXguraJDcWl{L=+U6H(=yci zo8u=Wx0MLlE$hl=5*iE3qhyO0udAt~AS4X_?ex|-VsOn&BqnTf!G@HFVhD1Pjp18- z?E;_hGBls1aQ-OE$<#!x5E3y3fmtAuLQ(Kh zJa*sa(!7z60SO$y(D}Q0XG2lKzIuGs@X!FO(yOsVfR~?RnI0kKvX>UQ@QY9vf&;mJ z_J^Zz;h6n~o#=f{p*Ll!M0S7-Xd`9?4|RM4A^t?SpB;;vB2S#ty^alGwd6Cd{y{iA z)(_+0Tz}A*S=D&_b)Ew77#7j9I6QxbG&;nckiyR=M|CDdMat4z^G%wKexlu?Cq0pr zu|n*NxJ@_@`?wGu;vy_z5>SVdT_g(y*vVMrQb?~B^iWrJ6n}G%1swD#00u)4RYKBj z=LpxvKK`FXM)5anYO$wSgO8(!X%bqi{ol_vu#Iqr03_tXOPFP*pqWbNkaw}iI~wXh z%LfQ@(G}wDGsJ~<{nydZiBEVwmTezCjlw;${)6nVNK;wh$H6^-5+C6Vj%eJ*PJs8T zQ?598;w#}<(Oo|akhO+-zZ2N75mhCdVjE9gPoYLsM3#p$6osrcS{&p3FBlO2pHo@k zp`+9k-zMzmy8xmCv8v zb*iUUp3ma--9{Wf1H3g0p&{^uZpiDD=pz^_6PgGM5#l?x6C&wCTMI<;gEb=DnSD}RzBv3yLqH4eNe|8N zK=ck)bQ|(nAE6;mQ+~vNvtkdR`=Kbyi>nd$_a1k``xlqX=JL%zNZ93LggN^phALjo&GlY>5%5g)mljD8CHgUXYq11Jw`bl=O05JDJ z3dAMD9~mA}bP9U$;leS_G%oatVxa^LG`2sm(fB_pdr78@P=($Au-=C63VYMC0#t12 zc=$3jc%r_!sQ4W>yDe^M$44QkMMekl1IHQ(odPvn-qgUZgV31cfcAYGp|bshLMaIeKERDYF;NB68Y|kcR zc-wxr9!y?J4H*#6@)W0&HkaTEE)An|6rUl>Pf+hy5s`Epx3M3rVa@RRk0j;ns__s9N{}c z@x4sSPR-%a#9KMF26PJE{_g^@&qfntqX`mLmNSJZ*=H1^Sg!{=uA$&FRYN7IdAOQQ z%kt<4cnBhZC?RA-CuyO(8$I_v(-Gt%X`B#;F9)@$lTIwELJpN?2h$Li&|lT)Kgz<< za0(>5mTC_*rmLI|Fx3<4hU4%#hN*kT*2!9~1_FmMkngs~72 zOJY^Z5HU10;5G=5p`SW1%1wN$b~2C<3N|k=Mactr>!z@^2;tT>p^cM6C_csnnyiIj zyp{2HlnPE8`2Prd`|zskGu?Lq7tIuB$~AivXdN}z%)Y$J0Lsj+wn{YtyHn24X@zdX zTP2x+>4u`MwLl%vxM)4L=YrOpYf8$lg@%SvC_!loNJTJ?J(w1^kbsE50If_UyhJL5 z2zgna&wW4NwGunC&-o)HBx|kT@ArLqp8L6<`+2^*s|THaoJHeo3J9U~ z41v}SHKNLkFxOM!c!mb@BABuY+KgrDFS=*ifRTtGOpU+T3WJrA)t%}~?dCeuxMKVS zBN!QLMYxQ{+ zKC82(W0^9rBsdQ4W~j@>LvAp-`oH9r`N zF($pDD$s>{NL+&0Q5|!WYTl(CQ2+c(K3+r5?RJ1ZY;5Y1XTFWMh!W;xER3qz9PI(%DkJCVMDD4RalEKNn)RVyCedwRkTU&$(iFOplnkII9FMP>4+(*5LlY8*|=l4rgL zLhgQ)rwUHk+^Y!pOmqD<_yj&;(YMTW1E_70@z{TzN6V&`1CpnQ3Sca{#K6|O$WLi-+W7FVw*0kA>SqrJf31omPoaR^#c=1Q>MXYhcc(~wfdy;NA;RgrEP zz5odox&hAD;p4H>Z{Y$Hpj{NyYsoydhUG&f@)}aQIZ>Fr%KnwcCa{J#)*M;DK5re0x=mevc)w}s&s$z`~&uX z!F3j}A1y-_*$N1?h8_t*a!}R#ng5nYi3>P(`n~Nz%Za{hgl|9_WwHkM!gqCDV4HmA z+@xQdG4_!Isu_VFq1=c@A2JqWBvw+F5iB-oLP~vVGdFT_=6nelsG|ht=`QeqLZX?E ztA6NX`evd2v|%5zJP=^auTw7-?-039`sp-i^U-S z&!M7)92xKud1!+txj#h?SghjTJo6H(?ydlM@Jxb^(>?>Z+5K0t7v@f9ZhEguw8hB9 zsVik;7e@Wkxk;`DTBwpR(eA1d0X%$Tq2TY1!F#aO73W$0dpE-#bEE{phq2U(zg+V> zYY?U>z^J;#hI1Yas~TCX7r^=gEb;y9?=f=R)rX>{AoF2Bhb}PV?Tcpxwue|s@Ri(; zmb^i)GU?ieQPBlgd~7eIXvfKm7gQ0|^JZ=plO?yqWD5#69L+v`98h?qVX`;M>~)Dy ztrLRy#EHG#d=UNz#Rr!GG*z#=Mt4=$T0Y~ouF~TV|93(+g2s-_OX6oKlt%|mcCy?u z)jzGcFuMN5<(FQj{=@gDeFnmG~e1AhDn8 zPL>b*IniM`lrM<`+&g7JT1xl<7X+QBp#q@e`F+d1v!&3I_B1MVD7S(bC5=H^Q9uam zf?@>U%nqSy3 ziD4Cnj7g~3sWZ2rT%&nwxkHyg+Ma2_Z+{_$RI3%7LB<_l@>N5OE8>}8W=D=O1T*rS z$vIs@f~7|Q9#5Nb6_CWD^#}*=Xezb#qGV4|3DrDr=_L#$@y0_~sqFWIR@I=A zSl}&fj%Pm!m|UxffESg$2^M;Jg}UdVTViB}&|->kQ^TI^E75R3Kt>?{*+)5n(rD6d z{BRnV`DouFfI3s4{C ze5vS(>xX7JE*oQlxojwPVU_F+5~**P+5m2<=8Yqp%E^Tp_A#hqS~$h%L)J^smv8^o z2cJdJyn6wrR6}sznm_I)r}@n#mGCGAgO*5CETbh=z`r*Tg*&%B>D6aoT!|pgehFNP zk}-#m?5EROK2efKdy)O0A$f!$sEEYsKU)Y#Hz|{xOd)4ZXpo zTn<7EhyRB(;phqaSgl!n5=ViaO~hhFB^A36CHf*;YOQF3j(L7g;jYLQXwp@Y za&93yqcYnRo1jo+oB8xJj0O$(C(y*ho=tIaKN1w*h&1*%yg;poKS1P8ZT1TXRg486yT^bExN!!P{QJiUj+q4VYGoSJBQ=3H6_dw`k_sY zO6qv`K=mtx**rrx03)P;#>ULh#z3y+8#ie=QUB z)_A7;9F+$;t0&J)g;L`VAnlgB@21Y!p5Jxx8qPHOzLu~JBjbrw#_S_vmD-fo(gZ=henh3 zxd^!pw$%~vEy(bv7uDLTQU1-jb?#%*)(yi;a|J22>vUk-rjzu=-`e$R2}zvXGig0PLmjuQ zQel@&6D%U5ce1xScZ&?_bQnpBep+xgH)K06JxTBUYboetYGmYcsE6jcuUyP}h?hb*=Vh zHtE5Q!DcMKW&$CQ+3L0M3Q@0*IihU;t6Ht=hIG^sujs5w#*zW-`*g?js}MEPfHx|z zavG$9wF!bnS~awY^QS*w^3~HFC(qoX&_*ruww!IG96wdfNSQ#h$t{~Z7B+*g5CKGK z@f0DyJDsOw%{|52Hk{o0#mWG=*>$rOU$o%r>4bO3m%eBc?I|>Qs3v@f13QJoS2G{G z@a1Y8*1iax*7x@P({~xuW#ih1oY!_<(Kx?qqBH@jfKU*5fVn1>t`>DlbWa?~QUP_) z4d{4ZQ%6~zg>iI|Ob58wW+Qq@^7k}_2D9pUAcMpVQ zev%pTE!sYTD?>P5+DvNHM~Y5Mgud_tNJ#J8B~1jj=2&jd0^kiL5JVV_5R2;k2J+{u zN1bxO(|WI~>x05n_{T-xidt)hWEl?DY%8vOe&4snsBa3}V75<5dP+9Jg-BK&Y1u6b z^RTk(hKR2Nie01Z0qTC|h!3sYt8n9*yTya+(@ZC|r+5|x_5U%368&(O?a+E4z zjR6mR;M|N~KPLjZ!r#F}UfSECb0hrYT@i^(yWOQ}Sj^0PuuMfPja_=~$_3X(jXh386 zkT_w4^`i7Kzm_^^Y*RR#3R!qq4g6*ihCpaev6%qS$D{w!o~0H35?px&Qg3Uk52E z9f7zQpzdXV4nQ!nO`2kYhxw75Ov%v&71n|N@%{kvpjt4X*`d?7k9<3eB@dj3N|#9# zj7%@QtD)t;8sgv*i0A@GA3yR2BUON@!kl0|J&iYnLPw!2C_4QP1#tnO{dB1>s&v0z zotVR*b3cHBQCuZt79?i0;Hl%3m_&L8@G|t1YQpkgFK;sZ)w%=HdLhSMxaVYd$?SyG zgb?0%oN0Dc65!1UwsGReF8g+6G!8*iEA}h;Smnh4mKaH2tB^V^Jdy=sEDDO|ESb`i zvl;X>f=XCg0ZC6Ui^NxC{V6t7zCGB%gdgRGbd22=YgLeFRU9 zWHHY9)-cnSl0&-3K9;Q%!Km~<4JGfM=A*a8@k*re9wb$|K?V{qGESdDRQbKC?Ec3O zLqD{!TPF9ukUh9(#C5;(Y(?$g!CV$UtOe|0R>J?=yB$ph`N`f&Y$}_H2AO^Z^?;C`;9kW2IaaiWb+e=(8q{&Wtbg6hiM@sIU7xJx+}5Tznlt^xP}cM7PA1 zAKp8fyLTC&wG z;{=d7bW5AxR3Yqnww!@Q83V6YcIxdDYU07o-B3GM9b)9hO54jW_2DiAq_<4}Ye3RB z%dU(f5U$)zuNaU1iVW37yjD4Ex}#uIuld}BaU54Z4CDI3s3SXGY#6zEmE2D>m(Env zvUqdEZCvaJpS(O*iX`0_v;Dx89Q{2Sbc7QY3aq>1^{72_Pwo9z`egQQ{}%=NJXo;7 zFu_VQPVO#_T)_0vZ|!^IhZpE)^x8xSPJC|j+6|w2T_S<;WtUFf_cyeinF%jc`kO9# zx?jb-z*c0qBuPDd&KN`YMUUHvGw$kg1Ge`_$NA12d;aE&M<`fc4~Z z-AmSP=)0pJu$fr=UcN;+Y^{#r-l-nLhjm5Lae=v5JYH}WEs*KZf17b)g6u~d%jIKU zS^>&jK~VKwez&YK4#rlV8Q|DYQmVO_a6_=P>Q5u0rVh<4$h(vEIWv3!@SdLU;ztg~_6qCj~7|lI0;z z5d5+jQw2L2A&dTkiM@BzZuT;KkK$n7dxBqj(#8b@1ea zLuCyi5%{rX(U0C+B_|9Ab$iQv+#Y@H`$<6}*r)*rJgMTPu>lqhp5U*e!bczA3y9mA zAhZ^jE0M#U*~5_&&JCAmrmB4ErUwWiBO{`d2A1FJPPS$nw1bhypXFjCOwk6Io4lqtKI9To}8C_Jw9J z{(GdPYHw;Q`tz#us_E}%4nq*Bo@Cl!b{MJP>MlS*U~54;WStGKnzvrC@Ku4@HF`DI!|q7KQcnOuS4 z5NYo2l}8&T-p8bACvVRFww!E_J{mx1#&|Ss#U=N`2(250gLw>heAvj`=TM|&Li}WL zjw^crZ0;Div|$8Ikezsvsak*j`}jM+075&KJH!vhHZY&uraW)gwU&JUEtfl6c4~j_ zj*V8!$f&*HJZi7i@{-QV=)Mr!)Uwa3$#8CYKZi&uXCrIiQsQk%`$_7#i>05p8m z^l|6b{1H83RF7g}2zNRsq5^i%g#*5Q3)$2qWu<6ACiP=F0n-Xd*$* zVy(JiKKt0Z7(QzdRW_0su2P{Xz&?Ap`8!;1!}uLj2hGQD_LLXtY!p!Kah2T^buT&H z+a$q0)pOH`hxn+WLu#KgDQ6u1}M-UfA--MCxKlD z7eM#Ye&tG!?Wk_*H;){W-*g0d=;6;zpqap09qaM*0hu+Jy`PYOS5!e)%iOnE4)(@3 z)RJ*r^c|(j&sSX;?t&TAihgnN+#m=aJO9nHOSuJu#+_{@X3niGpRrZBP9{)8p`#h3 zNa$$pE6J_ZucSGFhiGw;gO)44>A72GSDJHEP7CJ5kcVRtIi#hod;XD2fY4gdLkRld z#3%RmRU>rE5e;#7DNTNn(^deN=M|gM&S(uOPq=)PsvCa>1gD3ta6nDp zZa$4eDm^WUGNJd$vaxHfv1xJV(&i3Zu`L7MggBInlyhq984#%w_reN0o^$y+LLr-EC~*1~fs^H9k-FZ-aZ zKe@808@j!7en(yVtwcIo)T?5HSm>mT$Z&Ed5lVnT5Mk?QKRs(ErcLuanj_Q>ZZF!(oK?ZZ*#a=c>;dE^yDw5UuH3)zxJfo`2q|H*R%^qB)4$apntR3gj*7IPlS zN{;5;#D1q>Ap7VlY~52BQk9PRj*<6)&GVlFt$+|YU_nw|KH&EF2DD(l0kUj4at(DqM!n5+(C`DV{d!u^g z`9g&`J4MVk;UG!fh=o!rqwd2E|KPP@_O21)(O z&Voi9!up6iD4}{0IKtUDYzmE<1%Zmd0I$*$?h{f@U$#L5X0tG}1KCc2xq)B(>?fsv zunvQ*3>HgLOnER5!;PS<1k4t0N>m1!5dn773ECj~SwMcgs0{v6tNkp1BF%@k;xL#q z=sY+%e5Ye+fL=x=z2r8gN2Z=DhKpsv;TlwcMzf>*6S05#SFxQwPfn!eUeS;O4fL8> zYhP;cxPV8YWOXpqI5B_qv%Cy6h~97>6A(BBJRBfhGowC7`LT32+X!K`*;zh3nbJh0 z*t!;G#+F`$Q*3k1=0@_~IvRGTQvXDWI$C>Re1LNfRx`8ZwY8)zw9Gf-O@?nb`qd_K zX({TL+h1;AG@ep8$Q8-)LD2IXGO{G!zioO-Fo+SH_%RZY3K<|TJLz^a7BcjZstp#c z{SXH-NZ6TJztoBiILyc#RxcH$Q_0Lv<1?lUk_AT@m?(Ok*>jcr12v!%af?s>gnQ5y z!QdeZ)YcwSN(vdoNw5M$jo#ZZwZ=R`UCIo{{n^Vv!4A~z`5BVwTm}iM;&hN)zScuz znrYK~wn#uOZn!JULDn35bgLcesM>uAU_+p+Xq*GtVJcDr#!U$$deD$>&K^;z%=Crb z8xvN?f)IfY!fP-c^7e5}L=>!KXTVpbmUOdcom>`=77vqm|E@}+IzZ%>N~VLg#Q*C2 z0-fk4hkG)T%Mlm5hk%yciV)llceJ`3rF`aAQS8%4 zcl8WEx?r%xahz)2b0zAdQ#F)wNe7)y8lK%z{Um#7eRSta%qJmxwq>Ipwsx){*me{H zjERa68#(UL4(IjG4low#K-tr{<9l%<6)#|{FL+=i%m?a5ExlCe(o42RI%<7gFeKi>TC9__6;M?9TY zIXo4PToyOGpfnh^Euq+ek>MQcc z5i-!5n_Qy$dMSfTH>T}ad!0SM6=KmVVhgW)`JMfrs`^^}g}&j>*dOykG@H!??Zu$p zJIjW-uTmng=5;jP4-tL{Tl5Q>o^!0LDO5e{EH=BCo)^It=Z4uRyz z>-JyK(R#5x(SgY4P{Xsh!nt|+3Gz0ZbJE~9)IDwu>BON<(@vW>1as!n0;konCLG|t zZdh;SY+`F?2I0P&AIE?)j;b~XL>-$TQ%F4X9#RzVEkNG=2$|&GBW7 zokc%!Rza0n38#;cXszmX<_s^4o0Chgn5P{5D~LHs9)5Vhpu z)z*z1Q_cbQ9Uf2^G{rFm z;*UA#y7nVq*8?%uq$7q12uhwBkWdmp#vqS6^5TpDXksGM@YFxi)J3xZwRO1ghEEd( z_&B|v$3}nfJ)m?Xy{$*~7z2ACzHx8Ze8xnV(SRmfD`!+hSP{!tU>Ef3FkM#Qu5>R1 znP7R!?7=*GE*7Wzr~y_N3%+3+^Vv?7-GJCi5bJ3F)l5-j5oT!!1cp8^nk2IPv@@b% z-3ph*GbX%4ofMRiEJ8=7g)k}EZ7sC;2gVSiR&3{>jcdmjIx{TY11OEsz z0G4s6f%Ddawo)GGE^?30Ula#yf?#qYutWxE%Qm0xF0M6QvT*k-lYYQ_>IgZD!fbQ# zKKjbbg#ao@oAYQCVgtvWHcg~d_P2Kyqqqd^Jv01DA9Han^qy-og9}?!>q`bGxB5@}W0UPTQCz3LJzYrF29VAe=>z%@HG( z9@6e)AKa^#8#%|oQYHV;*i%u+xBAMAmfsoXYhYG3p1`awoO$z$S*mF(33qBos6ay! z9$`$*Xkqr3jXNsu@-;o3C!^%|1xRw2kh*@1Wh|!*$P9 z(&+uH{r0`WKit*onyXh5u{rFm{WM|UHf?(8n~(mctDB%ESHlhy)&Ci@OEaGR`tZg` zzfXR1%Zsl9Ax4$WM-&v)@IaGPjriKy4z|wQ)OTHF8QhD(zCb01lhxDPDC4r$#)-T&I99IKx+ zQ6kUh!+nu!Zs4SX4r54GEc%;ka#h(=Z#956T}dL|_w!giY=#!3>uShkMrj{CO1=lb zGXXm!a|;SwJ^pGhujXt-X!4^)z1+_JJwhK2#wQNlpaw;ixMAP`UPxL~312G2)5V+1_QC@q<`D-S~>y3T$4%ZV~^Q<^r z-a9jmdR{yMeadP9%aD#yvf*$svj`P{vPPsYIggtM% zwS=ndz6LtZ)Dwp1Vyr($X(?xvYFQi7umeDv3z-3)p+LFRudsMRZC+Zsqe)PBAjJW~ zDSQGHhQ#=Gdc-+3S-2TRvx z;EzOwAoaoa(cFx_+Zyp4ro83yWLE}`-v8In1p*2a8RUpUHSJZ4zhcNl;$zQ{QUoa^ zEk;6SL!@b>6dU-48D%%$RVY*$kzLof>C`O@0TM(xN=2&75L2ku#WD06ehrA`!TuUa zqx%0kW+xsIwPo5_Noin#d2k9U2B^y2$phCvPYQHY_RQ)zj6}l;umzBZZE&=)d@&g;5lZ%;#ZG0) z(E(;C8-2tP5&16?V=)@?uAmQt1Syi=j6|qCV$q~JkH>(L)##N8E}19mVsAa96!+&b z!U={ytzZ6$E~p06BzQ!yh}LY_u`~IUrMq8kp9?NQ>RUGg3JW`>ASi4e%d{U|jAtmj zH>$&tHFn!Y%E4B))gB;7mJoI;nLecAZJmmlZ|oXM00M?aFr1U>(27+ z3}t*Z{yIeO@Wc<%e|@dlMUm2%*su4@K1Cxr5)B@rR;6x8Gued-i5T7r(@2LH{G8na zqqvzvQN>AEw~|*}swp44(firB8~>lT7ue;JCGcDtRv8z@v>pWeUWuBN1Q^S(h&G(u z0o6rk3+7_So7pMQcPuwb=pMHyGG%$5Fi??#db z905IY@uH<)dGNJ-#91^EEQ$$3zsD`^z~MwhkQsD!ExUkfVrQXxg9C(q|JS2kU_+_PlnRs}g? zQGg%$Kh-?f$Ne6L^rD|6u`F-j78E;4z^)5Bi0Z9)Zc}VO6Bf#uj3i}K+qlr)b>oEp z4>x{mh89_hl(sqdw_4NXZB}$;B-a85T=`Q>r+x$!(@BMq!r%yRP(5ehbEa#R4fSAk zbov32ENhWt;@eN<9{JkFo>93WP&qc#;_pV+%S_(;Tjm5OCj2#KbOIeC5JP5)?>+MN zIW}S9Jk>^%_tg#+@FBgZOi#(&^ zO(YyHU1ddeq9k`V2n&oCCRZ5n1!EN>G(nfG;T6*&%pxy#E6K zVu(o@+#NUjzwv>-dI4>u_YE}IuKLvr%t8|R;pw3_GjnqTeto4t(7m9wKezMpu`jB;ni=hOgW8^)3g87pM)zZlQe^9V9 z_z$2gL6zq^wc5uGi!IPE8%>xvk~FB%`2Yb$cyy6bjwtkZo{)teNX5edVh9COGM?H_4gj zdsE^R0l0ouG%BHvDtl(_0^r_o9m|ttu7g3s2OPHn8Ex&x5=`MzR8;{p_}6#l*|pWM z0Bj&)9NQ{pl0b=&nYJRhppn1q8Q=Y7duWBR(p-T!Y^sc9ut(7S=4a7ffF35%4fp1I zEE|0&_uV(1-$#XN^ukzw{-F~@Jv47oG~bUIf(b}4#i33ljVfp_=M4~PASjhz%+mRp z+B8Bc@(cODN7L-Q6PFiCop5#2c_xBYt|{i>m{dz0<>dH{B;qJI65=5!*CsBM6;f@W zu+5kwJeh@p+{VJ;gw+>+*1m4=x}G`j9o-ZntPwZpxyF?Z?2wt)T z7=S+IvLJcU%V=~QaSOb04kQJH+F7i!!`ow$0@5&}Me^W9PSP z@dk6{n_am#IX2qM=9>4vxd3Yq!>{c(gdQ}I!^E4($%bs zu@>cr`)<#!Z)Go3?vqWSgy6Lcy$>I7i;<2+%j>jSocPq;tLWEVAfpcfDV2s1vzQDR zS4ll#>rnRq^}{O`Y#PX`d{t zBp(%yiznj=eA;Y;%n~}W8yP7GS&DfbqTo~)5~;n4)8vl%1hZ z1{1R@1#RY;dL$9a^iVcBV)&J=9k{OW!sCOn@y3&a6iYj)`RKEy$Iy>YE0?K~GDBnr zN^IODpdmGIla^>E}sm>8)Zj8O5&+ppbnB9~vR zL?8^T;?tf7dLKT0wY9F3!!<5!MK3bsygg(6?#K4a1P+&J#ufy>YJm5<=kepPToSpyl z)LOVJh28UcjtWi}c4krgxSc1aJahr%#a=cBzZ%2nP-aZy8ohUE;zX3I35V;a77@p+ z*F2Zqz#ZG<@g{^vHT0sZUlBt1%wLur%%1LAt;*Fcz|#7aK6Wa&g5{sMaL3=|Y9E|8 zZ{DJ=nk!cuMrHw}1jk?b=b1;2-%AnBYZj@#dj^}#PhCn$N^Qpq6e#k7VWi#q3n+Q& zWf{u$KknPw_aT8*eU?&Pk@5A6yLXAv$bb=yv$YMKlIQIA^5QIpU(Nj@7}+6M4CqtI z$k}{Wne#8Tg4h(N4=yYDl^M{YLwtc#9U}p+&T^3;t@!J67MI5+wHV+r} zAuEc-BLJWeGd&Peg?_2bvp&&r38@X*_?##{2jN(V>wae-Iw0y;fA*yj6)VR6`0RP#tual*&k@3mc*(3$ z!Fmi_d+F!EQK_31GZcB#HgAXk6}-^#2;JX%B8c&VurDPeckZqRoaTk!v4tJ#G2rqp zsC0$x1SX5;(mgN)EsYM9af%YMpJt;oo`gde6qKnsf`(U6r=7EyRiI8f(YTpUq@gwa z_Cb9LJ18f)*FupqqAdkl6y%e|V!ga}_!!iX%nnDiCnb9w4R?yqL-Oaj0H(pX<>2`; zj`yvhj42-6)D3qL6SM0n#CnsM6B?N%f>kujD$iL z{0_iF8j8$~*Y&N=J-Pr|F7E`2sMjWEI5}ixWD;c8!C&iDGOp!^AQQ!3U+FvSU`B{D zXr3j_SmALY&$v9hj^_qq5IAXHt+XLfeLZUuKpaI~t;j z{HS@SF3yOT+tNLM3UdLhc|UV)@VCkA@?p|Q$b^dhw!C2Kq90;T|6Tx9sWdM&HF+Y7 zNaOskN@i2~;_*T^AaO+V1caV{sr$Daz1M*NgysXkBQ%V?pn05HcKVfHw-AX%#$o^# zz-{#ZZ!5qhd_}mB0e?z=QJg=}_l{f)>cxKR>=eb!95V+110Ls*03$BA>2a)PW#Tt@ zQVq337gj-I`KVfnKyecUc)ZWuto)fJiW(;*}r>-|6zw-HBfpA#u8kG3% zcYJGn#2-0C9GARd@`ON$R^CICh0t?hUKkr>(1%1_)zY7aVRFeQYrH_L2#ZYD>8E-C z_w+({BQhYK4^x%KQuVb=vaOYTd-wP+(4+~4B+NiE`E0)8kt$L7 zHENZb1L@(%C9j+AT^SK;F*xH=jo5tMu$z4B=Mg7R!OA+^iPdA(L-@)QG*VKMLR&&H zp>$>G1K5vNNwZvJxI(fQ@=13EL6PKp`;V2iSm+RziU>HMWkbuN@9OiJ(_PHBL)c*Z z+CR~*yU@ArU#L36vUpga*Qo&bB_SXzaO7qf7C40Df7mAE;p0|(qoNl=ook%QrR@>L zaL0o4_W}3bp8=?v@plYQg^szgQi(Xru*{TS9#$bkto z%3=r-xN!rComx5su;x!dyAsGbrO$t*#_W7{m-PDa*lhdNe>O(8OqDQTMobb=m4O(4 zY+QBwAPEz6f&)NIt6^cYMt1C)b~q3pNYMBYCa3OkRY38>Ej&tNVw#i%^JpsKiJfQx zYR#>piYBoxQvI1U`!J$nJ!G#2?>?#ei)oZO81M)kP$hn(?95(Dd&9@dc?yh#w0$E9 zR6#v_q?)EPZ75{e`sB36_Znt`tQ*_;6qYizH`A9QJgr!G3b|aRrH+V%!)HrXK;zMYnq9#2OS>%ejEg2lA6K- z)@jxmB%e-Hu(ADU$#&NfK~3MEvZZ;4$O)FsKV5Zf-+Z}{a&e?KSY}g5p0if>h?0|E zV<&*azpFW*>w62u3e^bNse1R+YkZoRJ!Tt<%0|#BYkX{4ObI^Hh*o#TwCQ?e;_0?9$V3S%L#czf7Vb61 z)1EkNKwD@b!$E6{l)c;W;G`Cyg>3@uX~KID3sn%+NxVxJXK3}dWfVgZcV{b@Sg^N+A}3x)*jpAx;0&OY-oGQ*(^NSb}UvEhHgxi(f@n zNu&^+&ifA}vZu%RM~)Kcp?x30xek}`+w#l7f>mrmv24;n+x_oa{i=@Ptc8g5 zAke5+P^ z>A`|YkE0OPZYI_2+8q_7khmx86{p}4m#o1OXl=QByL_?&KRR)OImCJwuH|ULi$hWh zE}`|MTYC({IQDDT^2*V&{00Qc%IL)M~2GgjQF{L zxC*P20Z(gy1*cmspXv2b(EP%s>MyFZNWTmF$PP5+FgG#~`PFCJ8h%O(SQVM9OS?5a z&hl%TuD<)4HJd6+b`41s*uS6o{SAelJdUDa0G=k80_Dno4L0#RlXwvvMQMn4u-mic z-6e-kvD^%A*nvN#UNwfpcFZ732M@w|1l&?Rp!7u8%i^=Y606JyuYTa_GSHr|DJ>*} zw-B=DUS5SP$5Q5M`tZrQW+WQM6T>@)cQ5^!<#6jib@IzSe?sK#DJ?m}|Gv3E_!mvC z97Ne1ELs5jKEado;yNCBZBGO$7o;V`Zu`v;;`3$zYMhL$F*mLGP)$&XD(XufQAITF4%CPlx zJj?z!AAQZ48DbCcD|?ia53|jE+Kk+ZdK{2Vz600q4m&@k`9Wq*WmWYLZ2-Ka4Z<_K z1fmrwlyY(x51{^hVG>HHE2;Z0aw>{{rb*Vy%Qzb2cJ-{p2u-*~q+ueJGoY6L#)drE z`J_}Cj7`~rrc=#zPf0-1yZW`6&i!H=dgk}tM(YCG7qBT-pIRcwN*UIi;&Q-rL%JXkS+BYLUi0AG~j^`B)J}s9$np4RNk;=u*;aRT*DOT zOWh+{iJC;=Z#Ki_xhp4NgcK^0TTqT(ivuQ!DjDZu7i&?fUbPnEfJQ_)A)Upy7WXe8 z(e>Cmx%3#!sA%EFt*p{SM@7s)&lf0)DBa%750OU&Y&ndYA~+CQL{bTQ=F{y1eXC#K z1)HY-Km3`n9FK164Kml75L&BleEb~LmQDM2JV%J!)3^v#x(Iqm(5v2)keV!f5 zI}-z5Ws-ys5kDyjv`)DdJ2R(<-Vewn@L#S(Y=FTRw_mf9&i3r&S{N^54G6y=?;BZr z%KFoZODWvR-47jYYH0*Pys#Ax@WxAGTsUsadF82#z6&>5TbbB?;`2xp%6WDkZJf{k zZ`o+c-)2^d{v)TrdP~cNxHZ}o_h`NuVVPInJVvf!$vj~i&KJeSh$qUFo$mfMX;o+{ z=3E39LZz4=ud{s25rG?_s~NjLK{cms?E7WeuxP!$1}FGEL?^g5uPFQdUohs%v@5Sj zX`?KX{D7<*uvedutKn+8BDRqv37=q05?+F_Hy=<;$?g~mM^^*^#_6w$DB75rrwJ@t z2B|^mtizPGV@1R519-v)2?y!2OCi%aHCVMEyulvdTft>ih(5=)>y@7o9;2}e(U##q z#`!}(03&VFfx};=WAOc(NgXWlrnWvz%OSjjKQbEM)NJRJHaQsW2Tm?iPiN|vy5FKf zpN-gVm$_`G+2@(bN3jI?6l4ri5uaJ!Mgd=>pfu!~zJG6TgNWqbI%L4g&osZ0hSEdI zQ6N;zc9lJ+xbUP+d%_0f-3RyvJ7o=|c8qdL>^K(Z|FZH&w;}X%UXJNq)N@mla$=l*Ml6Z~0rQ9UW_2U1N z-Egf-1M+KwwH!5hiAu-%ngM=6MVR5HhREwfo0{Pk@uEscg7iuZlpp5iCOR6e5n8Cb zj!pG%~yDN z$+lMZB+5tuyhPD6gM#%(S*q?AgB-qn+_XJwQ|CW+9PybxE8itN77SzLN1boFbJ0QFu7xExLR^Cf7 z=|&#(nDd+P{&ewkG^gS&oZ0{BxDAyt%B4Py{KMa4_=Z_^!tMSwi@~_cQd~vh|i>Kth5^W;+I6nEYnb@Ba&pDD`*6 zngA-X0u=a^wsWhfi=BTQMi1o2``34zuz1l%6te()Pdyq@WeWE2jx&=hf zK2Uoa$*9^?RIDU6=nuEc3lvVpTudU0bwgeb_vA-41`6_wtC1CPxTvLp-D7)j@{XFN z)v{a4PQ99|+9+u{viPVeW*=#}W9iTkK2?;{%dH*t)(RAg`* zH{7FrgN3qFCmvT?zWP}{cPxC&oD26aph{tN9It=_y77{%jzWmM8k}CP$n9jOyb<;9 zQExq77^DYukaBQaI4AG5)1$UmZjOTtR!x%uy7``2${mU98|L;RFi2u4IB4(uEjVC! zdHJG26&l^3jTc2<7?$PH6U`teCc#vIcI+R6y-)yQN2RK zdjC44WrP9Y@%pX5XPx&m=hZK$4V8mcwpZ8>=9AMYHFD`o`_#lT)RL|qe<`>$;PB@8 zP+GTeALMP!OBvg?Q{SfdWr(`W8!d&bAoG0bY|Ovs9} zhFIBh)*Bg2I)1ce7xoFpLEI7lnbkU}g*coWR8|Y~mhd6iUm}SV!((w*P6$^PB?~N3 zLp9+Zx`_%WUN`G#Yf?Z(B)xeC3lEcdxOZ2wSu{Dj+pPdCy2!k+OQ9XsLn*!*mX@XF zGCb}tAZygnw>9_3_oW@vx&fu*^bU5sj)gtPAw%-q`8h2IXix6l)%RT$Y$^_A$Y@2P z>@*hbfA=MSZ(xF(V{`Q*0NFmZ>Ef4RT9XAG;NbcEDUtlke|{bke?TBth>xK>iYvnb z>%e8~vtdTkq)Oumb1RYxf=oAj8o$fZnn3=_>t}Lzu><-@5v}F)s{}Tl->2SH{fdNS zzAXq0`gY(rg(e@D1A#I+^Y*^8m^6)&{H^`6HdW!gkuiJd&gM{3-LeSdIZ{lJEXF}* zO}E;flnm3RW3M~YlIPgRlFmzigW1=*cTTw(0KRk%#_|GC^(<%1RaftXe{4Rk_;rds z=AmVcKwh0qad*#Ba%sqrHr$U#kjq$TJ*W@}uLrj$+sIJrmN3BQ0Ac9y;M_$tiu{i7 zi(RC6fBRgXl1)~3>D(I)hyG7KgAou4&~gFjGH}wem)=Db5h&v!!YNA0t8>S^apa!x z*1-bkEP!F1@_Y7z{^<4XoGG>cxjF1+coYP{$Id2y!yQ96d6c;^oX&>5`MLp&koM8d z_oyf2UaY_mJ}OVpyt?<7DOVEJ9J*A4x+EE?sqazOwW?9mn{Qy_a&`|D==QCCDnTA_ zGt;5We&_}DSfU4T=K2F)TA!>BL*~Njj-@Bd7C0ae7J>l6|1<5v1Ac)S2FIO(3< zzUJ3L@5fCrYd=@mZgZuE|L9M3SHYG+j1Z-v%%*Nl_RsjzxFC5p5#oA$uPCgg>OO%( zo_NGlMH4BTYV7DL*_v`j5AXfRxAzbpNsRIhQLp$UN}MsHtT0Xo^LBXcCtaQ>8;%m6{&Ap=@>{!xJ+la-3|Hc%B6upNUga-%c!m ziG5WOiav8jBI+~(!*Fl24iN@%3Nv>j53W;=P+52k=%W;?d*xnb@gt2wwIvhx0!Iee z3&!%^cGLJVK7vyTEMVG}a|=gr+e(dsPU47J6_BI%GGdOZjW!C6>TlWV>^?sc0P4D)BFVp3s$BhaulJ&uXNY|p|H`i5fEAZ1p zaYK0crTv5xPT3+oJnl_on#=Jk?1b;x? z==$@tgr+%NkArB(FaCY~1+=qmT~G1=DnuwO292L{bsrLmiJkBsMLUtgqLP0_&V*jn zm4uVCSKcMOBK~ty8q=>mre5CwQ+$KE!DulvYdR!|2@&sRz-7)jKSz9LZQJY(nCK7w zpDAMWm~92=3Cmdql)SwBW*!o<$z|c#KOVvl+Dhr(1nObuF}LA%3($~zBYY0iAbclp zUrpz>Pvpw(U*JwE)F6;6`^eWF_`60LTfKJR2k7UtW-whbiSFEKU=iSeEwqzaRW!}R zsPk*+Q=A09OZhN%ihGpwjgTsm%4Z$MV{c30jWJks>tUktlHu-^l)^k~k>rC#rp;-; ze8ke`{bIRj(QV-*$AN3)?Jx8;O`FUyNiYr;Lv7)J-TMw~o+bFmRuPxfaO5y&6D+YL zT*N*pQs7cPh)w_lt0WYq&^>Mj`p8gaJZ+kVLgEF&1`Q{sKR#kmUB$V%X!aNVC}ovR z1Il7^zM}eVw0JaS-o(IMl}M!dN^+CxZ_}rTy};8+v!C zRZ{^e`;tl@uKG$(u`F1fh7|@B)8-=PsNYEcbe(oG!a2B;JeBUxqY@2?$5C;f^H6124E4I z-A)=kPWECW%Qg^=+Eg+HWmBBpGik1a1ktImLGtM+_6A%V?1UJjX!F*(lr+yhv0(*K z>FX()pzX?GNkmK9Oa4JI5oU&*43Xm9wN-GTh_;K_g`ZE69Zx$d`}g93yfeWZEkx&fg*hcC|;v7O2v(eIRZt zatB`&1;uG~-kP7sjM69cVd+HdT)>3E!DW2h)uv7*CjkOgKYn#nc004No#Qh9INvzF9A^xv4<&;jh7 zz^=!}$b(R=_0;0i2y@)svaf4l5c+gJs6iMwkp)rrX9p*DXOEEc=(xqXgYKbE5!NTY z?fQ;7Y|iQn+oo0i%Rbv^{x^m>OQza$BHh!=S-1kkDd0<;IgW zPQJvG8>=VnG*+zgZ{nb^%xq18`KZqm1mKVOzhXX!We&uE#j%%5!b0yg_3|&8DE*xW zN6Ul9EO9c!fG}g94>gFni}DNsJS}?P7C;Kb$pc|JLZwOcAmWD~R{ySPCbC|GQ-r7924>BPHXS zF0Re&B4#^z9{zL$v~qfo8|@r8p{z}&>>#(3!whFMktCj&NmzGmA4(a)%@1RJN6hK_ z4y~Qd`hr@ptEr(Prr&KX0lvzN5XQ(8CAb(?hpo@?KsMLygOdOQ5XyQdDWKYlU42ky z4V-noW2*Da3*_3=?*TutP6bpqI;l+mf4Ef3x)k8hQ;v_-B&%vXALuKh6*Y`k8y$II zVj?v!w8N{gFu0n)k1+dm^R1N!_IK3IyMD|43-{=ZmG^$uQ+D8=YzEnNhy^r;X~Wv3 zhn$QayrrGB9x9Nk0t?Scq**)~0*TjjeY#}X6x{6Puo~*(jS(2!B+(BcNu1=7VHh~G zqp3HWT^}`W5L=jUXF79_(>H-0eV_C#NqexwN4BI&-FCH3-*#Q~3mqrl?I=KoqT8!h zemV*YoZjYK9$M&-6Wka_FeZ!S+p-2g$F6lkCI7usJtj1@kaumUR*w zFHzg>ZL1uMF}=)EEc*JwcV!@z4MO0<-bdM^P1$wwxoo%9+mD|-)^uR}MkdkR&|;3J z{Um3G{G$3h>Hy~D`TR>|oz<%0BwQ`Ye&0#Ni1Qo-sb9Y(I`T+BNp;#irJ^!f?FGUG zd?Oh=;|q`C_@L%_c&ULgdoGw}&y;| zfT}n}8g^o8sbi#*W8Tw0MpSe-&*DrUjRKCtc^L-96E>8pB_J$BB1jmuzMXh+!_ohY zV9SJK5!S^5a7Dm440&z@qrB2LCz0O}21aC}04*OFHA;|XWQ3)YQXXCbEMh&|cb$1J zTuUhcCu%#;S7y6N;`Nf=@A3owo34iIJDT3TPWKen!=nO7lgz zUHiJh?}!B%>@T|>Z5<4RjD)|U33+M)8*MG3<&0zT0J;v=Q{2bkMU<@Bl|e>E&CaC% zCzS zzFX4<&FbSG02A7vmh&~K;~u5{N9JAl8`1pla$E%L#tDga&JyS(^;NO0Q3$6de+%6i%vZ7QJ7!{bco~k)%{C(z*oFarX&K80&O?)W>Yc>qp3tc&w6)@H{Xn?M3!|SI z-g(Uw;95!3MT7^0vX|bS_i%N0L1WpLn&N2X=eLPu1Ic#AiN93l$+w9QO02$^BujXeT0kQwTOfNWc#UKF!abb%{EL}=o1TAZOL@L= zM9C7ot25R@EW(h)wPa2h=!a0uo81;i?qtfZmymFeD5E|qlT?d8HQV-9bEju_tBsEQ zEH}(l*y;J|Gi1G-d@7@|1+a`Zx?jVbunqlxXtQdlToIP*ReGGHk&)RR4dG$i z)l}Bnj!C_=U6bw)CgxtLryCQ=^#AeGbmM%OVa3~)-VdBkN1#$9p ztIUgoK`eC7+b=Iq$6wjh;_257>Fs;g_M=ztr~}af7@!XZ1aLyt|dQwEEj+bOUGjdJiW)vZ=i~ zoHESjGYKuK`iG4} z5Ja&^n<9DY7DnfG^!8$M@aK56vs#_X=jdWSaA;GHej3Nk-LOh-pHaYgqZiZtG9c1a zu+_vpLWs)eLv_(^T1kPzry}JMd@v$7+pI^KRsCf06~@yIh?TKwmTTgBrFXl!oDFWf|hNC|C|@DKnSNeN3Vx(l}r(fcy>?G zB#u8n6OEyG8l{5HJur^}0n`9(kiUjz6&bdpOSvp%=ru_jR1C{0#3CGYWS)3D&6^OK zOC4ayDhUqK{9@AJdJSmC1fbD^)ERBuchC{e8jaAIq_RQ4ygq=e@{)frxz4Mne0rE5 zPEo<+eTD~LWSC}-ayAg#c8!4 zTuMQg;q~Dxy~8-7DbNn%h>rkRlNIk#K)g@2CV@_(zq&qBH)UE@IX6#Se~@obrVt48 zV1`H&hfS42%Vkz#fRYMGnC)}>d#@ZlAenUP^$`utdAl-4l5jG%V@&#bBs>Kw?Xz54 z*>NsFf9oi|Y#*+jP1{?M!yp)b#srbI$Yf6)eM=%^N~Veo*rcv+)zcF@dQ*E?v6s|% z;ihE+r=&$Eet}z+ann5{>%5Ju+ixUmXRD4(o=C}gme9~;9Ds+lF68e8vF%o@+G-CP zQ=-F2gKLS--5GWLZBDP)JVRxy9>ZppMd6SLYPyJTXmCMWVLa^qT-gGt8n8#1FfVO#Z-GQYS*W-Io{0;lKK{60G#5&Fre7sYF$q}`YQ5) zL-RsCMhiN&v~`V)lwyE5I}u|}ApR6L{8IS+?{e!128gsOZ&4G8pPfw2_apCj&#czN zx#J7i`gb`)RK*E-b)CQKZqdz>pRT3+LmN{>y^*(^7k?yqjY>|B)U1qP=mBl}3&|*j zEsCW%zqAvJruuE5-C&{8H7uzH#P!P{$%xSXKJ3HOtx`{636VDSrxbl+4>QHo3wyHy z`N;*a9$PLQ2u+^N-EGJnFUBHLQ!GmV;tHP^KqR3BOkl8g`6$sZt9ZHQwPEEo68w4% zmMoTLNx#*ZlJ=LpigFhLldu@9iq09g70K$F$&fJOr?X_exAUZ(^^0{PL zVYCMCVQQ7k2)Mf4+GJv3rKE`ih>EWKo^>DU4+T(jbtEbD14o?5KW8W*XfHX_G+*`r z=g`az<$lcP>IW8P)4Qkkh%@iHz{Ws~fM}{sp|-q=L~aTvF{4^A^<+*6FToFz)Y44= z=)8?!h<&Ef5ktX^=olYOM1~L3eiLqVH05}RHiRQC?}Ji;CS1}=X+*!f0;W9g`B`;O<#%PRvRa?E~2(T#aiBi zTo*&$y(Q?s*FWI1zO)0PH`p)ai3)2$0FaQ=b?96}Z6< zSe9_pUZwFDNuDt7p*ANNtfpNvM!HX09Nb@d@S(K##~}DFp$u$|r6ot5gNTgMnWcJN zYG_w8B@<`XaL+`kg=Q~d_5+84iugY5vrS=-T%ab3=PvuJK!o*C{oFhS4VQS2kV|uY_4Tj z_H?jPoqvX)k75XFF|i7`WN0tN6#RH@h4A?Qr%d1o<9EkBRv&oSHs?xMol5s(& zAcT?#PK64gvQO|f@dB|h>SIf3rQoyWuTku@WrV_+qU4(~YTB2STH-LB=$86Gu&~hL z&&8Pn+*g?9K>h|p6KqO7&l8!1k%z(p8Wg1(YpxO!!&6%sRzk))$gWa890G)XA8JR4XzFx!lYs-52lvPjRs@Q`LJY_Jy|X`8wJ997 zl?$wny52_*eQRm178Cj@L6owVO?%Xh1M()FRq3|KlvxS)szYA%R~-S{UV8FfB-tms z=g+olb<(6J4vf;>AJ)AJ?@Ke1I5EPZr&H3rf_qX#YA83r?9Rfxhrx4r$9u(F=}GO7 zDTV*U|G2yNk$+{2hRfF)4hfPEvCz9U*ZPv#99?p$?;p592zPu%`|U5=0H{frPi08J zU<@l(-Ni^A`xHM68X*O#=g*w8@~?Y%Z&S8;J6)mjerC~faY8@-o5Q&mZ(d+x7H*D3 zZ)@79G`>zXcS}#AC9CZ6iWUjLI=_VJL~>aQ^M2db?S#Zmga&cy@?-FS?}wj%TO6fL zElCO!+u|a-h+Rp$9VmN-oh0{1ty@#?Bih$|$w(+!ct*%Je^c^6GVQi;%Ct|-5Ec2D zkyeJVf?*LH&bRZ?XUyF6Ue<>r(m?lM?!-pe$#DiNzmUSf2N{$e>3VgO@&z97tr+Ax z%ajThk0D%#${XRg6Z@?VI4sE2e+x7J#$60$N~K`YP^U3RYL3KIQ0IX$i9|fV4rj~{`9cDR zW;2!Su%8#_$X^B5wId_A0lgKInQ1$rd=8;~uzQg7`+IjDa#5E~!Ah!bM_l2A30+hM zKDKp%PT81D^MRd|gI;sL0JT3fAL8)nsVLykDHR5nOTyYQaVT=wT2HWZxaHce1=f9_$PR^zU$&c7Yoj=La*dOt?Zh0nW(K2#hP^ z;I%^IbzVUAT~LiF5s!#7`*d@<~i7+Ktr#9#PcnN<$a6d-G|@ zMNAxiz(3<}`D5LctG~3d(G07*=bp7J1JCs}VNnR5Q&eYMp_}S=?OUctfKHOORlrfL zcxZ2diB%QJquIk!{G}8{*a1i4#^M}(=9X6rcZ(E_W6?h#cMluMY-&3cz7%R<9o0h7 zzh&l9B7$uqB6;z3zZ0JY2NCkTeIYKEZIcOU+QCw9W4$Y$fJDw4Ov5A5t$Jh9924M6 zr`zZ-j>;vjuLmlIt({e8 zaxt?^w`Rv&D^U|uEzzmqQq(}bx?eduS$%Rx1uNgtD|VQAD|HcJBb%rpgbhQqscQ+J zE{_Fz@cJ})&Gc@eTF4epJI&e#+e&g$2(Ouj6}R_{@9i~=O98zl3wIlIs+MAZ?v9P0 zT&hFE0N9;JOMfHkdR|A}nb+BKnNdH?UcyqVVEd>${sUDsDH*$@ifmuW^s#D_hjkJ3 z)IG}54=I_~GdVW|F70&pmk+%%extYL`i_U1F7-h1_N^ejeX!|t585cPzmU-yPF%S> zN(EV$FoLQ}b&r#%?e%VpH5dv(tHg-%$&1A~n^NeS#9))U zp}MVJk_|Mfu2Sa^BPm(gcY|`DbOhnNY2I)jL3fmsspU`Sxv^VzTnm-0`QW3uYsJE~ zEFwgH>~&}Bl0DEg-c;6-Ykkt0Y|-tcLb+?4+h2m#&b)g5^+B^S`pl;_mVX|OUQIe8 zxU=9y^aU~|B73A$06I)$)tOuPT(b_X4w}}o)a|09kl{gyPgbZ`dv=e6h9)dAO5wKU_AA=;!2CkgKqF>QBNog7 zi_@<=a8a@Xiw_o)c(QyNI$M3u+In)8f9+Ev=?YMe8f2%13J!8*W0G8hr82vcnWSth z$YEW?MJtR}@YafFG$C}Bd~oSQ54@(GuJsgF7cyt;y3^wrZ@;EO6*U0n#Y~>AWh?1v zZy%<^%#?Q5>bp<0pMREEg)YMmp0FnY^7CLn+_PL3ON&S_oFXHw;$KZ-`o+?Mwn#KG zh)^+h-anPMh1194dvP+XrqI)Od3h`)vdcC{hbfn?#DP8?(41e%9s_=B9$L3#2dHJL zr)>#hL7UGmLDoqo^o3nL*K-4;y`uvi-BYd*9Jn=+B;EM1eabzZ?EA;oi6h;Xti9yY zsei!4sr?d^i$bdGzr8ZL=c7FzKL#n@M(m-2V~7TgyEeY4DK>)wt1&8+%iZdh1Q-y8 zS3#mZ`rDp;<Tr+!EMi!}y;Qg1>_&oWPLXd`~E9u<&TBYcZi+WsOH$ z@*1ac?ID8nEHRoa3DsvIcvEUqCz}*s-MW)l+I!JhD7MNqdxN~eTGHr^wFUtZkJ$C< zg@xp>WGU!q^8r1FZs+=;)#_BE!=uIQkkMl0NNcVhFOiAX_-@x0BLAcX*;a_t*nm(T zl)&G+a1RcX90{Yq%WtmW-0Mb}0(R6v!d1?y;jB9kEvX&$pbaJf3kC=xdNNMN@~lX! z2&VaH7fVHDGU3_>^fxDiiOW$ifNfweQW!&Rm*g2n{zUmAxG&Pt@aHEURDH^dvWr!> zNajdjo4ZB45WP5lLcIoF80QI%=;p8E+NHTSN^EqnRcS@}tV0wgs+z`$^E?;kMl^G< zDEH;Qdw^ZL2PXjOp@IY1hGAp&tDV82h3)>!>Z4=!Z0No6HaHJi1UvL4HoNoFDhh`Y z&R$*0xhRTiNWxPR|Kyg3Jnu%gYE%&ZS4Uv#?X>&TH~ch%1;UKSfwfAg8T~-u;buY z@m)D;`%Tu3YDu+giAWtu3Nk{A>cfl#2A~h(Zp)dh%zB|`iLZS_ZPoNQ#@*lC{ng70 zaEJ4cw|9MNNA?U@Vm4iR7E4VgE}orH(eAO1oAaU>V>8=m(tuw`0*#yQ_bQAFn`33u z&(ZdPSa`n%5(YroG8QEP^Lo^{87E}?ELf&6=pJEvL54fg$Y2MLJ%9&tvAa0vW$DYK zNJBQ?xJU$pT$Q6La=uxh2dPqw6~ce96WWS0Bh*h+FN6@KWN0#{+zWu-ucgk0N#q+1d*$RCJ1@XeU#(u{)}DPeX!*67pcn=V~aVCE^%d0?z6GCP!n zIsuSZ*2YjzI`%uZ_%Q7s3-`*MQAazF-h@?(Sb zForfVBo6vny$h*C8bMIdO56{YS3Ms4*eSjsmRQ$1_L^^Y^>$q9r4zk!xWnN}cNuGh z9Ek_6$k#-H7|0|09Gyc?JaDs)?buSTe#tBtM~tH$s=J>}c2nu`$T;j;cMbJofHnoU zrt+a9H4oBSyMj;<%@{|U-36uEngEoRvQ|8aEJ&kQKRxQ`*yiWIhN7>$VR0V<-2M0~ z@9n>sn#x0XZ@IP+fz6EiOS!s*y5c(*pBR60$5lZoTkWz_`wn-!V(j#c02OQsxSo6k zk6ZP%u(V@Sb?;BQX1t!U6h=0IauA++DLX-DTh(8ra&)QqDW^ft!!W55@hkHQ&Tg&B zm1Wyp*k8J*tL*(uMe6dzL*Vr>i^EvvLVo0fDS&Y`ub!*t<9)YFRt`P^9b%*gXkAO@ z^1kn7$7n636F;;t^`vVyX<3jw3!lRF;`|tu8Fi1ZEM3J~6BS|c4qEW>rY z*yn%Np6cw71Dx;UtS@km<=yB1nxW>$u*||rTFk;Ig@IqLK81!zYz0svITDbUb+aH@$3q||7-wWF` z*l!ueghiERQr?WiM!J)QVs)QF`=@uZJh+F{&WI5&KVEmtzfbq3&AA72s>{#Vs)N6C zSuHjeSqS{v_2}ABEQf=+M;2krl(T1*zy){q@~6+&8@KITQVrx7@k2iU6HuUBSk zT)SbAOt%3@zK`Z;NIG#L>N*MRf$Ly$Rz;Af(tX4)RvY#0R@5>dEiVaJZ@3C*;G%3nXkQvYNYM zv_ql5pk_MiFRB)>b03PVWy49r<6tdRg#4u>kpJCW(K4ufINv+da6xKI-XTCNJ~t)O zSih=^H>VZ|#apj`Ghj^tH%vi6cgPAb%F^~jW-30`&?CT`3@0L$3*J!vkCOfngVD8+ z>#r3!{AkC+^PGd^XDkMt4fiUj5?zTA-uaQQQ(DSSU7OODwDu9}v%SByjeK<|=tr&D0ft{B+9e!j! ztu5PqCu0ZVLx;%#V8Mc>R<>EuVobMgnGQpn=bUiC>MH3*4Ru(bV2uT`=nJ872w;{F z1))PcGv{`+h8$aD-b9Y{@NFNiXh*C@6HqM92htkiAwNm8&v6Fo;$7k#W`oa|LgQ*n|ezH1DUOnu$YbG7@10!!&& zHcw2W#<2CWm~w?K8TBP1E}&oWV59+}MSXRr_C$ZQN8c@p%LPBfA1mU03e!X(0Vv}# zKMX=sPshNy*k(-IlDRq-SD{;2d|VN3`QC;#!+7nHR(w2b5jstZtf0+I4zqpY^gAKX z%(U0q4jFJL(lIN@(aX6fV_2ixkXIRli>u(6mau>JfVf4TeGJVpj4pI^=)M1MV`m>- zWqGFijf~9{R^_yXFsENRYtDE?pt3r(j-n>CHHTHLMXLd{1x{hLP}D=Mpal$@)}>CN zHES%GQKUhTfdbV=LQxdb=`5kb2niHX7+_i&6FwpZk`NNIIlt?=pPfXlXU!j*gknT)T!#gniH-p_>{YFmX}R=58Y>>>N36ZTFSh$&6-DrSH=mUIVm z_tX(pW}EKctkuZZn0pruPSjRq4@C(lPly3ks{lw$c|f-#1gO$qdX~U6{KSZoqcSxA ziM!U25dM97=w^9Gg&=+-5|PRZEtIln_XPT4eKJB_*gGFUoev2y4o^ECzq z=}Cb%58&c$@HfOpjGOxZk>K^^%Gc`<0nT{(%^W_$}DoB+?3q%C5 zoP$u2Ir%f{tf$>xL02B2pf!1atr?H$d)Ip86Fq5m77Y#x@O29u(Gt`p>l?>LzcWod zzJyY`QT=7qgw8$3X1!A(xKAWl>03s3MpG^bP>hE#l|C@gwVj6=B{eL{g?GxOTQGST z<(+t3yt=#R`eD|u*G@+u^9+vfkO*cXi->oy1wm1UJsE&ra}D=Zm3%&4qx7r%&R0h$EZpcU%9`gO$pV{HMjv|119 z&N>FA#$j9qlV{*++~XW-A+N6Xy`auK4&M7LnUvguTXYb%2?BX19T>wGr6@#-Le*Kj z@-YNeH;C^BT(9t?)^5kH1Dw!EzK$Mf3k{D1v8&zB52g%hjM#a26X+wr9+)Jcb|%}@ z!Pa3{S&b1}wE&rj-(^xVoErkGFl(73)vC@@hO;hl3(i4 z0XtC|eAgQbY?qvP$C5iLwd-5!Zh*+(SC1;X1*D{+0sHMJQ52qlnY2r^b7S?pjgj}% zOC?@6K%2Ahwm<573v+H?K0xPAfR(vv+8d(!Yk)wa%6;rs!=)xf6`N9Bh1M;z_Z0qC zg8i5{7Nb>0BCVmFuSMcZw3mF%#)sRU=9VV4hhH}g{Ng_ zDbRs`I9emys0E5~1HwQAbe@PL4oxyB)Vrpwspybn=B2`~)Rr+H3ydA97nLRRY2akAuQDuje*9K5YFMG=j z+t)E@Y{b^r>|kx=9%-|Af#sOr($k_EpXaCmlVj6E9;#PyK=719_$*FK8S8Q}4}=Au z&`+j52`Y{uyb3UR(4sO`(bbbLjAm7?c2}ojK9l>TnKjJFP3P5^L%C{sL=M0T^$ffI zp6}g;QNV3@h_9RigV@+Lt{OD7SBlaWe1qYq?{94TUAo@Ur&xaRTP zl5+f=2lP_>hc&V;E|OqHIDk_1UARMYj#Bx_o!Ms&PB~F>>NxE!*m_Csa0ouWeeEZp z$BwDz=e=1kmo9KBbfTWOf-#+QdSgM$kXRw2xXAdKBr~0J3&W|%L%uo?o=lrT_F^+* zH`QV0<6<%5(oeAlxB_7o1IHASf>~t9FpXL<6M@_ek)VZ9MOK0#%+-m=TL5JsvQM0D zw2L0dW#WI&{y8sxa-_1M%JeoTK&?+kQ@DA;B2a}A%L?QudxY%}Qp7U|vGX5(9jUOk ze`~A#6&jkEh@WemH>|%5uKjmeY>^M%cCDMvQkHi+Ra)r2$G#N_JqQU|S6;6)6$aEkO<-;*fkX)bNzr!Z^r&5GZnSSIe-3rJj6G z+Hhc$$Bkw(2vw#FoC%6RiS(viNkAAbVB;*qxgQXz`EwC7a<5>XYI%6GHr1tk-~hEo zaKy;rtFDx@jhuslw~HQ_^afrNPeD!| zQ%Mn-WTt72x>)8$;VsEc21(jh(MNH3F8WAS3auND74LV; zyniJdr-Lo7n{6P@Z7WQm`9bBJPBpmLR-b3)T8yh2wH|0YCRTwt55iGi-?2mblVA6^&WImzbI#8hxO^^onej$V1!q)AC8scW$oq|le3o6j3B~|9B z*XskKd$5w=in9P-7*~*e3)123(W)ZZK>1ECqyQ0TYFcRU3 zfQPP%b_mC!h>F&9*h(siPyRg}+9ytAK-6UGT%NoL;bCmQ!oAA#0TxMDaU8n(r>MCi z)?m8*i6$j#gd6+=k00ZyiwH4iqg8Qy6^Nm!Ls8(6iw5*0n?J3_VD;Wg#&=s!SM$pl8a81I(_ApzQbo?uAvPJF_|#UJ7B}3dkVP#evJ9 zNmGyeARY^!C_gz(o|bU5*k|PqQtO9+VDYgnE*`}tvDd`JC3VI7SUja8X$25#q9-f1V zYh+XbO0nEx-ct{hy${Q5PQ&I&4iog!bbI+iXxcRxh$y2F%COHm9r0ZTayAAM^u$B) zkWRotePN)ru(G`JfhCPwf9JZW;3gQ015=AAYk5ZI~j)_dtcfP7omY-?IL^`!o4 ze(}g$=UlaJ+MF-Qh7DEukW?k;W5s=P=2nZf-eycP4}Ees?gY-02N!!He@xdxZkxj7 zsr+9^1@@_0S}v*7>b#K2H>gDDirpAvBi1GFD0Dguq#(Q*~2Sdq__c zoq9Kms4cE|@Iq!JH{eIWtjSI;q!6HS+m%9$sm6xED$EkO>876?CS-7wgGdeK&lqp5 zHK&i$g{6eq`JtKej4BOb+>bn*j`h{a(S4g|s%U@_(lxg_v*5F)ifRboR6d}>TLDI` zYfRhL7MV=EIZ7X-Xk{mutVGVv(MM=Kn;lDwQ8ZH+m07am|MizY)i4w_wbbj@-~D>G zgVvUibE_BEt8Pct;PpPj)_*KW9+PCaOCaqf-=pn?W|Kg>dB7*vyBw{RHYo|HPIt92 z$RE}!IfMWlQXep-*I3V(%PVylM2(2{N4YC_E+rxCkbUQ}k$EX|Aj$o%kqzf^r&Udt zaJ~GeY&A3`9|3M~LL>Fh=o1ma4O4_B6o#A6(!PYD>OMq9-ZpG=@2~6ipY^63crKdU zJV7Ng6*ji&Iiy+%G;!62^H)CcPP*U(F zKPRNkd%6t$(EH6e`K<_8H3w$l7b=I+$H=5oj7xc)*)WtsAMn!7AJs|_t4kZ~b{ON5 zQyfOBSe*bfUF0H~Np47lgeJ+@rj2);2NIt4_0z0d~DL=u2I$mXDWN7$&2zz4PH0GscsPbh+R21mI zN=?sA0|aEdRP}{0hHZ6M0xHi?%B{hhQ9(6n-oXSRO~8-PW^6rhX4zjYUm(iDT3oT| ziqs@-4js%lku%?)vJ&Nuyq{3TaYhW>z7s}dgA88bD=B@I0wDOf$Kc3lRUr`CnVKI} zz{m(I;&?PewRAp3(pC_bOd);#nsm%k-1F<~uFCz>Wa}M;tEa|vWbk%K9Y6&(=3v(& zyPjeqj0D8Y1W8=Badg);<&RLQzh``P2;P_fx?4crjSqB(8g@42h#R)2WGu7uur=RdDL#QPfF5jV)gtO7yYel{ zYmDf8kV1PlpRJbAU=BPj4|9(BbsF?I!5GMj&(akA$+jzi4K)VvCa(q0Y6+UGgnK&P zF~}40cZw}#Y9e!!1`z{i51%gp&oA{bus*7ISDGqB#p~aHyTWpp3egcGR!jotI&tR= zxY^bz6+U{|<>NnW+XR%8a*MGzhj`ykYQ2DpJyx3d_l}oRA#Zyi~sw+rlBl;=x}4h zd}zsd2A|0<5t)K#T>5WYAY7B5lF3bJf!L?qKlZw?95j#>UQpmpaFrL7uJYIM2=P4% zmGmfiYv5%Rual6LQW)vDD+4ntJE!MnEm;*r6%;4H_LJ;Ln-3L&MQv!mPAC*S|IlqT zvyTI6k!hyb+8kp9`Zhu_bprzVMww#64WNkQ>^R*m`E|jSs6O{VCtR*MG09l>LleP; zbrFabX&voD>8J5FHWc{G)r=EH6|iQEnCq8Bv#JHpge= zhz`#cos^$YZE07R1&HZ1&!9)eirE$Dcwif%$q}DCugr={SXEx>i4z{z5lg7ia{Bkz-ouh~!n0LJ=;OiMWM5QRNFv5b6pD#VF$EG*6x1sF5{n3Q0|t zp{GBd<^a>nfV72UepL_0K-|FIMrmkUNqhSipqK`oXJis*OCG zPJj~G?2e6UKvL}sw=tvR2NFMU6^+SUyl7le)1W;kvk~k z({{z#U$diHNyhsd_6T}M{2%r}AfM_cj^}Gc@+VCr=>GEY+&`6_Knk{RQej_F<%8of z0WZ<#BlY(*R~7p=P>y$|tL+n7wBlu;+m}UmtAVWLlJDtKuu&>}pc;n8uG0fwEID=N zq>Ueir|SLY{yq3x1eG@4Wgn6{J|!^)_$h=}4{AaedeBT%zRZ02@uF{Yzb=!jE^971 zvWNNfP%JAOkW1oPHR!MKDLCT&#Z6xdD9Cq?9n7l6#Uhus?@icPBy<0dpFRGMsn z)=se39fHY%i%VVEFfjA=t{1i)dR*G;NfE^5&D^(Q8NEKFdUNys1AsA}3?}2aQ*~|i zpUpBkOdn{otfSn6SQXvon*OL>WNIy#g=b+^^-B1N%^{GEHci>;yk{}AfRIoar%0Gb zG0QHStavJ;Jhv~Xk47QLTdNNC<}EU#46Z=cq$Ws`pprxcXtTDkvg+vV;8U+qW(eJ) zsy&DA+`;Auqv4E{g*Q3M3c6kl&rJAr9lF^j`!E(cftZ+nCqlP=`J;S(T%XXKeOjc7 zXcZg*iv)?hhU)klI5E3tvF^HJg9l9JH^Ku#FqIj~t@kt!IN84>lwHs(%ubS{0t{t3bBAgy?`fQ@Tdlyrk}F|F zljrxlT-5*eC-mR^nUq=()s)3p)OS4frZdmtt4)&~HjiT;J4hF7OI ziOOvuNJEJ{cK%4A4qR7XR~2Bt%N9IDOjh@cA;gag!53js3z;*gh8swnFkrUe~3!?3`??79k z`{YxHCZO0P;-neLc`tRjM`7*JrPx*MWTV7o#;I^afM z1DDHDrxIQqhbGFedUG1rg6l|$cBRI~U0qnbHG8C^ZQF@f4uO7i$@3e_A3d>misldv zSUnspHEvnHtwhp8s3Bkv6~N zOj!GFFDEOX${DR3QV?EpPg(?h2YG@6i8}9ikdSv;4|Q0Tn|pMb1}>_wO;4$j%BAM# zJr|MRdQgGEVGN&yWtDCP>yOi%@D>q7@()hY+Y1+hnWXFdqUVRnv&FK7gmBJM?#Z^F zAQIjC7KYnGl{6wf9L$b7t`M8qodc9* zA@X(^Q_siQr&1Fm8XcyA_w)vUJ&XSrl!v}33cR`(8H$`GU(cFTz{S{HVt-#sWY2ys z?5M{^mI7o-ykcA}XWFW4cvaVPP9k*cOslJB9EJLp~U z`FC|Q*=G~V*K7xS*!781m`z=|D@_)#H063%eg-ZilQmb@sjTXFfEBWHCic+OMb3Yz z=*aGRYc&~5%7=j8fwaJ-^1xh1T3YlU^&w#Pqeyaoy)|I!NGH`wYLpqqSu(8im>Tv; z*(*EhZZqUTyKY=w#B&NXSA2$AcW2*B!o>5s {A6#N|(fy|O2GFE9EBVnQ)mG5;v zG5KSL>3!|>!J~86pBORCB*(#qf>D72%J8Ega34$C&!goSu|l5|MI!m09x)dJJR?Qh zcZ-3lDH=QOVemRlQ7>}j9x^V+79>h0CW)|wAb2Mq7#yoaL@@narVs|l=5T{TS+zdvj~x@WtCXhWB=Ouk*bdSr!ChZX;jeOz%8)Mh(>?WxX7qw2!&H1U#Plq8BEF} zCGCnt#uJyV8qqRc6ST3tWm%3oPwg3nvm8eEQ}G}LaYfzQfwfpi25;_2c1igFjEa7*sY7YwN>x_eV6 z1K|5U?YMJlbA=S$NUW?!%*R!r45O2(xRhj(yDqK9I*}17xi{!m1noBEBPXKXbd2kpsGub zHqDRz2tdWnis?XV957;ChBu{_8S@5t4?)zOtrLkPOJ|zVa7O#9F}a#_s{-m9aQp1a zu8toNmy1ai8d%}5ka{>va3;nbDF84L73?j+^uF^yU_6Kn8CJVH7Y@1MQv&!~m9qp$ zRAr1jst~ydH58q4nFH5OzdT!4+t~Ja^rHIF)IQhC&s4iXQD`#^jRKT9a1HiANCf<1 zps3(QQMD|D#_bxvR1eg<$bT9hXc&HCS@Ff0wE)w9$_OzCF}kFd6~|o$xET4{M^oH zP{9iqU_cc+k`(dhHt^vOpr;Jrz(uE*%snQ9>Q?0~Ipxh}!L*eSk!IvskbqN?RjG)W z2|4#z2x3Pg4z06(f+cu-pD2Awyc06e_R0vl21B%~qh3 z=I^)n*8*UhK4Ugh7kj``d>V}e`P=0pg5up2cK_{3n5@sR(q|WgZiABm7CO2}WmYQx zc1O|JA<(`$b{r4)M)8i^qhG9xgg|YZDr10k#16kZnb;U=FFO;*8KC^1#x7-;g z7AhiM=(8g__y(cDO?g!#PyPH|hja5}k1Gyy=z^S?#A*d9kSaacSEXG-r&GCpy}g%r zFO`$wQ6Rfo%FXx2u}w&d&WkYlJ#G2vg|!6Aq_>Hfs%KSg$Ud{13iINtHtf9b#~dK4 zgC{g3jiMYALHKrAs-l_Bjw*DPkwYBt5-X8S3Q@3(U#vs$@S-_oPmvROx=2FV!lWuk z);?s;Nz4#&g>z8#7GFIFf>14Rx{j*y{hC__GWIe?UWB`w-`_`|Ums?Rp=QO~a*Ov> ztyHdj6)pB&Ia(^7oHb*~$(A{(Zdj>JjvR$o3=byxL z!O;cz1OqW*Qd8MC;9XP}DKqR_;{N5uP%`+~+!8>;w)3;O=FVwyimX}rAK+o8V}A)y z1*zaj*G-smqP9uSmvpxC`8OWlwY#Eaz=qwTTIYXk7ejPvu?7Ak5HRa{kl<-2lPBIj zwC>g%nLSY9LS|*O9OIGe2li>9Rfr-2u-2;N+&PZp5L5(W>T5KYfuB!7pSa{iR+EOfuFY!2p&}v)rm=!+X%5>x%#B z2T8Ervm{n@761E%%&yl>E@0=mny{nns+{1bOPbxRl(O3I@-w=sjWnNtmf&;an}`R< znU$PszLy3ae3kM-=$B->68><{w2>cn4_L|xy=#r~zK5AiJqwrr;Rk#j1Zf>kOo0F& z#%HR?+mn-YTb2NpK>OPb9`ZNruia~etU4l|{}B~67S42e84aQD$$nAvN($!&7wXRZntUQih{Ona?ZHnKweeM zJuO> zs)*5_4`<>XOE$6O^(zD2!236dGyWmQFc>vD~r zr?8#bt}$apr5reZa6D}fx7jc3+3;CDy6SL88^0vh-FPPf_d~fQZ6to;*jgSOTq56N zJdP#CXv^Re#laJU5i~+1-5^xcud~Zrj>r_mF*A^d7cVU7C?_nWg3`6;CQ0&eU#AY~ z$T#Q1*rh&dGVb@d%fh0=lKt02B9-jF+&KVFUpi<%2nuY(3D=My<4IS}YCNhQt%W-1 z{+FEe{2bXTEnn?c55$Z)OqYBwt!Q=Gai2TDC<&nhT!2-Br0`)69wEj0x)L?*U@!o2ckm;CLn-E;Re z)l^2pMnEe_nic9qWf66LnfgZ1=XH0MKeB5<%;FDv2~0KOYsEEoo-6)oR@*1{m(2W7C9VdGqNLD87o;1ilgZT->j?etVl}gmU zSmy#u94Z9H7hhl@=eTITWHl4Ux(4LFIZYZCHVUa=UMB345SRv#T|f6f>Mb}@bKVxg z&Vcjnhn|W86}sf2iejqrvTAuvRLhE=@kB$1^gWs%Zc}us6Z8Ljk>8)XV?S4QU`-!a z*di3HAeUuIvp)z+f}{zrhmVD=2PnH2B0Mq>?5G-%VJWD@G83Wkl+B55RWw3C`N@*@ z)63T$geX!f%N(4H*(RX%AZF|ce(7!dc{?VBV(mfD%cuX`RoT2xhOt2P+KgdsDXvKY zZ5**YFyl}$xIhfED+O-pqizDlp+RgR-L&|3U0Xk@pzwY&BK11=lPW6mOR3O2?zNX; zH!ctsitVHGtU0nPJX!o;(B z3z_uazm|y(SO8T_^jc$o?|Y0TOiSS9?Ec=^%fUuUVyfRsKUA7ao`|YTnV0;t@O3!) z&i?iMMEl5DD3MwlksQCD390L;-c?ryT$Na??l@OBGq|b+T;6LNTo@$xDSs(@g-wdI zp{i0H-ldA*AOZQ?xa7gcj?eO}a^jH#%3rlxSfY5$vY=o*Rm^GeczMo+b3FPCFN!w@ z8+YlMD4M9CPAX1j*3&2nN&Fy0YS>TlAG@z|305xF$2JU+8&TtmT1Q1O%19;ny^xMf zss#5hyhs3Wrs-dB`X7=Z^2V2V5+nH|ctxd+@}SS0bHUZoL#&5ne!Od}JR}<)S7%PD zak9E}oJzFlxFY)xRT+1}6ua4XsTV*^OgdV<^5f>izqH~62c=xdbd+h9>#hSOb<H@wEQ&<57rg*{erTDhMI^?gX)vE&ZrucwLhSbb))8ZIu`7I(@5e8rvfDU+}*gZ(dEtV zZ0wsWUyW0IxwGJ{X|C~zLD;S)1M~$va2E<=|L^8ZQmW(8MGzKMv(V6-D?d;I!*ylD zTx98qqH-iInV@8{LYdIz^NaskH-{K~$s9TVQT>ya#n^^BL*ZRcf_`u&ULL>)z&WTY z4gt>6Okq`1)zr>e3VcNf$`nM+gv=iG8ACb;&pq&or0p@g!RzboNSSVI$76TDvnJ3E zi5yDz`tv11B;0|%T|3C5cbQYCK*i_8*)wsh69`r{?%^Y=_9%~%EP}W^;eIE`Z5$EM zZtviVU|D3=2a{WmnUwsxG|wl9cfXULtC0O&R$Kpfznh|USy)TG-^2Lj7@M8v6{7&n zMi-%=(=)zZ9;fHAG$S+I3vD~G z9pphhzr^}K8TD3$1XxhS9lg&+jr#A^xtg8_cCrzfkSNF~Mtv@;X9d$PgQikK+GHe8 zN+b(D33%sQO1G!AP7=ieT3oe|ZX9F|4l4ol@OwcW8tldjh;4Z2=}4CROnDVd)SxMv z?T4|Ij~>n~P|3}Af*h>4kO_P)I?&L;n)c=-dN50G{IY-*RQe+A1Q0nEhH@woC7geprFAgp_(kz}#1Az;o~Gt`Gb!~Vpjy<}VK&(a41LHwyxs@ zl$_3-E^S~%wB|_}lQ2t#6d;}{oOXozE~aI^?vRRLKH};Z@3z~V7}2x`wkjrD^8v$ zInyQimk2)-)q~nSD#O}83KwaD>Mh*PNAhE!B@II_iBng68`UmV5;JaUu>Tkit4Jv; z0?k!Oj?*~}gBL+{kTF2Ebbu3cWUBRKFf!XXPG#ZC#cj7s{9|~Uj*YFJD$vJ~uyxo4 zNdm>fE4n^JVV-=YinL9&unREyr-8y}kIIZ#d)t8rm}^IH|9Ujv6(E^7msH zIHOtSk^rJ2kstr2D!Z;t#Dl||B<&C>*~_2bu` zINq4dfYP@dvPJbju?A+88MmAfc5kXoJkmVr%$cz&A3U(#vF!Mow|DK#4OO=PLgr5+ zsVG=Fe)*Y4epK`a@K_hA9U8b*^~HibaCSOa#Y-n&Obk;^zTnta)jR8^m8)5C=9A5L z{Nm$htLE_6!ubmL3^2c1^W)Et`e@mLihrN6;XBV_EB;5rBYmNJ{G>L0EZS%O`6{+kM!j%`&zybs7h7(e zTi4gfQp-qNem^nOaZ`R!8Hvj)@BLrYSvi5AU9)MQ@%YbMPRazeQmQ0dBvRXGulpxN zn6mLCdh1hKf#K8oW2N0~O(bA63>%WaE8c4G?;`|~2@+W8E2=I&saNFeSXyZ9<<1Qj zoFVH02MD@Ldug=--BGa_1x{jckS(mA-ICyjwyY;DE2AhiK}f%xJNZr>YYg!*brg$p z*BrT0yyBk*w{yLO&<=0U72P-jC$(@pkB_I?(F}UjQh1!;nzM`axKrH>A;9NJEv{yZ znCECa?GmMfE_RbmhsX zGLP%?9?OmUa|d9#nsV|6{cFktf7Z0NAb0(9nD-Vf_Y_q|kNd?vSH4_Qb8gEWyWW`p zw+Hb4SfoQ7r4<&Ua;O>_*^oQx?dc9*dVN!$y-{qm+@~9PAdkB8CS{v^pD%dl z)C=y~)Mw9?Gi?Hp7B|Dk%^yZ9(5jJ~4CkuNoiF_cH2*``2L!3rGWgBauW09_KNB)h zUokv|(zq=NyAu>=7fH2Qe${s9dshb4ZdFg*hj~{~>p9MU-Jt~DO03bXdN++^*u3$p zpIv;Uu;J#<%j#?sB<8AD92z%Wpg>f!&CxcwP3{fGaXc0j7f|u@=`oUB{DUqRvg?h} zIf|$rxJEK_^Xc0T=dNq&^ZTTsq-_LL^nM}OC9jQ{lX+y9PDLr4$)63Hv~kJ#i8Y}R z(2}B}ht-ccQ?>ueU~Kuz8;V}ux;=E+FpPZGdSQteDW)@TpV<9(uV2T(;+yY#`VYDZ zxNDt!pj_>jg*4Q^%~8Y-u=i=k>yzL8Sosr&7i|1y;}<6`C^$d&8`fXa{iCzP+RN3u z0DGZ|IeSSt1}9l=Kfj@KEEzMa`S(|y+tvJ4v}MMy4HUwu!s+Het=jx$8MaQA;n35H zpU^IdMjK^QW9s1xnc+E|PdPZ-)~w9JOQ|T722?D#%OM7cTR-JS)Ev>mCM{CU2^LtD zrsv0f^nS1ki$aZvs^fa*)pO_P%2#{UE{a}?t8CK0aA^ZW9L}xT>4(Z!b^RayVAqY6xo_3Botu!6+z-6% zd}G_c;K6A$pr@<-qot*dB^yROQ7fy#qE@vulR?({$VkYgWe;wAy!QSlw*U6PEiaw< zzqeg>eAI`dsF*h>(zo5f_+Pev(=C5hhME5%mw^p!J%?Ves9k)%81<8MYda@qku)t0 zcwgLhX5ejC{PC-MPRM97jHsJu!4Hqvx^U#>XTQH?$jA@f`t@^P+4%al*LN>p*4Q8F zyz11+F)Pd)bvmNE1|*i!SC| zQQPpx=l0)F`{u|Qo!L3_|CUm^Q>|^ss%6EOa5UED@hdm)X})_v$KT)cEE--ZXK3K_ zue{+d0D80m!;dT+sau!PCy-`9qE0y5+)(_^xL0qo;a5QyJqVcnW#x}Q`|?lc?cI}` zw>pgSV-QY5$9%4K!@W=Ov7eifTY69FSEl~YPcC_E`n8f21$nl-mpyeMv;TpDFI*-V z$3$8BAGVJedc#k@z5AViSn$}6uHpk-m%WYmgPVZ`aDhNYH1h9xk}% H!dw3j4N9GW literal 0 HcmV?d00001 diff --git a/tests/ut/data/dataset/testAlbum/imagefolder/apple_expect_rescaled.jpg b/tests/ut/data/dataset/testAlbum/imagefolder/apple_expect_rescaled.jpg new file mode 100644 index 0000000000000000000000000000000000000000..0a74fdf4d24fdd204269254e48d1f02dcc78fd91 GIT binary patch literal 150549 zcmeHQ3tUrW|3ACHfMA3k4R8nw$c9;5rVbPw@(#uj*ag!_YpJhpq{ht~h7Kw1zZZ$i zFr0{EYy&Ke-q19&Muva|!XQ)fvV=syyOL>^|K}N^_~!kNKKeQAaJC)JdCvFwKHuN% zd9LicV#HqLg@lD*1OkQ;z<=z@FIX@}B*1_0ClUT5li*J?GMPj+H#0XkZ4^rjbBZO! z+}z>?3rj2b5A0dLU}X(|fhXztD$$HYGP9zXQ%oPZryp1Jn4N{i0b4zZFdQS=5lD7~ zE8kxM(wQ#o*H^;C+m2XEtD%caZr& z!P}JK@wiWFR-uLK#Lp`S2^&wl`OZjKVQDq^C3^=)_Yod+h9`6Mn6WHB|KNXdLPEJ= zJW=%I*QQK;J!a;tf6tyHk;Wz_Em*i{@si}U_di&<>cjt&XXhw#*L<9}R;e!1Xm!OU zr5peIMaAYVUvAy@)pt9;ui8~zQ@h*H)cn)lpZD!=`SnO^+iyqPk9BmOIeYGW_wN_} zxOfTP7Xc&n+!pwG)BCam_eF%yfCBG}KwJbik{#J>n78?WAOYp=_<_THQZ4X_S%sfh zTDtlQPY;@r&}cQ-jrptl8F*`^JG;9ZTXA1^cJ0Ql-&YG}O(KAYN3z2vV3#RsIUPeA zY6FA;gaL#BgaL#BgaL#BgaL#BgaL#BgaL#BgaL#BgaL#BgaL#BgaL#BgaL#BgaL#B zgaL#BgaL#BgaL#BgaL#BgaL#BgaL#BgaL#BgaL#BgaL#BgaL#BgaL#BgaL#BgaL#B zgaL#BgaL#BgaL#BgaL#BgaL#BgaL#BgaL#BgaL#BgaL#BgaL#BgaL#BgaL#BgaL#B zgaL#BgaL#BgaL#BgaL#BgaL#BgaL#BgaL#BgaL#BgaL#BgaL#BgaL#BgaL$sCyjxL zyT>&U8zT+{*bJ#IR8a)i_ri9hNDh5dfmI0SzqwYcBFZCv=n7|>;R3XwHb5A7sxgqH zIRrYc=)FC$6Wcdqf%4hsj?oXLn1Q|Y>?O?YACBjfI$ofkn&g(Fq!WHkH9E#*$gymB zhQd7H3KoN<;C02@ZR=(1j*AVXW4eoJTU9mVtgr?6Vc(XVrSQ9i<|a#zQnidWZDG-m zzv~*=FYMSM^+}PaWmfc?Z`K=P`0YE~aqL8EgW;spY9Beath-xV6JX;Tf6_O!w%~}& znKM>(@r~2k9~Ql}S2kGAPTXf`AaHO#+E5!H3_LLm44{kvy_ynG0yKansZ{4XpNVPD z=ru1_!|p8LJfbbz-;u5|qb_YqOWV7agK-156}c=ZIaW);2_u4zNlH3`?HWw%v}&uC z3Pnr^I0`P2jyBW=2m_BC1CRky zW3P5ZKslgl-HhP^&g0z*Yr4WX*m%|{o<)F7DBCT8PUJCC_aw)O7uR8<@L9=?NX0(Y zDi&Fq{nOeOzC_YuqrCGAR%Yg({huLOVRt|%U=#eFu;+&mY!^wj0X9gsTr~8ef+js?TMsjL%Q*c`={?UrXOtU@sSJ;@adIn>Up15w}K47Hl zo_dE4@1+16;NK|xki|3xbxd+A(8*sTv<6mG=()$F!N!?*EU84d%BjjfuYamEq^{9( zdmeZLDP1$Kk!4JzyvWpi8;ts z$s@QHs)#Z`4+)lSGdsd7>~fi(9l$2cjMVw$G8F`O{HJBR)=xq%O5~zN7`O!nKGNdW ztvPJ7VPbznHdb4rudQ+!$|d=co5JRvmHIb?9gDD>YR0$q*z6QV%vQ{CRf<{p*2=g2 z{EGdm@s>OlupCN=3hPqlMp?RYLAW@S6QJKERWxk{Y(EKW-Q3sXMR711Hl`g~y{cLG z6@gxgVPesHpC4%aQ8o|%)USg}@W4b*l?qc0fH4uuyOfo93thbz(~>h*0!M3d`FLRy zYyQ`cafc5Gx4yu^1#zGikSyu@sL}rO?d%5TNTVP-?5NC5<3VEX*j7^O)@Fz_lG}48 zc00!~Lf<;ABMQRneU>Q{nDv}2p?Fzzm4lya+4!^GLjBur=j1$Laq%?aWEbm!4Wiu0 zC0Z-9QQ=|`)8G<@D)#az#Pyt)%Iz9?MpBi#k7d)< za*RGc9XO-BWVR)mQpWoU@hzoZ#I5aq8ns+yWj+q59|D(D2!6v%AO6RyVv~88iu|}; z6h$6;!G-Pq!`QUSb+i+G24nO-N+Z{PsG=eVkR+oRaSq~`=bK~T^Cuf;HU7~weH9Oz z_`RVxW}ZWUFRul9L4^X2mla)0^7B)XwSuh%6?v8%$P;{)6%ZbM`?4z2-!PaI*u2-z zPnWBAUbv(qEYjHe?h%G1x!t);NC2E5#tAuc(U*Tnev1GpU6Fr{TcG_^f1NAVtvT3~ z-asm~pEZ+N!6(^^k7O78@Rf2W#o?1kBmKl#!?y=@rN^W2Q>~X70s42+X>xlkj=t>M z58Ro6x&A46?DsG=c7HJ#Y!5lcW4z8YJeiEFNRSI6QU46`)o5qJD?$E8mPqW(aS7KdT+KoPysGyMS>J&&3o@kc{0iKwN5sEZ;c-a5?kEVd1V(ah_}+ST)>qEj5v14u)1z`!*7<6mn*bBQj`f|yl%G&9>aCZ6P{+hTb{*^IpslF}h6ZjErInCIJ@ z>2>;Pw!9chq@?5q&D8sti!yX_e?D=-LFVpwJ*&I$k7-t#<0MHDJ{c3Fr8lnd2Vun` zilHaCNMKicY(0{#7_`fxNZKW5yW@hipYc9As)v3Ulec6j2)m+qld|;8`PXRkaStH8 zVriW_PH^#aQ0Em>nA@ltf3;|mn$PDEUMqd?(ex1}IRH7jA9V1+HM!NTL{(N9#ir$VImS?P{A zbz5CT1xB?M$YWwlXh+uvmz&{p@&#OC-+Wu(#Qv@>U&J#~4wq3*%K4WQBRs|{%h-%1 zGsCizmHcX#*)rn(PC9E3UgQ}?%=-3V6$bitff=r6M3EeDg^JYNKon=?i$4C6$1$F3 z+VRT4ovf-6MDc=J+666rso>AuFWBk*v?Vrx7v*t&on~fX$IZscTIj6fl52(#DU;}j z8@%Fq=8+c6&(hC?m)HPDjqH(S?BAghfX@P1e^|m67PB)WzpTIp18m$!NTU54FofFX zS;GeKJFhFZX#Y(o;8+lImz~yVJxHW;I#PyZ)vJ4pu$)w3Qi<5*w=cpu_?8iZy!H<(2lta6JNcek58q7bZj^BP=$zZG&YsGfaicR1yJPfy2JRH zY71KHRfZW`ZWg*1g#ODyc`lxtmM||!z^5 z_^s9hS^@#PYh=|+TDda18#RK$I-=aEJGD!TZA@&4=S7-F;6kEr?U9e)8ER1&y`@ z(lVr;GQLl|(kizt-H=r3`9`^&@seW#kt*Hi6weuPk`O^DtWRy?n@vqKF z>2Zu7_Zb)I21LbZ%U`H;aN>BDxQ@tV1egWpEX3RhIov66?PU(+++&hIySU~89yVjC zFbOc^f)0)oonWN3DaWk-rG!k{lC`dJ<7h&$!x6bpvM@DAzC$+2IXJxDKfWU*>OyI_ zxjiuxn2{jW5!jhF&3{QQJskm@$l&S`>v&~f!s#x8n*tR(q#h_E;hE~oPb69SY2`&> zCNK^oWa*Z`k!5szYnM_*jEh|aix`Y9;WR?-uOc8&;^^rtk`F^UU0TM z1|%#jY`Pvxe2n%WP`lWzw#hC-rMu(eFNzquBNt|IDFSX}Q^Bq^NQqs8TQBU;>-8?! zFWW-of+e=d=^KHu$9Ub#imjr_V?0N*NdZ;3;57*Nn-xT8eup2w>Z%@OeS>)Fjl6NM zU`Yl;;)#hE-&+}jRTJfyR{-pA3`?wa@0uGs-$!MBl;nAS6T8oUvJ~rlC*O`&P0W=2 zWjmTqemShpD&fEdRUv73_9r1&2-OB35G4w)m{%{sgoIk{0m}04*6k5QPMhP_pkLr2 z$Yw}N21yiD=*N zx|OV5fr_o~?kJFs0uy@OwDF%c$1A#@=hGiqQ|~$SQ$QWM|8+d|+H!qgzvy*_^Poct zyPIwF7LY(mivJRCC(9+%UOBb9G)6ye$j1d0hrbHw_@N<%KA$o?TRe8jE>K4b5;k|p z=vtSwE!i@hu}k_xgD4?+c7a#uARp)jW6xb*YB?l6L;CuMMp5kETN$5yFZhY<)AG+u z%#2*x6jWJUlOH zZV3#j^GW49aRV1uk;b3Uw6k29g-f%&qu7oKQNvewS87LXPoJ^U>B8(h%d6>yghlUd zm-@t~H$EkVd@F&L1g*cIR)>IOFv-{e>xF9Ns?Gk{ps9LI$wxF-=^ZkkdV_E16}A6E z!oXLMQ}lPz65##+^oow{z2As{^&dU?)0&!YyeOa}oq8N3D+bDauzE>NwRxsRHK}ZD z{4QT^SU|)e=~PN-%`Qo2g5z3)EG~gDmGa`wKmJc+!L1v&5`U4aFZ9Bb2iD<9Do}=> zX~iIcIe&b~pQ-UIP&9D;aLt-d zwz}Tv-p^bS*n&asp;5tEY*23hbOgZAb)e_)crYM@ishOx!Ein`L;`(mpG2{NNYNR= zyB;M#qp-~=T@m#=GnxkvS{iO!c@daGn`NS_O0&wf!YS^8wHSH?jdEV0YLC!w-!x88{eL&Fi&RuD$BUkqpWynEgqqEd^@` z0%Z^7RzC54dN-x`Ag(daY6>EyFJr3G8k)TZul@RxRSG$E@B1X@GIy5Efi*D2&y??W zx=>0r_%NoK|0P|NFBTGVQaO&;&`8M*))+|rd-@-IK=~Br?u4PuSgB!+Qneze7>$CD zXB0&3{Ak+Qq%L5@S4T+Y;L)(+a`LD<`APb+OYm!egy_)W+LLe>q)yY*8d z$%XCqr^2?j=Nu{>LHax^Ey5+OZhX>_6$GOpQtwsrLcT#%Q&TlMBtj^5A)m*M?`K3< zW^$RG5#>Y5la<14)tg5Rt7mB!674RJ>zJ6@@`1g*wy@ml?LDHdD3vV4#&VxU$B#Z^ z`BN}L*TnVu71G17=*m;(lM(^w`h_o2xyuxq(qjxuutdUEiYbBm>7(&mvmRyyvl9^~ zeH6neiS#Z|becR*cVUI1&)vc6kJ|9|QISW}J{i7Y&?_<)7$mTa$f0TwBoZNTngW|9 zeiERNLhLU6ED+%n8F6^-fV!oU3Opz;&!G$^P4Xcw$sRn6IU-FC@~->_x_H+rB_ zx({UU#y2cu=8ElEb^VDv{^we)ip}#`ZcCdO`!A+;gQX{t6BS|&ZT+zg5(6gIJ4le9Ey1g+ivvZQ1#pp;LbJh{h>5daks-~<@N0fnRo zKKFB?m%s4ATgtr1$XMQ-35yPuVy{kHDpX&=221P@SNX5hgD+(EeBo;3=N)hJ^%2XW z@V!PdHb2` zrwd>=m`-F{oY!g-|W3%F0^Dr13MrV3KTZ& zsC?4pUqZE`F#P?t6Wv;k>sDG5dvtPZvuK6!=oQNJKxlMcT56zwba;6TT<~K%IN1d6m zp8pdRFl>Bvx?_ywZ%>JN!Pg+MMSKURa9_91)3?7dE)gs+8L&7eCTMrk37((Bb!!bt z5Y!g^2LsSgDrg3p?Ap{((06fzs6x$^YU8oDO*Om2Yz*Euv{hI!AZ z9}2sB9_Xa^Vvm^Itv3|r^DjV?Lq&|QRe^MGQ>rUye(qFi1y&&%TYf5%lB%3Hl`P(0 zGh$to14%i@P>S&_ov!|AD(9rMS^s@faWj`FXIScMm@?^a-<d&Sjy7mV&?V<1&-=ok6GkNx;o_kJ*Uv~q)MFSVZAORo? zPd2^ADFWydZnX0mty(rjwj=7zmAP?jR^iGJ23ub><~*kP_rY+Ei>n&0d$5}mBb6z@ zY%Umeaj|FC-Yoy7&NndKAeVT<29Dcta*}9wTCHyK4Xku8oc45mHXt%2!Y6-yhZmX} z_`Yr5M`4+xbKXv=gXHnXAcw98=}JN91cV=|m0hRTSv4v58;?6U=o01>e1tMrGuh z=m!!K)}J?q=#)@c{ML6XNyqy6o%H9YfUf;Mdl6;){drb2Kj}f6n{w2Za0%+tcc2Bc zfev^=2-QZc9*7hdaOi576}(!KN{d_;FxMW;YLPisd&qgO?OLq)=k25n62OUuR5zTW@9Fx>e)ON;W#_rt_}tkS}pbkiMVZW>o?$5&k`TO&&xRJ(}-&F3@P-`kF5G^A{TE zyFzz>c=9b=M(8+l@jg8m$ja|^SIWnv3k_^=WnK?qxYofFK_`U}*1M>PS@jgW!dZ(k z2fNKOWD`2}D!_D}+AlV&Sr@)zhZH1{D*-w6|KfH& ziz#5+zG*r1e8s*$n@S!6Yd;L;N9TArBj~f)j(R2>@nD*#Lwg_yR)ACo&R~GtaSc!< zJm3~x0}YgG{XhdIa1Q70bj2|*`&y=+draa-o2?M1$G?`FX$e*o0?`nwzz&Uv2Xot~ z)O`L<3NGW_dvNB_NQHX~5Z&<_sKND&=Bmy>#if4jgF)8+eOmv>V+H5*5lox{$pQK| zVe$r_JSPW`0IZ&+V-%X))&q>E{ocs91#?&xAH^@tNclAir5N3xBMi0M<8L<&fxRc5 zgF|EE1DM1OYs((wIh1B&szmOOSaq_XM)M`;_nsz8z;KW)?6y|cSK3PenSX&t2XX?5;091ZmJKD@wmnf%TE4pH-T#IZntMYBomhX@Jt>nKcZ0?~VG=4*zw| z$qV|+D1fdLDVAOgjD(p94^oEFE0Bdt!T2|545r(5jM4nFAU$>B0rN(X;6!&|6-g@H zOI~ZSaUTUI*8wFJx$y;(8J1vmC+I$S1~)q|S3e|c)i**0&-cx3J21Dv)6|Bcc&exXiBa`ym%SPdoU4WPG?klk<0VUUR z`f|HrW0|DF$T)BoND9C{;Kyf3h`#A(*j-IbLCh@;X;KEqNpkI6{XXZ+jb{CTr7pK@ zDRkcF&YXWtSRfTMzmgfKi$?jU)l`vE6@HH)8z}v8xk18Zky#1E0_NTNHM8#PJ9qtN` z==@$t*k6aQzk*rCK)+)29deRpQU8}N7^9&kc;7Vhl&N9@$^%#{0ctN>T|2@dDO|0a zx}z=A-(5VVKWp7R7hI+j&I>rN$iVGBjJ(l)FPW}|mr=>Z)BG&8GSM3=!Q_t%PDPba z)tOE|gonk#F~_l8_1V>9hJ}CYve~e#^eKU<0LSr zt!YAG=-ie3qir8yG6ect51fJf?7{Nx@Ft8k;`}%>d=rU(1*0;iu#T=Yp1gcas1iF4 zm08E>KJatUER*rq$I8I!;cNn^j!aEu;qxdt;WB?K3?pf;1sH|iE^(j}n5k21db|@B zbrr4nOS^2#M*lZVQyPXHC7;*6mAgb`_oeG**B7Uxb!#{2{pI4Bh64Y@$>o45f)@fJ zVI)FJ3eKiI8JZ^7{T!peOc*E3sjSpxht&z$?spbA{r{0hMQxu{{8>=gaUuxi^oES! zD80;OU*;=ck}ksV8E+*49+CkP&_<$GiLAVpli3u;(ecf`P&OfrA^FaRLsXBBc+`?F%cc4 zN;_^Hz>hSS;%$k+f3*~B!4CM!$9`7kll!6m$a-0v%X#JiO3dsCa@)}izMDF4b%xEj zgi8jgYOtK)utmexJ_U)+8-2NbU z?__z(BzUix*VH6k{Yf5hK@}%KA@W=_!?XUFpV|A%8+T4VE&WKUXggswm}kL}vMR91m4bIqctt9!SW7AF3U3zSw01)dd)yy;4C!;o1sv?R zFEW}dVvNIwR&OTDeCu8%DQC{vFv+laAJy7>_2JdN9`<#(XtONg%h&~@Dai}Mv6A|r zstFzoy(evCGu*m0fGa#@i$|>Ril1ntDPI#V!M~EVfVd8IBGzUouhHw$Np-Ap%Ai-{ z?5R{q7jI`yDw{Q4E_TPyzVr=4(KCIOm8Km#qHroc8aoe5Z5B!_eZ2FVg4;>>`^ta7{OSVpwzY7Jn**Tn%f1Dsk(Rpo z_>FYUlWmvr)(u2+@{twnt_X5kuv7KuG0lXU00v!MLM$kD1&aUCrZ;8Iz zJS*`62V+^s(L1u~B&QC0nYR-AmwlU(uRm=w*3Cwjqs79w`#j^pyww`oihyGTg@Ct_ zn#i_VTg{7eIm@8pVrqq++aY>Kb~&|R$pxJN6q+)$Vlc?(nfclNP&X!>HaiCc>;ZhC z)QYoS+oc=h5~R}xEXE6rW~uhXb&(U|E^pN}$b1XH+Gp3TlU(yiDV?0LGG`xUo4#Z3 zS{_hi^cWY@d9N9d&u*3~l}vY_HI))}5yl&up|XWe^_%77MvQ}oFSLFKL8?OjX5?dj zI(*CwLanP|jm-7UqOKr&V`JpoSzBohByK=dGUJDu>SEqJ3_t&8#qQ*+42IX>Ji@3` zQDBaX!>x5LU`ooQ;x-aBjHytiuisfVtZNdn&*7m&ww+XSY?z9%Cw` zVbEqd^00*L=s&IIf&2{yoe(!>Lky2zeSNha5^Z1y2Rj1C{@jpOImYY53WLhol^XSm z#9CziLhDz?^}A}U*s}a$VmL!W81LMRfY@XNmdr~qs&|+C^6Y!1u4&xozytR*+-tgz zcZ45STHLL3-Wpi31TdX9Bbi3-%y;d-xlB_sH~?Gf%6P7JC6283m8&!Wt;OarSxwgi;b z<;ps3e+}!E-s$~65Pf&K1b4pAoGxK>5>=fug~($JJZWf#9*^5?Fd11O22K-Y1Bfrd zI>`qqgu3UU2FaXzN-2MK+gF!K*}e2avCylgY6J-~2u{dJZ{%Q3bS?I+n}JWvw%C)* zTNHgnnNsRP`LEa*FP*2CgHbe`v?tx5J` zgJ^#OOJ|ps7B)_{trb@YCYeQk`+>XXmP5^~zlR;lwD2yFR8Q-5^HZ2GBtTnuEsYWQ zMA!tkbVpv#tYM_nCPOj(&%Tj^q}%p`+x3xW>%EssE;WXqn}0z?fkd%3>eml~Un5A0 zT=(8Gj z=m?Vp(RLqgzztch9fbpT{musGNSi>Vp1S@R;ib$e%=fbsr29M*Te4)G{zBm4$oJVD zTFeu>R8LA__zQw++b-PRotd810E`)iTfX7Oak1|hiS~b+c2-(JRma5>ieszB+S} zRh(a&kf53KTfkCLP-PuVfNj`9m@01Hn>)p|K=F!0oYZUb_68y_T{wPv*En5T8mKU4 zENnA^xC(S*OmF{|xyq>o86S6WMBE_MOrPaPhB6he3c6L6bal=j6RdtT123>#`)*uA zVql((lU+>?yB)`%_s}5r&@e!rtt|c{!e64&xEiuE zn*V7Yg4Is0kEcG#h~fXq@WOVA94fTbW&Vtfy6mW}ON35`% z|Fy8(!dSr8HRn3iFF|<#rfAT8j%u~t#U3&9=oD9Z6yKSHM_*NDx6Q+VpZOh*A&?DB z!Btq_gS?)|lljk~`cbq=zH@BIpSq9BTmp7@&J)|&vbwczXG&=^kK!8>e4~l311k$Z z&QA?`Gw1~#HAIT-ri1F;8#9G!J08V#J)`F&D>hXBA$;ZU!)5+OV&J;CRmobfcF0m#y`0VB4!c&peekxgr4K{42mDDVfgDF_d zPmSp_GHu*)ayTQcX~G-X;X%gJ=jXeahZcLGIYrVb4e_mW$HKxsPhm*G>?w0fUbsIZ zgb%^i0y*xNQg#ogz!Z3v? zDa-!k1FhG1`*`bMBrWH)sS!OH!!|x5NGmjDd^;4fmM(BZ3rQTDS&!9AMr=L^&`?O2BV0x`A}u z^P@kXSbYh>VUZO(@$6>*Ed8zs7snqe48rw#kmCJVqALCU`I=@7!tkP#N?t-@UyL$be1Aj@qhD4q?_cz|@IUn|D>_im~br;{l6=p>Ai3Q*N z{z}LeaecnL9x{e0hH+4Km4c9~61gg|N8)C+%K}Efd0NGZ5m_Hgi+zsD_Ok*c1cJ4f z)Y4_%+vioKZ%P-9jN@R#$O|;~ry|KIdnc?OMs~z_j1Y_JWzC95Mq>TnruN*lggFF_1Sjur$3_05V(vt)P@{6GbEQ7?`Mi z17i2T@A^t-44qJzY9DK5{hluOW7k8Oex!m5tiiH_XDLd%DEho0w3Q2V2GOTA3si=2 z?VXN2BIjQJc7~G3V2UdcJ394;*%&#NnS}HGtwwqc`Qm83wE3Q&70i6-&8xhbC=I&s@D>Z8#=Pphunb_J6d#04u>Sai~{;!UTH=!BpBxSyG4t3O}T z$(yt16wf74o}tnF1Wg3?Ly(5BhQpK}fq@fBu2RM15kO5ugLekK2Hl+z+aX4v%zz() zfAwA_-MU``34QPGjmiGsRw2{B19ZXhpml3iXIV^H0jMRlu-RQ8)yzzb8d(r(u1suP z)e2aHY=`LLTEEw1>@?2?HE`dKIbaZ;sBM==cl=@aMn?Lhw0XnMv%POGLVW%d(sbCP$%gQYCK?y39D$f^=mU8+}eInYyAm-pCW*6%-nCfF?W1pbb^#l z5S_A0!GHSmXkjO*M0b3sZaoNSM*VuR!{nDBu>Sm}x#FJfkQ*P*Kj8|hcIoQXdcZ)h zK+h-C6%(g`mhoJp?;|KO!C`nNXw@Lw5w*cX$AJ0q>TxA*AML0c_X}=2sexqfKd{Qo zHa4CwaZl~YI>EM97c5-B((EsYZUCW@mP3l$xjOCLW?6xmaE>i|`%>dsyMoZopoh^G znqn&A3>XzCBA*8-LLs(V`#^9W)9Xrbxbi$xe8~}`7f~7-A=+0YgVPiNufWX=_^10= z=rT$1@*n&snKvu;7%xlDLr+7rOtjEe6si6XEZu@nu-cvm?t+}!Y`WV!F~yzUpLjDq zje&LykR*d5z>aCf5)9DFD=P5A8eD-Xt^zLY>26GPef{(KaHAc8XH-*eOgF5g6lygt zL)9|U_ALzR;Uh8AhHgB(e%K6_D~=U)DI@ZZHOEz}m_>fSeoJE8c&13bLIqKChwi^5 zTsr486Ki2qADrMB_3;)7m^UZgnRlaFuXVBk3zLKefCY_uI*CJ|4wPNh21GanRtVVU zQDr>`)bL<;kwb&a=6KtxT*mAwU7nMPhxbN)(q@Gy)RZ`Ku z(@Q$gaZ)(4_sq~c(rfwfTHt4HeL=k!hF)*}z)-u~b-PTIo!4!dB2RG$R1oa<#KtX; zu?+rMKIXI8Dw3zax7PS)(?n-Mi0@(N7;ItHdCk5cqd|NHdv96Nd;98_$6(XboR&LU z1Rcu!JYadQJ4RcitvUW}nfrbBmEPX)bi4-jQCEOmfC$5KQaPCCrp_^Awd0fU=tM7v z_q&;wOhf`ZaHdWDVPmm?i;;+VAZ`mnxamF0?m>nq)2yB*o-hC=VRS<};Xyi~slX5B zr&kq&X6xVU~FfmZ*3E#u;<}NrzJ2dOhY^DkB?;2x@1($4OHh)SlYOkD1TNgw{4Ab zG8-%IW^N)bx|kSVo~&GeZz4gnNq=4~hmMiEz7TewXE{~{%*kMQ#0_wX2vP?Va>&cz zH^FXr2s&PRy(^kr(|d?y5Fy+%A@l*=WBu+3^F;Z?w9ktLY-`V*o87~Lux^~-grEIS z80bUTJp9zl3z>5RD}W^h=w-+dgKU6&`l@C@`6aNj664TYJBDE06L{crT3fNiVrc{5 z9Y)8c#@{lI%7$~r zK^8Z1aOGA@P^;OqJ+3##z8Sh7qH1g=x-EZ2Q#THN|tC>ij?Uvfg({ zkr%kKfE^&gsES1FC#z)T^t9%t6751lDwv}{i0RK4kqmObyJ7h z6nwg@u6Gw0NPtAgytl7z93^m{R(;$Y2pS;WM8|H}CFTGUZRC8sLFWMdr0zyvo;@zoW699w$EC03N{|XfHU#kLzLZC1e80>#4Tul`QSA)Y~s;gAh z)c-5WXBw;2HU4|?-yr|{Rj>*KqM{Cm!T+br|EK1U2+&al$3jCPpp5`n2L#ap{do%* zD(zJHAM=#P{%-++Ay5?#me5X%Fh9)j>_r{ct_Y8&y(U3ZbJ(zj!d_-haG%M-irR0>$S5h;1S=rZb!g_J8@+2WUV*%E^Q101m*f@Kb{+tHfQuD|3ut zAsREO49<<77gB_#d8s<1bG$JKWP$HN)De&mBEts)OT1jZI6{+Vzq=SDbDe3{0DL2R zeUMQE8LKh#k(d9w3SQbQninvV_peZ`fUtx7a9eNmO=F`_7x%NT^9D>zii-<(ytAzz zPH5X4-$m{#y;`>vDv=Ail2bep-&PeHU2j2IsfG93DbnQ%zX~(0|&EekN`nV2@`F##+9wF?|JmtExM6GZuo2CBi}(c>U=4><)tM* zjQlMa>)fGdEL$Z3U*AYJf`FpXO-SH}QJm|pjP{{CpECo_gz*VV=6&BtVfe*MUtJt0 z$m;LkvrUc*)FUBzoy_V}9s!J2GyK*==fs9N0uUEQmUM7#*FMc#R`=mG3j+A20A|i{ zya7;%?y_~BRLaUteX`LLsa}?!(knEd)dAZeJv@HaP2actq#_)nQX1ys9t^k+kDN&8 z#&jfQ%vNpww{4%!4fG6mBxeWrW3nWND#s|t&gdeVk9O);J0?$r8XkE zfYF4S*s~klAq{siOenf<9g*#DWCs{D__=C}xo369(`TDKM|Hit>lsZ7VO!T06M~(3 zj2sO|jTVF*ONMWvTV;jpi_mRx#A0OCh1iNgzGVW@t4rV?fn<6-J9(B>HmJ++O zhs(d4Nd6jRj5suu#1)>CXHX7&Ly3)EA$wJwc5wIwu-mR7Pm!j^TDR;gJDyu)9ntNu z{#E}(G`{o)pixmz!tvke$9DZw4dQhfWE-ALbWpFh*lqGFza`a2zxGiheG3_q7e;?D zVlE7jRoUdWPzUNUw;*6TjRm8$nd&ngCu)%Z z_sjatH@8UP1Ni)MO&c=m461qDH6T`yZeCEetJAm2Pi6bt>#M%nc!#@t=a&k=Ho0c| z&_aVd-Bn0~7c38|iE(GviJeaC(^J~qp$h#{eOhMC==HsK+6LxFbEzgKJkNGuUK)!t z2ibA*ckXGPPU?rHZyx|N#Xerh?0?zS&_6)_YhJXG;kh51v&`7$1zj_rGTsNrG4m9# zr1f#aaBZsGgyd4V!|3DfjUgcb<11ar>uSf|&B}&eLK`Tuwuzi9TpTWKSDjcHTe$o5 z_-q%{1gdd<+R+cCDg4>-LsAFt%k2WvVbsFy4G{`-o5}`&qeGDc%+IO1)#|Dw&A)RB z!lO*mg8gpVy{U|pcVst?hVVo~m+k>`6%@O<1{e1p0nAv2>3ngw>0WDR)gBK2n`%k1 zw&zp6^dc*G1?P`>X8U_vKrUE1qgb>#~^`41v`?L{`hH@}_ z4Z}T0hLWz?yL$bvfoMbT+yLP;R!Yu12DR09mE2hBqn*K}h`J($LE#7P54s5H&Ho6`CbaU%1H>3ua%~b)6 zY6feZwRSdGWr{cROdg^oCmYZzj(s-s^8#?97%=$M6_`kh=`hII3d~ypL}@9ZsbNFJ zeoSbyamLO@%tZ1-1s0d;31Imt8*69sU$%)9Lz}0y@3}lz%+y zU%xeHG4n!OX0@h3!@uJR$O5FscdtVn+P**BeSBy6moa9uq9}8J^Jyt(yRG37fD_<8 z3Q9cqPrwg}l?3k$V$@=M=W?^iy<$)9U`*H6RhL99ibhq`N)SkYaRX9jR8m9{p*>BY(A&glj@Z{L!huCdH-Dzh1dzB_D_6%?)65BMxV? z^&%1Gy(Q1I-}?+F0uuYt1cUE1bK~v!PJZRB@3$z^xyFs^_Y^AwgdvP=pwN3{K)|-2Db_B@>CfP1{W~x-iS;$O$e?b( zJ_xj%U#gjKA!DqcCvo!<#^_Wu@b}*VC;kGtkyPb{n%FHhE=EPc`~GD!d=YvO5{yof z@o_m~Z0OdgHst|HC_ed`!KT*VbCas1eZMo&Wasrg3L1ZD^?GJaerB{k<#4^abIfY@ zD)aRa0#>F^X~3v7qeMcW)EIzQ6zC_;Pox6J@$U>n6;kBLReyfg`#FiLmx5sD0U&Jf zZA3*uHYn@duxA^~kFC}uas0%_h@aJB^O&@Z&o?K#o`q9(gnogxbhf$EmgvjgXbVTV4cJs(zry z#pEH)8b`$VCz65?!f!^!Wq`}(o!V~hWhU=m4(#I}np^b<^$my5doE==q_R=9-oULelg$AnG30SA7*&hb1XbucQfD8=2iBl@yZwxR3rIL0%JVSX3^{xqrZ_M?$G$ART{{z@pS(b*{MiyoJ zM%6EbR^2+t(nyO`ag?^VyleLH?twzD3Aut!)FVu0Fq!5|{v4$hN_QzP4)*3~1d_&9 zZMHmZe!nrwPQIcL>ZDU##I6bMliW{PoCprvy+WTpSb%dmrb=lZxZlX=E;HKRZt;dR zH{tEOt`VQXzy00Jx~0bND5M(Q`p-K4#oiveOd!N&uo=^P(w&|7)cPNHOPuV#T`1!O zC`xWVD<68nT`D~kpiRPY1=xCdy9e|f!kJPiT9wICG;9L&$BQ+J!_Q6To=0g|FE0qx z<68;HZ|E(GmIX3y&jYQ3kNtfboUjSm*e}p>#VR4Aj}>5W4@${eNs6ybPs`c zjhL2^WeR!bMK2)ASLP@;kH67f3mI8(IN}6$cC_O+<~1bLqa=`zyp{uBhh?9Vwgj0M zy;Vu3BKxwtLt)hB>AnIZ4YC}@4u0>ZwR6dHPxINF-7L##-R(>=NV68~{swvJRaCeIJf7yY!&eT`kLlEP zh9)ArsFhRT=FLQx6};<6|7b|Sd;<{}ybj4o#|gBHi%}CF?@Yu17d3{p8dFV6L2B68 zfw>;9e1Dgvaf%z_wbn+q9&h7Rz!(Dd)#*!oREFG$IQV&G=ys*^D>HfOi9%vqbG>N=|vrbpH(9)MU+*n!} zU!{WHoG5}xXZB`ii-7sz$kI_V;*bm;&TID$n+0n*PEwX6H`w^jv&gf3@eyv8pPqzH zNjE<6xBWfshC|ro9_WiZ<;|C#6UsSEV$@xyLc;N_ zki8*qqg72-<$HJBJ*CpLzcH#Yy7Uy<%ep*PuPE5s3^bDPjaqa&>H4(ez08}ad&3C} zF3O8aDER|Kh)4J9rD0-O{{2f`w;a8%H=*qK?zGhXrx@F93>*v#H!dAvY4&eGL4e!u zp1mT`GdE(4FEaWaQWT}MFP=#<2shf`v;5PoWNc}DGjxP1pF^&`+evyMH9$DisoU7h zS?=cqVH*|u-EH^ZqEHk-is=@Rzu`|NaapVm4ISR6^t2qJ!#NJ%ASv4rc6n`zV=8d$ zQE8(&`sHcj${*lJ(CPl#R`Jum0}unJ>L$CSkLfGSE}Hp3&<(2RxLc z6_=IpTCOfO>0rYw0)tK3M{`Bht8s1cjma$p@BU{jyElonnM^!42n10(p<%C_&toHG zkrDEidEu3M<+0lMZ9$-7a(Z}zulQWJ-9oKth=-OTfi=9m543&58pFBjm)fR9seog5 zF1+MhkgWUrXrii=yCgU>006$8WVBVxPY;L3NUuq;UW)6yP_VtGO5Icdb^`<-!k?Xk z=bGM4OnMn~H&yL!?F42t4k`XsMTbWO$Xr}_yojG!|Sdz)7_*T7knh5+e6pS zi<$PY4x1JalOz0rlN4Fszir!@xoHmMD7`T*V?d@;=&DSXFo9( z4gSIEvXrBbH;{Uet19I`#id2sB833(J^yR36^Em(YL}@+izI=BzTt3rgVp4t)q5Xh z%Jp_dp;7XQ-+vpZ8t?G*rWM}c4s@N~tghEx?L?V^@b^U(nISDH?x)p%J zP#bevFtg^Oa1vckSYqU}$f@gt1gbtjltW74hGIhKttkDOJr=-DggjTBQ{(KphijRk zqy-AMlpQ)(M7h)dPne@rK3B}-_cW+D+SYQ09%mhERZTTNR>$<64SNO;-KJ1vE|t_i?;M)^iTo-oV&}#@20=!JqGS_75iO$ zwRw5|X-CkBze%QMt6+*diJ2BG12c9&|4b)`x%W_SP9k3Nw-Len8QsCrmA)qcB(B?& zNrB=kLO`9lPJ1vsH_pq>ImrCP-c38hEJ$Cso$*kU1be8Br@iesCy=2@q`hQSoyh5_ zsLhxB!mAs#KHh*Tkgs*9JO;r|+79VGtR$NF`A4q!XMFeJ=LL4~k0nkWx1D9|T{G7t z!?Z(`9lg8ei_#qzkn*L=VZYpn6MIEOqH#xwD|a| zBWdbtAU$J)N<>??E_ybgUiK1)$_#&N$RK;TS*Ee*QoggL7}X?jN@{e8(}V`-P~ZHe z--_>*p?tIpW(2UxLtZ;!ghrrf_iVW#X%?R*1b7?daz<;Kp>;$Mc$Oi&ZUsgO?nm_NXLse89oN(4bH^Tt_$Pu?_d zO^k2MNXvqOr2a&K^txZTUNy;V2h?86vHz^+@0gEd;*Mw2;9-6W33pGe%{gi`#kj&Q zDI=pVgX*O^IYEYjeAZzUn9;=e6o;LuP6w}@U?q}x+B+<=w7WeWv;FG``knw|>4*ggi z9z_rv-d4$gnefu$gE4E+*ot9DLaQBtmy;9Qwo1c`5>cwh?&sxsKsSYfAN5sluJli#`O1Nh+m}NHi+mIq9Z)IO2jx1A$_B@8G=+0Wki#M+Tr_ zZS@7m;WMYx@%=S(0~*hE(u{*$z69@Wgm#_=N?~(+X{a-F`ny&3DImu4W--s z0p7WQp5!4ewu6ZpKi^>1%yPS=tWa4GzY=o&OFibh>Ha=7@$pJP07StJZVFbvAucn> zcJs?#;QQ?2Vn?#NWv(sPc8)xeF!3u$@{Zj*1rn%i-Eqp%kVcLR;dRXyWRTHV|L+2o zRo7q7wHm36I5sUH$6m`rZqvs{(r&t`De07f%ze!q|GCl~t1YhPH{RSzzWo*V+D(nw zG(DkEj)pu{U0`CTk;q#1xR{{~NeY{qpz{J~--9sruyCEEM6g58M{WYnez~jfHbW!;h<+Z$p*MvGUDrCch3&>OpN)9) zx`win@gpfl-5qRX%PMUoW??+n+w7jKdU7vRU!14YNJpA4WzCI~5eVT~wM?72j&#GE z>FFlqeYD|UzvibYW6U{o(U@f)8?2|N9v(BA>zUB!viUaqYebURk@i6V;m2^=Ar#|_ ztD1yYsD#F-m3v--lKy&qOIPjZbJdJiz-)7`y? z0=vDM**E3obO1$kphchRS^MFYnFFh5Fj4DRB_Jw%!tG*9hs$G2XMv-j6i7|cCF6iN z0|jvAv`Q9(>UOF}z-tQv@@l!Z?O>>1EqmyiFp(iDrn1c+s7hQrDUC$Tvd4NqZKo-c zY4K9bKAxlqXYXWwk4c_-t4=XS+@|B|*H2}tk<96;_4VI^bh9rCI31ZO1fM{GP!N2k zqTut!RC7jNoE1{owOCMRYn_Y5HRNsH?O z%}SRei!U@d$_;6bxgRZ_cU^J*sgtu%14t2xH`*ygVfXZ6wn&_u(oGG86{6St7s zB1lh10RQMimdAf+@W~k1OIzy%&4(kiV#U&d!SZ{6pBo(&PIBKrUud=`fDanapPQy` zA2KigJ}-ckQ#akRWpPGx%;B5!tM*?O>5=B=|9MWZI2%q@BzPxzNSQ?A?~f`*5bADZ}-CB*98^2I`>zejaP)8o|Ahxat$rQ}G;3dnrjx(oJ!@2| z$87f9^Rd={tXwr!=*EFBR~JwMP|H+{PAa|GA&rI7u6I8v`N$hJ_Kl4l1HIJ_b5<2L zMhOgtGWPgRDbMpjTR_AqFr%W!Io<&(l;ybnrgj-g;`k$%{(YC{>~2zc8gxyrDbXK? z4`L!lIXd+ydqqljXCYAKmz0zqZ{+NT=bQSib%*BtD5y4+SE&?g8^ikfpTAJlGisQU zIt`0Re?BgpY(hXyxH~OnI$5fqO_1&TO40?Sw&-HZOJ2k0CS3#fZ`{skb7oR5*X`Pp zFq^bd(M)kZDe$2t}W}7nyVFt3q4G{1aAOFkj+c`~j}HSlK%7 zO5uqhn~S!pn$XAFA(ARGZF)|11M8f8c4cf+Yf((n1remb5Pyt{%g9TIKgG4|zr3y9 zA=2SvLHc%zhZ=M^sUh-fZ;m{D%Na>rZ0Tb>-EqscU#e54JQe2#0$TELVvkvZ`Xl!n z>8i(z;T<`eQB6$LNnU$?Au-kAU0pKV)`b@MNebaJr`=3K{|#7`{rkJbD*{lytRLd|re z65FDz?mV9qnRLG*ZSjZ!UCdO)7`Rs%g~Fl!5ERlHddlewd%l=$e3eP%_4D(~AW+{A zlNrs}VMcBg5lGakUW-hLOKfWHye9y7Zs1x+jofs z4>1`%anXg>5`KMft8?kdS%pw-lqrhKWeZ&z7_H(zz&99aJo>K;yeT7$ zgb6sXpS=S3uHl4gUw%?w#)qc9xUp%sttGX|MES!b7I(?EkLu-xfNB$YxBq9uY^-qw zBFW*+r|e@n(mMu{y2lOz@UZHdpJZ^9=K*+$v8wK*Ki!7gLtf5i8v>o&(I_1+Ebec5^H%7DA~ z-kz3z&kC()G{8yU-m}S0qy1h#;zL{T-nVx`e1~#tg@yzfm!#Z+YO{qPaw98&;(7B5 zgwt7Y$^*dgU$i$Nr$WP79uO4p87C>kq6x@Rm<51I=>Evlh^`kvu}Mj^9?X{2rb}$G zKXsD)Z(^zk$Sd;PBsGaMltifmCv_q{9jR7u9z|*~X;QWg`8VTQ)SK;a&yJjvAsUF7 zuF9LIn;@vMe_**_o#uK1`p$Ey!EIG!`K6;l@>DwXJd&+u=j(k85>8!KkNlXPaJy$% z9r#BLnC&_h6VMeEBza#~+tniQEweb(rT>P*4+7&Soh=)2hk!6d?6|Gp)2q)nGQQ%I zk*}^-e`9#AcDU`P6GrtJp3A-B=W}yMLK5d+^pV$Vw{)Eswh^`}IX`QubalM0dQ}rU1BRzIBRyvU?(q9w^L=xjWXguLIwPI5{Nmsv)K$_aVLxI&PNT{>Zc)(GBNWzyjesqy801dJ#{4y?&`%$V-G_P#hc zLK#X$J{_zXx^w@$b`50F`AZ|7vNYHP>G7~A2s<0%wa>Z(Xx*@_+L!SuD^DX~uiway z?T%tvIz09PaI(1`Rdf?URY!DkI?m2s#bVh6tas#v{^#;84_jLw&!62IZxR+BjVHBQ z$?OblJs_Y=_eWNi$(x>qyT=g)tM?`&)X=hWm5tvC5`#((J5Qd}{pkv1BU>;-UOM53 zX}VTN2I9`$H$L2q4J>VNF)pt;<)p|8=O^0itsR5AG}9gxloxA4;6`gh#m{6+#X{I0d@KU}h67xhz3bwT{FgtjkFW$ljb1bv1bs zU!hsE4b^)nPo+;)eBy|&1j5yVcg0nz_%f2jca$6DaC`Vo<8X)kKR|gdb*drqAI-2g zdPj|SJalgdHt;l6>U1A%=I4k<4wnPJyj`hL;2TYPYgc+Kt$^-);DnAravhISKi}-e^A>$Ivy< z@S5exkuZDD#-qiH?GL-)08nyQgLjtruP!+^=$>~*4@D``r4+P>?7BYKP>Dby&Wv=ReO?Y-w{ zFDnenPFH<^47>LR4e!qL3it!)NFEic?1}px8~ZChd-I29#({Rsov+A975|zmK%Orv zp66!!VhBe3s^yj?O1Ycp3|I1_Nx(4Y=jlsv0z2Hbit2Hm;*O78N{On#SnhmyQ;J23;rD&cfdZfhfuhr$LrRvaFazPv2th-yVji zWrl{B8yTfZ{s3iU1V+Q(pvwy&PwGBs7%sS8&%=CZfqXSd5y7x&;Y1Bt|mSA zxK_#GaF>MYltFGv8P2O%!xSVv?Z|f=!|f?zCgcU?7ttS8@2pBW;{JEidI(w!uO4D! zrsp8l3EZxs$G-O}mid+S#vICV;mqV7Ox=Y>Wh4)&Z4?FCMHW4Z)DR_fTG=Ls)mb{v zA^~%kol3RTq_#097ME3fxy^Q7U<-z^Jiwl`_lJgaE$|$sQcmUe4OB02>w0B*I|n^~ z=V=r5_#5JU5hNGjv1)ZH0u)VxX-7VhPy|NL$|u~<Sew0oCe zb&|;1x06P~dkwNwmLeZbtc*{hmlHnF%U?V9>2U`l1+Yt-_OLuu8&UbejDrM?Xv1^f zrn>4l%IRZb0ZO0^f-+Opy<_mEce!1<+mc?Af{d`748`5lwttRt{=sRS;G7YDxorVA zDkP?_$yoe_vo!MGpMO{kL6o0p*RX7=9!ZVYF6#wC7&TFK__em(N4 zPTY~RPx~-S>qsVZmL0WJUG(@iJGsf=U%RpL1D0h-(`ImXa))z}_f$}(lR@6^zt=Rj zs7cV{UJLcp9qDO#`meaboBqjix?gKjF|qTE*e}*a^(@u zGV8T!=NgmW0DXKHHsJ!3k$6(_R`=pGU)@22(>W^4=sZdO*S{jZq3=_}cI{Mx98! zh=AeJbrw!ih(=b(?tLnvSOK%2gfss}4Ts0yC_^-$E({p-6g3^P|Nc2DEVg9b)3wPS z7JRjONgTry!Z981F0&;^hV8HE5Q z4K!wwiD-kN?8~RO!Yrxnr`a0A^3O{8*~Z!IYP#(lP4#3ty)0YLdwNZ7o7V!{@8z4P z-q#hFq2xfG84T_-l-;&I25i~IYM)+Hj+90~y}ho2N)A6RY;zyk`$j{m(;r4M9p!Gp zB9{UJAo(5Xe8NP()wNps5?dj*s+krIv2`4q10e)yYH*|I#M_m2@$FFVOpkCo!RRcHrk6_Pm5_lc0nb^b0(}=SI z(>VF8?wiK&C%!L_*67dW2C(mC4*Q1fQ4;TN^e+p(jRP`*_=rEJad`EMS;2g{K<|a@ z^T!+E_d`|K^9w>$v^mVuZEMDkmT(F-i8`>?`tN;@L|=n8T456;4-GrA4>-=15GE5S z_=PKD(FT>bvK8b?C+L145fQ7Y=dv4%a5t2H;>OL)D})+qeJ&0yS8FvG!>69`d;bfH ztiWj6W5rn1Xa-;ylP@ka#Ri&bVPd@ZlGu**kh5qxKWpUEB1ldAbMvhXXP$r7yBBd6Z|X9Y-h-WP6MajrHdS(5`$|4RyQ&YptY!mjHT`GD8uszRrT zCzm^4ydT*!r_92?&|H-9ZM;DrmL7zwiNonXlc^!AZB&FQY^UaoMrYd(KVMm5XF*K@ z&T1nxz_QnBCE^F}=Y6k{b!2AYk0)oc z5r2TTG?u9#<{R?Wc4AwMmnYepfa_0}#+=St8v*12l`W>)0%lv7O*&3Ab*$D6?xeCY zarmOtpiP81X}IkO67Eg3?G;1!V9*@$R1_)t4*KSEUxX=$jLjr3r>geZ zdjQjW9F?*cfDFhZ!eerMn0Xz(AZ7Bf67UTqjHVLAZb6e|P-B!*$prjwRcafr z4b#53EP5(+=j!K|(o87=!|Z@_tg<#_~F2C+9?1{a~QmEX7Ls=uXQo5~2Xs?uI*|B)+n;)~}oP zRsOowUh})DTZrWa8&CTVV*`@=S?<__@~nwBd#@^K2@@N1MaVmI zk2R#dXU!9Y1PP3-?TdiHVsXZt=GT0(axvYe>|0a&l0X!wMpU59;&mv_yGi{(fxWw{ z|4dPUK}wldc8=<>Dbd8BvVr4Ez|OVVD*5hF4oEb9)P#vM&crf$WaUx9G39EQ&1zOB zPSMqYw`)XSPf}soUbF#h>bQe?R!RhPAUcNNY8^Rv4?N- zr4)#9hq?a(d4*2VFn^P}W%oZp8#s}g53Vh$bJh`;nEb`kQ(FtdpM6Qaa_q)bu2&t~ zxn*AV)sz&qLO;PE#>o`I>0D*ZiRwL(H$R47_QqJ;u21vk%on%xhtY;vqqmIKdC>gZ zN7Gfyvil;Wfr&Pq5qyC!LgZB$Zno01oVo+!FCdlZez@Fws-NH03`3szisziaGwfyW zQonrqlExbbfs|II@^s{`-lH7-`f8Fpjoz%(WFQxO4`<&p$VQ6@@EO64fCpQYq~QiP z;@R${4lSR#Z$VOomPt8GOHM``brD7;??t1Rmi^-tvhix0bINoT8qDmhE(l8FtVs*a60SB!kj5X~WF}(} zzq62o{6xHOL`hq8J-X;io<_C!4r%=T+ec<$2cOA26Zb!Muu)t4(L4gI27lD5;x@pY z=zmtHIrwbd@(&@2+!h9NkbK#f`5Z71$n)RjT1fryl9O!ZVmYMEVsbbN;p}o~NWgiC zKE1`vl)mH9x3(UW@Eyj(X)J#WXio}JUPkP7x_@6sOf2ErYQ!82NG2~;#XEcyp0D1%3+F)tS(xNxDi^n;JlM|R%-&odYsbMcQN2xDf_Ic$y?nAOiq(K(&jk8jw_<;l*p?RFSrA}R6go8Q#O}d^zR$iv> zUnGc*G6Oywh`i-C)c-P3`QBSCk>@$U?Zb5wc*(pVuyk^qVqMV$Bz2EFfpeNAc3!ok zpilm5dmHf`$$oMZB|BlWv$+lLLUNhd2XY2|1r-!-*osf!M+;P%#(ycbs(`6LW~Xba zSM9~|l@lkayMe@1oFj-LKCXaNy*+UQHR%@d$??Z%x<)!_FKH+g{Lrx#yc#?F$csoM z7Ke+C%3<+k#ygWT>X9ped4VAf2vUEro^PWY7aQYkWV}*N)F1!FY6sOTSzr@^qznc4 zs%yEyV*mF{!Q^=m3yAi=#so@5stf8%4}$J z5jJ6qcE`;Q6u|%5)!W-)k(9`+muiiia9FJUt-k5|n_Y=fcD7$)Ca$!P;vS+S5Uj^Z zq8|gA(UDgiq$LKpn6lihx~CFXJEju+Py6S8Sq=C9rV`fG`W<;YPg3&wyWyH4S*xz* zUnH%{BRJY9CFieW{ZYt|SA);+un#>??caZZS^KZ4ebKM} z-6b9~3sS@$IPTXua`x`&SAWHA_yg4MjobXyY?<}XdOsKxTKmdk1BN;08=?OyJNjx| z_si`wtIcm6$^B$em`iwsl0V%CS;}v#F`F@1jc7jcoit#0(Z}(4W;WIIx5>BoAIFrl zwUr!BYS0!QYEd9$a8_O)CW64bQY)uhz=hL@WX(!;)V9(1Q2#qwQ^0 zcHWfqTsU9a&%AEk{m}3vXA*0WU*`Y&$(PT)EAdkOv#&8Nw%sQdwN}<+20 znUuJyV&;bx52F_SmEATkYRzog|Myu)@wXp?!WM277=cCn0`FFX$%A46}E6lnit0%)Hvx+eS~f{s)HI0T&`5drG&k|gZB^q&#QCVYBvqG3?|r1zFbnkL3PiW>^2Pj@%YXs{+*XDYQ@ zPpfzhAI&^USO}a)CYM#p+ve6P26ytp~`V5=F-HRxEl&qxqWTIv}5Lv_2NUtj| z?a0Z2C1%3>vUks<)9{pjbdlX95fnjE0dUf5QrTI(y;f{TtgLf$vR}`S2K9P(v*;;X z58$rdaIya7uXNM#_77|jae25M46q)T5~b}s0){yeg^iz*UKK(T5?02*ICa_ukp)$H z`lk0q*EN-a65t~q_E{ID(uMKV<06(i+*dpc zP7)Ou`~gzRFbaaWBca!7NAgljrsD*+Al>2i#>TZdrj0J28gS>2>!s1WoW4`##Ypux z7b^Y$QF&YodI<`4jmCEOqtC96G6|AwqwOcR2$~GCRIdj0H#e#cD<;U7*LhGIl~T2F zkcl5+=@(t?)ZW^-X+~demH0{5News4TE2qOj==XNr0)z^)0Jr3K6TH+8AbW^&73gR4N-d$f~Q-evg^_eQqu2vexI4!sCb5 zoA8>2_4*XCuZlV5L)MaywmUgWRFpk_-K3oAKwC@pF%>B`-Y^O}zVti)_j3RRruz8X zf$3Uya(xHM1g*f>6vz$TU!#xbtSdQlMEciLm$M^o{S~3c7)AXUy{63~?q&hn81y^7 zsc_vg7IxadWkH6#2jNT^+@`MzmHPUgFTfZvw z_+n^*9m}iy(7f$qRe72D+Sk?aYW;Xb)>zZ|o`Y=SXwIRE*&p13!YzJQUmdLgjVX{b zActii^}As=Qb8;|6R3X;pa2|@2vqm7JELV{2Pn^QsEzjW#z6V+op8tm%6}b^tbF|?;e1m=02zT z=qTa7hLX`fvih-pxqjW8i%Eq+Zviyi@yAd?CPALCQksoK`jeUpN?YN@`b=`xQK!(a z5qumSFd`ZcFP|%v6si#Jk0Cu9qpUnkkY>_!tupLF0l#`BNJ)wxNuVAj-{q+9SK+o3 zNps}NiX`;#zKjyBY8^$Zi7KOXk^j4Y8Php*N$y*LYtw@C z1bjTfvGf1p4$!xiFTOjthmy+Ed|p0YPAYJn`#tlTJA9CR;qK#?^*I&O>-;(JvE$`3sb~q`NdDq%pNB}S zUizz%^ZB3d(4kA{+hvp6dY)WruIB+73cFB|oiyph-&ZBU`}XeK6*rg>wV!=wt)2TT zd&$}A`PO)u)-}8M(p72e7K?uW>fWfsIM)I7iT$_Pc0Kn>!s5XVvT42Jbo8DYY$BY4 zoa=qEo&2*JjSb8stR0ElxR~-Vc=A_T;R!w4otyhpJX20d2pb+pNg2vpS-LOKp4V|6 zZu(Z+d(lHYw@yYht{5Y<#qcdp;IFlswQa*4mw&w(59@eMu-{eCyZw?Ddhyfp^7zXJ zR*&)hEj1$herkVLyjk1-LD6}~Q~iH&{Bm*aQ53lsMUj>5+NA8g_lT^l>t$nb zD_qxJ6_TB@b8SNQzPR?@$@h1E|KES^_na`9c=QF{NFnap&->wQnIAk(H|Q&vL50 z$ZlW2Qf}aO?Mjg+^SAY-?m6$W4SC;_pcbPL(*(HghwfzIB_UFC9%9x)*9QEZg2_dH z6>Z}+dG4^+MC0s&imJ`0F((uEYTlHsytmZU?$@;E**$h$L-+%sXMp6>Z8Oj8KI#Xl zV2hkX*5G7npeysi(7Cf2{`IFC z?`~z8Ikj;5w4X(iwPJ6xAR7oGh(ySeS6T+bo=ndu)4GkO#mb{Ut`P&my~8Zk!OLug z^^ZMoQ@@R3{R~7D_S0=X1Bm^YbbF-v~~-x)JFquid#t zceAdT-UkY8m9l8@T!PJR{DV*5yQ}vA1?A9ghH{~k_47r|FFWV zN|{kBn~NXA!U;0AI~KKv6vcteu2280IsEwLJ^OBRqf?-bwOcW5BbQdmGkoUBa)=5j zuI15|0JA!$LvwiD>|yc|PSI2oa+urqCbQ`B7_nqXA^zkT$Zf2^jjOZn`jSPDbml zewJ-9<|S{T#9;^mo>f2syVA%S8fgHQoxgaOBV#^9>u44@Q zLw$QxCYFW+I^RgkNHQ2|doXrZd2#p9Y+$(XqD4q*S>x8U$Ib&$OrXzqQ2vWn6?;up z&bZ2-ldp2IPDk>cByxJvwiB%fxPn_Co-wdSSA?&^mMX_a`b+-9Mp_h-&4S}CQ#;4X z#~tdg!&)aV#?yYU==DG4h>ITi9XUjqcd~-S;E4=sK|YO|I^UagySEP1NK9EX!74f0D^d~(& z5{ebyrnB~9o+Tn?$i)MWj+c!deiI7KBSjv^Qw{0K?2vP!+;_hMSZ^K>k4sK1y4-gR zG0{ZUjSNj)Mds%YC|1D{MHV_b12jqM)#nn|sGo|pZ^_8gm6nwm>yP`RCr2~69FO@3 zGhZrk06Yqo6>mDTN}p3M`I_AL=wL3?ZJ3RdNE^_1v!0R}QQ+usrK8pEkBkyVB8P^* zBM5Bksn}d$_%G43hCy~rgHDKX#sC);W$(@I1EXdzqZj38oMZg+=XST72KiWRn}(b6 zn^={Kfo3fQQuL~U;N$V|WGxm|Z`?$5lpiyFsH~CVlCm9fY&9CWQBMzcj8oMO-F~YQ zAxGSxYm(6`Mk_6A{W2Gtx;;4>r4bJ)g%0geLELHXb~KP7h|{KEDlXn&F5lqu^%Uwh zG05Di>qYDEPkY;7 z9By`O@D8wmp_4L4$x#{RVw%ABLN zJFh3M+#~OLkO!!g;|O(g;T39s6vuTA1wx1GAhJ#sk19zq{^uJc5(#VHmdAKGgHqH& zZCp3wX&itK?yGy}Rxsln3b!u|)mgVM46NQ6Y6LnvvF@ull(KGx7}%I)+=&-G z_U^qhR8E^2SLz{7+bC+s`3esUQ8)>*DlgIuq(hZrQ&F)Qz{|1yY(4tXm#)$WXm|~` zM7$Ur2R_zUW>4y($17~y>8KnPA>O@Civ=2;lA{+Q;x)KdQ76mTghUSjCC-XPd0E}~ zImb;Zz@NBS;!Fc#7h?d-0DYCwnAzOy(mFu$$OsXj_#_P}h+y}K=P6UYKd9~3gV6rmMJJ3&-+S#a3 zlsxi&6Ry}Y>TY9_IC(GbPfzVTeNOqI6(eg_ZgJyHG)%sBV`L?A~vKK88f5L8CT3onH zYQzZh$ng3qw>WV5DQ{3hnqT8seg#!^e)4+rZuj1Elt^h`1RwI0b;)@Rfeqr$=eqR! z^*y^e*F3u0k$>pqH@}_?kADdFwRvU~tn)3&^l7?K4^Gbl^IEYKS|uD*rdKL1|2^m`D85q9iX+fx zSs=${&pGPUnUA@Ce(n|(kc~V72zEG}`Ld`?7P6fF9Oc+~=QD+KH`tB8$T zUZk3x`aT~=%S}s#R%uZ-@Uk~f7`!s4IaKOA)Z|S)aA7@S{4y=Wg)&w1Q}dI-OW@<* z5}Za#k*1%Hc=La~3e1XGLrxql2nco5v*yjH=eMo;f>wje-rQf-b{VZ2-Ftv83U8!g zol=2lY`0(QJDT%Ww+MDGOc%cZAjxI?;8%xZ75Z)qyrcIM^%B*?6*ZYdio!=C+ zg0kod&wc}a$%_Tj;)$P8{JE`k$JOpaHcPa74Enjj?@UUe*>vFH5*`X35Bv{6yKAkF zR$-2Bht#*qS-3(WN>8lTE&eL#-3zXeg{OLW%*yLcs*6Rgmz&H<#i;SPZV)GJr`sXQ zV+^xl-8EDsp7(k4mV6nsCYleN(~CY=uJ!W!A`n-`UtX8SUp-#!=a*gUkpW)4{v6cP zC#7y{^PE3*KJ&Nx45ZfJ(7Qh}Hn}wp+OxTsI8^gSzwB4Zed{`9<&I|zR|6qoZ$bhq zvis-HK0mCD$x#ulRPNadJ109`g8zcuNKEGatVOZS@gx^szYDwI^51XgtbnP(vzrwy z9k)zB5A@x+de@Zt9E>J^gSH0TvWS3T)eso0@@d5@>(4I8%hGKLVEy7 z=A8QBomV!8be%M+)!NX0aR30P5aWA}D4m`qRgId9wpYz?dbHd(y)JvJX=dX_HdnL3 z?P23>wY*gJmX~L%r^FN8R%L5@F2qxD8Z`S<9pg^&4Kj$J?`HvylzjH zx9c5pJ)?y@E&c-p>kmZF-Ps-zINb0255W4va7owD39b6o382Bo$`NDX%(Qq&$U0X7 zgpJB7CDi_jIbR z?sM_>nX1W19jlMfY|+GMoFNtmVW=)|F=ySPYFA1K^SEaJSMtX20B?%iuzS#0FY|Bo zM^nNiM=6wWq483&Gy1flqiWW=GmI-rK>z)n#g}4C4OZ1pf9D1}+I)j!zIlUD%&wfz zV+YGtUSpBhv*OD?Tu8$RKX1>j-p3#uj6=eh`>iXKjB7ubj6TWxQP-$^U^_GLP5lA( zt=KA)ewC)gpWiFi8OnK#2#!=RKK>Oc1uW6_ryEIgHkSs5m=&zFxK)}jQ_e7Oq0qzv z&W}EVZz|Ij#2bmC5b*%5SGzvRiKAEw6`4(4Zuywryg{X)nA9lsQp8hVG54c#JNRU! ztT9yHShwW1?bED29B&}U$5I10N+5Kg$5w;jLVh0zARLrx?5RxvDx!`hUyuwxFn=!HV!+!~HX zn7k#Gx#pKvFme7%jGo4;(nyUkIJMoh4#mHIUwh!5x9SoFz|dxS+*%ZscgD}X$Z#i1 z^3&HjD^j;_LOLDgQ4F<#)e3o#t1B30M)sCDmsk_BCY!(hd2$lZ@FlQ#fV*xS%)9j$ zD)oavO24l{Ib)n08;bIDB{hbu3D4+^!hz2m0A++m$=+aP9> z%xDAeoUO+~dmn23H2@ltbSpNUo|VQK`QFR?5jtP=qz(#bONqq3_0J<$ZdO}JWn}eI zAYuLIP8yL%As;dt#mcCdNH%Sbo)Ovxxkzc9n~;dakT! zq#bfwJ6B6JkUl3ohYdmkgjA0mMGa^@VG4&BwT&FDkr`vUx!sSP<;Gyj=2YJ^RS9^2yn@vO{b8 z2$8&wAzM98d}D?(BPJ3eyXS6{2ootzw`=LuOFiIwT#Wruhzc&xoGy+WaoZOjEBp7A zhtz_ZQ6~Quuvm-%6#(ha>?4@xwSkBy5ui%0`z8ld^PLhn*P~aLcq`SRA@AQ-E}w3k z&RwF%F2hkv>!P3!%~vA=scpYNZu0cvI0@SHHnQ_;K9b%DhE^`~~ ztr&=R{nRx$dlEeYUqPS1MDo#9;ivm6-@D$UG;dlk>$!vo(cZGz*B_!s=vz`5X)t~o zJ8ps206?BIm}#~`USm;M1qdmRZ-UN&*m$sY_h)M#KZWg15GixHS7=R;3CRap_Gww_PPvE2x7~l zHB<&!iEa;15COKl12MOP2xGUTGr;r075;Mevu~nrLsQ``mnL+2t$9DEabGDf{HwYdu*ac-gae| zz3qx^qV9cab!dA1()Dp`LFDA(rHNiaR8yUxg4U_YHLQ{^u^> zL*_wSQ!mT6^SkQTkE3UUL0%Rs3*2bNuWNHrVnHMb z(%DZwEJ6ELA)gccuHcFl{&$4DRMX=7#L?xENwRxqAY{7We*n3QEh1N$wc{=u(}UeQ zlFT(CLO(C2XT+~%s1-U0kXP$)&rRb*9uC5e&FyxRZM>uTt)?c=-O>L52edT%O!I z!P_5Gm)aa3iT@Z?qs{sdH}Uyxaf$Q~oatB_ram!80~KD@B{V%8MF&r@oZXb=w$(}2 z;PILr5~F<4L?mI3VY@@G);&zi_5*?>;8ZJow%V4gxs3AKrI)>Q=r2zxGmYJ$hLe#S zA-UcHapNznFU>CNTR+BhzNb+4IXv0iw+iDLeNu$BRSV8ttD9yMEOkuQ8&M1LTs>0h z-H_>UL--dTOJXu&A0$S-r7e*~`H4_%X$*~xrn>gBH5_5FD@l7Fx%I+<&!9B_^w4;P)AET9v%6<*ky8Srs`QtZBf zhey$@iRdxt{v-wrA2t35fPh||#Vj2;`^D=~0It4(#GHF$hsX;Cw!r2vKN zwJeU->=#;VU-;}+eb>U8@3m}feq{`^w+kX*3#!wsJwqW>LuqRJ<-a&dr!QUhG(WQ1 zv)>757wq~>??>4#vm^DV>9s(bKj(|TLrt?%yG^P+3XS+lx;b-4FCX~rxj7uPu@SwD zslyF(bCdentfDnniKO+%4D@iRJ&q@RZx?o-hM%roEBaUp3amORyBgT>JoI-J&`fqW zyTX>2{EI27e{0>h6dXteuEC{sk6e27S$0zbkWOfoCXTdb?H~Y@iDe#%W_Uv!f+Yx8 zehA5=PGb+KXF|mCa{mYTGZMxUW{Gcn_Fm}lM+{{X>+W_pXslR&LSOMqhUD*Vy90pChQCP*$QTc?Z4zFiKZ(E3}v;{xKQ0kPAV4AIUbZiGpv+@U8Xe zJI`!Ut|x@4lfTWX$_1|gjhFVWhevK?gM7F5W5E98C6yluAc<`zW6CS*~>4f z;h3-rLlh+2SLY|Ta$WI*wXYE+4X%;}gFdnSe% zXhv9W=R%X0YS#zXppN~7FKSD;vg^n1@lPL!Jov2H!}(+l8HgiZRn$^G=)_Tadsb(; z-3&r{%?yH^k`&Z0SMsWx$4@?N_y|7XLB5w(>sgf$b65pl+(<@0dG9Y5cQ^TIvnVpQ zdl>^h?G`o5_@%T_@^rSrsYTUka}t2x&_$%~WM(6+>Joh}0uQ6S)TTDAws$fOb;LXJ zn8q4ddxmxX(IHwR!z4Wr;3)2ZZ~3d`JhX>nBj&wXN+0mG$wE)O@*7S1E#@K&mxrjH z_9?y`KEUAb$1>Qf9xNgYfACdH;S7KRQhp)-;8vP_!W)OxAMcqO>+vpz*W~Vomj$t# zTS_z^vgCA4w~Tp%vxDEa395yuZN`C#M9}4{-KyBuTR%h(^*gt#f;?QzDe3;a9VoH( zL%6!%A3yVds7JXR&40u1#vz+EgYUOS*6PNpFO}oM^^|f#s&*FtT3lo=FTPu9Y?yFS zx$VMx97ZUUcT7D8aYwdqguZEz%wgmAmr{}$4liD>NE6WDfyjweb4C`orZ49bMrYyh4U9s_C*E6%!G<##wfX##@@NUu!soVEUmHv@D(H- zCr^#&G~6;6Xf}lpSEw3mHtzKTIMNvGQrYBUm`NE_l~)>2WzRg@Bh5YE%)xYcl1LO} zP0*>F0>BFspPXsvwXBKO?gpu`^Go7sO9GF@os?*zsWI@8D>DblIWW0AiOwc24|njBsI~6tcu!IVoJ@#u7GIJuo?GB?SI=XGoKZvx%6bk{hP)#iLaiVflbEuX9IdR zS4LkunXs`*2U7}RsSMuZY?OX#B<(l$gsU+s+gYlsf9}pNmG1La9gO?b`ikeiF(CU^ zzo6=*KrWi|U9{W$xFlT=pim~mMjqMIO$0jyHHccMRE}0FJ`(NgM#Bii-%&6_f0k%; z$)9*xTpIc zKuTqmKCYM>>Pzz&U(P&M(@0rnEr?*4h@swXU>(-?ppC=Oi^jWgldTe}i^#ApCJ{MJ z>sj=_Iv=TqjFMoGsJJQw#oYC33NgS@okU`DgfNc&_C$y16jvzcP}bo4$&gn}{WgUg zi6y3dSZsDXDk)^w4s|Tf?()bp^kW55ByOc$`PSg(o$yu2l zFfxhKu${#*faT#w;)=?gvMchaUOaZ+XyqgGb=}Z)C``K>)_gF0Ql2H@9SuH(Y{uRJ zp#1>5+m;ezBwqkf5RthFlX#t%wj=_=ifaIB=xjWx>v>!O5h^o};sjv3h$e+#9waZ2_^p<%-jj98QtpZ+t>GV{aU?o@&#J*Q+NcVEP? zE^&5di3Rfzh=wON(YI$x5TwzNn4#**-_oNnY)0b zF1U6rdB>r~7uF=Xdsb%OS?8DhH=w)=X_Stwtb?=x$fexkClJ`hAF3EoWD=E<6G*d~; z_nA-%2%}hD@{sdx9Sm?;QQ~lI`2DHo58Eoyqc0JtiMbO^%F7o(h47yzicxhf z(ibdtDzwy3!A;PAzZ*Ttu`$8Y}!0E zvMEyGd*n{uc~$zrG4yk{Tv+K2_8D*@FE+lApgCH_=JDZH1*$J9-i&r~bi2Qr=Wao;gB#cFPc=kvKLJOe} zneIQ5e#5@3QCj0XS`>UAdl@m#&@)RPfG$MSkdx8k6%OrHgS5_$+rG1nr;pCK>%4+A zX<*3U=|c=)+B1kKCQ1GFLH4pdPNy#9VuKqr(sF{CRrtI8WuX;1t5>u2F7Bn- zm_5(P)6??}a>t;IPDDxJXLgY0*~hQRX(aS&if8Ivu1KtSvK=lof7(iigQb%*Bc@$3 zZ<3~JX7FKGhPLaPWKmv28r9RbWOPWP%?TSplBuAi(lNuS>dE|P?!If|W%GkQq>!NJ zi!8xhDK=qx@GU=WSGlI0AO)GBU@=c2VVl|cXXoM!{Mvb9PPpg1Tu0Ci*f#dnZ^;Dj z{w~?#?(9Ig81B{`UzYE8pd@k1!SYz&pepHjzSBWB$(>11$PeF3^sPk!-=@V_B`F$D zGC*Ergg{p9t&rTn4Aaz&#y^YjK=t_3-+A$)W|rP8)OTEuvV+(!KqE+0$Yh`o!PiW?u`sIwCL!vbot&K2}CwOPGenS%E3g59=T3@2|^hz56a7x z2dCpHNoJcadCq3rhFR9l!cNn13CIfC=l-m$XBk&s3C&yGhoevxacxmyWMLj%M>Mt* z!T^%|<#C>YNZ4smGfn-r>f{3s-P6}2Q3&-7sw~8b}5A1Z1yvZ03gQf0_3*QS$@>G=8K5d7%J`VdLbBeRnA%*5XiGC-xU@X)k6-pd8a@ z*Zh#Mf#86(3VcIDFMG+m2xphsc76@8FnmvC+fpm*mFoA~HfQ(iQS+<|K4O_Sw$q%# z;yutF`6cAik}KA&@q5&IK;kgU#ZoKJwCnBbUakNcRsyqjiS6)<2J3)+RwvJ9VmC?; z?*U%spDsK-!>VYemeVJ>uqob8DNuURb~OM9Mj&WWV!^I=BsEQcV!Pi?s4bj6ymMp8 z4Pew181n5p94;d;*Q@s4vZ1h1kEKL1N60$3W;F%+2;TW??6C*pa`;M?(tfgUpy6n3 zv5xFZ@b;@>!zrlaaho<`L=>(35k7zLmAUwKjmE*|6Yqdv>@yTAay3cBf*8EiIC)gP zTANVlE^OOR@N_E#9QeX_l&`mo6ZNfWIQd!J&dN3VZv}LziBPS=aK^Yj4bEnP0WD&3J!d? zu>Oaq+MUu-!~_sJQnZ@u8M!Dw`7J2E_lcJkWjh;Bl)*BCjN;E${o-Khgj>vMzgb_1 zMxdzJn^SU&Lq76jh$4zz$4_=X**C1qb+6XyYSZ%hpXM&`9}PBHAOU{9Dl%Yth3Ce< z_ga5{&~0~sSSE4-C^=EE3*$OXGotnPzG*KxKp}ki_)DR`kGo*x$`K}U86{PoEXCQE#!K2YJXBfm}B$lF`KjDsj zgo3h3C*am75eV1ot8qlya?>rH%}lDHYhsy(GfCnwOGmi(N7Rn!u>5V}xJbZlTST2o z!*UiNypHEB|9^b=-o{lcPl~*!-aSdHrruXi_!n((z9ydV>R=bxI0uw7!6eq*3norJ zpY+UgO8!sc?YZDA43ZV{lTD@l1dnkxlgbYu6OD)-wA6#x95or%#d$g=LbRPqL{Cgw zV>4Iv=*8S7YJQ~s=wXg3{y>7&*YCOUf2EV@jMlb%Xq@=H4eyH!M-2U*V5^ZM??GnB z)kH3$^u15rm*b(MSH1C?x!=_}eq3=B)Wdm>8nhct*3liuox&_ zd&r~G2#*L=hi+cSS*`Bn?o^Sliq({KOo}FE^oA(|kb7iILm;lhk(}93qy9cM`y7*; zd=v075`}=}+L#U6^!8(=+)i|*yTm-C&d#OvXpeOc@~Rl`nhU=pPxwRaD(Ki)q=IhJ?5{dF0pEG8BT_U zI;T!Pt6F_LyynTS%t~UcB472P)cenjXEh`b@__?74%kaMP0+X~h%4E?yp6Bgs@D0fSm9!GcksSw6WIVU&m zU*2^52dGo~e9+%5(vlnCj9Pv={9=em6hi0T(&vbYVnoP*Nb{fRPM?g+@`W-l_nDpf ze?SmgqxOl(R~eJ>0@CdZA_x0j3^X%A{T+cND;)IF(_moj7_X31QBx}i_q zKNlW9e%@^slB`u2rtN=L0(Oly(ol6sQs(%%`HfMdjT?2+B}ELaQHc0_lm?uilzlh( zw)gTfn2>B(b4SJlpa*Nea?Mh#HPEAY^=d|1ENAce{W+@X5xh(Z(b0TTDi@vflZv2N zOGKof=p$l;ynAjX=NLqFoo((+?P&C8I!*nxu3hNq9jaujBP4@VcEcm z+ti%R0?6TKC5-ME6cKnhl#7-Wec`BDz>ZVA+@Zz*fS7cI=J&F4B_f1(02(SOZDmvB z&>_nGZLOoEFyom*XfIB0e72Efb@ew|3{)C!CFh?P6}Gvr0mP67_w2}ye6m<#=wy+d z$>q_)SL!L`eKG;y=kqx}1$EEMuyiFL1JqQ=DV^RiI=KOKO)Y5uZU4%n4 znMhdNXmZ?u=|yjk#JV8sr|*eQT42r}zV3jwwy56;jk}W)9YMVck2}=ohF+sAWrQ-5O>GA@ zbu|h^`G(dQiFQ=ei|MJ+sqL4Vcg<1A5_+`FK7@^ z8o|fjJnp{w72v-_t)Cpp9(lTs(zt9!r84HX^QqgZdrtf|d&<6E=2e8GKRIn@qXpoq zF2)`ToU_7F^sqsR=57!-2R2PPME9PN>dJFp3BHG zFS)iBi-6y+Mc)T69k_Rr(~z-2@#EvxzLCe{hkCRDdxBz$`w~@UyT!eFBJk0V`Kkv5 ztgkzeii#Vh=y}dujGwOoL?{H6*bBwDs3fBq%9uyu7a#i(Ve4<`Szm?*iNPab=H3)k zP`P)f!Jbwhn-&@tuO_F)0ED=cVVE$HcO`?R%#{s%`jW~#B$J5g0M@j2By)RWa0~-sMai_G+=~-UVOiMH>wKjg*wm9(F;|n!LtgS$wAvVA~2^P`KANRrqE(Py~dY>PyVylpe1tevO`c?&$S!w{MNx zc0hb`_HgrJGX9OqxbSI?+8RZerj}^?<&dHa5bOurMG15biRPMj9#u@AUO!Cz58&>U zN;ptIhK5?(IHPR+EYFj^2%&aj1WD6}!^B9gZlD|twXq@nIUarkzuwWmZ*~B#_qGH6 zg_!0bET0Gp?fj-Ze(33S8Hry7PR2N0#BbDA+*1H{7i-(VfKdg+$T;-3LeD2U@tJ>USNJ(iM0bA%=>lIRzL2fQD_Oh znfguR*D>mX2*AUTZ;&z`ODY)m5S zS$e>^(M;d(uh#FF0-yOR!+u(7LcSF_`%$j^@NvP+WnL`E)?yzZ_NIF;hVa^B27@6F zp05v9%h9F~fBe^OqNEH3(bB?=?I4ASld4Cx z^OeY>FE$U~{qcl-Un}ms2$KV)&6PZib)}|kTciJtTjb3Hhp|vZFV~Kr`Zs@%q^;&} zCRNiMQgs?no@}A>v0I$ykz)fTXl^tTU($-#e_+6DVu#R@*=d z*?M_S*?3W+DX1TaM`aQ#znnFt$@}Z|M#LjyfK`(fzE(xX7F*lC$!WJD`fMYib23B; z3R8uGcPEu;Z{L2ty#>~K-pKaROM5XX#@MiB`#QTr1~FCv8mdF^SoCie0dOa=ydk znyhS$C23!xy$WFI=~?=FNvDkGNsn zG67JU5h8s;4iNg+=_l)hvv(M|J7^U2T% zNCgd8J?EYIk@!oAI886|WhvThMy-OcUjKVF>tn|IuH)#wk)rpdLhEo{uzi=8!i1wd zO{^MNbO)rDw!O^jZprR=Xk=P-H-8Or*i@I>8`}R^kD6h7(_#jr*Rw-!;yZ|<&?mz$ zd}m-<(Bx^c?ru(NprGl2{0At0ksTa$i{{cVIjtSIPs^ZJ#1Sg6eQ@MglC7ZcKCCO{ zI%o4k(OIkPEOQWMA>Ur9F+4umReO#O@_UX~P2>&opG_j3jt)4g%>HkqkM7+R<6@ZpIsncz=?O}>$F#8yZE z=%;p|*LzZpw$FY_{KfL^rmV<49jDok)LkKz@XNdRJ!%Egc%>n?13Jd z8czP**`p-*5I#wr^pnl(r@ozh^&U&@Dl&ZOhV!J}_1wQNb!=Nvu#pO4ZQGw^`4k2H)Sqdwz2mRTcIsH-fiz{k^Zp$tFQ`&9au$V~a{YGU zUon3X*p>7_;aOsh_Q%!X!G^$j=+eKCNtA~e+wYe#Q$GZ$zj2neE7{-t^bfmzNOt`E z)hEY()d#AtyFxp zb~+d;Z2jIZjwkbCpYdfCgXN$2kuw=K45^yJW3~n-R5UYz_n|#n_TEcG=^)X2g&~S2 zotq4_P(O8_>wBX@VTp*BWM36i4K;94B&*aj!W%>3OyJ}snp`V4n&^~a52O8Np5v2} zi6?$fvu1l|kqidzsUHp<5K~r>83kgDJyywVABM5(Z#{en*NxGaG^EZ1rp-C~W4xHyD^_g(=^<1q1*H z719sPX2=xo&J`Y~8UeN^P_+j7KLL>U zIIY)i15>!k2ic=V5s}H*PG)a%qR9gy!6;)mIU8R>+=3yqYFw5cKka1eWh<*R5X8AJ zos=|?HhdJ%6$)+4k594=?o|+7h)ZgPBx-rxnj`1I0HfgNt@fKLe*Lk8xh&SGFvCbq z4bpgukkPvauw0x3pcbjIDjdA(k|~gWY2q9%tJAH2^@HKO>^sTc775wvIX*riAnpD; zve(3}uT=sY8GUY!1pTGFFKF39&R(S$7EBZlu)oJ+LH5TJi5YCP3}q)J)k>m0@~?~N zEa^b6_T~cqVozBG>DBNkF*)O?J(+F_(1@{RJp%?0M4~V+K2h%|Yu5y_ZQzVxM!~Pz86)vk= z1gcA3i?X5575+U|C*lmwBgn`T?NVq>Dpr;DJ-N|GOS9>U7$#dT@dP|@-Dy(dcI^%b zL@YYcMv*vBDkMm+bMvg8L-WYD5S(o;s2|i6qhX%MSn<=mawSnPM{UI%;BXOqjfRQNu@J$U}sEBzF(JD6r0Dl$41h(ZRRf3L%xBW7fy-ZSb+JiKmjen!Cn!AE0 z{D)T?Nw*pwLvuvaupMG_?f9n!R(2(!fHWOhzYa#_(eu%&@Xf=JSEHmGZRJC18aS&I zcb+Ie1`pw)5z72Y00s$9(JT2(8c?meSjvuuFf42VZ~SQQ6Y-hq)m!W zdk1aX;1fmH@F+uAP(ccVqy}9D-}XSo;=wb_^l=qzec%S;F=l#^!13_DF3KrVeI0{e z8gx-jgf)pwQu$4g`_!~Bu z$Pc>$0fEGJrZOF8Pws=fU0x`TXjtO~MYb!Y0c1^%aQ4eR*X2(4P1CBo{X&SnyHFBX zQRrR!wc(j-h4=4PRNLM`$Mx}|S}s0((f7M6Gw)ezD$TecUh>dj%|0BD)RAN_hm z0e^3SF-v@&|2Gk|)@ssLR`#9iSG9D9#$x^>HE-Q&nBwnG|4oQI3;|= z=3y5n+wJ)LBb(-2e^a}n2e63+4xif>v3t_)wnn{30opvJER)?{0*StY_&*{9MR}pQ z%j7Mb%4aUnzzt^_$6(I(&E-KQvm1xjb8|_fTAHa}w^V=)Wal@w+h#j6n>95p7CbS( znxzMM<)3gA%IrK=Fh#;LV>BD|*sTxcd?<8%B;>p^*7UedxIybn*ie$T{jANH(;or% z@2Aa}fnQ*A~a?fvOKS)dCKdM;L`h`Jgvz2R{H!8t~O6U*hqw$J`@5u9jcNn z{Q2QQ2g-NQty?Mu8JW&@){cjMv86mQR|K)gtasU-ioJ1J*;1;BV}sX4$N7pi{5vc5ob5gp=G*(`E>9eUT7_>Wz^k5-o4w9x}O3gSbOG1Q(q z;QcG3@!yMd%{gE?HN&)W+r<9>GM*Tsvo1)>V6a@A9FB2bY2wXB>ibce_TKx*HtQQn z2^ePtBypshd@7qSYmi$+!v=-3WT1Kq?U_j1J6$VK;$EQKu zxl6$iLZ4jlGfnJs~Z+1oQCWUK;UHK znv&Vldu3~-R{(*My#;yi5h*PXtNzdZ%2PY(d_!Y78zd?u17nObxb*~k{{WsV6U6XE z93R?S+C)qT8?aY6=N#n!0H3Zs)%nvShB()BZGfcoQS|N8`P4W1)R$&2DlkwWg~Rn_ z<0K#9`U)XeTQO3>I6i3XRi5tmO}bmVNAhK8Tor$risz;=#~uBvKJMhiJkYsy8{av} zB=^U;{c2mA&+M3PWiszZ#B(-Su)zdu>4I_Y2tSrx%L6k@A;fQN#rB---S={RNBC7} z+0zO)mys8Yb$K-n1lrnO+*(K=+LOlM$gIjn;kXb8IU|FQPW9%xh5h6%@V)t&VTnj) zh$-uUIL=NxpF_zt-YizvJLH;AHW3?*q+}T|cIVqTA46Pby=JgX_fx>A5BjmU0GxI9 zAJVm>$h14C(5DGp=dU&4Ea7LJvltu@273Bcy)ORcY=oyJJiu^I*Pfq;)K*0HFiEMj zuYe;_v@7HuI^c@AZe@)5D1Jt6SL^kVOwh~Fk(tpPkQ@W~r;*TTuPcHEl)vIKN!36V;-{+6UuqLW)t>;Nd zKtlVRg+2bBm6xYEVwPA87-EwMETzORik`Y@!gs(TQN8vZQg-vB))@b#SuIva&H! za0W*n{{00$_UYt|NhAEocL&EzjC1)Pt}8M-sf5oA@)eDbL6hFSI&{6$l69`eY8$R@ zoD>R3{vQ7TC& zj7HB3NUkC#H`>`#gN%dF=N*0PrZIXW3AJk#Fc3zrxg|1Gp1kzplJQzdJjTxBfDcS? zarDQhdPSBqFO;}Y4;f*{_*8nO*|mhoyg7UYIO&eHv|_GfCX&>a%KA%b)ZtVyP(ow@ zfsb7Jocq59y36JyGP8E07qmA9Rd=O0y-?3B!;^ zQ-P^#&nOTnGwRerjB;i2G_x7%?(^j*(gU!>W zl1ZX+iq8lII3(@_5uBWilZx}H&LlHLptww{5!d|Uv~+ECES%h5+BA&<7PpZlA)z@R zGJ)%Y2qT>5-nCG%!&{d69lWm@xbg`A%-?h#PeO5@rAwvVm}Dxd%N)3hR4dbvMtuln z9`%W3u0yR^t?Wncuinmc&j;(s{Oa6SjRfjs4UaSCVmg2I>qC-lHDlj?7wgv+n*Eji zw3}qMjxr@x=)mxQ&tJojw0~uvj#7Lt_?@J9jU_Q@cULNtMygl~cS4gz$0r+6)N(hX z=Ku=$Gr$-B0A{*JM_D3!U`W9ndW`=7o^f9N@O#7)-e3O!V9wxA6u?3YU^m@haoF?v zAFX!b`Kik2_%}s>>-JXogr5+82I#lb&AR7LhwK+X zws2zkRP-Ee$0w-CMIT9ZhK$X+MQWB!$M@U{i8BvB#;a zQEZ0nD-gh7{vMQ>80qvqu}t%mBbrY)cJ82ZIjtcUwPZP~XgS;`2AeE_ct8Q~OfJN2 zCyHT}0LVG^rsR>G+=X{k8=brNq-hmS4MfabT^hH7G%ZF{2qaGO=%l%I3WGsmZE0|4DG<(i%;Yvz8oy z00*TZ7~-Sz#tGZ>6ypB?HqdYY&T1U>2CTzb})-rzowPT`dqsM*GGoy+OcfWhB5!95S9M>o6#ydFW%y*eX@ z0|T0kA3T2$r$)>GIKk{Hqa_e6M!{Po_VlHZ(Z@rP)0%}O-r#indee-E4hvO*42mOc z52y5|mH~qL(*FR3F{LVSQ}v(#%(&yODIQ={V>v#D9+e*E0EIZuwIe#I!*NjNhCIsb ze(wXWI#3_w%K@KEa6Rd{;|IB>$l*vF=hHPVeSz41E6lC_Mv#^tI9G=vYO2M$aQwb-j>BTEIsB|90#7PQBJxw!l zCm^1EX*+Oe&N=j|P<=!r$RkX%U{bi=q%h8UlhYK?<9@(9Q(jZ$b4QuhbAp7{EBD<&@;H>Bk+h?@m{?fGn8f1o|JBJpTZNCK-t&zaiNx>s0Im!N1E(gq4u>+bE zIx_Mye_HEG$V&Y{?g}~W#RCCK=boo0sH7ybWc58w034ohIN*~^Vc!|Y9N<#3l>4WTT!qJee?B~YZ>!-aGuFv4s~MV8rzLfJzeTCaYF499&@C>cjKeIB zB@Ragq?TX;`sZjF=cyR4Q}K1gcUIQgDQ%)v0HGNw z3h+7f$gd;5FkAS0RJ*&kEX8Ar)c_dt9r({)fD@l;{MK@m+dqEE;<-7R*6Z`Iu}ht2 zO^!EGK}REuDbLrQ$3FhmQ^i`{!}z{?NTDkZAqv>eM#acHo<~opC*GjHx=SAsCA1RA zB};WIQHFl+AZLI*$myT0Q`9EBxbbwhP)1fl`-qr;LCz1i-c}rXR&q@weH}VURZaf@ zD?JOrmO}SXi|vH1v>267Y=tMBa&gy=I_I@r@TQNcc*n$$*uuVKcIfg+vusFX5ta-0 zNhPty+>wr$&3XR-_Lj39kke$;8r|6Ck~xa(QaXB&I&|uKVWa@^pR z!N~*>fCu+QdLMw64;Og#ZZD^AHal`rKrN1>uq)341J?(TRMw}&4JnA8(&8mnlHCMR zuF=8I%jieT+m52V8u?WU)TdP~E_?K|TG(t(kn}4RS3?^L6Bi>g^`n$hlO1N7IVk)AakhX}qhu79^Yq zyuqC3@dxu9bDl9*{5j^^OmE@}w*o@%G7<@PBPY1e91+uw^`~hH&2*BOmv-iD%mHE! zIXrQLkH}YzJTz(bJsQ~O&1-04YEiwkoFq!%&U-3EzPY)VJo z109E{72E2%WPVi7Is)c4QG%?j2m`0kpUd93>)6s8h*jivLaL#+oNzIW`hYQ5IHdF* z7P6{}$qZJ-?le1-3}CSHB#aY-*F5BBHHoZC6qe2;E17x+Jw|x?pYz_GtLkz<{z188 z8nMiAoGIy!2jf_po#Lb_T6RAm$`CjiBOi}HrDUI3lFujWxUc!W1p9gTnzQBFBQ)Q zs%5%ZogI**o4LjZL(k#Y6?D{;(X2U?u5niJ9m#7WaPCw-IL34t+fpCsI9JXW>7kmGUN@sj`<$kR%Po;BS$FO>m1-RIL_tA^5dVa7mR`z zxQ$sAhXgkR9RC3ItLY;}^5~9BR_3*g?M~_xzA>?EWO+mmG5z1tu(hX=??h3_kUNa$ zB!Bg*Q&yZvStDVzoP+IHw^&fl7b6_(N>@KH+VH?ye*-*Z1&mElE?1h9R;W1M5Z z`R!6PHva%LMn^mj zp8aWUWZt=nxC0@Fe@-h%rnM(qlEUk6a%3t_ItJ`&Jx&Pkl)A{dBY9o8+w4CNPg;A- z9^lA>2XH%BXB3PhkzEY2#qyQ~x||+^)bm=c$(^}%FeI~%8OTeWzc2A8UUE6(*wwU} zFD~BQZd)cIX5N_C;G6~Lk+g%JIV59@mR>^}zQ$rzPCy6cKK}s9p%@HG5EKA11$M_{ z%Bs0$?8#(|Ocw-YLBn*&?e*$vJsVq-O4bum2$GmvAhN3up$sr_)N|L0Y;rVg#X&ha z{Cj;WtHAzZumyJRh+d%F2)?Hh>5mbNE*!r{2dJe(ZolV_|N;FK<(hdk;cS6}@kCC1sm=QotOB zBmtg3oi{5=#|-0Tbb7bK{YhZF{>!?|k(|bo4>hRlJVR#JPT^%$49s}J##Eof^%d=QGCj|SA5J7P ztC=F?9i)t8euBE`Hzt|$nA%*mwd#JJf5AI^V5VJPMoXyP(_agy*`QsG{heX3G*z%%tZWaHL~2-#u~2KU`II zn>-G9;Pk~b%H=^EdYVpY6YsQ6)3C}lupvg39*wRRpN~HJc-kfC_AQQmnoYipK*K}h%;}nkSqbza9POemvJJO`Ar{*~o3eDGd z;AuqQkP6^ulbTNbNKjo%D=0rOY-7~Y??cIP`PFdhcnmW@AouCknv{_)EfdJ*_Y$jKQv z6xJY|D}lyaJ^T0i^!m|cfV1X}yP2|S1gHQ)2{;3<^Qe5nO!pjDhm!H5Tj- zH@_fyanzjBJLCdiBk786?1A3{AQ4l}t^m)uhDRWBeRG;eV5ff4cmn`thG3_b=x}I1 zGUX3GzSP=_)DLrM3!SV0w2e-S-<_Ul6Z;jkXq@BVrk8J~Ftcqi7nVB0_}XwooM z9)gpO%Y)of?;${3k5W6*b?;OG>`&%s zCP-`$F^uM&mW-6ZS$iM zBfrgq{c2hO91KthBO!4i4@)Bef_bbU4ox zgX#_sYG4Qua7G3?3IH4)_~(ic6Y_N@08mI)0G>}u3yMruVA(7&#{eFbhvvcjMF5aN zY=0^gWxhoNB;y_X)=jxPBSfqeyW4MOymEgUKu$A`X-FgH1mcZY2y>oCy-6EwLy|ghDVQt^dUeeMXaMIt z^rQr#fCD2L9eYvoNZ9n|kYFx4?&K{f3$&QWQ9ue0I3#-k=}S#jVna6s3{$G&;H1Neef9&9%8JJ9*(8Y&@}-zyWPe}lPmOlQ6l7pynqnwi63oXHtUEjj08y`*^rFfb2`EXpl+4dY4Nj%n5eIo7Z&U?ig$J)GB zRb5$PJ4QxI6M_7zhw-h@z0oY8wzLRWR!zVzclW~|;yH48`fxE;F1$*R&O*-3B(1&X zkQQJ9*YMK?3Nf($uwGQMyfdMD~038+ZiFZGA!eKp{%evX;{)F(*C5k1Ig>=x&xS38i3C6@K(COa19nL~g4rGJnG|+CQ`E$)2p;QbrKp9egC+K!^u)50u_Qb8gyDmP%V^v*!wusmdE zk_JzW{9ApgYoV=gGjARi*$xkszJnj99CMDfqvIbE9}sGjG=RhAsKvYdyGD3i6O0Ud zbK0<5Lz)djKRO~lbuY#-$m?I7<1x;gl?pO^iQndZ-W5irMP9eY(ndduBe^&6YB>ZT z48?|X@6>*E%KTeHa_gu|6n|)JmVg{HD32xAKX|Uv%mC@T(z`zzO>d>a3*0GG^NPw= zFvV8{kPCMsI9{h5V!XrlgPt|jFD>mC%}ZRY;RiE1pPPVwZT~$8(Gv;EV?ORPkK)y`h^J72;-lxAFXgYtZ+5E z?2Im5oe*>fo`n1IJ?pQ&)K=F^Ym*S%-z;iJb{&7ut#BH)u(4e%O)9oQ3arO-;~Buu zxas{Xo`p9#`W2SalevLB4>jtn%!}r8_wqpXHIJ_;ww4n2DvKnZSy>z?Zl@VN2mqga zcdbk7#g6oql2(j43S|4h_WW~>YYWC!8eXTSOJT0srQB0(but1%5z&-_FvB>($mH}m zHRwkUovwT7O3K7Bc$-aYcCr&IM>$fHBd96~`LIu4#<_i0#iHUydsPjVU}gY~{DL^n zFb;YOqph@wFYd{X9k@lv$N=NFpsf3gl(ExHk>*Abow0WWjz_gq!$LC9=#3R^N?M)g zpCP!GVHkl9^1yO1ar*jKCwZ$htjzpJ$%|+Qalsku@A+c43bodqb9W`tNmfI$Hs|HX z82VQ-Bz1O`sJ!7522wf>=VZ59#*EZN?{57+uEK5vT?w z-I+lLfHTHs$0$r$G!nKg-bsLgckJ=DdSoS7W*S-na8E4Gr-=aW(# zlOVlgD&U>~``GVJhTZKVig{7QcLezi!wSR@4n2C0%u?S(&;uEG-wKVG-ynuS`UjmRLL zK|N~JaYkW8RoVw61638N%_&^Zxlpfzoc1{CDzla)f)WAlM@qeMZl*yRNCDl|hH90_ zSe3~+CxOPLYs*ph)3wB=c@HdkI0LO}T25!Q5Ge$ZtQR;cR1k6Of0!SR zMclCR5daDRVVv{VKj+e=(sfCUzEi60P<9^R=k?Dt?AE3di*d3$j|JQt%~IOUqxqjZ zNj>ln@~?5wb+Y!Gq_;o1P_9&VTng}Sg!0RG;um?-a@xjPFc|Vm?Z!Pi*RuFx?m428 z*(J;}V6qM|j{cqTT{N#4=y+L=yE`-AKW0rveLKZb*y&P(B(EFVHu9%=kyIYsf=M6_ zh9xRNkHFXCm+Z}{7(OT1T-~!7VYyakQ^^EvJ^PIQ zb^4?57sRcvd>!!h=KYUvWA-VdCoGKv7>IB7zD%F3dYCy%Dct<~h+%4~N1++lZddR$ zdFprrlh9UZ)EJV70ey#BYhE^TB`3EhwRZbPe9iPP$#^aPX&X81=}{)3Zq4$)N~UFu zj5BfXP+%kO2LPT2r6)GTYCYC+$jW}B6peQfA0vW&4QFp|?tccS32?Z_KDnjMA>G== zvvzZkcXOSrIH=m?fjEA9WYuMxX~|-F$69Bcg!12sqVC|6dlp21le$cjbJyQAGpdeH z;Z;oPxGFbu&{BEr-A;SvkohE8GcM9celyis41HXD`p9d|BeJZ2e z6OcH^CkC2s6p{?N$4~yh=}3ob=vjGGkmb)La2JeyX`$2%88Syqo}Ql7Sk5p4^PVsd z`TqbSlkG9=CC)RzBbqMRoRqJjYtIiGL|B20j2?Q^4a+D50>m8QjX-COfh+T5o}7IJ zA)K5Z19f3QT1Awb_hTS|pS)MEwK-%`ySEI3z#^&%)!&sF2b1Yf^As-c5kt20O#|nQnDYuS-p28my8j)PZ%AAS2_nPJ2KF?Kz6wSv%sYn z3%4ve?~0~o4U?QiffU*0r?g;h68BHY>)A%!NLg0^x;B=c2u zYlwfH65)tA#~=j^`ZHS%RI4oxOTiO}pF&DPWjm!5neY zm6-MN#^N!PP`ltTW3=NuxB100%#(0Z{13b_-|J0EKv+UUF$Hp-r@y8t9l>n39CoNy zR|I8=0D95~W7MC{k7&TMvK2y9k^#xbPMH4y3PU2PAzPea5_uI|yJO{IdE{b~CAlmQ zxTNOT?9wM!Dh!N%>CYn>pn_8?$j1j6s7sK$N|S;xcXX$2DoPb2Cx8#NP1uQ7cwe1| zzSGv2vM62q&Ilhbq5lBu)N1U*JIENx=eOlZ2Omx-0zkn)C_1s=`cqb5T(LmFj;A!m z%CIQ7CAmM=fzUTnK+Xa5pbvZ8`*0hEBoHZ}VBii(=}MrtAatbUx8c_{=l}!$$Uj4SaY;79jN}N8KCYfjOU*8bJMj;LIa+qanC?= zKm$1qti!Bd;_NRYAuY#Yn(_ zcr@%5A=u|96y7j7=e;WAuo*pQ0qo5fE1&mhW_{l)agIId7nydk&I#*J5h%yZc2mINU}tjzOszDsto% z`@AU>DD!c3Z3bIm4?ZE+ro@~yhELwL&Wz_!Biu*u`5;6ISt8CPYcuwg zj3r!er z;s@^Z&pZn0ygj4t@M}#QtZ36L@W>dbRYDATKIN2ifm=Q%xmUNi&74IpBT}!NGMo{N z4&6^b{;x?;PLgkPDYZ4%Y>pC9V~01!0>VIB3fRvLY(rZ&SGgO)uQmogB02FF_V@Gy+9 z{IB}l`3@kIspI{xb$j`yti3lhd<7#}Tw2Zs{G8J;)L7Ll=($sGUXqmru_+WN` zNe7INn~(mrcPdTNaK4QAtVE$4XSCg;<$n;tc^;wU+{q+Yi~&tXf$jmvoZ`#uEf*^(P+Yx!)G}eYJ0cmO8b(CM%Ea`JfUxU5 z69S~3c_b1resA_`pt-j?qsb#FZb)n|QP^?Q+#f?(Z!N|m)7o{O*3M><7#+z-ov<_vaa|Ggi^B?iAe%r&%MFJhKvQ1e1~n0EQb>4Yz6=#Y93*-#9Oq`r%xD6_4Y%U~@QZwcNjsPS70I$-#+0yr&kEFw5y|o7$E2DeG zZL7hht;6ZA*62eS7v{kuIAS_>=Df>Q)Mc9Ib;JNN{ln0K{(bA(H3%O~&}U04n~6O8 zVq|dR=HL}jKe|{BqZt{m64xbEzGrZAoE(rxQ$;#yOWilhCaOwG+Z1NMxoJpgkVHui z4?sI-a3k}q&3U#&mLgKP(i1p6G#zUvmYrkWImq2a!;u~+{N)*ScGb65lB*4p;VAW2RLRSnSdETHVNeL z2Vg4?Pt-i?Ki^3s{EpFuJZA&HZV%`yc)`jko63uvE=zr`tTA~=SmOsfvIxcqJbqr7 z@0!JzTed;wLy}l>+4s*({uM5&)4jQkV_n4Iusa&(??_30&J+Tv@`6AF{{W43Qi8sx zHApm#ce^u2^ExRmati+d3i^Ng>zmizY*TTOa&v{<(0`uQZ&21&2mxssSPg&-Wcu;P zO6GN)S>>`Y%s0ru0Knts$mIUKSAHIyqjTmh&^X4tc@<0P9ylKiFoonJ!WO(en8d2{5V#cLrQ!laYg-G0^8Dp1ULiHtttH zFdQGwz5G8fE{1LMM?y4aSlxu1yu-DL835Ep4%#t;Klp z#d$5Pu|;#Z#LhU7Ez2NesX8`=)mpJQM7WSs$2H8z4%v?cy0#Lm1ar0 zxnC?`5T^u_-=W9U8t?o+EDG}6yA@^tmkGG%u_M3J+P8yuG%_E>#6a1^(X9;?9YE| zl`ZpUC#MuO%JqEl$4aNYiN<*z{L%;AlbzZ3uH}B`$(IGxM?CbV$#i#Qwm9ikJj4o4 zKmhkW4K>+g8xR0T7{xFy3#k|a4}Jr8r+umR1z9;Aaao0d2Q82~^FzCYIBei?%>urc zG#Y*UG=tud%#pYTLD!CYRTPnd6E>IadANG1{u=mnNB@Zejp#Bxl;0Ex{X@6So-v4AmyNm6R!N z0nH(q$prT3D7q$G(&$YL$-^GJ(sD>j4VM~V zGCE*z!DUs0Q%~pxhe{=#yIAj zX3K0)1`lD1N=c2Y7QFA?ETebJ!2J4BiI-qkZoO)$n3(aB2qVAiPe)c=><1lq=~S9z zw3%qmNFd5F#~G&?PIo&U!+=j}sIsb!!Ox~?MaDAQv+J4(ak*{gU;$EkaxqR>LHS4| z^U!rQP|nPEV2pmWGxNGlqz=cW0`_dQw*gn@CY&cwg|KjaxvH;oS`k?-b#+W&(fN+u{(x3amV@XNVN?KoZ(J>pI&M;nC>Lx zkEJv-Hyg8#xoT+y429&5IH0aniJEm$kTOqd9&k$@a!(nm(mQMeJf5Hq-npQO21{|r z82vi?^XQbs94AW9XRX9y(3Imkcsb-K8O7M zC<`lCu=2kfPZ-Z9r>!-lmm7T6ZaCa}8iD34M#4c0llA`q^^ew^vjD)5PI`g;C>JD4 zEg&u=^D+)H1u>_SB%hRe1I0qO3|NNSf%kAa)1r_vt7n!O6bq4DlXQW%vFm}6-lR!G z94ZsWdVlrm2-&6iPfU@>qy#bos6p%s@CVj_~7^4HOKquF| zDzgqZ9CyW2O4_lA0RtyLS}8zcxg>G#K~jWs+qFx#Km<$$9tw;i)cazO*qlw{#^ z--nngzKJZ#WwT- zNK=5vy(l^DO+q9Dk>84N7#PVvTHRj26j4P09MRf}C;=NKa1BY9UP5=^k4hCJa8A_# z^dxcr0PCPg>OfY*X9Lo)pTubdt#Qr!vK1pF9!)uRC?Jf3NJ@ebjOW&w=BF40W72_?{8#gMYW+5nTG*;! zcO(z%UL&hWxB9iDt#0HEHR0p0%PG&}`?1&5*SBl#f3fGuCvjX4qXUog?_MK)r)u_7 zUF#4|jRZPsM+9C@OmZ1rqR{^!#f#Olhre z;=i?bb0SL|<+FxdxETBoVf?GJ@h*oD*ON+;CXLKcym;E|30Uw(I&s*a&%e*cJkj(& zWU0xAtdn{){{Rp8bJx5L3{!X|67m*!u0f0jI8dPDAm9ja%jUM`6Y@*X8_c$n8<>?tIJ3XGcr4k;13m$8dPa!9SgPHN5S6 zY`RF1!GhVqmNUrRk@Fn#NI4y|lfeSJ@N(q2!Joy~oF7A@@Ybf67T;$?JNGlQH{FlE zayS4fU&z-&sUwpU?si|8kZ?%#9XKeyC^lgU0PTyEfoXDD=gbo?qD(qC5A^o%N$p!N=fc~UN#bicA~WYwe&o{<4=Q* zrD-kxooP9_jx{Dmwr}36utBkK4)Dy3R1L#%;PG4xJ_ppar;Y7x!^5iEMJ=H7SurN; z`A@lW4g+;K`2gxjub}lkA-q8(+B=eCMDpZbbF}VJj!!2ipK9kkSD;#Wg5uv#(qcgR zG%<#SqXP=LRSTXu5640Y7_XPkatb)glCK|&r>|eY^l*8tT2$jjy*mDfiTH-!O7Pgd zw1z;*0TNsjAI~bXwlVxUZKMS_Jn@muar0ltrf63_A<^Ix$*9~(WfR7LwtWu(soHal z@sW%g>wYfkHda~{%+C})WamC&F5nqimy^Km-uKxV|KC5u{6;}sKJ^& zr)~xY0qM^q^~Xx_eO~%YT|~hbndV9Y$rjaE+hzzk$0vcw$;y+O^SQZKFJta9EE}yE zN?mPZUs$=4XeVdXnaB;c6eA1)+yivykH;ST^H4!7ckgiOyNKbHv%wks`KL{-&wm{A zNa`LXQMNO{9Xs^^kU8}jsx#{`S)gQe$Wgcj>B!)ZrE@geKCcgjohZ1iS=9dkXp>|l z7p}m^_j$l0jDI28yr0GIXqwcj_W-W)T!p~(!1W!oT^09?W?NZqZ0+}XBPf~3e{6O& z!}#jzGkiCY%*Y*@5(vQ@sQ1n~)nTIJJ$jy&S%jTB&gLxFGF)DUxj`f`$QWk;o_Y-Q z`u478#rpec*B4ifB6)kVr_94;hjHJY{p&|ux}DqJ4n9ei#){03pc_=k#qB$WF!zqzT zJwq|%l6~uz@ioWqS_=rn5yKWd;2uszcDJY_iRUe^Cbp95a*ZKS0q1W9x?8Ki zJ{-vx0d}Z750`=8k-+qqs;TE~l0EofBLt|gjpI%X`+M&@QDfCd21(0v6#Z2OR@ z81~Ofxpi+NNs^-BBq{avs&@N`+(7Di?^dm(CJ<2L=Huy9b4|Mrcp;hwR(udXQJ!lOeYVkv{IURD zq^~T;xxuZAoH!C~0F#W0%)f}201?-ZT;jK)tj{wN+?|SwTgecKS)&iRRhy$@8OPF` zl1U!n8{j$=U;yMEqcld*!tByU7zIBr>~n*I{&=YFS>l0;43WoNeQTuPdZL}{$gUW6 z7<`Ta9AMV>hof7VqQS}D56XQQ9G*IV^~GXb#TwW>oyy_Wfr-{b^7YD|$2t8y@mqci z4<)jqG5InH13Qj$k9=`j*M8bMIUJ9@iq4 zKi;D$F@WFS0jbN{c=BA zv&@u}S3P6lW}zU}c0mvB*k#CX;vGl61%8%(z}lnD;S1PvxecA6OnQkfdGzEm{g9}HZ3Wb~w(;G7)i*c#4T<;YWeU+|@NOHCJys23v{J%FKQLNd*QdsPv9zun08r5DMbMm@XI1e-5zTz%n> zb4K;*Te-m}tyEi}$(6|&=sSIBKF~^m#(gtDD7IY$9#ql5<2|^le`yL1G6p%vtt(s* zI42-+ngm%%6^A4P>6%c(INUQ;eE744a0uWWon>58@BhX(28>1t1sOd$q+v87jdVyU zDUED&mw+OS5|A1oB_b&)i1g@|mKr!hIs`<%|MUAl*u(SWyzX=F6W8^=UZi?O8El>e zOZ|UUNG`s&jogr_Izq>cI{{m1g&b}(^I$3)3*n7;t0YpGj zPRy=n1&T^>Bj|d{Fhl)p<`mJnh&X)mBql+}$jmfN2s-ulWef7c>= zL+3{g-so-~-G`l@m1XCN@~g|uJW_LlL$KP+1EAg#z!W+M3~iNZ1{HsAM#>C8Bxci5 zhrqA`hP;|ldoOBX%(4~{E17NK|B=FD~O-#v&dVJiXId9f$>9 ztZDABP0x0ztKob_I+Aga2V_M0rG#9l7tHdd?s$~(c+zu9c?faj8f_6 zlZ64n^RCMfbp8(H9w;yQnW*;nhqhr!d|bJ3XT^UJ-DzMbEM^hCJPF`u_f$a9)JR<- ziVgj5gEBx~VW2z?m=kJ~QiNY_MZ%LU4O7wJoj&*hL2zC;odYu+j*$Uo6wQ=UJi|O+ z(kX*Wot`)JR5rNpMQ)3$@wuhT#=U%Li5SG{9=WVhxk@$h34iboHxNC88O6*88{HfD zTjG;`8VvM&Y(N&qjHQ%>=c;nQcd%iaH#zvGk9{u(s7J$)+AH?oQ=~yGS8E|guzh?j zzD6)EBOi|r*m8w6_5w64iFjpAbI;p_`|wLd8(3-$-j*rt;4|^?|0Avkjtr0D8O4J z{?Dzbg@(_FRNwMG+%b_=`!Soa6$BHVZv`@(iP}Y~BM0~Q)J4Mjc|paPpW8eT2}fc5 zQp$NEkTgmGOvmBnS+WspUmlgd_-+8Yikm?23jqA?{S zhHuh@7>|jkPQ|zg)RwYQ#Y?<1a6^z7w_xik3JpR>%D6@lw}IMO0n@gjuIig7s% ziEHO2A0oxupZ&%#w4YphXv8F27(`gE`&CIX$P0u|)soF(@i0Fx#k{$fxIopwN4IuB z`BW#vzv-JHp6AEF69dF^q#k>_CxZw1BO{AwJP$g&5I2aLlnp+a4NfoY>e0KZxc@-S zQ#Z*4FOsS}ID80;9KNP8mKf(b2(=4No% z%j8k)Zhc6*WMic!oZ(G+!kxkyEKWIpILHp1;_Bv5X$h%6vhd82$qy>rIz=)|Sx&gb zu-}kvS&Ta#cIa@CXYc2|vuYE;x$UW>i+*1BQ$?J^OVJSm($oZr%J4)I6RXSN8}rr4 zt$u1-`Lv;GZQe4nMVB)xXt8G4hf2)tH!`Ji*YS2XGu;&D7rncS4nz=qUK&K!SbTZu z9`ttKTuWrl&kV}Q(rz0KeRjJ~mVkWXr^H*ST_P%`U;f-E{xw(TFZK2`Bc1*hvo66$ zQ7NJ)XjyZO^?d&Y-#{x^j{UwJ)`D1yE&uFxN zY8r?c)tNexhY>fwAuZIYX&1G8E@4h7>dN^>(SmlEJoe`+qSw04iD^Fb0h2EaGUaM} z|MJ<=m4C1aqipeVm^shAiXB$UTk&bD{mHjTOL#ZUKavy_aG+d0-V^MU80?bzgM*xd z#&k&d$P|r$5+-}K;CU2{*Xx3OYTP4#s!uOGyngb5-j)5G@B=BfjFnPv|1{!9c!6ha z>y+|t*J7%uelZ5|!`GCp7fLE~Q{N~!{T}*NoCdY?O_sGj%L#vsa;sfgTp8Gbt2jR0 zXg!s>YF#c|^JS0vO+}D*CfOg@V+}QV`K4_Brn$NO?I?RmGAPa=HLZn0EZBKwPTvpI zp0`cat@b5RajlqGdFjYoW*o{>N3R_C;>okI+v^Rk$SzScK`BO%5qa>IsRs0;5mfu@ zQH7Z>U*6M*%BlA9W_`@t>+R6vW1R92&rsQu`7ViQc#l2RyNj7V=XCT&?t{4}Co;8fS&RR6E(SOjI` z--a9g=_A^F-HeYn0oLDR{{ulV$G%ds`5i?O-cF-c!~GyL)ptk?5PAS+Y5h|}b$nZR zeEEL)iEm$NGQLaBg!(XCDQf>&#j!wdW>e&| zcxK_`TD3T@TjUKV&j3NDC1zHQ^_;azFnHFFR#qxyaT-WDalYqkWXgjwy8w>teD5CN z`{mbNUVEXD&+}vsJ5l^T_LaAP@x58bhQ$xXM=Op`Pfi;)LwgF2vEQ8gIEKi< zX~l&2iaks7?PQuk5n(6FIP6bmY)zG6ssl}Jf@G4SK@oI)`sklPN8ab^afLd#=2p`k z%_aE}KEmYOGV1S=SF3`Etsha7fsC3j?*MF9CH!l3%wQmePs`)!1DhI#2ZI$bZ&? zswkbPz}X=2ug)+v=bevX}g35&sg^PQeQMG8s#4zh5(trw5fADS

K4`yS$NH+nwl(R?z}U(UrKJvQ!X02QUY?^d02Ee6mylP? zuHDCjU9f=9@EL1OsXW`z|3K38BHvQ!C^u^=c3tqPRQG@6s^jv4-fgzoYndUG0+@$57|ooG^?F@rW|c$zzdB*ZbKfO1m@l1|S{#%D9|X3m zBYvLYa}+-Cq~^Tw{iNkE_py)CKZ(s=1X?6N8~ih}*!3qSkg*&qd-h_yj8akpxCU)3 zhGC?n6=u7S4T{e{uX_&#IF#!e_il~pti_DWKXB7Z+n9dXh-=al$F!FGHBJE&I+Rfw zyXWIyXv`R84Xvf8PSZa^x_q5OVD%8oPvBW(zeT{ETi@IYau$~9oa_2zhPDDIV1A_Y=;cl2D%K`EO1`^xub&B+_2wI6u!QlZ4tO$obDOK${4 z!d#b5yS%=EC3x(ujfH!}Kvsop)RyKgAb`br_UX|hMeBdOEC1f#`d;N9kV-tWO8O7P zyM}2|2;12RSkM0&;!B@XkKm!`Ij&F}#=r`?bJF!TWh9={2yeimN&2bXkl9k3iuz9D zSAB-3gRFZI;zoj-Y5r&?9{SMj=tUW1do@*raKnAFa4!YwD%1O9g|^Zx@pS7Vod#DF zVklYGohK6W_X6ej;la4_?~QD>!>u7`{@Qo717mWu(~|r;4W68k9sLZ-wzSg$eo^%I zO`yTB=@p)wDzZ4y;z8l^8_v_D#uz9lPFLQ)30)y4AoukO_AAR0Tn%)N8h7(LZe}{g zf~2)o#Xc@-{kfii{Olwv@l|Lo>H6C`_`eOH@jW%AHiICRJcFsA`V9ZhF_i2Cq*+*l zVOgIEsnVVH)R_S{`@!rrh%QBL)0s{p(OWdKTt23=kQ5a-fS*1svaa3qWerhqqd9B~ zKrTs4M7-j|T7uF;9lgB5h-HmlvTHlA#Y6#|^F$R8e-wHnH>DNu5G8=fBc(+l^FYf> zx9Oi4nyA4Xp>$j(Ke>jDNj!ZbMl;m&#T)v3D3Uwrg0jRhn2hF0qBbucl74CONKX%Z zZJ)=iM{O^6J)zJ4ZLqo8S2Mu(t*#VpvLXsr7_5%C4Ekzb5Hed7Xk*ba8+!* z#elWy`6yID+Ii1345SRg&7A5C0zy;3{CNb0GVJ6i8@BijjE4h2I-?&R1d*CtK{y@- zQEk$32+QR^S7QH7jGh`HrX>BqHS$#~qZgEQHqW9dc6>TPWv)O^Nn7ma61qF%oah7H zCM*NHu1PgTjT>bS+)tp=+)KPR1`}i$6rRXN^ZB6PNss{ZdOs zEUK=KEEr0piu{ExU!s%ql4%YhCib}vnN(L7R~JEUBd=$L4~=bQuzF{Xn<9Hl67O*= zfa+Z-<6$VZhw+#Bcp>3e@cpashR}J!^8+GSL`Z?YinfTl3hdfwA>%8Vh^(QyG6Je( zsY)g|$fkgHc92zSx2S=Tk zs~a=v1?`zqb_Tz4!eN50rz;|f5Q5@~R3S!{q7ZL{;*VOYLgZ{>vL$$&Z33*q$Z*c= zg0DRsrBCeOCMeF>vii8Mmkd2*^9qro6t5-R1owv+ud}wpsX+1@B~#>9G`x#NK6yry z8le+@rH;T#6Y~GSBmkNu-tNSh)s^!pu3ym60sExbKSCRK-p2LqQqG4i_GwuD!l3Rx zD2^xDHbVBGQ8Ndp^{v^IaL`FC@+XypoppG|ZTiuL4#$AF+uF@Pu@-EZPtFw!ck=fq z1uwgQ2*s!E1N%BiujEe;i>0Bxt2+w0{wz@`>R)bB z$L?)2G^^%+qt+ff&X{Q?<&YpO?ql<=S`=v+IJ~i7)gC?aP3x;#XaeY=jCPBaUzoZm zL}VR_uq@}NvaIbI*~8v&@~MVD86~1W)5!)o9EIn*0F`Ho5`pGufD-jf zPd~nuEn!B?!Mx`S=oVHfF7T!@`6#X)qxhGrbUD|4s%lFpMKkTiP(1!4Ng-8Un%jS% z7)u4t{#jYi_E)NJ8(|c1+s=lbZTgyZwo)pdxpmLIPCn@HszFbt^6kEdxOSq z*dzwjS;1g)X)$EWJ z`Gqsoyh6MD7-p9KnR6{advV?SHHKv3xmpvBG#1{Z1D+Gqud&>pGfVQ!>7QM*SCjiW zNY%m&Wog+vd}b;6^7l^{ESq`qKB0tT%m63M55|8h|92{|66;Q%xM*!&_VaOSma)%P zOG*n#Q9MYL^D@3X_^{~Bg|{;mk5DE*vN`MgL@we^U^ONDi7&f2Y5uG+6aOIv2*-9| zFP^gc@R6*~VtJo}wD!?i%_pW;6eT^LV)ziT+C`BiEB5biBWa_?hx;zC$9pDs?gf`6 z;qJM&!i~MXgfvECW9Up6C{}qvb)9e#%CXRKEQ@SMfm!Xepc7itGdp)gBySUs7~IUT z+er))$feC5r^;2({SwcMGx87FD+cK9WykklANuKpxE!qOv_3O`A+1eNE~5_LOcO(} zQPliohEn374YPhNzmvL5UpMEbt+o?-A;t~IeC9BXR?m)KRD9esAmu_cuXkL3`0!__~{Tw-d8=+IcWthnMGydH{lA9w1|x`2Ol}xXY(q z?D2Ob)2AZl%pcM6vf*!Q4&HDEM0 z(5!r_KM%n%U|cy_T?G9(glU&nS@9BI)qYMO>C%ggKqzy;c9vIXj67_t!5gS_#Ui`S zqLI}bu)LvLmZ;$DW_HX0`+cs{Rc5(4Q6#6O?rePwBl#&oNJP3#T!omGm70H3ZU(n8 zhnL(_NPam38|-e1&$!`iOGvpx06dGjom-Q$xsYWZSL22W2w}3rbGKUPU0)9DQv{ui z!&7H25EUL(_Fo$)1zUtkHg7ZZE;hvJ(saw2L691t(W<&s*ReS30xg#DWoIu8*pt9m zG=Ga?gU1?5iUU*88Ja3Gu;w>sQ7`$PF5eSm5PaLRiRt|FneIy_ahlQJ`!8wkb?x2K z&tJyz7R}1EMS^PI5aOHsE1PYb7&YDUd6KqrIk*XF^L<3S(5o!1*JIzxpVXV!4TXIt_<%AAMX8+J&ZYLFz3*3=#Tx8{$Rxd3oP z(8dITUnFLDq#>**+Z#EstHdHxV{~^bDWQVuRekS^sH4wPRBbIVpv^J--dBaWDUeCl z{kUoR?uXM;reR)E)o3djo^N)6F=p;o_N1%mTrgAM-p=Zo1M;ST z%fyikU+p*inhBr!%41>S@T`3I8mAA@$Rk*ASmmh8GSN!7I*J~mvc~9X?uOO%1)sTm zto$<6*Y^8)Ye)1854FTdB-RcKz4<2~wed@ApFDC%RwUdNP=x52YS_grl^Xh^5iv^q zdW^*PkH*~&UP;OapKP@%ocD}$Tfo<-NOuwv1a^cLt70R|KRt@xku8yE?R8{o*zv+KVu8RLi9zYKjjR0Hi{dEoUHWr2_ZR@3tu(iBO3q zZ(Q5w{E}kI@ZR*>!P;4=_=#jD^aWXkP*h8?J+Em!m)J6x+*cj^J2L|=QIbO7BreU& z{FxBUdVs)H>bI1D3fEBI1A^WysP63OrJodT=z*>y3W67^_a2J0%Ruk|s@uJA zdZAbDo1fXiu5@?Qy{Tnjew&blw=U0Dq(`RGV~rG_E!;M`YyET&a0i3Z);OqZwhm4G zgOVK<#13myHeS^k|I7V#r_?};4HoVh=~2$Z~UE6PilQ`_9xF)Ju0 zqu73JMxzT}W}wy!s=Tuz_EDV?U+sD-b*|`Hmfh6+@xGlxn7*W-wA-pE9G~boOLu=i z&v)D~*)97aTE%!J=OeC8N`3bkjXxfQG~yZ@96o?doP7>FWX$4t@{M{HlZAfon9isq z{rzgAS~4VMr|*laaOB^T69jjuRn0po(m!yr-h^`_+-lwi%SHY0I{@+ZYaYeui%^=QZYG*9LE-{kEV8M zbal|7o-^mK7dtCo%+@1M$@A9siR#_dz8=D(zCQ>(5q}u8z)77xV{|oAa?fF|)Z{+L zBl_o2dAFwU0B$ufNC*>`lWS|?`i23*nj|w8iBr-K{>;_6AXaY}9IkOxH;_d@YYiGTKt1ee}}00vAI_6U)Q(`4GTZtavXc!2HCI7iMJ^-5)G)Gg1o3AZu{I9Va8 zG_}_m7EWB%@Jer_E-f(#8XfXy{|UtqHi9khe4C#>MP&H+T)|bfNWS)F$A^u_LcX6L z8qu!5rz+BAeFDW9KvEMEaXHTxuo0GeF3``B8u3|B`8@x^7SYs#_vec5#%tEKM=Y9~ z_3(-_0PUDSl+ei-=3DeWp7H_ict=hnM=RES8Fz&^&|4GPn43+7Z z+d<}5;NCB=jFf|P?kAvzDm!8R8#(E!t{?nT?$%5669@gzO`7_iTfT2m-)`ztaTb)& zxyeoaG5HS%)i>-c9u|Q&)=-!juzer3Xa`bWJ7jSY7tL1y(QD>=HtQv3Svg&k!r6mdOq%fo3eCoA=q%1*HXCIxZ zg*8C}FHbA{wxN@^#Fk`H9pxJIJ7`~->?7QcaeJAyG^jlKP+LbT*Soq@hgxvNf4$q0 zeoUGeKwJT+lAN>JEtu?aOwKXuo#!jt;XHFD9Pw|B3w}pR57Ftg1MWSCy$PSi4)@n< z`1r&u^{^VKQD8!btI+R)jiA-H)%W#hOQNhcWm}xA*WP&#sUP%E0>g6p5m(E^O!FN( zkIBq0twWkj{4gT#P6uWUy;tCzh2wW+7K9_^e~j$z7p=EAOv{6<&xgn<)n49Cz~gW( zW=4wU&kFBG{juWgfL4V@{UeVm>!^9rtWkHv`8D0>nBS(PizELHhcehBFENbdpMG7{ zciLV59HnH5L3(X0{t4|q?$0YURRrzacf_vD!&hJX8doX<%AN>jHw7-_a6yL6Hl;LV zWNh`DB_lSH6MYATt0jaO#(hGZLg*@dYukF$NbQCgJGR3ThYU7Pm*}^rD(K_uPFmXG zTRAOD>QyIU5wzRn|6U9!QOld(kA}CrrgMf#M__CAOnm>&?|bT2&_0g#P_zno7SHB0 z$FS+Y|IER4BdWsHU(0vOBk*ZY+{3z~0jboX#JdBV;eOEW?=ZJxHNRpvHk)04F^FW0 zbxf;u=K688cU5ytnzwdp4@OMHdH6q&y=kesr`z<>xdNnDye9re^?t%j;mGD#i)xd{ zOe4xI_?QMr!=^*IwWE3V&Ew0u4|}&fb*;@dO?A*9E-(WPXrUpm!U+8Lr$0s1sLAEu z>ps(eaL|6p_J4|RzCUYKsL&s?(dJD_pIKzTIWnmO zQ58BTrWOj_2)rzO)ciGf$r2utromz16Z49Z6S}wB6E<2vE zc72V?A(vCb9G5<+`31N%^+!O9h2Valam!QbuPrSz_^%;vShVT`bl6`wpj#1GyPZD2 z=d|aXpP#9I97W_<9Jg+67LF_Hv-wyjHN50=Eqfi}$j(4A_DwsuCUca=NB>rK2)_Q1;gG*)PQ{ zEmNE54L)6aEjq$}9bS1n0_2NdDa|>9rHd?@!#9zpUVe7`@Bp(tu{)gXoMd;LV>)jX zPFG5Mz9P1?&9$m1ZM-(dPUZ$%nBFj|MkUTE;NBVwp$#%^zl&m7wHG74ynWuRJ<8l3 z8Y}ceqgq4q?Z(@r%zn+hubN0&-nH|a9aRkn5_nm%)DMoK{g+oi3B%%vf0azS5BwO>OygH3HT}riPFe^Xtf(V z+}#@9*;hzWGxSjvoRvS)UO2#+VKW%`y1DzgrQJ63 zlHWVeP3P>kD|x*2P8V56(H-pEYO-dC=X@|k$0OHy)1Fw^+&VSePi;EKsyke+h5P-X z{{8h!O_vb=iOVaL&7eZ>`+lk&Ye?$tP|c-zZ^!-Q--}#Z|8inoNuouQJI3eEpgb!b zzc0-lC6wZGgi}f6?4}(AQqc@v|A7`Nq^1=0B`4wjsV%7x^x0WT#pba|m6x(X;Xe(W z8R4tJxHX5OQbs-&c2^kfJFg55Z39n8;-9lmg^?rzgWF;JrWsFh4A(zs*0Sw{$H?%I znI;h>{_N_zOmJKt0W`R@)hDS($r9CSoD$x^s4e9*zj>ar50BSMD{7b*Z2K;ifcfCnj()-} z;=aE-yKq9qvevlSp8kEfT>yyWrcxgeQ4fYSO1`Q~$3}GLRR_&uV=UhK?naAfauz+` zGgAF1%bFejT=_dbXHxHB1@ZgV6bh;yy31-AuH!VRME_6Zfw|qE?k?p7+w|*^%8aw( zXLDSCbEiuf*@{#h4$4nVeFPElEW=FcQbc(I7=ZRb)_>zp)PO>Ku;+3XrOvT7@}=iI`k6lz=4Yfzc@V&i+a#T9!$^;lAM($ z4ye9g=Tn6is$dlW8R$(PBYRwkk9sCoS@Xj_K0mI&T@dBI(G}3GL#81L0tg_NQXvBMZC* zvp^5GatxNDtT(AhKIw?Z4bz|kaiyBmF`Pjzf#aNcKYSjOyR(}tLYsi(@roYDm{kS- zf<-WUzOuYGqF-TG6;;k5K6oyvY-#Z>IXA&1Yz+`@hPhQ*Q+SbSlqrqx>A*^t0C6{v zqu@f&l}A9eexTgXH_SgcVDDC#%ic^t2m6H2L0QM5?$>p(O>n0#H>PPs6cv|i$Cgp4 zs1C4>WsMS1olYEuZQth*6ztO=6W?{Qc+I3a&XX4LpsL-I@DAB+L!~I_3oO}=gy(7xfz540Wc0wm_#L6Qtt^%)t0+aeSA*LPs4quBU^paI^v>> z{fa4vUR6pP_W`yQ5(RZNW$klDhj4^(KP+wBuCy> zSLlcwkf)LzWIP*#aIZLj4ZYA?sf;6%-Fuire%luOspURzCL;v^<`w*#e zxXP9QCKUd$cQ_lZj$E3fi>xNrqY_U5NfRg(+5#n;Z%%gjRg^| zJpchOHDm&W-Q_h>*X-;9!eP$WkOj|XLhRA+AkfK#D`vypN@u5F4xrOm`Ks_)^5v^NQGzK=?( zDvW8p$LGqobXHkr93fqYR}&-*Y-j!aR$potr`A(Q2XWAcn}RS;%67?;a z!kZYGqsNPDig*B$=@)VNfX*`jPbcse7Z_hnfJluq_VtGOW1^DrPs(Gj{F`4+<$Sf} zC01sjQt2lWU{A>~^4Y=gfs^#0e}bpY{lirTcHNz7sgo?|4*HjUOP}Y$qtM;OMk)q+ z#Hg@O$|nE%yClUk8o+%PnoP~XX)cUd1L~Htn7`liScpaErUCh}9_Wi*5O+OzTMAO^ zh)><$*H_KML_Dywx6bfzq_B0C_*C_MIOUv?vy9UAP)P6Woj<%po+{5M@lSD;%nFW- zx6t=j!xJG8FPs9hJgJr5#uRv7mSZc#8cRnTm2S)@SqohUI5yXlz#@WKryCYwc(JzzOXDx4hNib2!`jEX(FwAxplwG;=Xer77Q-<2V=?0HSnZ&&CvmaPrjr_V&LC%btmx6M5+7`L)u5=cZ2#5E&v@$ zE3#BCo#B4Yh3wlh7-S=L6G}B? z{Jli;G{NvYdeVJzNy>X}7ZnoT^2ni%a2;Y$@x$i^)SM6*M#x(h*tgahMOJH+68ML{I$<$;^0ot^Ox@@}9cyh87-B-*8*!C3u@%Lv{FR8d+yj|N`LgBOhN^!YX^DmK=|{jr?EXEHT+yBcc- z^rJuqP$R5AXeaeB0iXYL$(Z7qZbex|6~lMNP($K^4S{^c^Ub7Z6!o-2G=*#DptH3I z-?JL(<9pvO5Gyapz^sBsylkgC({=k+fROG`Uh@snSZm<)u#eY0=QP{-RCN+Kl|Fx< z%%J4fGmamnYmKwjeaTq00sW+r-!pltxZ?69ZDpVra5~EsRe=+XTbmOJ543Efo zS7hIgycPAp72zd%F?KRyQ4MvSV*(_*KB{cZPyqY(XXl^09;J%q-FQf97kyqaebJ!J zCrcqOH|v#8m86}|3&1+g@_sivi>5O~T;%+5a7V`r{EeI$|FhENgnB%M_mG&kQmNwa zw^Bi#RoIb|%+;6%RPeb{#b+BuY{8qHL`m6#MuDY4re(_aOm}#@m|G6qF~?M&JocR< zgvxl6BF$lIDPjNI6<2#{l&@Z-o~SE&G&6tOWez`Z-#legyJ86aeEEWLCmpjoR+l6> zUX4wgO04l2uQ{JDYkS<5Sic@OK*BJFj1D`V{nYn{>+NVL2_OQJcUHph8M+7a&%2pR z9r|;f9B7wJ641wi$EfFOn(@qS^aaBt>Raq8i~~ED$wxJXuQ(aD<(jHv%;OlGUD+d2 zXSU{2w4f)|>bK0ig2y)2BU}{nrEg}qMZ!<-j(b6is^8koFtU+ta-3)i0K`6Xfr_uo zW7ag#m8ImiTLp2OmcTC~RW1UdL$dEaFpgM_8SHzooZI*RqDze7Oly&Wf-oCj4(32V zKfZhzzAA!0r20W8-R-|jEil+0gsXO+d?m}G$lW0I(pc~IR`jkwYZ!Gfwl&woe!cz* z_VBz!+HJo>SxY}FE%4LnSYON__Zl@UJY!OiQXF4L|E`JOMoX~ni)L)lpkl@&Uo^7i z9@S^mdTZB(eDFrux3BtNC-0eou7cQDLX0_IKO1fWe;G-4<4@!j^ahR3F}qKc(|-5k!BtDVAd7n5C)f$9T~hXxxsNmR=bF*tv$ z_}nSp?*JD&>r-u<_hQ6{n1`t+F=zn%9`0E=+IeVvoL~DS#30 z+`RQuXmy3$5wWQsP6RK}TSM5_FL}yj6N4`*?2JKJGkPCn_J^q^HMtVrI zV9xHWy)hhACo4pbK~?kNL)NhZSerQ& z7Mc@f+Px`y-bJ^?9v2}?a)E4oOX~g9wwWg(MGKv`iPHQuxyC0*+F2sV+>^SRj)~b> zo;E_h5Kqv5wBJ50$NXE%K(i9SUfGIXlNFYT43x7wc@%XiW*FwaWt-HA#AftQkIP43 z8gNpxLz?QriDm&gBSqvy44XL_Kym%%Yjn8vMw2PW`~wc0^C1WakvY^q9fLv;zrN%g z$x++^?e7`EbE6hD$Hc4(c1(RcJGc}52Fu?#juS?q@ses@Vv6gyYG4^S^L_}VZ{Yei z!}>ej*mq&vzP?uW7(+g=kulVAEPs6x{9<4||-N(OGp z+0DRydR(#LtObgIAJ6y~X5D-HmV-ufFIjJf^zNJ5K6M#q^sBxJgG#r@xJa8&cr zhu>)n4tX)1I{*{Yi>E_>wo>r-LXtknk076}Gny`1B;+sdhVN&g5Tng&5U!LQVK+lN zPoqz0YR5@H9_cMZcPJiWSFa2s2>HR+*mCdHrSev-CJuT7+yUB749npDFu5NV!muxb zZ^nTfCw*$#I{;fX!qV;K$9o?#qA0G|{$SlKxOrY*KXkY@)wB-EO;`)Qo_O&F<-l-- zOzxq#-X@Q5r_cux?M3O+gsurtP`4vZhBMNA98+hm&&!2oQFry_?_%0>0+g*ajQS!8 zV`8w+Pe)n&H-!=@&Y{iU%o7|b7_|l%`;`VN9>=)YXM44g+<<~nAUHr-h;g4 zio&7grf+(ZzRTF7v@!bJQ9P5ZR)+n7=R4{u{gbQT$Rm5v#W@#LqQuD4h%(|Z`E*D7spzu05*g@`zS`#B$shihSKbl)Mo%eDqn0F+GIr%o_qBc# zWDf3}7t7m?cRP&Pwk~JiFqvw`B@x^wzxZpMB2tfL5=cNXz?49Chja8n{NH zSEq7v2Lw3InO&C#<#HK2A(Aw0lMkkA%iU9uO zSRT)+J+4|IF*p-vz4q5IMK;II-{dMzb91`Ik1#p-qb2S>8iyZ4nrlJ3o=^I0KGXY6bLJ#U{x^O3^TFxevq?{tf`uec6vgN8nL^&K z>!_^RQAQN8k&DXD0-wh3wARAK63W&)w8{aWQOYL=f4@<96<4beI!}IA^0-`GCk+)n zCfgNwI6?mp!@ObtyFLpvO6p1jh0UMS)Tjc`qtjq!_PPl6?# zhJrxR*X^_5Y>YHTN#4KIcy;T z9rXTqQMHcRG0{qL_~&F!%WRx@XL1kh9NXlB>Aj_&+x=?3aHGQgCcMK+v%{vlm3OjZ z)5|KUm6^Ku?wA0%b!~k!0AF*PeG3z}7U^h?B0y;^6N`zDsXD;WGS&Qp-}Fnn^kQOS ztmn+13-<{ro;NBHVVxvCv-G3NW73%fMC?^$*my=`md$QK)4F9RT8;$LwAj3+A0PsC z@LI61VNU7#*6qMC*Gj6Ngi)6}K;UzF7frD$c6KG|bd+VN}sX{+{`JawB^B?d#5aWCH_4JDUM6ZYly&UsU&lwX`FhuME1cebd0 zDk{;$)HY^Obi~u+u7*jWhB|l_Qti%4`Wm65t*Ob$Q(jYf+;m}8u|!X2(`>2fMw?9P z%z?`V4Y4xYIg{42co<4xT>i<6iz`(na5H$Y6KkA!wv+tqM~umWjHQJ8XxG_`%r+bo zrIlX&_ga4>jqT?UEuXeDfPM|ypOfsq-0K&0fAD3fRz2Yk-&0%7EY3(}WvH^k;O6F! zw_9IRKcE-7@mqw^rS)M)ple+DY-_=tol~v*7oXIZAoSizZ4^+Hj3V~?OKs~Wjpq37 z$zu0A!ymEo0}i@ndU^+Xt{rZ@0a|xFFyy`;eZP2ds|ZqYLd6`3fN&zEe{KafU#HZI z1s^;bN+BLm_h#D#S6VlFoA=7RlN{3? zrr={Ll=|G@E+a~-hn}fcwqTk*z*2N4^)Va5ffV^Y=3MEKf>OR}(n>j{o+&(={Bbk5 z^z=dugbGdz#O_=hdr7MQoLgiu?h3kWoz$gOSLYDk-qmY84Y60_EFK)p; z^aYQ%!nc2a&E9SQ)%YNv@nrC4PQ4_+p%xnL#nK?y101S+f|GErbBQaq_IlEHWF1zI zDuf;iH86gKx3~TJ8_u8ysgo5?#y>lP!6qIbSKIPP(6W{g?H`o!hm2 z2mF5=orgc0@7soBRA`l2tyJu&-CC^#G1?-iy=t^IYwx{RYc*n18bNETy=SQr6lqbL zA|bZgv%YVB?|+bda_93r*L|Jmao7q?6PR8%2sval;6FqN`IUL#7)TpoH$%AUk<_Z4fFdbgsH(}Tg=f(+q<$1=tUr+&+KW&p=sSe@~TbRxeU3@X{K+@1gF7_dSlRMaM;<)Qd zXhKKo05TM6=+S&>N4h#SaM#*k!+?-VRpPaOBm6qum90QD z>Jju}65c*El3{K^=Vip5WVc6@Unrxa%ATKVYSTebyUW=4l)KL}^@jvngmwf?igt7- zxU(*u_L()c3Wj6(sW}fu-!wqdxn+pggJM9@^ADltB|U1HebA8zxEf8JcaaU>opCMa2(mr#mBq1ABW$ zH>%+9+j8JLJqi(JgM)L;i~*eLDT184I|JAEX*^!!@Co-rM-X4t2K(vN-+Rud^8z~r z;S|X5KpmU$=hxV&c*$JSUpIF&S&es%mA^rt3+R`h_beG*^%2*#DXCY29aiiQG60Z8 z{4^-}lu1Ve*vB~nLSciP2?FQZ9!CXogdn|)w=k~}Dn0G&0i$4nmxHWZ*&os2%>TR$ zFV`@^T|Ewi*kVUoVCkSR=~Yy(oa#*`YW}1sN^CRy8d~`Bdx@UcybeX806~0uIcz?;Jp+K=AWDj>*^XJcE_QO8`41dMc@t_S;vb?%TEt@%H zVpiG9RW2`#LaQ+R?JNb-&Q~nZ=lV@L{H>KbNKDmj<2pouNi9?Jdnvb95;YXBK)e`# zw`l4HB#9%hE)5kLz*IO|8Lx>;{Kmz2hL4Xd0wT@pHx>bXU>1*epOye@k)nfxImMj; zWmd~dvf9uAe1)93IFwcrwWW1&XX)auW^eS1Mo%>ueDis5Vj3Q!s$P2X9+Di@4}=RE zg;ce@Rn3R5>)ET*t>VP{pER*v9}CW)>HP72%j3so9xXoSI_|1klG>F0%Rd5-pcdE`4s3dQXCL! z-j?qRAfi;Y{sT;w7j}v8_;KxQkWe5n!y8FUesYF_QqR+WJT>AM#i(`7YoBju5fDRt zp7cXxIMd3nJ<;YChdbIP5D}Kzg;=TzO&_L)e>7?oloFS@6Qq-P-2o^kFkn8y6Ck#q zDD0elNcZ_$*J=6bA2DA2{^gRUyCI%kNG1$vKYAUIN7ohlHj|%K2Z7V-D6PP%uy0K@ zHo?Lf)n3!dk_aerdL*{93dx&ks@N>j1A}+13)nzxo&bu-)A2|o>Ne$ST_O(iJyMt2 zs@=k_LmK9uuRU?0LtajC5H-+8>82p)OodM#0nmDP^QS6-qmBz2{K&Kw#5~(WRtt_v zmt1;8Ab@j2Y4wL41qDU*7qjrJn`&>n`UEdq`l%KL06y?byKL%h2elc1nCy_(ULSYA zpM_^2u>%+}$MLa{m7f+`qO~z!kX?5ts=iVNb#zw~%L+ak5g?|{6E{>Lb{@d3v1~^@ zs#c&r((^XEJ-dp8_*Nf?=&FdbRyP(Bg@6WG&8>?&02ba~8YSngr*(A<4AY}Fu@khq z%FN8*1AExI%*p47EgZ)Lzyl-W`(;CsJ;stWt@(W^?E`bf9OMD$dau(+^P5oP{1Sct zd&WUbxF`whqUXtkg934y&SG2c@?h?sg|7hIJ!;(1=Urv!eJT7fOU@%aq~Nd_b87FW(~Fzwy7_ZB@|V`Pputctnl z`-^;M+iG;e={!9Q8fz+~vaOg>v*9DmMy0}R7`sa?eJ^hF$A-IL=ui_Tg{FEd8l&Yf8#A#M@G_HipzB12#RQqONdC+pW z!u#NW{YO24#qWKbNN7pNY5^yW$X{Y_-S=Tj^zJh{K#Pu%;L4!rI%1;!N!V@458cZ) z$I(Ap!w5L&B{-0D_NQY%O_yu9xR)MsQ>(G=9eSJ*yi0c)95}UVCMOAdA8z}O;D%aw zQA;)2U7YI9HS$>{{eOp|}`m2*c0nR|H&;hc;JDtP=4Z{7uI>! zNeZn!n7rV$-K@*^Qsq*gkFG22x3;f^+G2L@DXsl!cxVz%5Cnh^f2qPo#o*F2izxHxPykVrqA1vuYxJ!YeqvwnnD|2|w-GLCT&wBnM z>F!HY*EfRC!rXzN&XmR z2Awj0kCPiyc7=NrmSf{|>qeEhRhXAj*uZ4n1S9&`N88>D22QUz?{V6gJdqhh(UD3k z?qZVt9%H0FWkdbK8^!GB$+jswzI@foD|1mTwK7a-ixU_{q+O;N=Ih32+7~^UFjGue zsY7O+!=8t&8+SY-#X!q!4ypO8J#astBjr)OiE|tTS0AU!>M@QXIZWro=6fg`ZCuSW zr?s2jREi(#Hcg(Eo<3>Fv9h{-%YC4%~_HwiS``(TBd)Jh}$glT}ksJkf-|`<+yV@M; z{}AYIqRoKe#?L1;mDz6=I6}rHvjgyL#;UxGrN*P$x*JD*uA60UE_|33=OMavd_Ta= z`G~46oBqkX@=0(L-zRjq2PSITXf(UrtKjW9g{vaN$E8Sx+zLB>7t{-l_=Zu5{el2A zZ?z?I7rJFs|KX32q|LAfUBWkB=qc%GVUYb@jfF7mP7jH|a!%IO59(=r9>5o=Pom>> z#miZt+1bZA-G2#z@oLN%=h1fsW#$%wvT5KF*=8KDNegOr8Qa%CWlk1Jw%6x#`h#yW z2l5OAu*tPXPtli|EAvf6LG96!b>63(Q~Q-jIDf5NCTsuc1CBRCJYAH*g{>4O^1Pk} zwf!EyvSMI^|D2v_eQV6zxh2_KdLJ3bcXK2)F;hN2Kmi;nn z>AXAVFRiWroBX|N%|Q2tL?DB$@9?>W1bGE36lSEGA_R}%rX}omFFF}* (diSx6r z)b5&q?$u_OoeBf9h1Ueq z!|+(Z^wA$WWT;U$uYB(7^k3A4yge11K3V$QwssUZRJr^}8P>_t&#`kNQ~*577x#S2 zhisHc8{e|ThHx-uIj+yIdx7H#x94Z(fLGX$;>i8yDIZD~9Uvlv%2R&+BAEQgK20w_ zk+G}?-Oe{&Yje2^K|nlW@i4tJcxb~!`V4Vou}hK>-1#5iQ)Sr0bP-mYXUQ4POPuQO zz=JhUmyXS^8z9PhUeRpE#T*T_)2rx)Rw>ck;r@AxN{NNdUM4wdrg}xz)!CAlKCGO{ zqmB(_b@>9)0DoAyxED`vEr&AILE{}zuDmA(7n+<4(mrx z1C!6Zi=GeKS=mvmQ4n?*6csp?o(J#_F+PtBDIXgYq$Kl5Eufv*KPKBbV9x010o6(jX;G$l5T z?7%Dc&3+}ZE5lD_@|#n!NsHdWy8)~bxA6w)JbnCo-0oX_`+1(M(#wc-8tgaMhHcI| z)%Od>8$OpMhGVp~rFf!(0Oth*32`^W@mAt&HgEsu?ndVd~>&RT9xjjhROh( zNfkjFQFY7nXa7 z8!MAAs!l-cQseYg#5!ZXou8Dhw9R$12cGolG<{?_L_n;kDzs{W`qiGYOvWCjKb~Gm zM*AE}v=ad*SB_(~JBu%^*{L^3pCXF}N!IOrnO(<}q$SM)vBbBdl$ax9}H)HeiMom*6EYeu#%+r$IV;wV*o)T~&*625^S0S)cCSNtT*8*ID z#9e|HRy|p9k-U1z8dTDKD+-OlbF3BVeDi-K0)yu1xu~qi&@1&~Pp5u+!!p z#h8L0r_@(o8|pF0{V6%`BBAJ2Qn!+${zG$4NpE<-|M-9a1lEGczb-Hj8>HYklHFm8 z#YwS1z#d7k5r6}u^j`rV@1M7L2sUk$AkfXc{{YGAuAgm&=e-6f9+ZI$RoJ37RI>}C zkMW(#_648FSF|Fj=3i1Ekr$zy>VcyMDKzfXfE3m99js}Q4VUwLzyQZY$ucLmsy6oT zKf#4N=+;cV4P*QH?R&g8VG8dNQykWSC{`jz14277$KOKLZA*&@(VTq!MvK1K@zFGs zOXk?CE^;xmB%1!PQ9-Y!Rh(9XF8}$oOcMP<&(W2FjIKn?&36`LO>28E3U*CRsFvcV zVc4;vdo*Y&0IC;%byB9Bm3CJS^W}f7FGXWt@o5X%BmBhA7b!v69)qhz-<2EuE$rqZpy+|c%nZ?|R2_Bi9BxI+Uma?@wTIUK*6Acp zbqMfYb{$44ZYYwitFgldHxT4`T_osMHl3JsU|^8V7_7UiUsW#Nw4OYJca?qp{M6bH zIR3Zb#sHrpes>H_HRxIpu5jN%iZIxby{#;}3L_Qy)SniM(13VT z6CKS{!~*HkZx{Sd06nM=yu>h6s=aY)c-0j?|X(feZ9Q2jouj$P=+g zpW}grk*KSSLuF=)YFa$yPsv5pQs6Hobs6(xxr zQnGU6!$Zk*QhBvT(evi6tD>t6{gytY>FvHLo#+7(PA}(|7fDBbRa!zux&go=_qDl* zYZh=E`DfFi9$Gt+l|3rSV_r{OG<7jzU^%q;%be7MlbU)rO9Mt~I>2dh9Mx6gi??4+ z3PB_cpu}xHK_J9Oe{6+@1P+&Doqnl4i6Z5 ztT7+KWLIz%r|Go)R3!em0PK~V4)!5N z3N2zD1~cJ7^$ETU_pt^>hu>>>o#;#u>pjsIsGup~^5F~D;`W>(Sg2qBuFPGECUD}j z#rBhm{F9g7<+#=bg{?X%{&8oEuhI?++;oXn*EN$>3Z-nhLMb9GTYsSRL^HC0NJ zemOk@!@84gU~&TW{KsGCY;QjpeNf7OZlkry>W6RUA&qob*ob6y73qgli z)3r*-r#_=Qn+%!~KOX0$4y8OmmRXQ+`Lq=g8I%+l1WV2eC&Q~->y=#S#$N|M zsPF!GU5!GxR@_wQPxPPddm}_oRxAJan5X%?pss1k^wzBdUX=$-f7~oWF5=g(LLSGQ zyvLLlj~+Z7U|i)lcMciBg29&VLC0NR>6w)MmfJ2vY`Nn36Vfw>$zKu(Nl&YJOZw?S z?1Ci^Dg&EC)h-uo8jtSI8P+tmdX<^C+b>5IJd-c}rWK}jWx&BY@#R0jyEu;F^4KVO zwpCe0zGnXEe_`t1r`)1z0v;cj6}Aq{Ww81y;ViEoCwh4DxR-T*KZuc|J^ZJT^CH!h zow4Uo`P6=^Q*hDY{zB{PnOWxKGfOleFEz0;y=ybzE)b%&Amg1tg#fD>JX2q(vNW9}5ER-aSWjv2=Qk@62e=Vra`Z+w>mch| z9SV7{G4eC2==4Q-;ee>g3P$XuM_Ux`=lzxv$Kvf2Gc_S9t~8xI7c46U*c*h+FS{Q+ zW-c=iK$cvZ1zAKyVc|h5|%RC|zl;H#27JX~7NGK3XKb#dhOC{f?QRAjmm_=1z*Lxy^O% zKgU_I-QZoG_Cn)yXWmNYuLYI&&N|VCYC0N1fBibNzx?(0*SC&zqZQO_0C(na#CBNEVcES;zhko49{%PO

A6D47GH2b&)* zBT?d07@-f{U}?oynj-IWJ`C@qXq%%C0&v7vX)D|Q&BgP>ibk#yP<}{|)z>dixdPtm zjEQ}x7mK99^0JF5%3u9+G@C*Ny!ip(u?Y#04MR^bnf|!{{7{Y6&OA2JobI3aRFP99 zO=EyppuK=T#Dq?R7M0^qdXgkt+q^^kQ`-zv*wU-0d(Ld$KWFnIp&5cwtqR*XE;m}P z?Dh4SE3$GA?k6N)M9NNLMbx^Y0HSnp@7jgY27`!au0p84!+?JhNLYTih8H9Bi^ynY?9VVGY^+ zk?`kx2UW=bnCg&D!=q0*A4$&>k_IYFgp@N&Z6m+O>0bgMJM_h_UTv=Cl++_N1O{Dw zt6Z1Dgi`N`f?A56CW09#NutCcOd!eQg6+o9HN^xA(mJzhx_8>NdjoeP&b_sO)L)QnQsbEu(zedXdlQPZ$Y&wz=ET?I1f?7r%IMvFo) zFOj*S@s0N69+TnBxr?;%&#hG0N;U;mBnPfMlyE;D+F9)7>T7ZG*s=2Srp|t3Egdi? zA<$locZ>Y-R$KgOT>c6EvkBR!5MgOIAn1Ek?ZhK*b%7eTHN1^i`C-w6ScFn^=gjgs`QhC=rvwaE0X7Ka;x& zx$0GH_3v6-ffj;4xI_x+feAe|_zNS<_(HX_iHh+M@|e{BMXAv!w9j>S_vx$fdFCOJ z=u`YrA+5^J7`UW+tqL>wd_(lZwbc#W#}3j&O@dveKLpirFENe6JrcJJrAsu2Lw6;# zt+i3wkKp|QW$pFcMGdyU-NzWBVSvLwSNnRsibC$8FD#t|=O;<( zCZ_=bM>w}}l2$JA%9G_kfHA*P|7}g?2s|BvxLDfFgVJV;rXfxhcOWokO!b1@gec@C z39N&eAoG=`OnX^PO+*tWQlPpSypWT}y8PbU;ank$3jcf_t^q-V7ukSI1(1Rm#}`x@ zst*7i4GJ?gW>eGehp*F7^GQy93%sDffkve%>MF<_9%-t2-Qi$6 z>Q|-bk*t{xG8O{x!usHua$>xgBQs4!&E5wdDL|g^zJAcnzmnOSzcOd<0pW`I<`ORb z^e-ZmyY#lHd^zX@<-S9NRI4H0R}SVv$dn%W=rlCgW;qX5cfM=&Kgu`%N?z))ivH)4Fp)D$IX+Jj<#jTV~JcRMbWg=?6E|aPwEoF zCN~wQhexJTQ#B%xkCgV5K$LbB6n;u69=HzSSLq;Gnj|{4lP$%C`C~b&NCcSnFkBF! zphXvk!>jRGed_EAkh?GYOGTzpMo)lVt79E*kF5X^lNG7lB{Z4s(D5*KgG6h^8|<2( zEmewCC2a!$iQ}o;P0&?Rf8>7hB}$q3?(g4`sCJIfdR-;P4;o6kjbgk_2gNCs6clJW z)e051FJ$P8@l0f`#8ok$@{Nj^$ZK5-71gSe!bD1&CjrE(Yvz6*Ue9ptd@cCK{*xsw z02YETK3|I#fBG5kga$8bPo@#j*)JI7EhNVi54Rn{ILC@cHeGJm0O2*~zs9o#2>bQd zRG)Qg1@zR&yt=?TEY;Gqz7JB-2jV-5OD^niA%O2rnSQ0VsjCtp^mCe^kl9#DnW!^q z-eSO9D=4hypq6nz2<0`6uMe=ugY)kl2 zQnx(7e%f)76b-geS*;*(P9mrZGg>YGJIG$th`2F-qxS=i+2-RvEstUbkKafVaj~!O zx-r=M>C6ZOEy$gpj8lZ~GyD$d28V~h;N`(2?#bbY%>!l z{PwnG3JXpoRUMe9C-%%t)TV#pqu+NttDqAA|<;8W<1gi}yv^DiHoI&iGVx3xEq4!SM-sfoyyvdz;*d4>sv zy2&-Qd~PE|Fq$t7}i(BGM^aVVww|h;GJuz7!6|jc|gsJWcb5V z;@^pA*IivSoBik-8Q|w}kK$oLaF;Jop-95N64bwbdDEz5vU5wa-Q%;_oI{P1X~@w% z25s(?1@5%-PYB9VeUdh_pM*6Nb(o-`33gp>a--zoPx=3?k#B*Cm(2d%#!Of6O zK|Nu1!#s`ks}C7B%^VC{89?YoV@QxS*k`Ry*HNW? zaP&Wb$B$Psvwv70N(ZQktNA8=q*LPZNHsAns*rdt%H#h+x8>hf{`@_5$t2Y(t|{$( zp&`89#N3;DhvIt-<>o(lmXXapsaqM8goD5-p$sbj)ws&%50f)r4e}5BlPMa?EY2w&Uo7>HS8*{rT{pDtQLzDEo@@$M zFqyuI%XcZzI9x)aeJUQ^ecD)bXv@@G-jcaqf|7vUQ zsl`yKyKgHRwb*W#UExF90RM8$=8H?}^y>=pWKB$*NHt$pya6 zuVZ?-sOkX@FK_f6qFau-x;S2`I|-aW+${7R7kQX`@*&g9n=-a5otF6k+I@ixrK*-w z5AFZH3pctoSV&7yhNXAm9p0KpY6ZFuaIG~KrQE5keKN8g+#tRKta@QVYvWxR_Oih! zy*twqs1TIfv;Wv&qY2_6Ehs7Yb#ZOg_Rx53P(%y(?R@K{Z2l)B+XT6aJU(>gts8U$JILSZebbsJi+If?UWp1molcR=P@G39u4>TB0V!&2r8@4|i zweIGWQUw90Nr4RSakUC z^>6Lt@u1fd;%Aeu=IU3Gwbm#aoa@J}5uY*B`San|+@q$SBO;e8Fw4B$R$X?aNc$4g z__QG`xxj|Qswg(gN%;epFf4S1M}8Bg$Lb z6>4kr!_zY)#zJA#VN4;7T=IyOEhNN|Kl>P>tkFcBH57x#)T^(qEikh&Q8<>)s{jVC z4xIeQCk`DaS2ejIjM|*TLy|u!rljU_#6-V(4nh z4`A99cNxq=Szf$J!7Vm%5%-!1jQ_68)N`&Ax!JnT0%`HNh|7(qyw+#hZvO#vzMjPR zOnX06<>3j$rwD7|Nw&t2o&n&uXWEV9ul=uQf3b&0w7G%cu;Zm`)t{3a_Lc4v4h$P7 z57dAkv0PUp6Ba>}qq($q1bsgQdFznw>^5{~d7kF8`pD7!7EXLTU^r{M8!j~Fd{{ow z_EV$05yfzSczZ~^y_ynz1n}ng?ewBH`|0;3c7kOz478E`Ho2Jn=@CpNIp$Vf%g%LJ zx}$^dgVn28Yk|E7H_JH4uzIqo*$tWvUMM}a+f9u;Z?l#UiY2W;u4*yiUco`r=-|((4g%86N{3ILDgmzXw)DLnM-fE>_rGWk9MQk)yyJGu zzG&?LkSYDqA&v9ck?OA81=%^qsV*FVck_ra$h*nQO2HNU$Fzllauz$*vanZrn9eVh z;u(~=)UTUnRb`gbY-1T2KymJ6Qq7oGBNq~Al>XV~PEMj1g6J~c0o31#GTO1>A|x!- zoEbtTo0n+)%8jlpE~Mp;eh)tCL6g=Sxq&sV0bL$PM`ge4JaoEjJY}z0T&JlLWdlhv z;1@Z`wkiP7$%tt}I3x+W1U@P&6U*cHUinl)Yqtjl->4rG7kO&_q>l?H36HXd$Ij?2D`Bh5CHbPe3;v9nnaKdi1{`zS!}0_~;e`tmq&&mk;U|+0 zPXmwE!tbp+kj=`!9X!crt=@qm6}sjZJq{10HLn)rziI@f)=r;%jfLrE91PHvF)g)C~E?OWCPap{*F6iXa8 zilpQ_r!O?tQ)lJ^S5dInk@9)P<$G+I&3H;u5O>ND1g#bkTS#AsRJFz}$&Qa{i$uzt z_QOb*|2BbsORxXTJP*gtdR+11M;pP|Jn3Ewd(2poNkp6mwizXv zqI9}r$ZN0m=)sEkzz+7YEwv$=dl#)urG#p*ZytTa$7cO-$@=u}5i%1p?8u?sz}I0i zNZ|2Cw&pTv{R$pmh&usezae)j|DU0>E9L#LT3-2bYq8A!}g0CyP!->u9Edj z{}cxER#?y8v|%Yo8iyle~qHv`8pk0C4&+^YKVdcDZI ztb0+5Nu7#a;A8#Cln>S9118ee$HkUyNRFd1n%7aOzXEk6G1Oa!A@)vt?6dbAxkWL7 zXbw1B&}&qokV{s{svt1xTt_4MpoUme;!>L_+V=&Nw?v!z^H;cv)@@!L_Q0QTd&toj zO7tcA@+Voi0$IEOE>Hk)p{oj7T0@=XjnTx;u@PMHg@O>;h-?*|MeKwg7r+%l3Q(6w zfngw?c$uTm;s1}NhbMw{)OM!wu1e%EF;R+C#;|^%hn7XY>fcj9wqpui2%UprENP1v zPTHMq*03h>LM)p%w2>Jg%kg8Dn+!Q0`&G1|rD}#c)B~fy4ys>%Pme}{B+FeU5DJv? z$@3{qan9RP=9u|h>o|NvL`x!&WTpw$kD8cFRht(096vp;!Vi8JPzVs{V)o;#UpZB zi(w*Ces2HUJMTU4S)k@E9h-!H(m%)kG(dtO60uw0-qXC&`ReS$vZj7_ld;T`3Y!hr zgNG|xYc(IE4k@nGeiD3u?t`pVDI@M2{pK6?xH+h7ye%@mP|v938w(-$bY(g` zV3#??%;vzx#M4NbQ%Lv@X_hXLvtHep2nel{Gs^DWv*j(g4Le8Mrr9N0zInFQ$?l@e zr!nwhsP^1AsmLTbao)|xZP{%_j?Sq^{=u2VVNZ72*uN~kXcaa-E<93?A#y36)O-@%D08WkI1Hj$IR#%L-aIwz;Ub_VAq466fp`BQP=LzB!D zv)R6XIi*DehA8Isj(bHiL3WAi<#$W@nxBY5qPa(>65WC$CDEL3w)9_V+QH-deE75E z&hO4qy+ZJFM{J>y47s>s>DsKE!h0oKw+MA9P6#)v>_;N8v-VMy)LqN{G$1^06wVZX zrDVIRCNep)I@~TOe-2_(<*Dc^!Mcezn-KKq$yAoMe#Nju_bv=NdE#M=pbEFUKK^wa z!^^Pv`Xm$Qw#N0L}T>F-p%y6lD%iHB}e1|LRRC*JoCH9wNH&8meFQwP3^2JW&w zEvxQV^xP5~;4Pw(^~!DwDc)((`TXF9_RwgzHD4eJU=xK>Wj|*b-F&dR7o7F*iIirO z^M`aw?}4LrCr9tOz>n&+x<652cW1Ez0g*MFfe${hT?DtqLXSqFMSq+EUzgjADHjJY zG5RLf$BAX(&kS-&(ZRBdc|qG?t-`!#9RIqxqk|uW%~+X5V5`)J&B|L`hU9j?KM_gJ zfKb7^j!)U0%A}Z}#byR`)2NqKc;#=@*ZShhnF$4v&@G&9Zpl}TCVl?_MTi2|2|l0Y z>*r_gd$d28o=YbY$fvi-HbYu3jO6qA=BFNcNId#Tl(p8(^O&XsTo}KwcgQDB^{HHD z>>O>l&YLBti$6UIXejf=<@T@t^k_e*+ z7e6k$Uyk$L{%gM?FjK1s9Joq6ly9gPkR0f>4yadl*u60bVTvSKSU3qr9_b0QrLT6v zjh^T|qt~K__^#AjRinAH3gX1b1RBe6 zR{&3DZ=CiaDP3RGF8*;+D($@QT=5-1_VB{^xwKr_@nd0g>DfCmvLAeH!AJwi?*>*} zPm{4jVYt9wqE1erNc)*n<%2t+8(83C9_Oq?l+%r7+faGWP<^bH3?3EYRCIF| zUCLgKc>10rLi^dZhu)mYEAxfY6eDUGZfoSxsEl{Ttyf|t8@M@x%0uab_Zg6s8=>SA~j!FdD1O*vY zL?_<<9x%d`_z55e>%ZaRfML(s7g8}`%Beb7g?svguOZp%y76)|n*61iMQ1_7VMJTEB~0U&QCmwEFWnD3Mv8yMOEL{SR;x-x(Tyn4laf?w*lty!Svi1H(5& zg4qWp<6a~Nw${wf&i!z*zr{DeWt4+WboU&Qe*Yc_WyUU{t#^|Ih-OqJj}8msfMM1|088|z}^)V z^vcD9CI2C z@!w~dMH~TG{Y@u|<>IKsn4{J9mZ%>sEficx=hupFepHb@Yuha|_FX>E4gc<~Y<$Ui zW$^AJ2BCDR>1>E~j~DK-=OPs&5Dy3$`!<;}OG&|(32XuX9TrV){ag$D7lAmW*4zi*%js66BQBfdVgJc26FSIP*ivzI_UQ_ z!yWd5Yl`9yC1zV~GQ`Q_x0EDk3b80~e6?XPEnCzyzl*G8*JEn2^4a|)P6h3;yhgEFyZ1g$xQZ|MJzG}e+Pw0MiN+^~)2!T8HDpOEtPFtSVjg#IK ze(TE4KR@}k`lFQc2)cJvE^fVxn!Z+Ta;&0wBK?S?Wh=Y(iz}~Ec<}{Cy3RDsJz4i2 zA?>Qb^n3O6d%!CbJ6Up{wi}ng63HZVvv;! zML#v0%YlXLaLiC#2|`l%HJEsp!h3j|8Z_9Vt@{K5mNT)_Z5B;qVT2=U~2MGz+eFDDKpmou6b@$LErHCuvd5hMF*l1 zv6TS~r$5Y#>r;D`k@A8el5AISMa9{MIR4ng#7H?QVPaXp{YNvV_jCj(_Z9Q5;_%2y zayW4Jed&vb62z0-%F$h_QMGU}Y0Qd6cw2#>XiyNc;(v zZOg$=w!xqXs}YciH1YwwL><*uQwvOasmH zoa?c{9C@^*1X}wQg$Mw)y7)W2uBCoOq`%2T|Mf(S$a*H}FpyAh&ormmdq48tPA6@m z0MNUSB75oeW(M$42sPk4Qo-Jy6;ip27@Q5@NCP0gM|vdZ1_HoLDPNU%OSaTWkPPTL z^%vwZr$3!Nm&@E#1S!l+a*%!De(B~KlbTp{Hu0_&>4F;{f6Q34Yn~1w^5_2tcu-_$ zV=l{Rz<~Bb9+5`NF7ki@)~)!=_p@alRQUdfz=j^BrxG{XSs!*&X_(mSsnht!8LyfsZ! zFy1JP{O4k|;N(;3noVdIi@9nF>&_1`imk2d&~%}pOW#rpRo#vA>{{*S9Fu$O@PQH4 z2L~O{W89m=J~;a;io8*;pZfG8Nz?$0r{d$i`Yd8(c?%i#=Y{S8b+mCOJ)D}g$c2)e z6Ag}1M^H0Upoz(v_2Ey#QGXVoxbq90hz~Bf1yYGrI5YhjIuIZT`oT5G9wuYF2^cm8 zaDL*Ho+5oy8CNZFjG$p;N>7!mAiaooX#t@v{Gd0_R`KR3Cvh>^6Oq)2;KB&-dmhk( zJrFA6NFqmc;kN)l zQ5Uq3SSjOMz#HN}zkbEJ)IBGNO&p^HUMup549FjMH1 zV6LNu$AXqpC*z+(D%bF6E9BLjhsh_=_pkxjmAJ~S;4j}Q*lYIGS~Q$R$lzpf`|7>{ zIj`}Syr?ZjIf_jI9P8-_fD2yjf6QDEE@4kP{+}b4gTlFx@lK3g%pmrSmph z{0pIWlazhOj`O5h&`F|Z@+dX)qL2ygUbBS@XbR4o60njB3(|V@nR-iXLV@?vPY2-7 z;1q@-juv*o&0Bz%cT}|;9}J{ILBQ%_5ICuX3=-VF7WGs+P;zQE)tyq&Zjihh9LAs`9kOfwLeKvIEb^e~E45#yVl_oyC|}m6pBd|>@Ps)Z!Q+nj;mW)z z_k<)u(^&-j3(ReNmYod08zneujhW^$pUrhoVdy)P-L^4z>yj&08LKYdTPb)*oW@4q zKy0M%)T7w?F!!Q2??qpHziVvHYU8!~3ZL|~XZM>=k{#r#Yqcq4a9?5BH+i3D@I!3H zF?3nH&v!5_9rkbH#}%mv>%)&K;LMQo`+G>Dpp<$nhxok^pM|oj!VVClV_wb$ISIxIBJiA<%hx8kWk(ZcJEgr(iaiM% znJ0gDkt?#X)6o_|&`}9^_$kixYtyL>Np@ZOF1Y*vSxIj_VVGldYc+kfwOtjhaX6eR zEM4hIKax5Tsmh;fSo~>+FIOLaTlf5A zY&Qp8Zkk#e*JH}xqoFG_UlkdUyyCL)(Hv(uEp5lbRT@5%6_xiW&$e1G%$KYES6~ch zOZM>MQ$|}gdH<{5^obS$S009pu!@JQg=O{5)Ph3gEE@x|0dmH{|Hsj}_%r$cZ~UIg zoSH+_WOFX34oXH+y?uxfY@8P2P7X{t< zggdXYPP!C9s+3^5`Qi8HQEr`#td0PH$pqr?WA!8qnru5iwG%E|6YgM>AFp(7d8q94 zK~b(kaM}&`F2ApGe+M1HOAN>fx+Gk;(-Zt}rX~iK=A_@Ah0@*W%x-JBk0@7ElIiHJ zKCAcjrWE@`fsGe@bN6WE4uVMPc>Z%g$?j0&{w#a2H{DmZ96X;J27T!mQg5vHGa1+F~HFyc#Z}dIfWYsc_b`pYckY#ZC0`(ii5H}iJ_TeGe;iqEjbAc5#Es|eei;^Iug=EA8d_zk z%RZYNL%b5NIb654m|{r~=lUGw#8_&4zKxLX9jNM1aQe8~%$=y0@+!yA`@Aw&wCuc( zxSweBaQViuB0V}-7p(;i3l_uS?jz0(L603$mq(f#-5j^BbhfiNLK4^0)UQcD9{k(g z>R$4EbK-3?Tg(7ggiqxb1b!}3>{<5{n73$e(JPa3=iFjds(BmepOY;-QnJ+GU=MNjvm(VWrBA|qqX{9 zMEJx{+0sKI1HPB8pD(8u@M`lcyJ3}W<@ zp`9N6lE^)q#@xKQiE;!8l8y{=4=^Hv*oV$9 z3yH#z3!3*dQGY5)zD?^|9TCsOBy6CNoV%bafiPerB-l*)Eg#(`>svr&5@GrqpYT~e zLai~z{!fdyj(<6mooOyW^Z(!9`g8l*7$iMRFG5c7yqzrRz;;8-z04!caZ^GU_m1l8`C zpt9-SIMVa%Q^&`-ds^V3gyuHJ824FH$jU|UxhN|l zT0$hR$?*K&x`(AyM4_fVfte5RSvO)JXz4OOYvwza+*HPwh4ZPmy2zTk-aGd>`(t$W z<|w@uso*&!KsvF~SMcHXftgs*wLc$jak6nOPp3)VjMAUw3~2XB8@h1m*E?^Q*}H$K zrtOf>kHJIzU(dIui1T&yAZO`4a}kpFUh~(s%hnf5aj)2W%hT!EzNfx?_v;=xb#d!7 zA^!$pb!%nEZ!wW!>d9Jb14?|hUXooaZEtqO(}Gw*xvzDEz44-I-*clnVGz=@+-yf# zu)k=mVaf127q-f+?=Sj<1NcH)Mrz!Q{)e8UIMR=O*6z3(aHc9%H1?^U-<+M*J_?or zC=QZu4FyGO5Gd!?+azqkJdm`gh)Bb?8uV>$Jcil8P}RB5H-nqGSG|MoU>(kUb*K#x zFYM-BaXMfJ?XP0aUbzVbl10^TX@Kr0_i*sP#2Rjq*yXu(DW`%bZn~gmZ32bx9G^$W zHvz7eE}eAaB)G_IP?F6BjXn ziZ`$#-f|Wl9U|7jpCQyAo>m!3SGCT@!(QO{=O;P-^fLN%Ur(-TkyxkGe-7Jpo0>bC zQ60S{hl@Jl7}l2=G;~3J=Mv#?+BlfTm)8n{V3CW1i4$r=qYln0=kPUI@X*SGA=+A; z9O}`eV9-MST%D48l4CV*AQB=fw71@N7pk!>A0^!r6eemE*~!YJv7#-g7c6)G?B7?D z0d^`uzJ*--Jj5)oKb@zcl}(F7RAf8yc(FH%)mx|6M44l8D%Ec`(v;x4n!5jaik5a+ zR7F=@h!cCu!E=}owY;iZGYc~%o9gI{MMC@)9w^xfWOlI&i#DoW^qkz_K((J`*J23o zOP_aPWRJ3yZDR`tJov^B1!^2~;5M zB7#k$hU^8?GiM6gWcd5+1kh)$d1X5!HmN1S$-?7fC{eXWIOZXq-SEESDnXp&nc(m@JjRKer?S@(2Ge#WVqa`@5OC`tkz!@bv9vjh3^8qZ7yC z!?+i_LLtGYl>lYQsUn+8M5}e z6c0Yivz%;?w$NMnbq#krJ5pLD@GVunJz6oQ4Y#a;9l=n`^1rq9%w*7Na?~`h)$)+- zS|jpK;)zKYU;XORVE59(rUrej`x}UM&A;!eN!dkZ$Kt>3AY2$sG(xiddR&AOph14w zAci=It(OFQrWfSz6b_*&4D7CUv)4z>Esypws6_s2i3rSd({_S3ho0?Zt8B57pf9J08gvSpVG!$^3I+&fRF%tMa+12SHy@WKPdA=xVSDow z_*rABY8)~srZZk@x0Z)fbel{9;Gmn_FQr<;hf;wLULB@G^sW-V+~;5Zw(JP{y;ba* zi$yPhrVZQUN+FE9i+3#?E-Nvh3w~B6V2(pvl=7}#xeV5==K~xzv)`gyq(=@1@Xb@$ zUcQWwfwE0v6y8KYo9`28iA5AfZBSzLrXx!taVkEYIs3U^8Ja(2ENgteZjXXpxRP6b z{NOJK^k7{(QET(Y%XEHW&Mw(`j5q3LkT(PYWzv9KBn#U?`C|)sa(FL2e!|j~G&Hj^ zVmf#IO=CrF73HSm&xecohGRDXz!7kODl=ea&kYjd&ID)^PkBBXOj!x(`o8&C>tu!3 z@QO)Lf~Ft>XT2;-OZpdE{m&2Fo0<^o|5Y2{M35`pXZq$dlfF{s{DtMl^(f`nA@zZ{ZP>Gc4c+CLsfe!n(&dwbDqbh1d= zMyG38?0W6IOm-1Oy7F(FgBLY*D<>T5w?sdVQJs<3Jf4oFUw5(2_K+<5R%uQ&cwTK< z3Ws8v>`*=4hVB$>u~WvxU9qrHSE0t37Pdeh(X}P^N3Sg*1POcc*S!+$&37v!Gl*-7 z=`G{0YWd$i`&=sD$(@$|>?Xd~Pc(I|Dm`kNi&1?nY~xMwxRkx~2k%d5NDEMSdoVmG zIyoZU%&hj)2mcqJy}uGlm*n{GFT1RL2<^Wvs{_nBxQquN^iI)F81{e6aHw`?SH5cD zy5rk^!-j@GMxnoZl8Mv-4^;&9=Y6Ahr%VBVU0PpX8cD3hrxA`LXT3Q@cNctnU!^*? z+`XuKqxhV5Reb5=cxNUpIDq*6*Cuhlz8Z45DlcNBjQnb*V7%W_EsP%i@(kD+{#Fq_ zBrTy{w=HTEd{$9hQ7f;bh`zow1tsDJbi<8E}rKMgo~_u?BD zJv&w?9W|&g8X3xyiJhL*R7hcla|Fc5dmn?v%)#H)B)Sorzeh zny{+S`atg*!wsds4Yis;%lY^KuCGw-@rv|J-ip7CG0HCqV*LLB&JSPqYe(FgxW5#? zt{%AtEcdE{hV#Z++jHfZMa8Py)Dh`CIz(F!L0;#bTry1cTr4axnx8lMPu zyH|Rv9W_RNIA-r>ko^m@!Rd4(0iZC@VwRFG6s>s{6Z%oRchIN)7HjmePM*X`P@tE52 zHL}%Ri(|@a({GzXZ}Lply2tYW;J9D=NqXYGS#NE(LlL)K}DBUBj!N zC;vn%ed-B$;|bZvspqw{PN2)cWP{b1#p_Qm^B}*!SRzATWt<>;ZCg__uQdiT{*Ww&De>RI$%Pf$ zJZXkkG}-^lwHE|z?&T^B6S=j!4FIEaB`7d$&jn2ewY#rmp>T~VA+cmG~(%v5b z8~&a*6XFo+@ykWdyFaG&DC%td^14+JJ|Nxn#iiyUTSa@`ETX1qx`hph9zH7QsYXee z7T^8X?w6Ohm>M$juK0_v`rP;KR*LgI)4Vplaqi?U$1Rq-8n*hBLCp7t z^p)WmjMZ_Q**W~EFF1X<%`c7fz|`YO^|Hl_!U(+|fewp_BZ56&S2>!GEW){4p18OD z_&b;UMn&m)qV*d)b37=5L-pd1XV*T*s(raypPyW8r{iFi8F^D;yf7RiX{SLlO+Ds;|Frgg^qu2 zrQukAOS3L$_VlUeDJ)DpInF9^-z~T)41*Jn20o6#RQ+?EB_G^l>+#Ya#uKND0d6e_ z^&Te*vX3ta!r3)#v6E={yG@Hk^_C+K~V1u55W2JV2T53xr^u6(+zxQY#(#SQ^7oz`gr#@Bsc&R4hD z$&df;(Olj{6$(w3^KMo1cVHh~)-b=aw}4WVY`JVkyh$<%kJDAZrsjST!+--(b9H(T z*A3XZVkonO6CaJ&B9#+b=Tx7!5*#)ZiW~V%2_-xof5#*F>R@z6=;N3aZncLRL>JJj z(VuJL^PM!Fy5?udSyB+u%?L)3PzmDv<2qrb1;9k%iK&2nQPiYE$LXR1mr#dsB8dr- zalEAYIvm1C^44inZ6eQKkAVCE<1t)f8kU|xq&4y_y?hX&lZsq64RrY#S4-!tw_fY} zh8-@xpW1OZTeD{W(iraWE0sq~Xfb~DG3 zzlyy@-LuxdYd^L2mda=$5w@gEdCIt_MyZWYH*>6hlfrAn9OgY=juhXq-Nxkj-4+${ zWc>zrMjV;MO+;A|1lx1W{*@%KZfJ!RP{f|gwEM(5bYFtnUs7v5bBmLc7HdG?ex?6N z=B!}nW(K$+&wWhF#Y)nw_BuOr##i5SZ9pI}xIb^UgHyrPKa+2|e!T7R+XGLA(erQ* z*!c6$*2Itg^DSt#ziA@ggS-MPPp5Y!JBTs80x~wjdWXMsGUK$GqOKKnAi=O3m!bZg zfeRX^p;d|+EpSCp2YtoY4PuWEbbl%EhR)sUW6-=vka6e>nekG-7$g&$k6_Zmo(2!J z=hTRDC8K?Y;74@*Pn+I47E-NdZem5*6&MtBR2r&`i!0J7&Oq(5Fd=XBDM*BN@p2{ zHwnh&f@1Pqm-~#5A(>*QX-N)X@HV_g#P|$P#7h{MdyH_U}QI{?5 zmbf6PLszhkg2L&A+rNM*Z-{%e4VB~!&QXwy2n#&mBA77Tf{3oe9R?z@t{JzSH>-I{MH!@80`;g~l?E8)ov{+8=d*#hNauFyPM+_gelsNOeR@l= zghcB(!qHWm9ib3co_cUScq5l4!H6$031~yYI$38qm1Eu52$6KV zcF5TqHf1gBK|aIDR^@Bb(QGFA-ZaJ%&(oy8k(xrBY6>=yJ)n*M=#yzA)>Xci06mE1FDr z!~J0UEZ6tYNocLW?lp}CHzjSx$XNkHo{N%WT&FypBmm41fV82nMvoem1I+z{tx(&S@Ck=0IrWVvKJU1?(S^per&HWWIHMRg(_=(*{OPeO(tjXo}D z%khDjUnNX)287+*G>=}imF)Ler^M`Qz2jPQN!fLLv@PmDC}8?#O66EF&p0R-tUBPH z??|TolYW5rhI;$yN7{JF@W%cwOWl7gRwlNejUqFZCyCxD7>gv&cx}z+NC$ex_Vh-p zb9|S7F7#coCQYEW+~2-T%+j=ti{Ni`c|vI40)2R6?z_-I(aax#{^GUW2U1Y;X=3jAhmX>Aqh^%Z^z|=pKb#GU8hmXu4uL zTJ^r{Y0t*&Pll4on%5sh;ZpgB>C4l@)s^d`c&cBff@KiLkL#Q|IkhRS11Wt$CsQ2s z4V%u+&-*Io)ulDWafL$*>3~)jNN#_dabqdl<=`~k-D1(VgG)V6cSpDcqf(CzJO5li zWy$-eV>=;gbu1r!!|%bCts7@+N3GcN&rPX|D~*)ouydEHIcDy}D2_WlN;w_3I(5so z^X(Mc9B9;C`+O^A)cq}uSQVeY$clJHp>u5vuYS6|J8>hi()yw3b{FlGC7j9@7O!fW z-O8{PF*n=$DE3cCh;;f;7$volGvkq%t@r@QB-BBEwxpV_FfBj)bdeYr`|%BarH=12 z+E7DV4EAHyW^Kas%J-OJdun@K@4f4_2e-8;07+Mhz+~sCClTPvZvqV>)!A{#L-e} z1qP}8@KK^fH`ZD&%>GmqwQx?V zJkK-;q+>32TNuH@CKm-cF&C!1+*wJ-^PsCfdg5IfLtFF%}-LAG&NEs=*iK zs~0`^Mg{H%{s(X!g_?C7gR z*;cvJdy9k$=W|TqXUo{meBoioM!&tvPx!B&9y zc|U`yK_PcOedB2Y!wM|*lj2y}vD+O zN2DK%-=bnbNLH`LV!c%B!ZmF8Xq4cIw{-4ZAhRR zqt&uSr%mfHgGqGV$man(NLd~b2EO=^2&YxzdwL!~K?kEJxUpEEC*)Pq+ApimJTVyw z5Q_rp;D%o&yX=rm@A7?4E99ZAJZT4A*c#+DbuSL|+bfVK7_}$dZPR*JzQd^T+8y7W zf_8$g6a{CXA$SrrO#euJO()v_IppC#aJ4IZk_I`R$-N_A zW#v@XUvzuo+<=fS`k-4`#g#WonCwmrW93EJyf^FXz4OeTO%ta=GFyJkEIP;N^VVj* zBqTn7=^8#w*6_Sg?dx#H|A30=K9fD475IRB+vis@mWdr|X}cohxou+b?R2;PH-?w2 zxG@ljDBldpcM)8`H_AahF8GJCi5eqSyT`<3K~Rw4bi}9gSH%T>e^Qt{n;E>BIG%ml z%k||}0=CcRuJP|^%OFwtmDiPu9JVs9oi5$=L}p5&^%MI%L0Nh~?A{q8UDMq*fzC;I z7J@JJ%9xr-#)+nywBKW9E-gbF3)0(HIvNSuC_1$8@ffs|ga5%NJb zL8l!c>zTSHgIyQfIot*mCqgNV0vrmtzs;M>ArWAY6RC#S6w+S@*{ixOj`3+Hi$8`k z4x79!MOA$kcZ#{(GOK6*#;W?}ME|~7V4O%i-qbGNmY$`#Z#=|eP0_+ zFY>xWmFpEzl-WYC>DlxiM|&yndI87b0dZF%WHunQ$VO_rFOqXRzMm0`D~jWtQpr9R5z{mIx1$yCsT7w&Vh z-qc~jnAfxK9+`dcBfw_`cBRO3)icz*P|07>vuD4P#Sx1VwiXP$`swywIyZy}&dCb_ z`+0bxauBOx^@>ji<)ZPM#4@Pc4+n@!qr3sDABjLAzAQPM$=3Ok=-Zh{%!;+0rWK<^ zZlC$;(QUNm$(p(?b4+$DH4#W!Qcr5s{ zVNA58W9mjA+Ca|e9gm+9Xu1DcSa#O0wz%7#Y+{C&oI@|trrt^vmd^bQ`M@^BB{k1v#c z&dKrbQULj|O~)fuzQ#{SRf?_-^n@mD0F=lc8SN@a0tX+2uzA!c!t*-&p==5q`obsO znn6mpT63|Z69TT;DZ0x4Dav$EVEW;lj+#${8L1l7uDyt157&#R*J7b_Y5lqzg~p-Y zw>VDA^3svKF6FJUTawG3{#;DTKwQ$A1H#+t`PRKWhwXTCInk!kT7&)m7cSg(Wk-cF5 zvgSK^FEMS|fiv569-AbEZ?1+)FDUwLCYG% z<7_Xi`wXYTB)h8R+hqrfp(A}~*jwZzm;Rg;0Y$WWf1~VYzPSt&(CHmP7r@>gzMk## zIG^(2yNUcTft?2LGr!|3UIJESrPELbQ(4((-eKVk7=)w>m|Ux11P*^td#PfA2dBsu zwYaW3NzFvHA!XZFCy96-fCe*)4ocMpjBBvs%WZ6u=Bz5u_Hz>x3=1+K++Aj65P_t! zHRo&m+w=#wlxLGa=O1)IB}mO+jZg0gua9A^QJqh@FLePSVKI@WOfUlX!0q6?JBtP? z(H>p299b(8r{3zR@J)9xvJ;Y?R}83DBwz{n9#?GTGqiGIZ?Qt<^Fl&Gpg_aotq3nI z=2gsK1%eKoR1M?Id5r3rbe_iFJ!GSON9J^*V6cnh!N;yV~Ao4vO3Cz?0jB=BdF#Rnp&#+j3>o;m^! z0xBLm-HN&(M(HCuRE{oIyClB?BVj8N6_3+~MT0h65kClt2meexQ_hR)wMVp0xD zZ+|I}TWHgfCl;V$^*A&6CUC|kS;<)(zTJkwCh*XBByD;;COPS@H4&C0QgNqQdOj${p!*IBf$IZ{ww`^Wu@FqctVG7W2PW z+b5MY@Fe4`Q55N3=alOitGBEu)zHS_itATI9yf`jw*tQW2=-X<)ptYnguWN$U?tw{X+}6Yg|AcyhV5%3pzYbbhP8m5o zg;S$@j2F7IjIAh7eLbfAM4*8;KKf*GeS8Io!=u-)?NH2dvEgKqiZ7M+8nT&}YQ<8! zj8-02TGaiRbjcDHBqG82ye1DAad+D4Z%it_CHp?+ZW8>!de<`X&QuOUr?)iUtMTUV z2TDj+U-7xyb_{B@g=DQuwXZ0_@DcZMxhQv8!0eVTpY)uaGeA#(&7JW$)7?Epy{|FK z!Z}+bsycvR>CqMT4y69?+8(@Fn3uyWu1T@@HaT)7i38hxX8#mbvWeyjCi!0B70sfx zh(Ui>-j`2zFC|47ooNr4CymRi**G$&4Ao{|bDEqBE6*|FP4Xjmq0srU zI>OOW=44Is)|X`8DLpE`WuNg0sdwLqB&cr|jC}LaA9YY!WtuSNPJeruf6x^izFk3KgCh{#N+->?vT&=j)U*TgfECk=&lc^DQpx}xTX6v7iZZ;)|4G#2U4%9 z`E1Q@E`N(^dDjthHFuHm>rc`nASx!giynos4rp}lAS_TlF zxf+K^d)aow-J%$me_Nas4ui@97wUdtM)=^_9+iQ)N5M&_J<&*UI)7Z~1!cR}(`})~ zE7W%i8{fN=>CxZAN+HUCGBwH`tVB=U(v5_2@R5TU`OW^`Iy*{Dt-d-W!OSFIlPinC zwSx!|8mhs4Go1!^lYU9vSN#`m5cL~$tr*87W8gN1qt4N>65%&nt&V9f2xT9WJ0I#S zZX7n5(>eA%;paMJ08eImDa{?0cEs1Tl*}2im^xZDQhk<~^rs|4{AyP(vss6cL8zU| zX+#PaOM8EB)dMT5FabFfBf*r>=^Y)j5unh zq-M+0f{8wNIho!hQ;rLDcC*(#Aiq}2xaUF_RQ-%~tL~g#CWl#nc=E+-bYes_w&d{0 zq#S&C#eYG|MX$r7T=wES{*M^=R+xdSjxKbeV@V#L8gF_v*`6cAom;(kE04JuG*U22 z3%{jX#P|Jihp6edYoOhZf(S=1B#gS*riRpirB^qZ7jgg4H2I!}y-!_`gMyZnv_|E? z5W!KAlLp~~ET8fekPd5M;%gN4$LVtbJB}JYQpPK+HBolQ_sF?HE@(l!+_}kD6v0X) z`O@Llm2)Qu5^Aaiq4z2yL`Ph+j-=RQCA9;XSWBdz;hKt(Ma%`*ain% zY|CN)#S2J)DBo?hPV)a>45ss9c&#Fbcn;V=)=~1Dw5J3!cs&Sv8YUIDJ3FAo+oY-L z39%yb7xTFR7Yg#TxDo;L^9DCjE zE3)-FeCq=rc->k3-Ql9Xl;14-mX_=gZ7NSZk$ek(hv(iQ54-c$r=C6G>JELKqIxX< z21p0f>>n_ zLK}OMUQC(?P#M$wO(baGFF~naEbZ8@t2n|`BRH(pp01ssmy8lh$88w2kpCb2VR1U_ z_fN-QnzE87Im%GuARF(%Hpw1uEtLVsWPp~jxU8KwuNh2EV}#~LD1G@;w?05rb{0p| z85!Fd4z))<7$znkqHZr$L)G~>6&;%coL7ZL;MjdgeHw5sO*j-_cv}M;hv`XBg0`16 zWuSRaYn!uNg_3QtL#!M%bg)?ev;u(h=Ac{hj4O%SVfrTAg(!+IPi@EIU@9Qr3_dg_ ztDYdU5(h~GrOiUbC>&=56x3Urh2k?o$ABX^GP41xNI{KN3>#rL$D;+Prg!V=wxfDN zzJ{7*;Krom*IaKnWkBsSR!wx)TB@Y#n3)ksu|bVXH(#Pw6bFFgym@Jq&ojlC;!Md- zdlnmq20`o|B~D{`c~M{Pf{HOpyjVQ-*7M`-<*7{RB;=|^2V;$0a41ivpQ2A(0AdNf z+UPGZX>{)cI8-J~%Hu{YM-uMa>T+b83;?>qAxa49``K*oWq@7Bcq*o8dj^>RlZqaq zqz%^#tUfiWM}5jv^iltT(eF37Z2Gy$uT2RjJCsOXzB8RSOrQ>?NnC9Jd){|J2kh+S zAbZ6ZV}B$j;_5mVqcPs0YXHSZG1E$SQsH2N|P~C1Gz(_XD zUWw4$Le*~ zM>ea~c7tj5D5nWBUo)dG4}l^hT?3<`&xn3+L#k54qg@l(+q6yF&PW!y-S|=TR&`5f z+*L+{ypu7@@h)*f!2$lbD)MD77jz&%wM2~d^#>GeJvIIa;OA3IB3qcNylb55w(_~{ ztdNKzP7f=tDh%OXi^2**{9j-u( z!ww#^kUV+0DCur*EhI4GK1*&__SoeyX{~akYG}Ypu&GW*8+s@bWq5GjHQCsch8%&} zUJwh4{<6fvn9R4kx&^deSquLN4(PBd zo8RLlII{+-Iyc-d;muGSdypj4%rh;FzI#U`QV2Bu!R!l5!!HqI0t4MI z7yP@9^DD*7$-qzg+>&H&-(k9`)P2WJcW`_=gxBQBGxg??YH3#rJ3ow;QTR?V62;0C zs5;cOYpSEZUO4Y$DGopzt(QDkcwsB>coJ~e^lk6KMIoOh**nR_R)-0G&es(` z@-&>|E8JR?j%mLk7Uo~<>$6B;ks#sQL4|`>3QxLxb_yQ(_s!=Fcqzd?jB-tELJu-WO_+shagh-Bh;YCZkI!0rw&L^K~x=vEDdFAzx*b81^!CN;z1#nGcd1oURwMGcQ)%hg#Qdmb+gu z)N!6Gjq)$t_-Afh>pCMg)@}T>m+^#t2l}4m;glZ}tzN3?SC#5JQAPXci#sa`i$t~2 zHuoDzFHh_#9L9GQk^DG88?%23@J+d%^khD9*bdK)aB~pmC~Y@)ul4yS5fnwqO|#OT zB@Wvc7735ug9f~xc06RQ39aMlEt4^7ybk$aGJZH~LZ}HT76(Br=8neKO2AZ)2IgYD z=3l3H5C@u>kEp2+%L{8AZ(ueSB}0@dpc=-z}*U-RHY6dw)SK1sm`A z(%iT?9EL0Dd#Yq=V}p1cJf^5`d)?$RrR$C-z|N%&pHH@H*MW)G7MWmJM3MBmgP*ag zm(?oIBD0OR{89Qo(^7%Qq3Jy&4@aQ_8l+b2L~5ZLIVmDW2joh%u5K6ZqEO42p+qN9 zH}X2iV99QdVp&bca{R(?kH42>W+ljc=*6?Omvn4646Q9NsZL=P=RKw#KGkz;gE2}j zkMOz|c}!yQi@G9*co=Xr^TthF8m)I~qUKe%JCi!RRdkN?5ZknDVImXFyrz6KK#gDm zp~=N8Ry<$2ig+A1bax=tlFQK1Cb9;Bn%Jfn2YPD{rKhorwoj9VW0LGj5E65@Yk16w zKwbMhL_F_cP&J~J7_g8vvS=xwsL2W)2r_%hbS@CB+bP(U7jI=?8LDaas#M(bW_n6^ zswv-e^^I|~auBxde!v9aPVeO1Kdo$g^W<;16vW1V=3NCQT}rCz>ua9U$zW*u%h@nJ z&kHJgvnB4d1*!eF{>fHNK+xXm@uF%1bZ#Q?v^@;!id2jAxoqa*%|HQnHrWi`7P+R= zAqo@CyWtT(^!lQzKw!IM46`@0&G`lEPveQ*#U1RJGA(_=CUIwwoB23TD4h7sf2T;A ztyQm)5s+W*HEcODsnip^Bp;rD4v}zc?&)9;2AXZ4x@;5MxrL>CKhLj6(hU~Yc`o}D z*%hB~LbOSQJdRm>-Jkwe1$F(4m(ppBR`Api2?4iExFi`Lly33J#u`sU^T2q$3mGU#-k)Zsl4 z!=sv6ny0LBVa4twfNF-R)1u$kVC(xF7IhdM9AJ$Y`#C65>U-JgAPJXyrKxdR|X+|@50 z;wX8Ar1zz2SHV^|=3OYU#lhNX@O>WR^gZ1pzfU;mXg8QWKqMNf)9wQ7f5P{=sW=`P zffk3OIc!}z5waLa18iLrv7I&q;Qdw-acIMn*)OKO-^l~Dp9NqDU7 z29dcZV0|ZRlHPrz3BahR^~HfC4`wrhV?-^uW;n5ND~s<~IZTS_!%cz*3WAOiYS*HB z2AcoX9qnY(%@G6(@96vdnt#|5G|blIB-ze&KK&6M7l6zWqy7XqIgBJFChcH2%i+Y5 zzgh-9(^KK$P3lo@BohozhOZblRZo!y21uK!uf@Hyf@8$pKqGwX%Xi<-2|pub$;)(@ znFLVNhs+2dM!C}vH%ZlLnPkZP`k*>3HNQ>a0#=}$fUn%8i>q0n(v)`kkUwP4&l4(=N^)QX$MIsgNq7>;^uyT*xj-sUUo-O+pZKA@tY3D|8#3tRsegtk zg6K0!emf{Tkoq-O#@Kp08|oB6$b&FCF(`tA^F<$jtW_9J@&q#ifiqE06X%fpU!xuD z$_^{mF;Ks8#RAIi{CkbrpvQR386M;Nnnl119O}5iO_Il16p8jR;SRi{1$3zGa!qP@ zj^+6>7;7?8xQJOLIrigqmF{t!4km0=@haK4tQVp3rsweZW1yBSe?jn>&%uyg<((+o@44CW zFZpIZh(M^S;U-QG>JGI_bBFsIjikErbuGatY8u6iJ$}cR%-4NCMMexf6l7*l=OF2k zok(KMRI-0V-j)?*B$yfU3w54IO)dewjeI8ZheKED!)>RkJ{VkE(9?}Gd&*zeqg`dS z^&jBp&(39>z~h*TP4xWrsnIr>C$2+D(6p(@{stStXF2+HlW8QX-8&9Td8PFj53<%lR$PPfGi1rGI3#w0Z+zZEU8>$D%DGim z>yt&U(26f7PWlbre3OC2>+YF7`sMh0zG4)mvD%dDE+qI4wrsSRS0R4iDsT373!yY; z%XI4V>ch3zu2ee{pH$oKSA~0j%Vn7Nv(Nux=KpRaV^oupj!31E#X;27s*1;XuZEjve zd!MNiSW8$DYB%>Y3gYHZQ>(;fdpH2KMpmb`jZ~fS=XMSs$JS&mj|rULt!Oz+m)&!x zu!lxW{-fJ*-m=TL?!EZe+(^0cv_s`}Xy(_U7*2N-bz@ntQ${{@gs~7`U(+Dwmmu5w zRAacL;p^MS>~bwY_&`!q{Y|3mf+RARDX`j(15$z=PrW*wfI+SkNyD-6Ei$5Y&3xT* zPrQHW6|klMB`c74E5SF4+gs|x(>`_!=)5ik?TI%&+$@IFm^6t}S$ZVTPe|Mp@JVku zrfub8U33?{-`t;FM{B9SQ60Dfdg%9qg_iTs1)M2?#|3lfIcSQmT{D;oq!mlce3gqQ9 z%>WW+?TR3t1^ISZD0<4ce$GkC!mdY`BB_FUPZ)H%B26aA9Ys$t5YVW zxVboaBIR#NW73a6!H{G(*B5Wc0U@rqC*tREk-xswK5QU;2{OJC_@K!`jTnjfH1jPXj|N#2 zg`6JM2(j-`8<$PszEaQ-3j(+uSvXfe#8e^=06>Z7r2kGHqRCDm#H9(ofxgREs4E8A zqR!boTVHb#RmiXZ;2AVvw>nLfkKI01@MX)$up^|~2yW)Vjyy!g92o+S3vPQ$@+aAu zMC-CHJVE?1@IvP3w7@qYn}YN(pc9nsvYax9wNXZRMSKGPR6ajOB<91qokQO9LBHgr zRVt!erwlj7_-sj2#YGG!zr58~g-J_||K!A$-q6EJ!k-56EX>ww3fK_p15PU`0EIo+ zL@G}Aih<#r?sUdl>=!wR>Ck5Sk9Un~Nr0RZJ5J@Of2tv|mz_JH_pb)EBy@C9+XMvn z&ps?0f+{;S0M~=$hk^&tGtl|UZ5V$nW|R_)4j zJfVfLt>w!Cd7oQ%5LI~ZbeICD%-|5)=h3o6oDtufjh93pVp4c-is?8WM@97@Ga%0d z-t&MW$6151}S5Y!OFqxH($fh;`D0~vqQ@uX=Ni{!f07AmD^?hf% zE3Du-F$p+t&-{NKU3)y!`~UyUFvQ%ZCc}zy>*5w0#zy9n>qtd)gmO92Mj^sblrwWL zX_(Bd5{e?JDQvoILO5yZE3)MNl3x1#VnAGov{C&6~1Gyby>J&Gg86(dN3ABx`@5JMo5wz86MJK6ePr~IO zqGGF+0a70uuzCY(d6!FWh|&}p;c+wjP-Um80E$y<`F*d4`Ih0=)g7Ok>1N_vJCUk7 z4@`A5$!5Kn+3?=44SIPOdgbl$I6E5;GljT_e%zyj)`oU?ii=CfO>q%fe0a=w*|F~< z*&kfQ%~u{H^dSy14Cx{>cfR}?!&3YcBCqs??!lL*hYmHNc5ZxaNuQe2f-yjL5NG6^ zEmw+8!bLDVLN8y``^%bEdldedt8}iqQ&|f$;l=2Z7Af4M^p!;C#gj^ZDH+~c6FW%z z*YA|r>E(r!WwiOx%AR#@Uz4Sm23v-d-!Lc^d~B$gD~fPPz;(Bv06Ei_#zwMD#*h3 z=F-X=NNwVSuA+g(eWlS#;!4jXX7oILYVbr?l~?icU-u5)RPmIb<|Cv(n*0Qi!hS@N zO+rk4S^Yc#?75S^@m8oL|NO6`z~nN=vZZqOOYM4iX^qQ3`7@yD5O&UM5OBZ>Umm=_ z<~2D`#TFu}tJ@|+Kd{A+B590{>ZM!4E_m=mC6poqwNf=f;4AEw$;e$MN@{5(nR%uw z?$(5)5!1VFd`!Cs3xBy+9HW+}^8B85MMjwe5x};e@bV@4p|sCa`WOh`6cD;K1BKz! z5asltBqBuM4kmR(R*r1;uzg3O*UQ*Eve7JF&g?QqlR;mRAUF5Fz2JIr*++vZ=&40K zn)u@;s&+bzOvD2j%PU(|pCu*s1fyy4)1W;5r2U8Q-gU-prk1Qec*kGpY!p2J=p z(ta+CA?21e4{&&9o1TjuWl{2-6^6I)k6>MCUXitDlw=s@_jxY%eB+dXpUN7#hJ=OB zXY>n)ukx#p$C&*LBRhIZwhpY%uFB4kW<=tpFf?W=FKCArMIo|USHT+}==4^!rQ z6S6vDJWaA+x`+Ezd7ZfG=J-;h2HS32a~BebMA5~326vHLD2ep)f<+BV#)B(JR<`fh zrjpa~P>B6_X5961Is1)Nu(*|{d4t{Bg69?`uXHmgI&h41)7dQW9S4~!8 zILN672fdtU5;nhv+n0`tOvqt5)pipPfAI)mZ|<`W9K7!V>se+7R;qhH>yr6B08?(o ziS2uV)yrs^o+G$iyx|*g^hE?g+8#}VH(1?Ee2LDuSUGAUdSn>dwb^y-SZ})Z+=i^` z?J_b=Byn}T+EV*9XZXFv<|^yzMKLZCp>)-*jzX1%^%d5?1maJkxz1k*^G>iS04f3JKJGagY%6KKzf0TWr`9UP4xxRB$vyPA{ijMuGe zmPMqBa)ov$PlfBpsD3Lu`1E$}Q=*1fm`4sz=e_03ct ztJ4qaMCvSR!T=Tqg9HH*0e#P>?WhRHeZCU$Oa*O+r2(5gnt@kHSgjS7*3=NsyeD^$ zxk2Z&V_m={*WBV}PxBw9Ff7-+#rzrw{kb7(HFYq;vaKFnu;ZO@$O7I)|jVfzVE7;an{dbjm)?lRn z&8VFaKi!>m67PuyU?_O+>y?G!Q?(Hi2>7?mvIQbWSf=^|5Oer5PJg|^l9td?&u3Dv zquuP4KV-wsx``~?YKWudY1`3dI0|0EQ?VHZatO=t?ciX9oIZX_8xq6{9r!T5SUw-! z56c;(qS`7y(7IJ;^3U6Mz;zyd*1DlygLsCL3#ZKCCDr9~R>yx;9O+I(!tE-&_n6zW zr`a6uzx6@*2fYSHfbHE`_?yDBQ=KNZ35s03tsU!}hatVI<+skL_)iGSgG7kd6TX2k zOh*tptHB2ZSG${|JN;ahg$UrfFQ`z|N&})fVJo-lz7Y8C1Pn?(YZlKllkO79j>+K} z6iBEd1AETM%vUgb8iWH# zW>T1GJ7Z~L`q|0UNgIdfQwv4dGA>T1`YuKav-6AjZVm_igqBv!88~+AWZrt6<~S{C zd~3(^ve*VrJf6}0SmBR^5EZJt^YG>q?_x^bc>H%|zSI0PDIn~BKyaK4@m3cK@^LYT zBqmjEO>|J+4l;()%eaDZ5gNn)dKhf++W2VJ^#q1mp;=nEkcnwOp7)ilPw9$Rtz5{N ztyIlmW2SI_C4d=ha?u$iyXjMIv4h5Rs&fL@9#vH!Gwd$msJC7ZV|^#|7Ux z=g6id{&w4Qs@C03{d+q?T3ZzKr(ApZ_#Pf%Bt9%_O#MSLy`NPp9d+;?cx_{hsb*^r z>1I)L_NPT43oZt41j^=hbc4PmumtId-4&tzCuRh{Aq0B-&g*tp&Cg`ZqTUB|6JeUQ!vv$ttibD3qj)UI+^{mVX>}^I`Hub`+G@L@ z12VhiS#b8F1CLq zk3X9!E0R-vygLXz@-C~_aLF=Po);dU+mKbwt^JwLx1ssn^c*<7JV!#&6_wX32Ti=d zl^rjG-WFg!6Q`#V_Bdt=WLON4t8r#!UVlV-CL&JL_+Ho3a6F+egCPWyI>1F{_A%+m zk_T%1saZnKV2PAbVJUV6)WjL5^&EPN=U`W+3T7TFqjQRyRF^$ERSZV$1y@%qe@P75 zL75NQK|o+vf^P6_g995}4xwkZk7b~bR=~3GrY1YUbkVANlOer8S_>TF29Gdz7R-);AHeWpIn}^ZGGJky`tqr(vf5GM0;7Mz!t(` zhSU9^XZGy(dwa8${;>Ri=t_p>Q^n0FX`kB*bhMhX`&7h#Dz>sY9`2o&JC|yu$kBZ7 zzQ1og6(`^#+&>4kV>L9&uU8#l(UKLFe8UaFu%tV4Be2i?T~c3T&z77UhBMV#LX@iq zCe|!6?LaE{{6De<#MpP90SfU+_fI)Lz9My9wR2@0XSV2WOgNo!luF62M^p@#|I)E; zFAB6jUf>}V0D)h@n!)26HO=(= zshS9W-+fK{;Y2Qrq8Gp1|r>5@Zu1Gprc|9#d;zobK4_Yb?`P=ySy>;13tV-NGtbUWwQAg*f~FrP-U z9f5s!4q_-E6o23!8(Pm8K8lH8T;ne>-+kB>5CEn@IUmfMKW5rn8S`aF>7wr`v0B)7 zhsmQbU+i4ky$=JuA66){9v0{&ahy*19=24%=+r&kE7rL^`_WlcD{q97pt#J5>nim) zgW{g&OSU~he<&Qb$+q;iHVIFTS}iLZbiAjP<`wX=N;*WO_Go|Q3Nz)eJQYxC&5~k< z$}fHsp_W`gdZB zOeXVQ2Ax9HRvmd_AZKYKCypIUB9m1r)Q{NW{JnU)Vpk9ocO)bp=VXknOmB2)%;agN zx4T;99Wk_Ib_XcnXNO7IP3&VoQ36T5B=aS07F5~?XXnz}0)th=01{m6Pout67?9j0 zLrcKS)fcstlkjJOr0YrcLF}^Q1gfmGOnd6Yqw~aP^-K67MbAx9(4MNje;PBi_VkJJdYZW0?W|W zEK0F^cNXL4_B^2R(j700HvweB>YF&2*aVSYh1yI5w zBm1XM$VMDD0&{{UBFXym`YEuGHJFrJFhB<2up0$IerGb(@^LzZ11HqkQh&>#c3U|m zcuTgl9m2Db8@0~p4NWOT4m~Jy9=p!QRRQPNyEhqK`?zJbPhJOF{S`uynOs!+P8Sg# zEa0h3o(I(LS7@7CXBvF#OZi**tee3{RI&Fi5f>hrjLT5(b$bsLGvI{m`hNTJwyPs= z;p%B7Eu;K8vna;@7n8CKPXGgxcODrn>?Ck2!#2}Rp!B$9JOD`?O2cixitPg1858m} zTmtg1PGtf}`Ccj2B-$-4dTC!SO0|vR^69>DJ;7F4I_fS8?E2KVbGOT)WMAkVqu2k_ zcx|o+5vCFq09x4}I&7uAl3`~;CTL6Afc79vgc=^>j|$gufM$oVWi`TFbXX||e5f$v z<`l|i&%yXgSRWPm;4((+Ze%`(4U|sLIU9;FeFIS4yAQl{6XCu$_n7YqHlYqZKhY6a zhljbhOZz+%(skewOXJE5)AlZVCIuSW5 zdk``Wb}g&r&$@Ht6IwFv^;Fxl$EijO?jL9|S2bWm1;7O{7YSCGubtL1-&^?~RMv)< zN8XCe&4+2aGU#>1ppj-&celyU8;$&y!x+|Rd9p=W=OH6Lf~LGDa@fBDrz7xx>*17h zWN6Pz@0Vh}8Z+-Tm-|6-;8pUG9fB||GX==R&bfbLRcfKs@SxVM_j?^}eSg*oldl*q z8#xnl&{`!(h{MfvYEzoA9NQnd2X8k+c`%aL8BI-SYvqTCMA`9I<(U`q8!ZPA9fz); zjW>Qmz19D>{IEB$IudGMc6#dkmt3l-jy|=YaG;)?y2(pKHwg^Cj;jqi zaOP!Z^iRr;v81gJNm64pTs9==)@^{MHnF#`E2Fs1_*)6{!L!#{z%V^_Z<_sWvGssI zX$_jOJi=6Iz(qj@M@iw!`m;@SO@yGM10Eh$*935)7`6e~Nou`KbcFBbQ4JlqTw z{&>dU@XF1+xGp9{qCpBN?K%m|>GE)>)votdP<3%J)G0>}$kRs~B7Kf`9`d2HBb=g- z|FeW$QNGZZQm|{_p`Y&%e#W7sRJZBM`8;3i-#nwA@L(yme13Okk8ZsqyXgMHXhGI9 z5rXigTHb{cHd-3&++JMS>`~LgODKaaxL!?Xs_os;q))Pd+VG`^Ci9OZG#C3}tB;#q z$fqpZJkFfVEm2m>g*de*FFWuse9YR~j%Q!`FV%Y(oPGPb)6~$mEFK_sgV(=ECutnH<-v#{spMR)aY#y*1=oF)nG-+ z6ga`~?l<_~p5wY<)QSsT>Okx~#8i52nGwO|#Dq9MJ5bv76aW(0$ce=?2$zc%7QsUT z+1q2B&KU!1ZCxgmYUj=~T5$1d=-h`SnS;I=+;p?U8k!t(iqxin*TMjx1L0s+&xKZv zU&k`#4+RwR`@0LGGdH3;Exy}2687cl%&}F?-4M#VHvTfe&e9T!XYtZYZ%9a?Ds1+I z!W>K&tCh^J*x1r>6WxgDFLLcAB`4t!t5$cIqEZMd07XrJ_=3fkyl>q1kgUWV3I{>S z#$zqAid~mX+T+FPErFE&{MiBCuMIsAOX_juW>6vo3F? zT1$c!wxQtg#>+*5ODRv-A#I+i9RfQI%*xok4}OiBNf3@rL40J@hU>=;^DfgUjjdG{ zfCvMwX$6&D4W|sD#(POrHt?H&1D?B2<3E%Q02yCSgEBuNBL?_^UD&pj_*4;I8Nn9? zg^{_CRRUYEQ|}f*H=9cAhc4Cl3gQ0)QmbJ(L;+HSm2_RBVgGKuWofk)pq^fq0`<$y zG%Emc1xOO^2@786j9sSh!LFCh7|I;{-W+(t4YEuYk&Y6U9o}bd2NWohk>Qe^3wW7p zg1hlX993QH!wjh=E4_eQn(@)IePw|&QaSzeJ<8PWK{fLnyo}22L7(DToB5IDYYUFk zNx%VAevGjQHXO?}fVWQOm^l&0%)uTPj>4d<~-Nd|t z2|0Rbu*w##)}3qa7++u3f56p}?dgJmA}w+`8a0lVmoKA*wOhe=%e%^&aAett?iAm) zY#0tZ?Tob$v1>HGMITe$q#Guf!T6QTi2fRHv*LGIYtI(6Xsc=uGn)*Lm``K|$p%lD zzcE^HKAhsYxxCSJ!o*Ww*G}F0&b#Eqq2zXm&cQpHeykVKxL1Dp{FaF3U#sPHot+lN zEvpZcRGJ$VTn}dOXgQtO%`f<*%zz|i$_eMpOiM3-9&CD%tbFiG-t6?8QyCX~xW7Tm zzxceKvgIX+ErMiB^P+)i`oRr#%v(vT#@#0`YQF9DWB9;S95$scr(pwsnE5q~qDNB9 z!~VPm8|+nWbkttE5iMtS05=Dnyz!p><-;%ZbDA6Z`<_#`V*Fcd>z_*-bjYHX%uW{9 zIMOdppsu1ACud~#o6>x%+%w^TDyH&0F7}jMQ+X21fKWApRQ^v% zTnH|W>kwA>=tdE1=W3@-4wuWvu+~J5MEnluz9tHtD~Q!0!7kx+3@0PAdWJ^JPWVW5 zrVswx83rcr&h|(jcyLb$Vt&~UEM26(s_hxLiBI}P=?6A(-$H!`IJrr-3~KVsy~nY({RbYck<8JLt&AbC+0{>T5z5dK07WZuz}dOp?slh zyylAc>o0l}76Dm3gC#+Zg+sA@VTTa~7qO+Z7bkz2=eRdC{h+FCJbsSHd!SWPUH#oV zdm9T`u>LoRv2Z@_X6N4H%&zTL$*yIxaDQx@0^Qz ztT56z-P2b=c7O2#mr;#&k|~M%J-f=3>ONj_`>N-r zsARY|CVKp2S5E{8R`~0&ZXbT?zdyXKd+p}kPG51eqTC)yKU8vm8aM+!6ft2Wb0_|g zD2z)LotReSjID`nq+3(x;m#G7C_J%Wyfla*otPc#_r%!NN7cnp-`pddeWB5+P3&Nf zi1@l1R4R-3zORcMoOgjskyf;LVQW`v}dUdqL8q?XKi?Uj3L><`11A9AuQ z$Ax&6HSDSFICZ$0%${NA6o)17jsLYyI!9rnD_6l-rfS!r=9@sD8@v><_uZR#5IPPetLo)`&IkMLk-<>w~9%v%9T<$HY;MN zyIj{0o}g&|NC~fuM2b8HKXIZ`GV9WMv)ok8^_zjpJHUum5{%PlH_?!B8jV%FE)Jju%u=WQxl^!brXF zEiRfVP=IL|OA8a-)&VKz5@Gdy1QGY>IEe*$?@y!m-a!sJoR26n+f4Ar8z;$r{#HO# zAJNPJvHU_aOy{3%%J1#qlRi>RIpTF=po{z|VZZCFi|xA`kJNhukrmfUT!Jp# zs2!>d2#X2!vS24r6Ixn*jE4(W-UU(Sf^}#qp|8Pyas)$kuoQf*x*$j>wXOTnt#!-t zTLW*kH8#IA&JRWlBEYdFc+V$wQ()Qo&=b?$6IZ>@pb);%oE9vPhzPU@a1*tCP^ney z(mR!Ll?CardYAL|UZ(i<_0ov!r$ho*RBelbhZdRKoj^gHHFH2VK^PH$R(ri%;PQLL zA`Kut;K_rNR@I!1p0t9vWQk`MW9KR)Ckek51RO4cL3@nF(Q*c>pcdG*!uwqQAYt61 z3$k=;IACE})+H#kX+vc{Ezb!;TeVoWw(K}e6C|VqI&Kou*B3aB5a)k2sF-%+I01f| zKWi<+(wPJQp$bsPi|x6Sffi>E4Uo*`b*^d57C*RwecN*Ma zG>fp2`%7|RkTDs#K`*|~6p#P)!ic+Gk(wj+PCCwDJGe$eR2N)OoS86IYWz7mPS>q9 z#q(=idrt5VSH~Nd5gm?Nn#$Lcn?`^Pu(cA41FTVp^|9X^cM6v6KhhV-9FQ)r7Z5Z_@O-AEL z@-ym98azhem!b_73x{Ld(dJRYDZa*Nj>^Dpw6us_1y0*oiT8bKk3|mX)Aj|xLNPzPp(>BW7NWr6#Col0s*&E#e?31`5 zw!GVK?~lZnb)1?O6EAQ;^QBwnX%K1Q?;Z}_)nIlCyYJ^N#HQgePb#s;R5x zc{1H^F@tAW>Hm=z5_F|)6SWO;T2U(Y`?W7ffxJ)II9-HxKwC+ z@M*e)N&3)V8QwNr&eAcBY~+GdV$U&UkHYGr=!xDXv_{ZBG9QGv=T3vO&6+h7Ue@dzpi&*5vfhIfy5CKcOaBCMsShdQsUzZ*S z;1A2uas<_wZiLr4R-E>(yb+}iiArTN?9b7n)vIlZ!)bD;`t_QNj9Hfxo5^gr+8L)@AtoxOD(+GtFPg{q90>$CJ#V2wDI+SZe-U zHNOS{`Rn6JE;q~-GwYPCxOc%0I*^T7fwr8{pN!~;%?>fK**e3vDg;iE+rDpb&Mvzf zD_DA&hQ-hX5U(!=e9-W%ny@#3OW@A^&=!G;|CDQ^iP*0jcU~YPtH5?v-3#=H-e20- z)>dIZxPp%rh`^ll(U2zcleWz`T_vJNjvR;i*oAqcvE%3R5NH#f-N{qY@zPdV-Ty52 zvM9XS2d2tjE?-x#c5YDm$6syRz^1J_8zD_M;7pT_T#Qv>`@MaAHpVk74bTmjIjb$M ze;tHw0rVED$hr z{4%ZBvl2JTrOu=hoTYrE&8Xo<9`nqt z+wC0w!Z2ArBJagQq%;6a0c?HG%G8ougtdAl8KpP#C}Pn`6u_fx%8yqAb(ze8P4E?W|>)(z>|tB zs!zOkokpkhx`o?|5k!}XQ~Xci90sNMzef6>xY%Wf6xbxRq9SIO8WHqneU@t7xyCb> z1}|dM1L5$iyL4z2=u3GUbwm9AXV;`-ACF)k&Ya< zbkZV}b%4Du1tOev$*e|vypt&>2Sp)keD;~@Cia6y%B$P%n(_^X3yaz z#FyeCa#l+9roslS*T$<&7*s_rp9vBxCx=;Uh)?5>eLZrYc0ZwXvP)Aj$2oOarueq( zw2Rcnpsci+TZsb**X?m4$wBnyuQSExlr^Ol>A^u5Rc9bClwAO5FXRw?`UmL*kZ00| z+(vD7Rs0PF)hl>dNr+o>#`$EBs(YY0-c%+?I@hTz2rRHR=tm37L>ZwlYBW=9-|4WA zzSlZg07|UWGB;Hefc1~wlOiaye;VMWW<0-VLRY4p%f`c`mS+nbfg754HH$>PXnj1m zmHl?~uQGI`3uG+U?D@_wvTKbaVCR)y`+)#TMoHqzSglAyZ5b*aRkYKqAQH+S?TBk) z0+!+?$^F?@Ql>|B8#SzMvjlRx;IUJA19h_uvNphFqZv~v2kvOSi6I5jGF-HgGe&cG zIHm5FtP@FxBO3U^8Lf#*lhmB(vb>b&AQia|VyimnB&bYVHe|tJg{h!onVL*+4BZzl zP7XIuaI{5-cFw@mQ@~bIW!`E3tsSefeFyA;OGz8|++DTGG|bk}6N~g*`gco0hn@G| zDOzGLeZvjXq+G`ATOTO1p(wbK^D=LRUs&0Y)bKS&j6y^TSxl|_jvNdY)R zv!0*@yp)gCI~eX3`m(+NEKGNU3~SHC#Gl#8b^m6W>jvt08d)1JBij!HIlcY`fDZEJ z*@$UJFvz%ErzRCCv(TLr4D_qu;jTE!=>4QUWQ#z=o6E(c3)9ZLm_EZOUxd3YTDIo9 zEy6AD!z+88vJ)l_f~)Q346uH_yEJ0!=>LE?ue;eXk}o<8S_fpI7#1&jtt_rdMK&y0 zx9^?wSdP_4<5DEn*aNUoc#~T2x0u#PjhZqHl_+Dp;ITvpre_I^oR-XvD~Emp{^h~R zn5p7}9IK<61=h(=^k$%4lZX#`ZzEo8QS1Hn>WWN99D5CS_ z<)2vgxba?{=_+M_ChztEFqhyrRb^<=@3Z{yJj?pMa;VMTFdvAYDjYK*`=R{Wq4y!H z>$CO*STJhSO*B-+{Nkppi!Au)?fs|de?XZFJBvC{$uP({Afc6rm)>}(md^%H3Z70( zCS-;}=rH;W(<;YBfaNWU+RGB;_hf)UDexU^D-Ar{ucQHR=2x(&9>-M7MbKR@U3j}Q zSkf}-fs6e|nKnU8kBd<&lVE#f4|HmCu{(3zVlSZ0gKWAHF~${Wx!4IesJ*D5#?}2q zy?m0{(NtKbol{c8DQQX-93l{m zH&*%pnP(wIQ>^%|r_ATRBzx-ClNhzH7X;B0qb@P1YZi1JdUB*BR3jTJy2|W&8k%@b z+vjWD=!5;#0bheTkKAie@{Cbm{$kQixEZ)Woqb3?4WYleUpa!ZN8V~h1D3@*uj=-P z`kGRTDjJLy0AkEKQ{p16YDCk&X?kez{(j0jfjj$qd`tJbA0PVE?9%=EFpZBK9AtXI z=G1|XaaD+$N;2(;D+Y8hHb5|QoU_FrH@_w+J6P+x;!l-HC;oMKuTbNzR@xm+r_#_g zVb06`wDj?%jzxao69;*j>_%|;PS|^a2#1pqTVEgiV@>)yTqGhD118BuE*wv5_RPa- zOi9P~mf6rSD^uxd_~FtDN874{UZ9O@P~+d|(vO2tdg^Vf$1LiM!W)hGhX(@S<`MWF zS3hyLg}U5;xM5Ib=(Pu?3yt+`g}gVAl7LlAw;QLZYqx??2QRDj4jLL~Tn$X*JBs=? zp8X#n>%LlLQOjmfwvHD#foUWh)LOadMe7t_*N@!IYh_Z_Q`WC*M%t}{-E(2kN@xS7 z%@Z;SDo3-ff+A%-GZU@h;)3R7Z)lCraA{)Z%bNh7gyyy)w`(@V`t(LapnW>Q|aeTgjX)m7Fi z{`jNtcwYm4nWcKZ-0(OPm%6c2KfZehFG|&yRh7|se}hF?@IWj?!g_2wEie6CW(XI{ zY1$Ks3$9DQNv8`rZ@YRhlq8&950)*c@^#qsAj^s>v|V-Q;ik{rMrX&2SF4<6EU3nT zm#3MyFw9V2+{%nRiqJW4K;2Wx61!P)=J~QODrW3fpr6YQ%$@#vj0(##Rd|iz5she@ zJtroA-2s2|S1TF^ST7{BsNy=-5}lOkZmLS1k96zr88JBf#)Dn{PlMZ@Wwt+qV>s1E zx?BYIS~V?z!Ch0)8zrZG|MW!rStlW~U0^MGcK20^Z8pbP5(rg>o9GSw4|8n6JZkEJ zitY;^tg61w>~iV6@?dXvheqgqo)NzqFjaUBj>*x~W zWB?t%Q|3Si7SSD>LqgKKOcxua{d*I)LyWg4&IdY;XH)~Q;t#Y%mvqzl&L2jbWBHO= z7N9}X?tg$!HMBytN`C$E3xhi9=CUIJFo|Ca2mLqhd?aSmox$*L5FMi;3}l+C)>$3V z9B=D^(8e8wgnPexs2@XDIjc-j+;s$NLI}i4xsJKPM#rbS=_?>w6RNksAW=rC1|4xf zXjjht%lTLU;4#yy?Ksp?X=-=q_Lm%-IOTWoFV=;6u+Xb*HxJ;iFvGm0J+Cb|v`e>9 zC8zP?E)plb0Upa&m5s5+o8y^cz&M@Km*x~8{7=lHtNa3- z7Ogs{yJ-Q7_AUnv{7e0fv&`irt=c)$csLg&L{@GIILJ9_oX+)YzOLm~g+{*8%w>j~ zNcuC?jQ(<4jonA$FYwQ*X>+Z~SXi<%Gq@g>OxrPz`mDo7ax9xz%m(w?(#2Ag;t%cd z#!a^%)Mk7=$w*GN_OlLBGX}gSG$GkRT0k+uIVWclJfwi^D8LT-()+kBaL}emP+&O~ zcfz+Iq%v3{EaZA7JTseY9*LVS8&++DbQrimSnpFf%ytFs?+=Q;D&(z*=$eD2&8N;) zzcjwN1nXl$-vmwf^hc>qp_|iMR&Dd#V`6>!K!X>Rz z65dq@twIQVjD~pr;tkeB&S2izYy|z^&%~ZR9v#4Cx$B=#OAKiG`a{LDlRS@(Y*n@n z6JwxT{NLbsvK_R(!dzeT)1J370|-UGXFIh0Sn|!NHDlVN=l-alyoi+&HwE^JMVF2q zQ%B{hMuubh2j5$lW`}#)@Y2xL_43?vH2e=|ai|13P&I~>*&GGLP#wz1oj8-_mdjeFRfyzs@70B*k2rpr;OH1$~5xE(ii1GJID=dvEseygB=X$Jod0&5untM((2Po3byEs~t;4u-P`&!qZw#q`IHEF+O zm%)U*dwOT6RE3xx<+M5}N$h6Yjbb!x`=M3wZ;#!V!b<^o$fv!}_cCvH%JkFrhGBSi z-3$p`_^7GBGV8v@b83R4=sp0Rid%L#vHww%6dcDNci2J5JT?aaX$bMUumIq1tg&P= z;bvX?V*LlxBskkEc9TSm42VpXUpZt782pIozNh&UrTV_WZfGyrfM$1$R2$XSf$HZ@5VgDBDP1N^yH1{KL*z&FH5LAhhy!J{Z*Gtzqyii8F;0 zkM(dmf*pT~+3ckYBU?E)cCJ3PhrvpAh|lincGdTEJ?Z;!!eHWsQN3UOua2y2qr`>o ziY!Ml>wkv_jE(&-XbOka-Tw=*@=+Rbw1gRyC@HHwEWdcJgQz@2i_epvWc&}f zcAMo~_|$F!y>a`p@5+3L0@eO=a&`p2<;>6>k-(~s;A~NyDW!=F@7+&qFWr(QPzjn> z)xqS%ejV;%UJN;IrWJ1k-O|jcHh_zF88i?OIOO~Xe%MfL6YqNGoKvuxdAj(00o_Hll=G`7J{M6wkN;4Ud5WH(`}foJ!CAUdLH@%$X>t7r8AA5}!xW9D zc9Y-^&usqYr{i~#dx>+!#_3$uQYv&&(-Z!?I&h0FOI?`~7K&ST zMX=x9dvlKO^VB98!m811*yWq_aM&?D&ZPZg#ssuG|1a`k8if%Lrdc{l3H|pTvt|3S z6nT^^XM=@vT*)hEL7Bv?XQLZkN(e(0$Jjz1%Jxk1#Ka;>6!ufqBxm9pS?Q6rVXr5A zd;aVo#8jf9K0`jo-5xb6d<4L78M979Y|T{^pnHk8D==|r`tzaN7ClAHriwEX7V)6C z&tc#xp3rpL%_YiIB#!`_xn1y@@&7N<9Rat=VWEV@u1*7gm4N{uDI zzVS^sno)St<_H|}SXzR6`0MTQv=Gh|SDZ}X;u83du(->oAQY-Sa80j&YuRxRd7x0) zbz~FOeLMN+-P9m9^bG`^;O%)!I=YqZW{1OkhRUUX({B}GRmHITB2jWiJV|D Date: Mon, 29 Jun 2020 16:34:39 -0400 Subject: [PATCH 154/254] bucket_batch_sizes must be strictly positive, 0 is not a valid batch size --- mindspore/dataset/engine/validators.py | 4 ++-- tests/ut/python/dataset/test_bucket_batch_by_length.py | 7 ++++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/mindspore/dataset/engine/validators.py b/mindspore/dataset/engine/validators.py index 9857608c19..2a0bef3b42 100644 --- a/mindspore/dataset/engine/validators.py +++ b/mindspore/dataset/engine/validators.py @@ -643,9 +643,9 @@ def check_bucket_batch_by_length(method): if not all_int: raise TypeError("bucket_batch_sizes should be a list of int.") - all_non_negative = all(item >= 0 for item in bucket_batch_sizes) + all_non_negative = all(item > 0 for item in bucket_batch_sizes) if not all_non_negative: - raise ValueError("bucket_batch_sizes cannot contain any negative numbers.") + raise ValueError("bucket_batch_sizes should be a list of positive numbers.") if param_dict.get('pad_info') is not None: check_type(param_dict["pad_info"], "pad_info", dict) diff --git a/tests/ut/python/dataset/test_bucket_batch_by_length.py b/tests/ut/python/dataset/test_bucket_batch_by_length.py index bca30723e9..4436f98e53 100644 --- a/tests/ut/python/dataset/test_bucket_batch_by_length.py +++ b/tests/ut/python/dataset/test_bucket_batch_by_length.py @@ -51,6 +51,7 @@ def test_bucket_batch_invalid_input(): bucket_batch_sizes = [1, 1, 1, 1] invalid_bucket_batch_sizes = ["1", "2", "3", "4"] negative_bucket_batch_sizes = [1, 2, 3, -4] + zero_bucket_batch_sizes = [0, 1, 2, 3] with pytest.raises(TypeError) as info: _ = dataset.bucket_batch_by_length(invalid_column_names, bucket_boundaries, bucket_batch_sizes) @@ -82,7 +83,11 @@ def test_bucket_batch_invalid_input(): with pytest.raises(ValueError) as info: _ = dataset.bucket_batch_by_length(column_names, bucket_boundaries, negative_bucket_batch_sizes) - assert "bucket_batch_sizes cannot contain any negative numbers" in str(info.value) + assert "bucket_batch_sizes should be a list of positive numbers" in str(info.value) + + with pytest.raises(ValueError) as info: + _ = dataset.bucket_batch_by_length(column_names, bucket_boundaries, zero_bucket_batch_sizes) + assert "bucket_batch_sizes should be a list of positive numbers" in str(info.value) with pytest.raises(ValueError) as info: _ = dataset.bucket_batch_by_length(column_names, bucket_boundaries, bucket_boundaries) From 5276551953154291eb047dd01f5a0e331cbc5c53 Mon Sep 17 00:00:00 2001 From: yangyongjie Date: Mon, 29 Jun 2020 21:11:54 +0800 Subject: [PATCH 155/254] add Yolov3-darknet53 model --- .../deepfm/scripts/run_distribute_train.sh | 2 +- model_zoo/yolov3_darknet53/README.md | 132 ++++ model_zoo/yolov3_darknet53/eval.py | 328 ++++++++++ .../scripts/run_distribute_train.sh | 81 +++ .../yolov3_darknet53/scripts/run_eval.sh | 66 ++ .../scripts/run_standalone_train.sh | 73 +++ model_zoo/yolov3_darknet53/src/config.py | 68 +++ model_zoo/yolov3_darknet53/src/darknet.py | 211 +++++++ .../src/distributed_sampler.py | 60 ++ model_zoo/yolov3_darknet53/src/initializer.py | 179 ++++++ model_zoo/yolov3_darknet53/src/logger.py | 80 +++ model_zoo/yolov3_darknet53/src/loss.py | 70 +++ .../yolov3_darknet53/src/lr_scheduler.py | 144 +++++ model_zoo/yolov3_darknet53/src/transforms.py | 577 ++++++++++++++++++ model_zoo/yolov3_darknet53/src/util.py | 177 ++++++ model_zoo/yolov3_darknet53/src/yolo.py | 437 +++++++++++++ .../yolov3_darknet53/src/yolo_dataset.py | 184 ++++++ model_zoo/yolov3_darknet53/train.py | 338 ++++++++++ .../{yolov3 => yolov3_resnet18}/README.md | 0 model_zoo/{yolov3 => yolov3_resnet18}/eval.py | 0 .../scripts/run_distribute_train.sh | 0 .../scripts/run_eval.sh | 0 .../scripts/run_standalone_train.sh | 0 .../{yolov3 => yolov3_resnet18}/src/config.py | 0 .../src/dataset.py | 0 .../{yolov3 => yolov3_resnet18}/src/utils.py | 0 .../{yolov3 => yolov3_resnet18}/src/yolov3.py | 0 .../{yolov3 => yolov3_resnet18}/train.py | 0 28 files changed, 3206 insertions(+), 1 deletion(-) create mode 100644 model_zoo/yolov3_darknet53/README.md create mode 100644 model_zoo/yolov3_darknet53/eval.py create mode 100644 model_zoo/yolov3_darknet53/scripts/run_distribute_train.sh create mode 100644 model_zoo/yolov3_darknet53/scripts/run_eval.sh create mode 100644 model_zoo/yolov3_darknet53/scripts/run_standalone_train.sh create mode 100644 model_zoo/yolov3_darknet53/src/config.py create mode 100644 model_zoo/yolov3_darknet53/src/darknet.py create mode 100644 model_zoo/yolov3_darknet53/src/distributed_sampler.py create mode 100644 model_zoo/yolov3_darknet53/src/initializer.py create mode 100644 model_zoo/yolov3_darknet53/src/logger.py create mode 100644 model_zoo/yolov3_darknet53/src/loss.py create mode 100644 model_zoo/yolov3_darknet53/src/lr_scheduler.py create mode 100644 model_zoo/yolov3_darknet53/src/transforms.py create mode 100644 model_zoo/yolov3_darknet53/src/util.py create mode 100644 model_zoo/yolov3_darknet53/src/yolo.py create mode 100644 model_zoo/yolov3_darknet53/src/yolo_dataset.py create mode 100644 model_zoo/yolov3_darknet53/train.py rename model_zoo/{yolov3 => yolov3_resnet18}/README.md (100%) rename model_zoo/{yolov3 => yolov3_resnet18}/eval.py (100%) rename model_zoo/{yolov3 => yolov3_resnet18}/scripts/run_distribute_train.sh (100%) rename model_zoo/{yolov3 => yolov3_resnet18}/scripts/run_eval.sh (100%) rename model_zoo/{yolov3 => yolov3_resnet18}/scripts/run_standalone_train.sh (100%) rename model_zoo/{yolov3 => yolov3_resnet18}/src/config.py (100%) rename model_zoo/{yolov3 => yolov3_resnet18}/src/dataset.py (100%) rename model_zoo/{yolov3 => yolov3_resnet18}/src/utils.py (100%) rename model_zoo/{yolov3 => yolov3_resnet18}/src/yolov3.py (100%) rename model_zoo/{yolov3 => yolov3_resnet18}/train.py (100%) diff --git a/model_zoo/deepfm/scripts/run_distribute_train.sh b/model_zoo/deepfm/scripts/run_distribute_train.sh index a19c857936..fb2c3db17c 100644 --- a/model_zoo/deepfm/scripts/run_distribute_train.sh +++ b/model_zoo/deepfm/scripts/run_distribute_train.sh @@ -21,7 +21,7 @@ echo "After running the script, the network runs in the background, The log will export RANK_SIZE=$1 DATA_URL=$2 -export MINDSPORE_HCCL_CONFIG_PAHT=$3 +export MINDSPORE_HCCL_CONFIG_PATH=$3 for ((i=0; i Unzip the COCO2014 dataset to any path you want, the folder should include train and eval dataset as follows: + +``` +. +└─dataset + ├─train2014 + ├─val2014 + └─annotations +``` + +## Structure + +```shell +. +└─yolov3_darknet53 + ├─README.md + ├─scripts + ├─run_standalone_train.sh # launch standalone training(1p) + ├─run_distribute_train.sh # launch distributed training(8p) + └─run_eval.sh # launch evaluating + ├─src + ├─config.py # parameter configuration + ├─darknet.py # backbone of network + ├─distributed_sampler.py # iterator of dataset + ├─initializer.py # initializer of parameters + ├─logger.py # log function + ├─loss.py # loss function + ├─lr_scheduler.py # generate learning rate + ├─transforms.py # Preprocess data + ├─util.py # util function + ├─yolo.py # yolov3 network + ├─yolo_dataset.py # create dataset for YOLOV3 + ├─eval.py # eval net + └─train.py # train net +``` + +## Running the example + +### Train + +#### Usage + +``` +# distributed training +sh run_distribute_train.sh [DATASET_PATH] [PRETRAINED_BACKBONE] [MINDSPORE_HCCL_CONFIG_PATH] + +# standalone training +sh run_standalone_train.sh [DATASET_PATH] [PRETRAINED_BACKBONE] +``` + +#### Launch + +```bash +# distributed training example(8p) +sh run_distribute_train.sh dataset/coco2014 backbone/backbone.ckpt rank_table_8p.json + +# standalone training example(1p) +sh run_standalone_train.sh dataset/coco2014 backbone/backbone.ckpt +``` + +> About rank_table.json, you can refer to the [distributed training tutorial](https://www.mindspore.cn/tutorial/en/master/advanced_use/distributed_training.html). + +#### Result + +Training result will be stored in the scripts path, whose folder name begins with "train" or "train_parallel". You can find checkpoint file together with result like the followings in log.txt. + +``` +# distribute training result(8p) +epoch[0], iter[0], loss:14623.384766, 1.23 imgs/sec, lr:7.812499825377017e-05 +epoch[0], iter[100], loss:1486.253051, 15.01 imgs/sec, lr:0.007890624925494194 +epoch[0], iter[200], loss:288.579535, 490.41 imgs/sec, lr:0.015703124925494194 +epoch[0], iter[300], loss:153.136754, 531.99 imgs/sec, lr:0.023515624925494194 +epoch[1], iter[400], loss:106.429322, 405.14 imgs/sec, lr:0.03132812678813934 +... +epoch[318], iter[102000], loss:34.135306, 431.06 imgs/sec, lr:9.63797629083274e-06 +epoch[319], iter[102100], loss:35.652469, 449.52 imgs/sec, lr:2.409552052995423e-06 +epoch[319], iter[102200], loss:34.652273, 384.02 imgs/sec, lr:2.409552052995423e-06 +epoch[319], iter[102300], loss:35.430038, 423.49 imgs/sec, lr:2.409552052995423e-06 +... +``` + +### Infer + +#### Usage + +``` +# infer +sh run_eval.sh [DATASET_PATH] [CHECKPOINT_PATH] +``` + +#### Launch + +```bash +# infer with checkpoint +sh run_eval.sh dataset/coco2014/ checkpoint/0-319_102400.ckpt + +``` + +> checkpoint can be produced in training process. + + +#### Result + +Inference result will be stored in the scripts path, whose folder name is "eval". Under this, you can find result like the followings in log.txt. + +``` +=============coco eval reulst========= + Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.311 + Average Precision (AP) @[ IoU=0.50 | area= all | maxDets=100 ] = 0.528 + Average Precision (AP) @[ IoU=0.75 | area= all | maxDets=100 ] = 0.322 + Average Precision (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.127 + Average Precision (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.323 + Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.428 + Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 1 ] = 0.259 + Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 10 ] = 0.398 + Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.423 + Average Recall (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.224 + Average Recall (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.442 + Average Recall (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.551 +``` diff --git a/model_zoo/yolov3_darknet53/eval.py b/model_zoo/yolov3_darknet53/eval.py new file mode 100644 index 0000000000..6680b10476 --- /dev/null +++ b/model_zoo/yolov3_darknet53/eval.py @@ -0,0 +1,328 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""YoloV3 eval.""" +import os +import argparse +import datetime +import time +import sys +from collections import defaultdict + +import numpy as np +from pycocotools.coco import COCO +from pycocotools.cocoeval import COCOeval + +from mindspore import Tensor +from mindspore.train import ParallelMode +from mindspore import context +from mindspore.train.serialization import load_checkpoint, load_param_into_net +import mindspore as ms + +from src.yolo import YOLOV3DarkNet53 +from src.logger import get_logger +from src.yolo_dataset import create_yolo_dataset +from src.config import ConfigYOLOV3DarkNet53 + +devid = int(os.getenv('DEVICE_ID')) +context.set_context(mode=context.GRAPH_MODE, device_target="Ascend", save_graphs=True, device_id=devid) + + +class Redirct: + def __init__(self): + self.content = "" + + def write(self, content): + self.content += content + + def flush(self): + self.content = "" + + +class DetectionEngine: + """Detection engine.""" + def __init__(self, args): + self.ignore_threshold = args.ignore_threshold + self.labels = ['person', 'bicycle', 'car', 'motorcycle', 'airplane', 'bus', 'train', 'truck', 'boat', + 'traffic light', 'fire hydrant', 'stop sign', 'parking meter', 'bench', 'bird', 'cat', + 'dog', 'horse', 'sheep', 'cow', 'elephant', 'bear', 'zebra', 'giraffe', 'backpack', + 'umbrella', 'handbag', 'tie', 'suitcase', 'frisbee', 'skis', 'snowboard', 'sports ball', + 'kite', 'baseball bat', 'baseball glove', 'skateboard', 'surfboard', 'tennis racket', + 'bottle', 'wine glass', 'cup', 'fork', 'knife', 'spoon', 'bowl', 'banana', 'apple', + 'sandwich', 'orange', 'broccoli', 'carrot', 'hot dog', 'pizza', 'donut', 'cake', 'chair', + 'couch', 'potted plant', 'bed', 'dining table', 'toilet', 'tv', 'laptop', 'mouse', 'remote', + 'keyboard', 'cell phone', 'microwave', 'oven', 'toaster', 'sink', 'refrigerator', 'book', + 'clock', 'vase', 'scissors', 'teddy bear', 'hair drier', 'toothbrush'] + self.num_classes = len(self.labels) + self.results = {} + self.file_path = '' + self.save_prefix = args.outputs_dir + self.annFile = args.annFile + self._coco = COCO(self.annFile) + self._img_ids = list(sorted(self._coco.imgs.keys())) + self.det_boxes = [] + self.nms_thresh = args.nms_thresh + self.coco_catIds = self._coco.getCatIds() + + def do_nms_for_results(self): + """Get result boxes.""" + for img_id in self.results: + for clsi in self.results[img_id]: + dets = self.results[img_id][clsi] + dets = np.array(dets) + keep_index = self._nms(dets, self.nms_thresh) + + keep_box = [{'image_id': int(img_id), + 'category_id': int(clsi), + 'bbox': list(dets[i][:4].astype(float)), + 'score': dets[i][4].astype(float)} + for i in keep_index] + self.det_boxes.extend(keep_box) + + def _nms(self, dets, thresh): + """Calculate NMS.""" + # conver xywh -> xmin ymin xmax ymax + x1 = dets[:, 0] + y1 = dets[:, 1] + x2 = x1 + dets[:, 2] + y2 = y1 + dets[:, 3] + scores = dets[:, 4] + + areas = (x2 - x1 + 1) * (y2 - y1 + 1) + order = scores.argsort()[::-1] + + keep = [] + while order.size > 0: + i = order[0] + keep.append(i) + xx1 = np.maximum(x1[i], x1[order[1:]]) + yy1 = np.maximum(y1[i], y1[order[1:]]) + xx2 = np.minimum(x2[i], x2[order[1:]]) + yy2 = np.minimum(y2[i], y2[order[1:]]) + + w = np.maximum(0.0, xx2 - xx1 + 1) + h = np.maximum(0.0, yy2 - yy1 + 1) + inter = w * h + ovr = inter / (areas[i] + areas[order[1:]] - inter) + + inds = np.where(ovr <= thresh)[0] + order = order[inds + 1] + return keep + + def write_result(self): + """Save result to file.""" + import json + t = datetime.datetime.now().strftime('_%Y_%m_%d_%H_%M_%S') + try: + self.file_path = self.save_prefix + '/predict' + t + '.json' + f = open(self.file_path, 'w') + json.dump(self.det_boxes, f) + except IOError as e: + raise RuntimeError("Unable to open json file to dump. What(): {}".format(str(e))) + else: + f.close() + return self.file_path + + def get_eval_result(self): + """Get eval result.""" + cocoGt = COCO(self.annFile) + cocoDt = cocoGt.loadRes(self.file_path) + cocoEval = COCOeval(cocoGt, cocoDt, 'bbox') + cocoEval.evaluate() + cocoEval.accumulate() + rdct = Redirct() + stdout = sys.stdout + sys.stdout = rdct + cocoEval.summarize() + sys.stdout = stdout + return rdct.content + + def detect(self, outputs, batch, image_shape, image_id): + """Detect boxes.""" + outputs_num = len(outputs) + # output [|32, 52, 52, 3, 85| ] + for batch_id in range(batch): + for out_id in range(outputs_num): + # 32, 52, 52, 3, 85 + out_item = outputs[out_id] + # 52, 52, 3, 85 + out_item_single = out_item[batch_id, :] + # get number of items in one head, [B, gx, gy, anchors, 5+80] + dimensions = out_item_single.shape[:-1] + out_num = 1 + for d in dimensions: + out_num *= d + ori_w, ori_h = image_shape[batch_id] + img_id = int(image_id[batch_id]) + x = out_item_single[..., 0] * ori_w + y = out_item_single[..., 1] * ori_h + w = out_item_single[..., 2] * ori_w + h = out_item_single[..., 3] * ori_h + + conf = out_item_single[..., 4:5] + cls_emb = out_item_single[..., 5:] + + cls_argmax = np.expand_dims(np.argmax(cls_emb, axis=-1), axis=-1) + x = x.reshape(-1) + y = y.reshape(-1) + w = w.reshape(-1) + h = h.reshape(-1) + cls_emb = cls_emb.reshape(-1, 80) + conf = conf.reshape(-1) + cls_argmax = cls_argmax.reshape(-1) + + x_top_left = x - w / 2. + y_top_left = y - h / 2. + # creat all False + flag = np.random.random(cls_emb.shape) > sys.maxsize + for i in range(flag.shape[0]): + c = cls_argmax[i] + flag[i, c] = True + confidence = cls_emb[flag] * conf + for x_lefti, y_lefti, wi, hi, confi, clsi in zip(x_top_left, y_top_left, w, h, confidence, cls_argmax): + if confi < self.ignore_threshold: + continue + if img_id not in self.results: + self.results[img_id] = defaultdict(list) + x_lefti = max(0, x_lefti) + y_lefti = max(0, y_lefti) + wi = min(wi, ori_w) + hi = min(hi, ori_h) + # transform catId to match coco + coco_clsi = self.coco_catIds[clsi] + self.results[img_id][coco_clsi].append([x_lefti, y_lefti, wi, hi, confi]) + + +def parse_args(): + """Parse arguments.""" + parser = argparse.ArgumentParser('mindspore coco testing') + + # dataset related + parser.add_argument('--data_dir', type=str, default='', help='train data dir') + parser.add_argument('--per_batch_size', default=1, type=int, help='batch size for per gpu') + + # network related + parser.add_argument('--pretrained', default='', type=str, help='model_path, local pretrained model to load') + + # logging related + parser.add_argument('--log_path', type=str, default='outputs/', help='checkpoint save location') + + # detect_related + parser.add_argument('--nms_thresh', type=float, default=0.5, help='threshold for NMS') + parser.add_argument('--annFile', type=str, default='', help='path to annotation') + parser.add_argument('--testing_shape', type=str, default='', help='shape for test ') + parser.add_argument('--ignore_threshold', type=float, default=0.001, help='threshold to throw low quality boxes') + + args, _ = parser.parse_known_args() + + args.data_root = os.path.join(args.data_dir, 'val2014') + args.annFile = os.path.join(args.data_dir, 'annotations/instances_val2014.json') + + return args + + +def conver_testing_shape(args): + """Convert testing shape to list.""" + testing_shape = [int(args.testing_shape), int(args.testing_shape)] + return testing_shape + + +def test(): + """The function of eval.""" + start_time = time.time() + args = parse_args() + + # logger + args.outputs_dir = os.path.join(args.log_path, + datetime.datetime.now().strftime('%Y-%m-%d_time_%H_%M_%S')) + rank_id = int(os.environ.get('RANK_ID')) + args.logger = get_logger(args.outputs_dir, rank_id) + + context.reset_auto_parallel_context() + parallel_mode = ParallelMode.STAND_ALONE + context.set_auto_parallel_context(parallel_mode=parallel_mode, mirror_mean=True, device_num=1) + + args.logger.info('Creating Network....') + network = YOLOV3DarkNet53(is_training=False) + + args.logger.info(args.pretrained) + if os.path.isfile(args.pretrained): + param_dict = load_checkpoint(args.pretrained) + param_dict_new = {} + for key, values in param_dict.items(): + if key.startswith('moments.'): + continue + elif key.startswith('yolo_network.'): + param_dict_new[key[13:]] = values + else: + param_dict_new[key] = values + load_param_into_net(network, param_dict_new) + args.logger.info('load_model {} success'.format(args.pretrained)) + else: + args.logger.info('{} not exists or not a pre-trained file'.format(args.pretrained)) + assert FileNotFoundError('{} not exists or not a pre-trained file'.format(args.pretrained)) + exit(1) + + data_root = args.data_root + ann_file = args.annFile + + config = ConfigYOLOV3DarkNet53() + if args.testing_shape: + config.test_img_shape = conver_testing_shape(args) + + ds, data_size = create_yolo_dataset(data_root, ann_file, is_training=False, batch_size=args.per_batch_size, + max_epoch=1, device_num=1, rank=rank_id, shuffle=False, + config=config) + + args.logger.info('testing shape : {}'.format(config.test_img_shape)) + args.logger.info('totol {} images to eval'.format(data_size)) + + network.set_train(False) + + # init detection engine + detection = DetectionEngine(args) + + input_shape = Tensor(tuple(config.test_img_shape), ms.float32) + args.logger.info('Start inference....') + for i, data in enumerate(ds.create_dict_iterator()): + image = Tensor(data["image"]) + + image_shape = Tensor(data["image_shape"]) + image_id = Tensor(data["img_id"]) + + prediction = network(image, input_shape) + output_big, output_me, output_small = prediction + output_big = output_big.asnumpy() + output_me = output_me.asnumpy() + output_small = output_small.asnumpy() + image_id = image_id.asnumpy() + image_shape = image_shape.asnumpy() + + detection.detect([output_small, output_me, output_big], args.per_batch_size, image_shape, image_id) + if i % 1000 == 0: + args.logger.info('Processing... {:.2f}% '.format(i * args.per_batch_size / data_size * 100)) + + args.logger.info('Calculating mAP...') + detection.do_nms_for_results() + result_file_path = detection.write_result() + args.logger.info('result file path: {}'.format(result_file_path)) + eval_result = detection.get_eval_result() + + cost_time = time.time() - start_time + args.logger.info('\n=============coco eval reulst=========\n' + eval_result) + args.logger.info('testing cost time {:.2f}h'.format(cost_time / 3600.)) + + +if __name__ == "__main__": + test() diff --git a/model_zoo/yolov3_darknet53/scripts/run_distribute_train.sh b/model_zoo/yolov3_darknet53/scripts/run_distribute_train.sh new file mode 100644 index 0000000000..c6e83ae8f8 --- /dev/null +++ b/model_zoo/yolov3_darknet53/scripts/run_distribute_train.sh @@ -0,0 +1,81 @@ +#!/bin/bash +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +if [ $# != 3 ] +then + echo "Usage: sh run_distribute_train.sh [DATASET_PATH] [PRETRAINED_BACKBONE] [MINDSPORE_HCCL_CONFIG_PATH]" +exit 1 +fi + +get_real_path(){ + if [ "${1:0:1}" == "/" ]; then + echo "$1" + else + echo "$(realpath -m $PWD/$1)" + fi +} + +DATASET_PATH=$(get_real_path $1) +PRETRAINED_BACKBONE=$(get_real_path $2) +MINDSPORE_HCCL_CONFIG_PATH=$(get_real_path $3) +echo $DATASET_PATH +echo $PRETRAINED_BACKBONE +echo $MINDSPORE_HCCL_CONFIG_PATH + +if [ ! -d $DATASET_PATH ] +then + echo "error: DATASET_PATH=$DATASET_PATH is not a directory" +exit 1 +fi + +if [ ! -f $PRETRAINED_BACKBONE ] +then + echo "error: PRETRAINED_PATH=$PRETRAINED_BACKBONE is not a file" +exit 1 +fi + +if [ ! -f $MINDSPORE_HCCL_CONFIG_PATH ] +then + echo "error: MINDSPORE_HCCL_CONFIG_PATH=$MINDSPORE_HCCL_CONFIG_PATH is not a file" +exit 1 +fi + +export DEVICE_NUM=8 +export RANK_SIZE=8 +export MINDSPORE_HCCL_CONFIG_PATH=$MINDSPORE_HCCL_CONFIG_PATH + +for((i=0; i<${DEVICE_NUM}; i++)) +do + export DEVICE_ID=$i + export RANK_ID=$i + rm -rf ./train_parallel$i + mkdir ./train_parallel$i + cp ../*.py ./train_parallel$i + cp -r ../src ./train_parallel$i + cd ./train_parallel$i || exit + echo "start training for rank $RANK_ID, device $DEVICE_ID" + env > env.log + python train.py \ + --data_dir=$DATASET_PATH \ + --pretrained_backbone=$PRETRAINED_BACKBONE \ + --is_distributed=1 \ + --lr=0.1 \ + --T_max=320 \ + --max_epoch=320 \ + --warmup_epochs=4 \ + --lr_scheduler=cosine_annealing > log.txt 2>&1 & + cd .. +done diff --git a/model_zoo/yolov3_darknet53/scripts/run_eval.sh b/model_zoo/yolov3_darknet53/scripts/run_eval.sh new file mode 100644 index 0000000000..ff6b79e516 --- /dev/null +++ b/model_zoo/yolov3_darknet53/scripts/run_eval.sh @@ -0,0 +1,66 @@ +#!/bin/bash +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +if [ $# != 2 ] +then + echo "Usage: sh run_eval.sh [DATASET_PATH] [CHECKPOINT_PATH]" +exit 1 +fi + +get_real_path(){ + if [ "${1:0:1}" == "/" ]; then + echo "$1" + else + echo "$(realpath -m $PWD/$1)" + fi +} +DATASET_PATH=$(get_real_path $1) +CHECKPOINT_PATH=$(get_real_path $2) +echo $DATASET_PATH +echo $CHECKPOINT_PATH + +if [ ! -d $DATASET_PATH ] +then + echo "error: DATASET_PATH=$PATH1 is not a directory" +exit 1 +fi + +if [ ! -f $CHECKPOINT_PATH ] +then + echo "error: CHECKPOINT_PATH=$PATH2 is not a file" +exit 1 +fi + +export DEVICE_NUM=1 +export DEVICE_ID=0 +export RANK_SIZE=$DEVICE_NUM +export RANK_ID=0 + +if [ -d "eval" ]; +then + rm -rf ./eval +fi +mkdir ./eval +cp ../*.py ./eval +cp -r ../src ./eval +cd ./eval || exit +env > env.log +echo "start infering for device $DEVICE_ID" +python eval.py \ + --data_dir=$DATASET_PATH \ + --pretrained=$CHECKPOINT_PATH \ + --testing_shape=416 > log.txt 2>&1 & +cd .. diff --git a/model_zoo/yolov3_darknet53/scripts/run_standalone_train.sh b/model_zoo/yolov3_darknet53/scripts/run_standalone_train.sh new file mode 100644 index 0000000000..2597dca24e --- /dev/null +++ b/model_zoo/yolov3_darknet53/scripts/run_standalone_train.sh @@ -0,0 +1,73 @@ +#!/bin/bash +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +if [ $# != 2 ] +then + echo "Usage: sh run_standalone_train.sh [DATASET_PATH] [PRETRAINED_BACKBONE]" +exit 1 +fi + +get_real_path(){ + if [ "${1:0:1}" == "/" ]; then + echo "$1" + else + echo "$(realpath -m $PWD/$1)" + fi +} + +DATASET_PATH=$(get_real_path $1) +echo $DATASET_PATH +PRETRAINED_BACKBONE=$(get_real_path $2) +echo $PRETRAINED_BACKBONE + +if [ ! -d $DATASET_PATH ] +then + echo "error: DATASET_PATH=$DATASET_PATH is not a directory" +exit 1 +fi + +if [ ! -f $PRETRAINED_BACKBONE ] +then + echo "error: PRETRAINED_PATH=$PRETRAINED_BACKBONE is not a file" +exit 1 +fi + +export DEVICE_NUM=1 +export DEVICE_ID=0 +export RANK_ID=0 +export RANK_SIZE=1 + +if [ -d "train" ]; +then + rm -rf ./train +fi +mkdir ./train +cp ../*.py ./train +cp -r ../src ./train +cd ./train || exit +echo "start training for device $DEVICE_ID" +env > env.log + +python train.py \ + --data_dir=$DATASET_PATH \ + --pretrained_backbone=$PRETRAINED_BACKBONE \ + --is_distributed=0 \ + --lr=0.1 \ + --T_max=320 \ + --max_epoch=320 \ + --warmup_epochs=4 \ + --lr_scheduler=cosine_annealing > log.txt 2>&1 & +cd .. \ No newline at end of file diff --git a/model_zoo/yolov3_darknet53/src/config.py b/model_zoo/yolov3_darknet53/src/config.py new file mode 100644 index 0000000000..16831b048d --- /dev/null +++ b/model_zoo/yolov3_darknet53/src/config.py @@ -0,0 +1,68 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Config parameters for Darknet based yolov3_darknet53 models.""" + + +class ConfigYOLOV3DarkNet53: + """ + Config parameters for the yolov3_darknet53. + + Examples: + ConfigYOLOV3DarkNet53() + """ + # train_param + # data augmentation related + hue = 0.1 + saturation = 1.5 + value = 1.5 + jitter = 0.3 + + resize_rate = 1 + multi_scale = [[320, 320], + [352, 352], + [384, 384], + [416, 416], + [448, 448], + [480, 480], + [512, 512], + [544, 544], + [576, 576], + [608, 608] + ] + + num_classes = 80 + max_box = 50 + + backbone_input_shape = [32, 64, 128, 256, 512] + backbone_shape = [64, 128, 256, 512, 1024] + backbone_layers = [1, 2, 8, 8, 4] + + # confidence under ignore_threshold means no object when training + ignore_threshold = 0.7 + + # h->w + anchor_scales = [(10, 13), + (16, 30), + (33, 23), + (30, 61), + (62, 45), + (59, 119), + (116, 90), + (156, 198), + (373, 326)] + out_channel = 255 + + # test_param + test_img_shape = [416, 416] diff --git a/model_zoo/yolov3_darknet53/src/darknet.py b/model_zoo/yolov3_darknet53/src/darknet.py new file mode 100644 index 0000000000..4a2eb1de78 --- /dev/null +++ b/model_zoo/yolov3_darknet53/src/darknet.py @@ -0,0 +1,211 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""DarkNet model.""" +import mindspore.nn as nn +from mindspore.ops import operations as P + + +def conv_block(in_channels, + out_channels, + kernel_size, + stride, + dilation=1): + """Get a conv2d batchnorm and relu layer""" + pad_mode = 'same' + padding = 0 + + return nn.SequentialCell( + [nn.Conv2d(in_channels, + out_channels, + kernel_size=kernel_size, + stride=stride, + padding=padding, + dilation=dilation, + pad_mode=pad_mode), + nn.BatchNorm2d(out_channels, momentum=0.1), + nn.ReLU()] + ) + + +class ResidualBlock(nn.Cell): + """ + DarkNet V1 residual block definition. + + Args: + in_channels: Integer. Input channel. + out_channels: Integer. Output channel. + + Returns: + Tensor, output tensor. + Examples: + ResidualBlock(3, 208) + """ + expansion = 4 + + def __init__(self, + in_channels, + out_channels): + + super(ResidualBlock, self).__init__() + out_chls = out_channels//2 + self.conv1 = conv_block(in_channels, out_chls, kernel_size=1, stride=1) + self.conv2 = conv_block(out_chls, out_channels, kernel_size=3, stride=1) + self.add = P.TensorAdd() + + def construct(self, x): + identity = x + out = self.conv1(x) + out = self.conv2(out) + out = self.add(out, identity) + + return out + + +class DarkNet(nn.Cell): + """ + DarkNet V1 network. + + Args: + block: Cell. Block for network. + layer_nums: List. Numbers of different layers. + in_channels: Integer. Input channel. + out_channels: Integer. Output channel. + detect: Bool. Whether detect or not. Default:False. + + Returns: + Tuple, tuple of output tensor,(f1,f2,f3,f4,f5). + + Examples: + DarkNet(ResidualBlock, + [1, 2, 8, 8, 4], + [32, 64, 128, 256, 512], + [64, 128, 256, 512, 1024], + 100) + """ + def __init__(self, + block, + layer_nums, + in_channels, + out_channels, + detect=False): + super(DarkNet, self).__init__() + + self.outchannel = out_channels[-1] + self.detect = detect + + if not len(layer_nums) == len(in_channels) == len(out_channels) == 5: + raise ValueError("the length of layer_num, inchannel, outchannel list must be 5!") + self.conv0 = conv_block(3, + in_channels[0], + kernel_size=3, + stride=1) + self.conv1 = conv_block(in_channels[0], + out_channels[0], + kernel_size=3, + stride=2) + self.conv2 = conv_block(in_channels[1], + out_channels[1], + kernel_size=3, + stride=2) + self.conv3 = conv_block(in_channels[2], + out_channels[2], + kernel_size=3, + stride=2) + self.conv4 = conv_block(in_channels[3], + out_channels[3], + kernel_size=3, + stride=2) + self.conv5 = conv_block(in_channels[4], + out_channels[4], + kernel_size=3, + stride=2) + + self.layer1 = self._make_layer(block, + layer_nums[0], + in_channel=out_channels[0], + out_channel=out_channels[0]) + self.layer2 = self._make_layer(block, + layer_nums[1], + in_channel=out_channels[1], + out_channel=out_channels[1]) + self.layer3 = self._make_layer(block, + layer_nums[2], + in_channel=out_channels[2], + out_channel=out_channels[2]) + self.layer4 = self._make_layer(block, + layer_nums[3], + in_channel=out_channels[3], + out_channel=out_channels[3]) + self.layer5 = self._make_layer(block, + layer_nums[4], + in_channel=out_channels[4], + out_channel=out_channels[4]) + + def _make_layer(self, block, layer_num, in_channel, out_channel): + """ + Make Layer for DarkNet. + + :param block: Cell. DarkNet block. + :param layer_num: Integer. Layer number. + :param in_channel: Integer. Input channel. + :param out_channel: Integer. Output channel. + + Examples: + _make_layer(ConvBlock, 1, 128, 256) + """ + layers = [] + darkblk = block(in_channel, out_channel) + layers.append(darkblk) + + for _ in range(1, layer_num): + darkblk = block(out_channel, out_channel) + layers.append(darkblk) + + return nn.SequentialCell(layers) + + def construct(self, x): + c1 = self.conv0(x) + c2 = self.conv1(c1) + c3 = self.layer1(c2) + c4 = self.conv2(c3) + c5 = self.layer2(c4) + c6 = self.conv3(c5) + c7 = self.layer3(c6) + c8 = self.conv4(c7) + c9 = self.layer4(c8) + c10 = self.conv5(c9) + c11 = self.layer5(c10) + if self.detect: + return c7, c9, c11 + + return c11 + + def get_out_channels(self): + return self.outchannel + + +def darknet53(): + """ + Get DarkNet53 neural network. + + Returns: + Cell, cell instance of DarkNet53 neural network. + + Examples: + darknet53() + """ + return DarkNet(ResidualBlock, [1, 2, 8, 8, 4], + [32, 64, 128, 256, 512], + [64, 128, 256, 512, 1024]) diff --git a/model_zoo/yolov3_darknet53/src/distributed_sampler.py b/model_zoo/yolov3_darknet53/src/distributed_sampler.py new file mode 100644 index 0000000000..d31048ee9b --- /dev/null +++ b/model_zoo/yolov3_darknet53/src/distributed_sampler.py @@ -0,0 +1,60 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Yolo dataset distributed sampler.""" +from __future__ import division +import math +import numpy as np + + +class DistributedSampler: + """Distributed sampler.""" + def __init__(self, dataset_size, num_replicas=None, rank=None, shuffle=True): + if num_replicas is None: + print("***********Setting world_size to 1 since it is not passed in ******************") + num_replicas = 1 + if rank is None: + print("***********Setting rank to 0 since it is not passed in ******************") + rank = 0 + self.dataset_size = dataset_size + self.num_replicas = num_replicas + self.rank = rank + self.epoch = 0 + self.num_samples = int(math.ceil(dataset_size * 1.0 / self.num_replicas)) + self.total_size = self.num_samples * self.num_replicas + self.shuffle = shuffle + + def __iter__(self): + # deterministically shuffle based on epoch + if self.shuffle: + indices = np.random.RandomState(seed=self.epoch).permutation(self.dataset_size) + # np.array type. number from 0 to len(dataset_size)-1, used as index of dataset + indices = indices.tolist() + self.epoch += 1 + # change to list type + else: + indices = list(range(self.dataset_size)) + + # add extra samples to make it evenly divisible + indices += indices[:(self.total_size - len(indices))] + assert len(indices) == self.total_size + + # subsample + indices = indices[self.rank:self.total_size:self.num_replicas] + assert len(indices) == self.num_samples + + return iter(indices) + + def __len__(self): + return self.num_samples diff --git a/model_zoo/yolov3_darknet53/src/initializer.py b/model_zoo/yolov3_darknet53/src/initializer.py new file mode 100644 index 0000000000..f3c03a8ad1 --- /dev/null +++ b/model_zoo/yolov3_darknet53/src/initializer.py @@ -0,0 +1,179 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Parameter init.""" +import math +import numpy as np +from mindspore.common import initializer as init +from mindspore.common.initializer import Initializer as MeInitializer +import mindspore.nn as nn +from mindspore import Tensor + + +np.random.seed(5) + + +def calculate_gain(nonlinearity, param=None): + r"""Return the recommended gain value for the given nonlinearity function. + The values are as follows: + + ================= ==================================================== + nonlinearity gain + ================= ==================================================== + Linear / Identity :math:`1` + Conv{1,2,3}D :math:`1` + Sigmoid :math:`1` + Tanh :math:`\frac{5}{3}` + ReLU :math:`\sqrt{2}` + Leaky Relu :math:`\sqrt{\frac{2}{1 + \text{negative\_slope}^2}}` + ================= ==================================================== + + Args: + nonlinearity: the non-linear function (`nn.functional` name) + param: optional parameter for the non-linear function + + Examples: + >>> gain = nn.init.calculate_gain('leaky_relu', 0.2) # leaky_relu with negative_slope=0.2 + """ + linear_fns = ['linear', 'conv1d', 'conv2d', 'conv3d', 'conv_transpose1d', 'conv_transpose2d', 'conv_transpose3d'] + if nonlinearity in linear_fns or nonlinearity == 'sigmoid': + return 1 + if nonlinearity == 'tanh': + return 5.0 / 3 + if nonlinearity == 'relu': + return math.sqrt(2.0) + if nonlinearity == 'leaky_relu': + if param is None: + negative_slope = 0.01 + elif not isinstance(param, bool) and isinstance(param, int) or isinstance(param, float): + # True/False are instances of int, hence check above + negative_slope = param + else: + raise ValueError("negative_slope {} not a valid number".format(param)) + return math.sqrt(2.0 / (1 + negative_slope ** 2)) + + raise ValueError("Unsupported nonlinearity {}".format(nonlinearity)) + + +def _assignment(arr, num): + """Assign the value of 'num' and 'arr'.""" + if arr.shape == (): + arr = arr.reshape((1)) + arr[:] = num + arr = arr.reshape(()) + else: + if isinstance(num, np.ndarray): + arr[:] = num[:] + else: + arr[:] = num + return arr + + +def _calculate_correct_fan(array, mode): + mode = mode.lower() + valid_modes = ['fan_in', 'fan_out'] + if mode not in valid_modes: + raise ValueError("Mode {} not supported, please use one of {}".format(mode, valid_modes)) + + fan_in, fan_out = _calculate_fan_in_and_fan_out(array) + return fan_in if mode == 'fan_in' else fan_out + + +def kaiming_uniform_(arr, a=0, mode='fan_in', nonlinearity='leaky_relu'): + r"""Fills the input `Tensor` with values according to the method + described in `Delving deep into rectifiers: Surpassing human-level + performance on ImageNet classification` - He, K. et al. (2015), using a + uniform distribution. The resulting tensor will have values sampled from + :math:`\mathcal{U}(-\text{bound}, \text{bound})` where + + .. math:: + \text{bound} = \text{gain} \times \sqrt{\frac{3}{\text{fan\_mode}}} + + Also known as He initialization. + + Args: + tensor: an n-dimensional `Tensor` + a: the negative slope of the rectifier used after this layer (only + used with ``'leaky_relu'``) + mode: either ``'fan_in'`` (default) or ``'fan_out'``. Choosing ``'fan_in'`` + preserves the magnitude of the variance of the weights in the + forward pass. Choosing ``'fan_out'`` preserves the magnitudes in the + backwards pass. + nonlinearity: the non-linear function (`nn.functional` name), + recommended to use only with ``'relu'`` or ``'leaky_relu'`` (default). + + Examples: + >>> w = np.empty(3, 5) + >>> nn.init.kaiming_uniform_(w, mode='fan_in', nonlinearity='relu') + """ + fan = _calculate_correct_fan(arr, mode) + gain = calculate_gain(nonlinearity, a) + std = gain / math.sqrt(fan) + bound = math.sqrt(3.0) * std # Calculate uniform bounds from standard deviation + return np.random.uniform(-bound, bound, arr.shape) + + +def _calculate_fan_in_and_fan_out(arr): + """Calculate fan in and fan out.""" + dimensions = len(arr.shape) + if dimensions < 2: + raise ValueError("Fan in and fan out can not be computed for array with fewer than 2 dimensions") + + num_input_fmaps = arr.shape[1] + num_output_fmaps = arr.shape[0] + receptive_field_size = 1 + if dimensions > 2: + receptive_field_size = arr[0][0].size + fan_in = num_input_fmaps * receptive_field_size + fan_out = num_output_fmaps * receptive_field_size + + return fan_in, fan_out + + +class KaimingUniform(MeInitializer): + """Kaiming uniform initializer.""" + def __init__(self, a=0, mode='fan_in', nonlinearity='leaky_relu'): + super(KaimingUniform, self).__init__() + self.a = a + self.mode = mode + self.nonlinearity = nonlinearity + + def _initialize(self, arr): + tmp = kaiming_uniform_(arr, self.a, self.mode, self.nonlinearity) + _assignment(arr, tmp) + + +def default_recurisive_init(custom_cell): + """Initialize parameter.""" + for _, cell in custom_cell.cells_and_names(): + if isinstance(cell, nn.Conv2d): + cell.weight.default_input = init.initializer(KaimingUniform(a=math.sqrt(5)), + cell.weight.default_input.shape, + cell.weight.default_input.dtype).to_tensor() + if cell.bias is not None: + fan_in, _ = _calculate_fan_in_and_fan_out(cell.weight.default_input.asnumpy()) + bound = 1 / math.sqrt(fan_in) + cell.bias.default_input = Tensor(np.random.uniform(-bound, bound, cell.bias.default_input.shape), + cell.bias.default_input.dtype) + elif isinstance(cell, nn.Dense): + cell.weight.default_input = init.initializer(KaimingUniform(a=math.sqrt(5)), + cell.weight.default_input.shape, + cell.weight.default_input.dtype).to_tensor() + if cell.bias is not None: + fan_in, _ = _calculate_fan_in_and_fan_out(cell.weight.default_input.asnumpy()) + bound = 1 / math.sqrt(fan_in) + cell.bias.default_input = Tensor(np.random.uniform(-bound, bound, cell.bias.default_input.shape), + cell.bias.default_input.dtype) + elif isinstance(cell, (nn.BatchNorm2d, nn.BatchNorm1d)): + pass diff --git a/model_zoo/yolov3_darknet53/src/logger.py b/model_zoo/yolov3_darknet53/src/logger.py new file mode 100644 index 0000000000..b41ab405fc --- /dev/null +++ b/model_zoo/yolov3_darknet53/src/logger.py @@ -0,0 +1,80 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Custom Logger.""" +import os +import sys +import logging +from datetime import datetime + + +class LOGGER(logging.Logger): + """ + Logger. + + Args: + logger_name: String. Logger name. + rank: Integer. Rank id. + """ + def __init__(self, logger_name, rank=0): + super(LOGGER, self).__init__(logger_name) + self.rank = rank + if rank % 8 == 0: + console = logging.StreamHandler(sys.stdout) + console.setLevel(logging.INFO) + formatter = logging.Formatter('%(asctime)s:%(levelname)s:%(message)s') + console.setFormatter(formatter) + self.addHandler(console) + + def setup_logging_file(self, log_dir, rank=0): + """Setup logging file.""" + self.rank = rank + if not os.path.exists(log_dir): + os.makedirs(log_dir, exist_ok=True) + log_name = datetime.now().strftime('%Y-%m-%d_time_%H_%M_%S') + '_rank_{}.log'.format(rank) + self.log_fn = os.path.join(log_dir, log_name) + fh = logging.FileHandler(self.log_fn) + fh.setLevel(logging.INFO) + formatter = logging.Formatter('%(asctime)s:%(levelname)s:%(message)s') + fh.setFormatter(formatter) + self.addHandler(fh) + + def info(self, msg, *args, **kwargs): + if self.isEnabledFor(logging.INFO): + self._log(logging.INFO, msg, args, **kwargs) + + def save_args(self, args): + self.info('Args:') + args_dict = vars(args) + for key in args_dict.keys(): + self.info('--> %s: %s', key, args_dict[key]) + self.info('') + + def important_info(self, msg, *args, **kwargs): + if self.isEnabledFor(logging.INFO) and self.rank == 0: + line_width = 2 + important_msg = '\n' + important_msg += ('*'*70 + '\n')*line_width + important_msg += ('*'*line_width + '\n')*2 + important_msg += '*'*line_width + ' '*8 + msg + '\n' + important_msg += ('*'*line_width + '\n')*2 + important_msg += ('*'*70 + '\n')*line_width + self.info(important_msg, *args, **kwargs) + + +def get_logger(path, rank): + """Get Logger.""" + logger = LOGGER('yolov3_darknet53', rank) + logger.setup_logging_file(path, rank) + return logger diff --git a/model_zoo/yolov3_darknet53/src/loss.py b/model_zoo/yolov3_darknet53/src/loss.py new file mode 100644 index 0000000000..acdc6dba60 --- /dev/null +++ b/model_zoo/yolov3_darknet53/src/loss.py @@ -0,0 +1,70 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""YOLOV3 loss.""" +from mindspore.ops import operations as P +import mindspore.nn as nn + + +class XYLoss(nn.Cell): + """Loss for x and y.""" + def __init__(self): + super(XYLoss, self).__init__() + self.cross_entropy = P.SigmoidCrossEntropyWithLogits() + self.reduce_sum = P.ReduceSum() + + def construct(self, object_mask, box_loss_scale, predict_xy, true_xy): + xy_loss = object_mask * box_loss_scale * self.cross_entropy(predict_xy, true_xy) + xy_loss = self.reduce_sum(xy_loss, ()) + return xy_loss + + +class WHLoss(nn.Cell): + """Loss for w and h.""" + def __init__(self): + super(WHLoss, self).__init__() + self.square = P.Square() + self.reduce_sum = P.ReduceSum() + + def construct(self, object_mask, box_loss_scale, predict_wh, true_wh): + wh_loss = object_mask * box_loss_scale * 0.5 * P.Square()(true_wh - predict_wh) + wh_loss = self.reduce_sum(wh_loss, ()) + return wh_loss + + +class ConfidenceLoss(nn.Cell): + """Loss for confidence.""" + def __init__(self): + super(ConfidenceLoss, self).__init__() + self.cross_entropy = P.SigmoidCrossEntropyWithLogits() + self.reduce_sum = P.ReduceSum() + + def construct(self, object_mask, predict_confidence, ignore_mask): + confidence_loss = self.cross_entropy(predict_confidence, object_mask) + confidence_loss = object_mask * confidence_loss + (1 - object_mask) * confidence_loss * ignore_mask + confidence_loss = self.reduce_sum(confidence_loss, ()) + return confidence_loss + + +class ClassLoss(nn.Cell): + """Loss for classification.""" + def __init__(self): + super(ClassLoss, self).__init__() + self.cross_entropy = P.SigmoidCrossEntropyWithLogits() + self.reduce_sum = P.ReduceSum() + + def construct(self, object_mask, predict_class, class_probs): + class_loss = object_mask * self.cross_entropy(predict_class, class_probs) + class_loss = self.reduce_sum(class_loss, ()) + return class_loss diff --git a/model_zoo/yolov3_darknet53/src/lr_scheduler.py b/model_zoo/yolov3_darknet53/src/lr_scheduler.py new file mode 100644 index 0000000000..ae30454745 --- /dev/null +++ b/model_zoo/yolov3_darknet53/src/lr_scheduler.py @@ -0,0 +1,144 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Learning rate scheduler.""" +import math +from collections import Counter + +import numpy as np + + +def linear_warmup_lr(current_step, warmup_steps, base_lr, init_lr): + """Linear learning rate.""" + lr_inc = (float(base_lr) - float(init_lr)) / float(warmup_steps) + lr = float(init_lr) + lr_inc * current_step + return lr + + +def warmup_step_lr(lr, lr_epochs, steps_per_epoch, warmup_epochs, max_epoch, gamma=0.1): + """Warmup step learning rate.""" + base_lr = lr + warmup_init_lr = 0 + total_steps = int(max_epoch * steps_per_epoch) + warmup_steps = int(warmup_epochs * steps_per_epoch) + milestones = lr_epochs + milestones_steps = [] + for milestone in milestones: + milestones_step = milestone * steps_per_epoch + milestones_steps.append(milestones_step) + + lr_each_step = [] + lr = base_lr + milestones_steps_counter = Counter(milestones_steps) + for i in range(total_steps): + if i < warmup_steps: + lr = linear_warmup_lr(i + 1, warmup_steps, base_lr, warmup_init_lr) + else: + lr = lr * gamma**milestones_steps_counter[i] + lr_each_step.append(lr) + + return np.array(lr_each_step).astype(np.float32) + + +def multi_step_lr(lr, milestones, steps_per_epoch, max_epoch, gamma=0.1): + return warmup_step_lr(lr, milestones, steps_per_epoch, 0, max_epoch, gamma=gamma) + + +def step_lr(lr, epoch_size, steps_per_epoch, max_epoch, gamma=0.1): + lr_epochs = [] + for i in range(1, max_epoch): + if i % epoch_size == 0: + lr_epochs.append(i) + return multi_step_lr(lr, lr_epochs, steps_per_epoch, max_epoch, gamma=gamma) + + +def warmup_cosine_annealing_lr(lr, steps_per_epoch, warmup_epochs, max_epoch, T_max, eta_min=0): + """Cosine annealing learning rate.""" + base_lr = lr + warmup_init_lr = 0 + total_steps = int(max_epoch * steps_per_epoch) + warmup_steps = int(warmup_epochs * steps_per_epoch) + + lr_each_step = [] + for i in range(total_steps): + last_epoch = i // steps_per_epoch + if i < warmup_steps: + lr = linear_warmup_lr(i + 1, warmup_steps, base_lr, warmup_init_lr) + else: + lr = eta_min + (base_lr - eta_min) * (1. + math.cos(math.pi*last_epoch / T_max)) / 2 + lr_each_step.append(lr) + + return np.array(lr_each_step).astype(np.float32) + + +def warmup_cosine_annealing_lr_V2(lr, steps_per_epoch, warmup_epochs, max_epoch, T_max, eta_min=0): + """Cosine annealing learning rate V2.""" + base_lr = lr + warmup_init_lr = 0 + total_steps = int(max_epoch * steps_per_epoch) + warmup_steps = int(warmup_epochs * steps_per_epoch) + + last_lr = 0 + last_epoch_V1 = 0 + + T_max_V2 = int(max_epoch*1/3) + + lr_each_step = [] + for i in range(total_steps): + last_epoch = i // steps_per_epoch + if i < warmup_steps: + lr = linear_warmup_lr(i + 1, warmup_steps, base_lr, warmup_init_lr) + else: + if i < total_steps*2/3: + lr = eta_min + (base_lr - eta_min) * (1. + math.cos(math.pi*last_epoch / T_max)) / 2 + last_lr = lr + last_epoch_V1 = last_epoch + else: + base_lr = last_lr + last_epoch = last_epoch-last_epoch_V1 + lr = eta_min + (base_lr - eta_min) * (1. + math.cos(math.pi * last_epoch / T_max_V2)) / 2 + + lr_each_step.append(lr) + return np.array(lr_each_step).astype(np.float32) + + +def warmup_cosine_annealing_lr_sample(lr, steps_per_epoch, warmup_epochs, max_epoch, T_max, eta_min=0): + """Warmup cosine annealing learning rate.""" + start_sample_epoch = 60 + step_sample = 2 + tobe_sampled_epoch = 60 + end_sampled_epoch = start_sample_epoch + step_sample*tobe_sampled_epoch + max_sampled_epoch = max_epoch+tobe_sampled_epoch + T_max = max_sampled_epoch + + base_lr = lr + warmup_init_lr = 0 + total_steps = int(max_epoch * steps_per_epoch) + total_sampled_steps = int(max_sampled_epoch * steps_per_epoch) + warmup_steps = int(warmup_epochs * steps_per_epoch) + + lr_each_step = [] + + for i in range(total_sampled_steps): + last_epoch = i // steps_per_epoch + if last_epoch in range(start_sample_epoch, end_sampled_epoch, step_sample): + continue + if i < warmup_steps: + lr = linear_warmup_lr(i + 1, warmup_steps, base_lr, warmup_init_lr) + else: + lr = eta_min + (base_lr - eta_min) * (1. + math.cos(math.pi*last_epoch / T_max)) / 2 + lr_each_step.append(lr) + + assert total_steps == len(lr_each_step) + return np.array(lr_each_step).astype(np.float32) diff --git a/model_zoo/yolov3_darknet53/src/transforms.py b/model_zoo/yolov3_darknet53/src/transforms.py new file mode 100644 index 0000000000..c654a94c0d --- /dev/null +++ b/model_zoo/yolov3_darknet53/src/transforms.py @@ -0,0 +1,577 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Preprocess dataset.""" +import random +import threading +import copy + +import numpy as np +from PIL import Image +import cv2 + + +def _rand(a=0., b=1.): + return np.random.rand() * (b - a) + a + + +def bbox_iou(bbox_a, bbox_b, offset=0): + """Calculate Intersection-Over-Union(IOU) of two bounding boxes. + + Parameters + ---------- + bbox_a : numpy.ndarray + An ndarray with shape :math:`(N, 4)`. + bbox_b : numpy.ndarray + An ndarray with shape :math:`(M, 4)`. + offset : float or int, default is 0 + The ``offset`` is used to control the whether the width(or height) is computed as + (right - left + ``offset``). + Note that the offset must be 0 for normalized bboxes, whose ranges are in ``[0, 1]``. + + Returns + ------- + numpy.ndarray + An ndarray with shape :math:`(N, M)` indicates IOU between each pairs of + bounding boxes in `bbox_a` and `bbox_b`. + + """ + if bbox_a.shape[1] < 4 or bbox_b.shape[1] < 4: + raise IndexError("Bounding boxes axis 1 must have at least length 4") + + tl = np.maximum(bbox_a[:, None, :2], bbox_b[:, :2]) + br = np.minimum(bbox_a[:, None, 2:4], bbox_b[:, 2:4]) + + area_i = np.prod(br - tl + offset, axis=2) * (tl < br).all(axis=2) + area_a = np.prod(bbox_a[:, 2:4] - bbox_a[:, :2] + offset, axis=1) + area_b = np.prod(bbox_b[:, 2:4] - bbox_b[:, :2] + offset, axis=1) + return area_i / (area_a[:, None] + area_b - area_i) + + +def statistic_normalize_img(img, statistic_norm): + """Statistic normalize images.""" + # img: RGB + if isinstance(img, Image.Image): + img = np.array(img) + img = img/255. + mean = np.array([0.485, 0.456, 0.406]) + std = np.array([0.229, 0.224, 0.225]) + if statistic_norm: + img = (img - mean) / std + return img + + +def get_interp_method(interp, sizes=()): + """Get the interpolation method for resize functions. + The major purpose of this function is to wrap a random interp method selection + and a auto-estimation method. + + Parameters + ---------- + interp : int + interpolation method for all resizing operations + + Possible values: + 0: Nearest Neighbors Interpolation. + 1: Bilinear interpolation. + 2: Bicubic interpolation over 4x4 pixel neighborhood. + 3: Nearest Neighbors. [Originally it should be Area-based, + as we cannot find Area-based, so we use NN instead. + Area-based (resampling using pixel area relation). It may be a + preferred method for image decimation, as it gives moire-free + results. But when the image is zoomed, it is similar to the Nearest + Neighbors method. (used by default). + 4: Lanczos interpolation over 8x8 pixel neighborhood. + 9: Cubic for enlarge, area for shrink, bilinear for others + 10: Random select from interpolation method metioned above. + Note: + When shrinking an image, it will generally look best with AREA-based + interpolation, whereas, when enlarging an image, it will generally look best + with Bicubic (slow) or Bilinear (faster but still looks OK). + More details can be found in the documentation of OpenCV, please refer to + http://docs.opencv.org/master/da/d54/group__imgproc__transform.html. + sizes : tuple of int + (old_height, old_width, new_height, new_width), if None provided, auto(9) + will return Area(2) anyway. + + Returns + ------- + int + interp method from 0 to 4 + """ + if interp == 9: + if sizes: + assert len(sizes) == 4 + oh, ow, nh, nw = sizes + if nh > oh and nw > ow: + return 2 + if nh < oh and nw < ow: + return 0 + return 1 + return 2 + if interp == 10: + return random.randint(0, 4) + if interp not in (0, 1, 2, 3, 4): + raise ValueError('Unknown interp method %d' % interp) + return interp + + +def pil_image_reshape(interp): + """Reshape pil image.""" + reshape_type = { + 0: Image.NEAREST, + 1: Image.BILINEAR, + 2: Image.BICUBIC, + 3: Image.NEAREST, + 4: Image.LANCZOS, + } + return reshape_type[interp] + + +def _preprocess_true_boxes(true_boxes, anchors, in_shape, num_classes, + max_boxes, label_smooth, label_smooth_factor=0.1): + """Preprocess annotation boxes.""" + anchors = np.array(anchors) + num_layers = anchors.shape[0] // 3 + anchor_mask = [[6, 7, 8], [3, 4, 5], [0, 1, 2]] + true_boxes = np.array(true_boxes, dtype='float32') + input_shape = np.array(in_shape, dtype='int32') + boxes_xy = (true_boxes[..., 0:2] + true_boxes[..., 2:4]) // 2. + # trans to box center point + boxes_wh = true_boxes[..., 2:4] - true_boxes[..., 0:2] + # input_shape is [h, w] + true_boxes[..., 0:2] = boxes_xy / input_shape[::-1] + true_boxes[..., 2:4] = boxes_wh / input_shape[::-1] + # true_boxes = [xywh] + + grid_shapes = [input_shape // 32, input_shape // 16, input_shape // 8] + # grid_shape [h, w] + y_true = [np.zeros((grid_shapes[l][0], grid_shapes[l][1], len(anchor_mask[l]), + 5 + num_classes), dtype='float32') for l in range(num_layers)] + # y_true [gridy, gridx] + anchors = np.expand_dims(anchors, 0) + anchors_max = anchors / 2. + anchors_min = -anchors_max + valid_mask = boxes_wh[..., 0] > 0 + + wh = boxes_wh[valid_mask] + if wh: + wh = np.expand_dims(wh, -2) + boxes_max = wh / 2. + boxes_min = -boxes_max + + intersect_min = np.maximum(boxes_min, anchors_min) + intersect_max = np.minimum(boxes_max, anchors_max) + intersect_wh = np.maximum(intersect_max - intersect_min, 0.) + intersect_area = intersect_wh[..., 0] * intersect_wh[..., 1] + box_area = wh[..., 0] * wh[..., 1] + anchor_area = anchors[..., 0] * anchors[..., 1] + iou = intersect_area / (box_area + anchor_area - intersect_area) + + best_anchor = np.argmax(iou, axis=-1) + for t, n in enumerate(best_anchor): + for l in range(num_layers): + if n in anchor_mask[l]: + i = np.floor(true_boxes[t, 0] * grid_shapes[l][1]).astype('int32') # grid_y + j = np.floor(true_boxes[t, 1] * grid_shapes[l][0]).astype('int32') # grid_x + + k = anchor_mask[l].index(n) + c = true_boxes[t, 4].astype('int32') + y_true[l][j, i, k, 0:4] = true_boxes[t, 0:4] + y_true[l][j, i, k, 4] = 1. + + # lable-smooth + if label_smooth: + sigma = label_smooth_factor/(num_classes-1) + y_true[l][j, i, k, 5:] = sigma + y_true[l][j, i, k, 5+c] = 1-label_smooth_factor + else: + y_true[l][j, i, k, 5 + c] = 1. + + # pad_gt_boxes for avoiding dynamic shape + pad_gt_box0 = np.zeros(shape=[max_boxes, 4], dtype=np.float32) + pad_gt_box1 = np.zeros(shape=[max_boxes, 4], dtype=np.float32) + pad_gt_box2 = np.zeros(shape=[max_boxes, 4], dtype=np.float32) + + mask0 = np.reshape(y_true[0][..., 4:5], [-1]) + gt_box0 = np.reshape(y_true[0][..., 0:4], [-1, 4]) + # gt_box [boxes, [x,y,w,h]] + gt_box0 = gt_box0[mask0 == 1] + # gt_box0: get all boxes which have object + pad_gt_box0[:gt_box0.shape[0]] = gt_box0 + # gt_box0.shape[0]: total number of boxes in gt_box0 + # top N of pad_gt_box0 is real box, and after are pad by zero + + mask1 = np.reshape(y_true[1][..., 4:5], [-1]) + gt_box1 = np.reshape(y_true[1][..., 0:4], [-1, 4]) + gt_box1 = gt_box1[mask1 == 1] + pad_gt_box1[:gt_box1.shape[0]] = gt_box1 + + mask2 = np.reshape(y_true[2][..., 4:5], [-1]) + gt_box2 = np.reshape(y_true[2][..., 0:4], [-1, 4]) + + gt_box2 = gt_box2[mask2 == 1] + pad_gt_box2[:gt_box2.shape[0]] = gt_box2 + return y_true[0], y_true[1], y_true[2], pad_gt_box0, pad_gt_box1, pad_gt_box2 + + +def _reshape_data(image, image_size): + """Reshape image.""" + if not isinstance(image, Image.Image): + image = Image.fromarray(image) + ori_w, ori_h = image.size + ori_image_shape = np.array([ori_w, ori_h], np.int32) + # original image shape fir:H sec:W + h, w = image_size + interp = get_interp_method(interp=9, sizes=(ori_h, ori_w, h, w)) + image = image.resize((w, h), pil_image_reshape(interp)) + image_data = statistic_normalize_img(image, statistic_norm=True) + if len(image_data.shape) == 2: + image_data = np.expand_dims(image_data, axis=-1) + image_data = np.concatenate([image_data, image_data, image_data], axis=-1) + image_data = image_data.astype(np.float32) + return image_data, ori_image_shape + + +def color_distortion(img, hue, sat, val, device_num): + """Color distortion.""" + hue = _rand(-hue, hue) + sat = _rand(1, sat) if _rand() < .5 else 1 / _rand(1, sat) + val = _rand(1, val) if _rand() < .5 else 1 / _rand(1, val) + if device_num != 1: + cv2.setNumThreads(1) + x = cv2.cvtColor(img, cv2.COLOR_RGB2HSV_FULL) + x = x / 255. + x[..., 0] += hue + x[..., 0][x[..., 0] > 1] -= 1 + x[..., 0][x[..., 0] < 0] += 1 + x[..., 1] *= sat + x[..., 2] *= val + x[x > 1] = 1 + x[x < 0] = 0 + x = x * 255. + x = x.astype(np.uint8) + image_data = cv2.cvtColor(x, cv2.COLOR_HSV2RGB_FULL) + return image_data + + +def filp_pil_image(img): + return img.transpose(Image.FLIP_LEFT_RIGHT) + + +def convert_gray_to_color(img): + if len(img.shape) == 2: + img = np.expand_dims(img, axis=-1) + img = np.concatenate([img, img, img], axis=-1) + return img + + +def _is_iou_satisfied_constraint(min_iou, max_iou, box, crop_box): + iou = bbox_iou(box, crop_box) + return min_iou <= iou.min() and max_iou >= iou.max() + + +def _choose_candidate_by_constraints(max_trial, input_w, input_h, image_w, image_h, jitter, box, use_constraints): + """Choose candidate by constraints.""" + if use_constraints: + constraints = ( + (0.1, None), + (0.3, None), + (0.5, None), + (0.7, None), + (0.9, None), + (None, 1), + ) + else: + constraints = ( + (None, None), + ) + # add default candidate + candidates = [(0, 0, input_w, input_h)] + for constraint in constraints: + min_iou, max_iou = constraint + min_iou = -np.inf if min_iou is None else min_iou + max_iou = np.inf if max_iou is None else max_iou + + for _ in range(max_trial): + # box_data should have at least one box + new_ar = float(input_w) / float(input_h) * _rand(1 - jitter, 1 + jitter) / _rand(1 - jitter, 1 + jitter) + scale = _rand(0.25, 2) + + if new_ar < 1: + nh = int(scale * input_h) + nw = int(nh * new_ar) + else: + nw = int(scale * input_w) + nh = int(nw / new_ar) + + dx = int(_rand(0, input_w - nw)) + dy = int(_rand(0, input_h - nh)) + + if box: + t_box = copy.deepcopy(box) + t_box[:, [0, 2]] = t_box[:, [0, 2]] * float(nw) / float(image_w) + dx + t_box[:, [1, 3]] = t_box[:, [1, 3]] * float(nh) / float(image_h) + dy + + crop_box = np.array((0, 0, input_w, input_h)) + if not _is_iou_satisfied_constraint(min_iou, max_iou, t_box, crop_box[np.newaxis]): + continue + else: + candidates.append((dx, dy, nw, nh)) + else: + raise Exception("!!! annotation box is less than 1") + return candidates + + +def _correct_bbox_by_candidates(candidates, input_w, input_h, image_w, + image_h, flip, box, box_data, allow_outside_center): + """Calculate correct boxes.""" + while candidates: + if len(candidates) > 1: + # ignore default candidate which do not crop + candidate = candidates.pop(np.random.randint(1, len(candidates))) + else: + candidate = candidates.pop(np.random.randint(0, len(candidates))) + dx, dy, nw, nh = candidate + t_box = copy.deepcopy(box) + t_box[:, [0, 2]] = t_box[:, [0, 2]] * float(nw) / float(image_w) + dx + t_box[:, [1, 3]] = t_box[:, [1, 3]] * float(nh) / float(image_h) + dy + if flip: + t_box[:, [0, 2]] = input_w - t_box[:, [2, 0]] + + if allow_outside_center: + pass + else: + t_box = t_box[np.logical_and((t_box[:, 0] + t_box[:, 2])/2. >= 0., (t_box[:, 1] + t_box[:, 3])/2. >= 0.)] + t_box = t_box[np.logical_and((t_box[:, 0] + t_box[:, 2]) / 2. <= input_w, + (t_box[:, 1] + t_box[:, 3]) / 2. <= input_h)] + + # recorrect x, y for case x,y < 0 reset to zero, after dx and dy, some box can smaller than zero + t_box[:, 0:2][t_box[:, 0:2] < 0] = 0 + # recorrect w,h not higher than input size + t_box[:, 2][t_box[:, 2] > input_w] = input_w + t_box[:, 3][t_box[:, 3] > input_h] = input_h + box_w = t_box[:, 2] - t_box[:, 0] + box_h = t_box[:, 3] - t_box[:, 1] + # discard invalid box: w or h smaller than 1 pixel + t_box = t_box[np.logical_and(box_w > 1, box_h > 1)] + + if t_box.shape[0] > 0: + # break if number of find t_box + box_data[: len(t_box)] = t_box + return box_data, candidate + raise Exception('all candidates can not satisfied re-correct bbox') + + +def _data_aug(image, box, jitter, hue, sat, val, image_input_size, max_boxes, + anchors, num_classes, max_trial=10, device_num=1): + """Crop an image randomly with bounding box constraints. + + This data augmentation is used in training of + Single Shot Multibox Detector [#]_. More details can be found in + data augmentation section of the original paper. + .. [#] Wei Liu, Dragomir Anguelov, Dumitru Erhan, Christian Szegedy, + Scott Reed, Cheng-Yang Fu, Alexander C. Berg. + SSD: Single Shot MultiBox Detector. ECCV 2016.""" + + if not isinstance(image, Image.Image): + image = Image.fromarray(image) + + image_w, image_h = image.size + input_h, input_w = image_input_size + + np.random.shuffle(box) + if len(box) > max_boxes: + box = box[:max_boxes] + flip = _rand() < .5 + box_data = np.zeros((max_boxes, 5)) + + candidates = _choose_candidate_by_constraints(use_constraints=False, + max_trial=max_trial, + input_w=input_w, + input_h=input_h, + image_w=image_w, + image_h=image_h, + jitter=jitter, + box=box) + box_data, candidate = _correct_bbox_by_candidates(candidates=candidates, + input_w=input_w, + input_h=input_h, + image_w=image_w, + image_h=image_h, + flip=flip, + box=box, + box_data=box_data, + allow_outside_center=True) + dx, dy, nw, nh = candidate + interp = get_interp_method(interp=10) + image = image.resize((nw, nh), pil_image_reshape(interp)) + # place image, gray color as back graoud + new_image = Image.new('RGB', (input_w, input_h), (128, 128, 128)) + new_image.paste(image, (dx, dy)) + image = new_image + + if flip: + image = filp_pil_image(image) + + image = np.array(image) + + image = convert_gray_to_color(image) + + image_data = color_distortion(image, hue, sat, val, device_num) + image_data = statistic_normalize_img(image_data, statistic_norm=True) + + image_data = image_data.astype(np.float32) + + return image_data, box_data + + +def preprocess_fn(image, box, config, input_size, device_num): + """Preprocess data function.""" + config_anchors = config.anchor_scales + anchors = np.array([list(x) for x in config_anchors]) + max_boxes = config.max_box + num_classes = config.num_classes + jitter = config.jitter + hue = config.hue + sat = config.saturation + val = config.value + image, anno = _data_aug(image, box, jitter=jitter, hue=hue, sat=sat, val=val, + image_input_size=input_size, max_boxes=max_boxes, + num_classes=num_classes, anchors=anchors, device_num=device_num) + return image, anno + + +def reshape_fn(image, img_id, config): + input_size = config.test_img_shape + image, ori_image_shape = _reshape_data(image, image_size=input_size) + return image, ori_image_shape, img_id + + +class MultiScaleTrans: + """Multi scale transform.""" + def __init__(self, config, device_num): + self.config = config + self.seed = 0 + self.size_list = [] + self.resize_rate = config.resize_rate + self.dataset_size = config.dataset_size + self.size_dict = {} + self.seed_num = int(1e6) + self.seed_list = self.generate_seed_list(seed_num=self.seed_num) + self.resize_count_num = int(np.ceil(self.dataset_size / self.resize_rate)) + self.device_num = device_num + + def generate_seed_list(self, init_seed=1234, seed_num=int(1e6), seed_range=(1, 1000)): + seed_list = [] + random.seed(init_seed) + for _ in range(seed_num): + seed = random.randint(seed_range[0], seed_range[1]) + seed_list.append(seed) + return seed_list + + def __call__(self, imgs, annos, batchInfo): + epoch_num = batchInfo.get_epoch_num() + size_idx = int(batchInfo.get_batch_num() / self.resize_rate) + seed_key = self.seed_list[(epoch_num * self.resize_count_num + size_idx) % self.seed_num] + ret_imgs = [] + ret_annos = [] + + if self.size_dict.get(seed_key, None) is None: + random.seed(seed_key) + new_size = random.choice(self.config.multi_scale) + self.size_dict[seed_key] = new_size + seed = seed_key + + input_size = self.size_dict[seed] + for img, anno in zip(imgs, annos): + img, anno = preprocess_fn(img, anno, self.config, input_size, self.device_num) + ret_imgs.append(img.transpose(2, 0, 1).copy()) + ret_annos.append(anno) + return np.array(ret_imgs), np.array(ret_annos) + + +def thread_batch_preprocess_true_box(annos, config, input_shape, result_index, batch_bbox_true_1, batch_bbox_true_2, + batch_bbox_true_3, batch_gt_box1, batch_gt_box2, batch_gt_box3): + """Preprocess true box for multi-thread.""" + i = 0 + for anno in annos: + bbox_true_1, bbox_true_2, bbox_true_3, gt_box1, gt_box2, gt_box3 = \ + _preprocess_true_boxes(true_boxes=anno, anchors=config.anchor_scales, in_shape=input_shape, + num_classes=config.num_classes, max_boxes=config.max_box, + label_smooth=config.label_smooth, label_smooth_factor=config.label_smooth_factor) + batch_bbox_true_1[result_index + i] = bbox_true_1 + batch_bbox_true_2[result_index + i] = bbox_true_2 + batch_bbox_true_3[result_index + i] = bbox_true_3 + batch_gt_box1[result_index + i] = gt_box1 + batch_gt_box2[result_index + i] = gt_box2 + batch_gt_box3[result_index + i] = gt_box3 + i = i + 1 + + +def batch_preprocess_true_box(annos, config, input_shape): + """Preprocess true box with multi-thread.""" + batch_bbox_true_1 = [] + batch_bbox_true_2 = [] + batch_bbox_true_3 = [] + batch_gt_box1 = [] + batch_gt_box2 = [] + batch_gt_box3 = [] + threads = [] + + step = 4 + for index in range(0, len(annos), step): + for _ in range(step): + batch_bbox_true_1.append(None) + batch_bbox_true_2.append(None) + batch_bbox_true_3.append(None) + batch_gt_box1.append(None) + batch_gt_box2.append(None) + batch_gt_box3.append(None) + step_anno = annos[index: index + step] + t = threading.Thread(target=thread_batch_preprocess_true_box, + args=(step_anno, config, input_shape, index, batch_bbox_true_1, batch_bbox_true_2, + batch_bbox_true_3, batch_gt_box1, batch_gt_box2, batch_gt_box3)) + t.start() + threads.append(t) + + for t in threads: + t.join() + + return np.array(batch_bbox_true_1), np.array(batch_bbox_true_2), np.array(batch_bbox_true_3), \ + np.array(batch_gt_box1), np.array(batch_gt_box2), np.array(batch_gt_box3) + + +def batch_preprocess_true_box_single(annos, config, input_shape): + """Preprocess true boxes.""" + batch_bbox_true_1 = [] + batch_bbox_true_2 = [] + batch_bbox_true_3 = [] + batch_gt_box1 = [] + batch_gt_box2 = [] + batch_gt_box3 = [] + for anno in annos: + bbox_true_1, bbox_true_2, bbox_true_3, gt_box1, gt_box2, gt_box3 = \ + _preprocess_true_boxes(true_boxes=anno, anchors=config.anchor_scales, in_shape=input_shape, + num_classes=config.num_classes, max_boxes=config.max_box, + label_smooth=config.label_smooth, label_smooth_factor=config.label_smooth_factor) + batch_bbox_true_1.append(bbox_true_1) + batch_bbox_true_2.append(bbox_true_2) + batch_bbox_true_3.append(bbox_true_3) + batch_gt_box1.append(gt_box1) + batch_gt_box2.append(gt_box2) + batch_gt_box3.append(gt_box3) + + return np.array(batch_bbox_true_1), np.array(batch_bbox_true_2), np.array(batch_bbox_true_3), \ + np.array(batch_gt_box1), np.array(batch_gt_box2), np.array(batch_gt_box3) diff --git a/model_zoo/yolov3_darknet53/src/util.py b/model_zoo/yolov3_darknet53/src/util.py new file mode 100644 index 0000000000..1a3da99181 --- /dev/null +++ b/model_zoo/yolov3_darknet53/src/util.py @@ -0,0 +1,177 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Util class or function.""" +from mindspore.train.serialization import load_checkpoint +import mindspore.nn as nn + + +class AverageMeter: + """Computes and stores the average and current value""" + + def __init__(self, name, fmt=':f', tb_writer=None): + self.name = name + self.fmt = fmt + self.reset() + self.tb_writer = tb_writer + self.cur_step = 1 + self.val = 0 + self.avg = 0 + self.sum = 0 + self.count = 0 + + def reset(self): + self.val = 0 + self.avg = 0 + self.sum = 0 + self.count = 0 + + def update(self, val, n=1): + self.val = val + self.sum += val * n + self.count += n + self.avg = self.sum / self.count + if self.tb_writer is not None: + self.tb_writer.add_scalar(self.name, self.val, self.cur_step) + self.cur_step += 1 + + def __str__(self): + fmtstr = '{name}:{avg' + self.fmt + '}' + return fmtstr.format(**self.__dict__) + + +def load_backbone(net, ckpt_path, args): + """Load darknet53 backbone checkpoint.""" + param_dict = load_checkpoint(ckpt_path) + yolo_backbone_prefix = 'feature_map.backbone' + darknet_backbone_prefix = 'network.backbone' + find_param = [] + not_found_param = [] + + for name, cell in net.cells_and_names(): + if name.startswith(yolo_backbone_prefix): + name = name.replace(yolo_backbone_prefix, darknet_backbone_prefix) + if isinstance(cell, (nn.Conv2d, nn.Dense)): + darknet_weight = '{}.weight'.format(name) + darknet_bias = '{}.bias'.format(name) + if darknet_weight in param_dict: + cell.weight.default_input = param_dict[darknet_weight].data + find_param.append(darknet_weight) + else: + not_found_param.append(darknet_weight) + if darknet_bias in param_dict: + cell.bias.default_input = param_dict[darknet_bias].data + find_param.append(darknet_bias) + else: + not_found_param.append(darknet_bias) + elif isinstance(cell, (nn.BatchNorm2d, nn.BatchNorm1d)): + darknet_moving_mean = '{}.moving_mean'.format(name) + darknet_moving_variance = '{}.moving_variance'.format(name) + darknet_gamma = '{}.gamma'.format(name) + darknet_beta = '{}.beta'.format(name) + if darknet_moving_mean in param_dict: + cell.moving_mean.default_input = param_dict[darknet_moving_mean].data + find_param.append(darknet_moving_mean) + else: + not_found_param.append(darknet_moving_mean) + if darknet_moving_variance in param_dict: + cell.moving_variance.default_input = param_dict[darknet_moving_variance].data + find_param.append(darknet_moving_variance) + else: + not_found_param.append(darknet_moving_variance) + if darknet_gamma in param_dict: + cell.gamma.default_input = param_dict[darknet_gamma].data + find_param.append(darknet_gamma) + else: + not_found_param.append(darknet_gamma) + if darknet_beta in param_dict: + cell.beta.default_input = param_dict[darknet_beta].data + find_param.append(darknet_beta) + else: + not_found_param.append(darknet_beta) + + args.logger.info('================found_param {}========='.format(len(find_param))) + args.logger.info(find_param) + args.logger.info('================not_found_param {}========='.format(len(not_found_param))) + args.logger.info(not_found_param) + args.logger.info('=====load {} successfully ====='.format(ckpt_path)) + + return net + + +def default_wd_filter(x): + """default weight decay filter.""" + parameter_name = x.name + if parameter_name.endswith('.bias'): + # all bias not using weight decay + return False + if parameter_name.endswith('.gamma'): + # bn weight bias not using weight decay, be carefully for now x not include BN + return False + if parameter_name.endswith('.beta'): + # bn weight bias not using weight decay, be carefully for now x not include BN + return False + + return True + + +def get_param_groups(network): + """Param groups for optimizer.""" + decay_params = [] + no_decay_params = [] + for x in network.trainable_params(): + parameter_name = x.name + if parameter_name.endswith('.bias'): + # all bias not using weight decay + no_decay_params.append(x) + elif parameter_name.endswith('.gamma'): + # bn weight bias not using weight decay, be carefully for now x not include BN + no_decay_params.append(x) + elif parameter_name.endswith('.beta'): + # bn weight bias not using weight decay, be carefully for now x not include BN + no_decay_params.append(x) + else: + decay_params.append(x) + + return [{'params': no_decay_params, 'weight_decay': 0.0}, {'params': decay_params}] + + +class ShapeRecord: + """Log image shape.""" + def __init__(self): + self.shape_record = { + 320: 0, + 352: 0, + 384: 0, + 416: 0, + 448: 0, + 480: 0, + 512: 0, + 544: 0, + 576: 0, + 608: 0, + 'total': 0 + } + + def set(self, shape): + if len(shape) > 1: + shape = shape[0] + shape = int(shape) + self.shape_record[shape] += 1 + self.shape_record['total'] += 1 + + def show(self, logger): + for key in self.shape_record: + rate = self.shape_record[key] / float(self.shape_record['total']) + logger.info('shape {}: {:.2f}%'.format(key, rate*100)) diff --git a/model_zoo/yolov3_darknet53/src/yolo.py b/model_zoo/yolov3_darknet53/src/yolo.py new file mode 100644 index 0000000000..09cce0c97f --- /dev/null +++ b/model_zoo/yolov3_darknet53/src/yolo.py @@ -0,0 +1,437 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""YOLOv3 based on DarkNet.""" +import mindspore as ms +import mindspore.nn as nn +from mindspore.common.tensor import Tensor +from mindspore import context +from mindspore.parallel._auto_parallel_context import auto_parallel_context +from mindspore.communication.management import get_group_size +from mindspore.ops import operations as P +from mindspore.ops import functional as F +from mindspore.ops import composite as C + +from src.darknet import DarkNet, ResidualBlock +from src.config import ConfigYOLOV3DarkNet53 +from src.loss import XYLoss, WHLoss, ConfidenceLoss, ClassLoss + + +def _conv_bn_relu(in_channel, + out_channel, + ksize, + stride=1, + padding=0, + dilation=1, + alpha=0.1, + momentum=0.9, + eps=1e-5, + pad_mode="same"): + """Get a conv2d batchnorm and relu layer""" + return nn.SequentialCell( + [nn.Conv2d(in_channel, + out_channel, + kernel_size=ksize, + stride=stride, + padding=padding, + dilation=dilation, + pad_mode=pad_mode), + nn.BatchNorm2d(out_channel, momentum=momentum, eps=eps), + nn.LeakyReLU(alpha)] + ) + + +class YoloBlock(nn.Cell): + """ + YoloBlock for YOLOv3. + + Args: + in_channels: Integer. Input channel. + out_chls: Interger. Middle channel. + out_channels: Integer. Output channel. + + Returns: + Tuple, tuple of output tensor,(f1,f2,f3). + + Examples: + YoloBlock(1024, 512, 255) + + """ + def __init__(self, in_channels, out_chls, out_channels): + super(YoloBlock, self).__init__() + out_chls_2 = out_chls*2 + + self.conv0 = _conv_bn_relu(in_channels, out_chls, ksize=1) + self.conv1 = _conv_bn_relu(out_chls, out_chls_2, ksize=3) + + self.conv2 = _conv_bn_relu(out_chls_2, out_chls, ksize=1) + self.conv3 = _conv_bn_relu(out_chls, out_chls_2, ksize=3) + + self.conv4 = _conv_bn_relu(out_chls_2, out_chls, ksize=1) + self.conv5 = _conv_bn_relu(out_chls, out_chls_2, ksize=3) + + self.conv6 = nn.Conv2d(out_chls_2, out_channels, kernel_size=1, stride=1, has_bias=True) + + def construct(self, x): + c1 = self.conv0(x) + c2 = self.conv1(c1) + + c3 = self.conv2(c2) + c4 = self.conv3(c3) + + c5 = self.conv4(c4) + c6 = self.conv5(c5) + + out = self.conv6(c6) + return c5, out + + +class YOLOv3(nn.Cell): + """ + YOLOv3 Network. + + Note: + backbone = darknet53 + + Args: + backbone_shape: List. Darknet output channels shape. + backbone: Cell. Backbone Network. + out_channel: Interger. Output channel. + + Returns: + Tensor, output tensor. + + Examples: + YOLOv3(backbone_shape=[64, 128, 256, 512, 1024] + backbone=darknet53(), + out_channel=255) + """ + def __init__(self, backbone_shape, backbone, out_channel): + super(YOLOv3, self).__init__() + self.out_channel = out_channel + self.backbone = backbone + self.backblock0 = YoloBlock(backbone_shape[-1], out_chls=backbone_shape[-2], out_channels=out_channel) + + self.conv1 = _conv_bn_relu(in_channel=backbone_shape[-2], out_channel=backbone_shape[-2]//2, ksize=1) + self.backblock1 = YoloBlock(in_channels=backbone_shape[-2]+backbone_shape[-3], + out_chls=backbone_shape[-3], + out_channels=out_channel) + + self.conv2 = _conv_bn_relu(in_channel=backbone_shape[-3], out_channel=backbone_shape[-3]//2, ksize=1) + self.backblock2 = YoloBlock(in_channels=backbone_shape[-3]+backbone_shape[-4], + out_chls=backbone_shape[-4], + out_channels=out_channel) + self.concat = P.Concat(axis=1) + + def construct(self, x): + # input_shape of x is (batch_size, 3, h, w) + # feature_map1 is (batch_size, backbone_shape[2], h/8, w/8) + # feature_map2 is (batch_size, backbone_shape[3], h/16, w/16) + # feature_map3 is (batch_size, backbone_shape[4], h/32, w/32) + img_hight = P.Shape()(x)[2] + img_width = P.Shape()(x)[3] + feature_map1, feature_map2, feature_map3 = self.backbone(x) + con1, big_object_output = self.backblock0(feature_map3) + + con1 = self.conv1(con1) + ups1 = P.ResizeNearestNeighbor((img_hight / 16, img_width / 16))(con1) + con1 = self.concat((ups1, feature_map2)) + con2, medium_object_output = self.backblock1(con1) + + con2 = self.conv2(con2) + ups2 = P.ResizeNearestNeighbor((img_hight / 8, img_width / 8))(con2) + con3 = self.concat((ups2, feature_map1)) + _, small_object_output = self.backblock2(con3) + + return big_object_output, medium_object_output, small_object_output + + +class DetectionBlock(nn.Cell): + """ + YOLOv3 detection Network. It will finally output the detection result. + + Args: + scale: Character. + config: ConfigYOLOV3DarkNet53, Configuration instance. + is_training: Bool, Whether train or not, default True. + + Returns: + Tuple, tuple of output tensor,(f1,f2,f3). + + Examples: + DetectionBlock(scale='l',stride=32) + """ + + def __init__(self, scale, config=ConfigYOLOV3DarkNet53(), is_training=True): + super(DetectionBlock, self).__init__() + self.config = config + if scale == 's': + idx = (0, 1, 2) + elif scale == 'm': + idx = (3, 4, 5) + elif scale == 'l': + idx = (6, 7, 8) + else: + raise KeyError("Invalid scale value for DetectionBlock") + self.anchors = Tensor([self.config.anchor_scales[i] for i in idx], ms.float32) + self.num_anchors_per_scale = 3 + self.num_attrib = 4+1+self.config.num_classes + self.lambda_coord = 1 + + self.sigmoid = nn.Sigmoid() + self.reshape = P.Reshape() + self.tile = P.Tile() + self.concat = P.Concat(axis=-1) + self.conf_training = is_training + + def construct(self, x, input_shape): + num_batch = P.Shape()(x)[0] + grid_size = P.Shape()(x)[2:4] + + # Reshape and transpose the feature to [n, grid_size[0], grid_size[1], 3, num_attrib] + prediction = P.Reshape()(x, (num_batch, + self.num_anchors_per_scale, + self.num_attrib, + grid_size[0], + grid_size[1])) + prediction = P.Transpose()(prediction, (0, 3, 4, 1, 2)) + + range_x = range(grid_size[1]) + range_y = range(grid_size[0]) + grid_x = P.Cast()(F.tuple_to_array(range_x), ms.float32) + grid_y = P.Cast()(F.tuple_to_array(range_y), ms.float32) + # Tensor of shape [grid_size[0], grid_size[1], 1, 1] representing the coordinate of x/y axis for each grid + # [batch, gridx, gridy, 1, 1] + grid_x = self.tile(self.reshape(grid_x, (1, 1, -1, 1, 1)), (1, grid_size[0], 1, 1, 1)) + grid_y = self.tile(self.reshape(grid_y, (1, -1, 1, 1, 1)), (1, 1, grid_size[1], 1, 1)) + # Shape is [grid_size[0], grid_size[1], 1, 2] + grid = self.concat((grid_x, grid_y)) + + box_xy = prediction[:, :, :, :, :2] + box_wh = prediction[:, :, :, :, 2:4] + box_confidence = prediction[:, :, :, :, 4:5] + box_probs = prediction[:, :, :, :, 5:] + + # gridsize1 is x + # gridsize0 is y + box_xy = (self.sigmoid(box_xy) + grid) / P.Cast()(F.tuple_to_array((grid_size[1], grid_size[0])), ms.float32) + # box_wh is w->h + box_wh = P.Exp()(box_wh) * self.anchors / input_shape + box_confidence = self.sigmoid(box_confidence) + box_probs = self.sigmoid(box_probs) + + if self.conf_training: + return grid, prediction, box_xy, box_wh + return self.concat((box_xy, box_wh, box_confidence, box_probs)) + + +class Iou(nn.Cell): + """Calculate the iou of boxes""" + def __init__(self): + super(Iou, self).__init__() + self.min = P.Minimum() + self.max = P.Maximum() + + def construct(self, box1, box2): + # box1: pred_box [batch, gx, gy, anchors, 1, 4] ->4: [x_center, y_center, w, h] + # box2: gt_box [batch, 1, 1, 1, maxbox, 4] + # convert to topLeft and rightDown + box1_xy = box1[:, :, :, :, :, :2] + box1_wh = box1[:, :, :, :, :, 2:4] + box1_mins = box1_xy - box1_wh / F.scalar_to_array(2.0) # topLeft + box1_maxs = box1_xy + box1_wh / F.scalar_to_array(2.0) # rightDown + + box2_xy = box2[:, :, :, :, :, :2] + box2_wh = box2[:, :, :, :, :, 2:4] + box2_mins = box2_xy - box2_wh / F.scalar_to_array(2.0) + box2_maxs = box2_xy + box2_wh / F.scalar_to_array(2.0) + + intersect_mins = self.max(box1_mins, box2_mins) + intersect_maxs = self.min(box1_maxs, box2_maxs) + intersect_wh = self.max(intersect_maxs - intersect_mins, F.scalar_to_array(0.0)) + # P.squeeze: for effiecient slice + intersect_area = P.Squeeze(-1)(intersect_wh[:, :, :, :, :, 0:1]) * \ + P.Squeeze(-1)(intersect_wh[:, :, :, :, :, 1:2]) + box1_area = P.Squeeze(-1)(box1_wh[:, :, :, :, :, 0:1]) * P.Squeeze(-1)(box1_wh[:, :, :, :, :, 1:2]) + box2_area = P.Squeeze(-1)(box2_wh[:, :, :, :, :, 0:1]) * P.Squeeze(-1)(box2_wh[:, :, :, :, :, 1:2]) + iou = intersect_area / (box1_area + box2_area - intersect_area) + # iou : [batch, gx, gy, anchors, maxboxes] + return iou + + +class YoloLossBlock(nn.Cell): + """ + Loss block cell of YOLOV3 network. + """ + def __init__(self, scale, config=ConfigYOLOV3DarkNet53()): + super(YoloLossBlock, self).__init__() + self.config = config + if scale == 's': + # anchor mask + idx = (0, 1, 2) + elif scale == 'm': + idx = (3, 4, 5) + elif scale == 'l': + idx = (6, 7, 8) + else: + raise KeyError("Invalid scale value for DetectionBlock") + self.anchors = Tensor([self.config.anchor_scales[i] for i in idx], ms.float32) + self.ignore_threshold = Tensor(self.config.ignore_threshold, ms.float32) + self.concat = P.Concat(axis=-1) + self.iou = Iou() + self.reduce_max = P.ReduceMax(keep_dims=False) + self.xy_loss = XYLoss() + self.wh_loss = WHLoss() + self.confidenceLoss = ConfidenceLoss() + self.classLoss = ClassLoss() + + def construct(self, grid, prediction, pred_xy, pred_wh, y_true, gt_box, input_shape): + # prediction : origin output from yolo + # pred_xy: (sigmoid(xy)+grid)/grid_size + # pred_wh: (exp(wh)*anchors)/input_shape + # y_true : after normalize + # gt_box: [batch, maxboxes, xyhw] after normalize + + object_mask = y_true[:, :, :, :, 4:5] + class_probs = y_true[:, :, :, :, 5:] + + grid_shape = P.Shape()(prediction)[1:3] + grid_shape = P.Cast()(F.tuple_to_array(grid_shape[::-1]), ms.float32) + + pred_boxes = self.concat((pred_xy, pred_wh)) + true_xy = y_true[:, :, :, :, :2] * grid_shape - grid + true_wh = y_true[:, :, :, :, 2:4] + true_wh = P.Select()(P.Equal()(true_wh, 0.0), + P.Fill()(P.DType()(true_wh), + P.Shape()(true_wh), 1.0), + true_wh) + true_wh = P.Log()(true_wh / self.anchors * input_shape) + # 2-w*h for large picture, use small scale, since small obj need more precise + box_loss_scale = 2 - y_true[:, :, :, :, 2:3] * y_true[:, :, :, :, 3:4] + + gt_shape = P.Shape()(gt_box) + gt_box = P.Reshape()(gt_box, (gt_shape[0], 1, 1, 1, gt_shape[1], gt_shape[2])) + + # add one more dimension for broadcast + iou = self.iou(P.ExpandDims()(pred_boxes, -2), gt_box) + # gt_box is x,y,h,w after normalize + # [batch, grid[0], grid[1], num_anchor, num_gt] + best_iou = self.reduce_max(iou, -1) + # [batch, grid[0], grid[1], num_anchor] + + # ignore_mask IOU too small + ignore_mask = best_iou < self.ignore_threshold + ignore_mask = P.Cast()(ignore_mask, ms.float32) + ignore_mask = P.ExpandDims()(ignore_mask, -1) + # ignore_mask backpro will cause a lot maximunGrad and minimumGrad time consume. + # so we turn off its gradient + ignore_mask = F.stop_gradient(ignore_mask) + + xy_loss = self.xy_loss(object_mask, box_loss_scale, prediction[:, :, :, :, :2], true_xy) + wh_loss = self.wh_loss(object_mask, box_loss_scale, prediction[:, :, :, :, 2:4], true_wh) + confidence_loss = self.confidenceLoss(object_mask, prediction[:, :, :, :, 4:5], ignore_mask) + class_loss = self.classLoss(object_mask, prediction[:, :, :, :, 5:], class_probs) + loss = xy_loss + wh_loss + confidence_loss + class_loss + batch_size = P.Shape()(prediction)[0] + return loss / batch_size + + +class YOLOV3DarkNet53(nn.Cell): + """ + Darknet based YOLOV3 network. + + Args: + is_training: Bool. Whether train or not. + + Returns: + Cell, cell instance of Darknet based YOLOV3 neural network. + + Examples: + YOLOV3DarkNet53(True) + """ + + def __init__(self, is_training): + super(YOLOV3DarkNet53, self).__init__() + self.config = ConfigYOLOV3DarkNet53() + + # YOLOv3 network + self.feature_map = YOLOv3(backbone=DarkNet(ResidualBlock, self.config.backbone_layers, + self.config.backbone_input_shape, + self.config.backbone_shape, + detect=True), + backbone_shape=self.config.backbone_shape, + out_channel=self.config.out_channel) + + # prediction on the default anchor boxes + self.detect_1 = DetectionBlock('l', is_training=is_training) + self.detect_2 = DetectionBlock('m', is_training=is_training) + self.detect_3 = DetectionBlock('s', is_training=is_training) + + def construct(self, x, input_shape): + big_object_output, medium_object_output, small_object_output = self.feature_map(x) + output_big = self.detect_1(big_object_output, input_shape) + output_me = self.detect_2(medium_object_output, input_shape) + output_small = self.detect_3(small_object_output, input_shape) + # big is the final output which has smallest feature map + return output_big, output_me, output_small + + +class YoloWithLossCell(nn.Cell): + """YOLOV3 loss.""" + def __init__(self, network): + super(YoloWithLossCell, self).__init__() + self.yolo_network = network + self.config = ConfigYOLOV3DarkNet53() + self.loss_big = YoloLossBlock('l', self.config) + self.loss_me = YoloLossBlock('m', self.config) + self.loss_small = YoloLossBlock('s', self.config) + + def construct(self, x, y_true_0, y_true_1, y_true_2, gt_0, gt_1, gt_2, input_shape): + yolo_out = self.yolo_network(x, input_shape) + loss_l = self.loss_big(*yolo_out[0], y_true_0, gt_0, input_shape) + loss_m = self.loss_me(*yolo_out[1], y_true_1, gt_1, input_shape) + loss_s = self.loss_small(*yolo_out[2], y_true_2, gt_2, input_shape) + return loss_l + loss_m + loss_s + + +class TrainingWrapper(nn.Cell): + """Training wrapper.""" + def __init__(self, network, optimizer, sens=1.0): + super(TrainingWrapper, self).__init__(auto_prefix=False) + self.network = network + self.weights = optimizer.parameters + self.optimizer = optimizer + self.grad = C.GradOperation('grad', get_by_list=True, sens_param=True) + self.sens = sens + self.reducer_flag = False + self.grad_reducer = None + self.parallel_mode = context.get_auto_parallel_context("parallel_mode") + if self.parallel_mode in [ms.ParallelMode.DATA_PARALLEL, ms.ParallelMode.HYBRID_PARALLEL]: + self.reducer_flag = True + if self.reducer_flag: + mean = context.get_auto_parallel_context("mirror_mean") + if auto_parallel_context().get_device_num_is_set(): + degree = context.get_auto_parallel_context("device_num") + else: + degree = get_group_size() + self.grad_reducer = nn.DistributedGradReducer(optimizer.parameters, mean, degree) + + def construct(self, *args): + weights = self.weights + loss = self.network(*args) + sens = P.Fill()(P.DType()(loss), P.Shape()(loss), self.sens) + grads = self.grad(self.network, weights)(*args, sens) + if self.reducer_flag: + grads = self.grad_reducer(grads) + return F.depend(loss, self.optimizer(grads)) diff --git a/model_zoo/yolov3_darknet53/src/yolo_dataset.py b/model_zoo/yolov3_darknet53/src/yolo_dataset.py new file mode 100644 index 0000000000..45657db823 --- /dev/null +++ b/model_zoo/yolov3_darknet53/src/yolo_dataset.py @@ -0,0 +1,184 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""YOLOV3 dataset.""" +import os + +from PIL import Image +from pycocotools.coco import COCO +import mindspore.dataset as de +import mindspore.dataset.transforms.vision.c_transforms as CV + +from src.distributed_sampler import DistributedSampler +from src.transforms import reshape_fn, MultiScaleTrans + + +min_keypoints_per_image = 10 + + +def _has_only_empty_bbox(anno): + return all(any(o <= 1 for o in obj["bbox"][2:]) for obj in anno) + + +def _count_visible_keypoints(anno): + return sum(sum(1 for v in ann["keypoints"][2::3] if v > 0) for ann in anno) + + +def has_valid_annotation(anno): + """Check annotation file.""" + # if it's empty, there is no annotation + if not anno: + return False + # if all boxes have close to zero area, there is no annotation + if _has_only_empty_bbox(anno): + return False + # keypoints task have a slight different critera for considering + # if an annotation is valid + if "keypoints" not in anno[0]: + return True + # for keypoint detection tasks, only consider valid images those + # containing at least min_keypoints_per_image + if _count_visible_keypoints(anno) >= min_keypoints_per_image: + return True + return False + + +class COCOYoloDataset: + """YOLOV3 Dataset for COCO.""" + def __init__(self, root, ann_file, remove_images_without_annotations=True, + filter_crowd_anno=True, is_training=True): + self.coco = COCO(ann_file) + self.root = root + self.img_ids = list(sorted(self.coco.imgs.keys())) + self.filter_crowd_anno = filter_crowd_anno + self.is_training = is_training + + # filter images without any annotations + if remove_images_without_annotations: + img_ids = [] + for img_id in self.img_ids: + ann_ids = self.coco.getAnnIds(imgIds=img_id, iscrowd=None) + anno = self.coco.loadAnns(ann_ids) + if has_valid_annotation(anno): + img_ids.append(img_id) + self.img_ids = img_ids + + self.categories = {cat["id"]: cat["name"] for cat in self.coco.cats.values()} + + self.cat_ids_to_continuous_ids = { + v: i for i, v in enumerate(self.coco.getCatIds()) + } + self.continuous_ids_cat_ids = { + v: k for k, v in self.cat_ids_to_continuous_ids.items() + } + + def __getitem__(self, index): + """ + Args: + index (int): Index + + Returns: + (img, target) (tuple): target is a dictionary contains "bbox", "segmentation" or "keypoints", + generated by the image's annotation. img is a PIL image. + """ + coco = self.coco + img_id = self.img_ids[index] + img_path = coco.loadImgs(img_id)[0]["file_name"] + img = Image.open(os.path.join(self.root, img_path)).convert("RGB") + if not self.is_training: + return img, img_id + + ann_ids = coco.getAnnIds(imgIds=img_id) + target = coco.loadAnns(ann_ids) + # filter crowd annotations + if self.filter_crowd_anno: + annos = [anno for anno in target if anno["iscrowd"] == 0] + else: + annos = [anno for anno in target] + + target = {} + boxes = [anno["bbox"] for anno in annos] + target["bboxes"] = boxes + + classes = [anno["category_id"] for anno in annos] + classes = [self.cat_ids_to_continuous_ids[cl] for cl in classes] + target["labels"] = classes + + bboxes = target['bboxes'] + labels = target['labels'] + out_target = [] + for bbox, label in zip(bboxes, labels): + tmp = [] + # convert to [x_min y_min x_max y_max] + bbox = self._convetTopDown(bbox) + tmp.extend(bbox) + tmp.append(int(label)) + # tmp [x_min y_min x_max y_max, label] + out_target.append(tmp) + return img, out_target + + def __len__(self): + return len(self.img_ids) + + def _convetTopDown(self, bbox): + x_min = bbox[0] + y_min = bbox[1] + w = bbox[2] + h = bbox[3] + return [x_min, y_min, x_min+w, y_min+h] + + +def create_yolo_dataset(image_dir, anno_path, batch_size, max_epoch, device_num, rank, + config=None, is_training=True, shuffle=True): + """Create dataset for YOLOV3.""" + if is_training: + filter_crowd = True + remove_empty_anno = True + else: + filter_crowd = False + remove_empty_anno = False + + yolo_dataset = COCOYoloDataset(root=image_dir, ann_file=anno_path, filter_crowd_anno=filter_crowd, + remove_images_without_annotations=remove_empty_anno, is_training=is_training) + distributed_sampler = DistributedSampler(len(yolo_dataset), device_num, rank, shuffle=shuffle) + hwc_to_chw = CV.HWC2CHW() + + config.dataset_size = len(yolo_dataset) + num_parallel_workers1 = int(64 / device_num) + num_parallel_workers2 = int(16 / device_num) + if is_training: + multi_scale_trans = MultiScaleTrans(config, device_num) + if device_num != 8: + ds = de.GeneratorDataset(yolo_dataset, column_names=["image", "annotation"], + num_parallel_workers=num_parallel_workers1, + sampler=distributed_sampler) + ds = ds.batch(batch_size, per_batch_map=multi_scale_trans, input_columns=['image', 'annotation'], + num_parallel_workers=num_parallel_workers2, drop_remainder=True) + else: + ds = de.GeneratorDataset(yolo_dataset, column_names=["image", "annotation"], sampler=distributed_sampler) + ds = ds.batch(batch_size, per_batch_map=multi_scale_trans, input_columns=['image', 'annotation'], + num_parallel_workers=8, drop_remainder=True) + else: + ds = de.GeneratorDataset(yolo_dataset, column_names=["image", "img_id"], + sampler=distributed_sampler) + compose_map_func = (lambda image, img_id: reshape_fn(image, img_id, config)) + ds = ds.map(input_columns=["image", "img_id"], + output_columns=["image", "image_shape", "img_id"], + columns_order=["image", "image_shape", "img_id"], + operations=compose_map_func, num_parallel_workers=8) + ds = ds.map(input_columns=["image"], operations=hwc_to_chw, num_parallel_workers=8) + ds = ds.batch(batch_size, drop_remainder=True) + ds = ds.repeat(max_epoch) + + return ds, len(yolo_dataset) diff --git a/model_zoo/yolov3_darknet53/train.py b/model_zoo/yolov3_darknet53/train.py new file mode 100644 index 0000000000..0781cedccb --- /dev/null +++ b/model_zoo/yolov3_darknet53/train.py @@ -0,0 +1,338 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""YoloV3 train.""" +import os +import time +import argparse +import datetime + +from mindspore import ParallelMode +from mindspore.nn.optim.momentum import Momentum +from mindspore import Tensor +import mindspore.nn as nn +from mindspore import context +from mindspore.communication.management import init, get_rank, get_group_size +from mindspore.train.callback import ModelCheckpoint, RunContext +from mindspore.train.callback import _InternalCallbackParam, CheckpointConfig +import mindspore as ms +from mindspore.train.serialization import load_checkpoint, load_param_into_net + +from src.yolo import YOLOV3DarkNet53, YoloWithLossCell, TrainingWrapper +from src.logger import get_logger +from src.util import AverageMeter, load_backbone, get_param_groups +from src.lr_scheduler import warmup_step_lr, warmup_cosine_annealing_lr, \ + warmup_cosine_annealing_lr_V2, warmup_cosine_annealing_lr_sample +from src.yolo_dataset import create_yolo_dataset +from src.initializer import default_recurisive_init +from src.config import ConfigYOLOV3DarkNet53 +from src.transforms import batch_preprocess_true_box, batch_preprocess_true_box_single +from src.util import ShapeRecord + + +devid = int(os.getenv('DEVICE_ID')) +context.set_context(mode=context.GRAPH_MODE, enable_auto_mixed_precision=True, + device_target="Ascend", save_graphs=True, device_id=devid) + + +class BuildTrainNetwork(nn.Cell): + def __init__(self, network, criterion): + super(BuildTrainNetwork, self).__init__() + self.network = network + self.criterion = criterion + + def construct(self, input_data, label): + output = self.network(input_data) + loss = self.criterion(output, label) + return loss + + +def parse_args(): + """Parse train arguments.""" + parser = argparse.ArgumentParser('mindspore coco training') + + # dataset related + parser.add_argument('--data_dir', type=str, default='', help='train data dir') + parser.add_argument('--per_batch_size', default=32, type=int, help='batch size for per gpu') + + # network related + parser.add_argument('--pretrained_backbone', default='', type=str, help='model_path, local pretrained backbone' + ' model to load') + parser.add_argument('--resume_yolov3', default='', type=str, help='path of pretrained yolov3') + + # optimizer and lr related + parser.add_argument('--lr_scheduler', default='exponential', type=str, + help='lr-scheduler, option type: exponential, cosine_annealing') + parser.add_argument('--lr', default=0.001, type=float, help='learning rate of the training') + parser.add_argument('--lr_epochs', type=str, default='220,250', help='epoch of lr changing') + parser.add_argument('--lr_gamma', type=float, default=0.1, + help='decrease lr by a factor of exponential lr_scheduler') + parser.add_argument('--eta_min', type=float, default=0., help='eta_min in cosine_annealing scheduler') + parser.add_argument('--T_max', type=int, default=320, help='T-max in cosine_annealing scheduler') + parser.add_argument('--max_epoch', type=int, default=320, help='max epoch num to train the model') + parser.add_argument('--warmup_epochs', default=0, type=float, help='warmup epoch') + parser.add_argument('--weight_decay', type=float, default=0.0005, help='weight decay') + parser.add_argument('--momentum', type=float, default=0.9, help='momentum') + + # loss related + parser.add_argument('--loss_scale', type=int, default=1024, help='static loss scale') + parser.add_argument('--label_smooth', type=int, default=0, help='whether to use label smooth in CE') + parser.add_argument('--label_smooth_factor', type=float, default=0.1, help='smooth strength of original one-hot') + + # logging related + parser.add_argument('--log_interval', type=int, default=100, help='logging interval') + parser.add_argument('--ckpt_path', type=str, default='outputs/', help='checkpoint save location') + parser.add_argument('--ckpt_interval', type=int, default=None, help='ckpt_interval') + + parser.add_argument('--is_save_on_master', type=int, default=1, help='save ckpt on master or all rank') + + # distributed related + parser.add_argument('--is_distributed', type=int, default=1, help='if multi device') + parser.add_argument('--rank', type=int, default=0, help='local rank of distributed') + parser.add_argument('--group_size', type=int, default=1, help='world size of distributed') + + # roma obs + parser.add_argument('--train_url', type=str, default="", help='train url') + + # profiler init + parser.add_argument('--need_profiler', type=int, default=0, help='whether use profiler') + + # reset default config + parser.add_argument('--training_shape', type=str, default="", help='fix training shape') + parser.add_argument('--resize_rate', type=int, default=None, help='resize rate for multi-scale training') + + args, _ = parser.parse_known_args() + if args.lr_scheduler == 'cosine_annealing' and args.max_epoch > args.T_max: + args.T_max = args.max_epoch + + args.lr_epochs = list(map(int, args.lr_epochs.split(','))) + args.data_root = os.path.join(args.data_dir, 'train2014') + args.annFile = os.path.join(args.data_dir, 'annotations/instances_train2014.json') + + return args + + +def conver_training_shape(args): + training_shape = [int(args.training_shape), int(args.training_shape)] + return training_shape + + +def train(): + """Train function.""" + args = parse_args() + + # init distributed + if args.is_distributed: + init() + args.rank = get_rank() + args.group_size = get_group_size() + + # select for master rank save ckpt or all rank save, compatiable for model parallel + args.rank_save_ckpt_flag = 0 + if args.is_save_on_master: + if args.rank == 0: + args.rank_save_ckpt_flag = 1 + else: + args.rank_save_ckpt_flag = 1 + + # logger + args.outputs_dir = os.path.join(args.ckpt_path, + datetime.datetime.now().strftime('%Y-%m-%d_time_%H_%M_%S')) + args.logger = get_logger(args.outputs_dir, args.rank) + args.logger.save_args(args) + + if args.need_profiler: + from mindinsight.profiler.profiling import Profiler + profiler = Profiler(output_path=args.outputs_dir, is_detail=True, is_show_op_path=True) + + loss_meter = AverageMeter('loss') + + context.reset_auto_parallel_context() + if args.is_distributed: + parallel_mode = ParallelMode.DATA_PARALLEL + degree = get_group_size() + else: + parallel_mode = ParallelMode.STAND_ALONE + degree = 1 + context.set_auto_parallel_context(parallel_mode=parallel_mode, mirror_mean=True, device_num=degree) + + network = YOLOV3DarkNet53(is_training=True) + # default is kaiming-normal + default_recurisive_init(network) + + if args.pretrained_backbone: + network = load_backbone(network, args.pretrained_backbone, args) + args.logger.info('load pre-trained backbone {} into network'.format(args.pretrained_backbone)) + else: + args.logger.info('Not load pre-trained backbone, please be careful') + + if args.resume_yolov3: + param_dict = load_checkpoint(args.resume_yolov3) + param_dict_new = {} + for key, values in param_dict.items(): + if key.startswith('moments.'): + continue + elif key.startswith('yolo_network.'): + param_dict_new[key[13:]] = values + args.logger.info('in resume {}'.format(key)) + else: + param_dict_new[key] = values + args.logger.info('in resume {}'.format(key)) + + args.logger.info('resume finished') + load_param_into_net(network, param_dict_new) + args.logger.info('load_model {} success'.format(args.resume_yolov3)) + + network = YoloWithLossCell(network) + args.logger.info('finish get network') + + config = ConfigYOLOV3DarkNet53() + + config.label_smooth = args.label_smooth + config.label_smooth_factor = args.label_smooth_factor + + if args.training_shape: + config.multi_scale = [conver_training_shape(args)] + if args.resize_rate: + config.resize_rate = args.resize_rate + + ds, data_size = create_yolo_dataset(image_dir=args.data_root, anno_path=args.annFile, is_training=True, + batch_size=args.per_batch_size, max_epoch=args.max_epoch, + device_num=args.group_size, rank=args.rank, config=config) + args.logger.info('Finish loading dataset') + + args.steps_per_epoch = int(data_size / args.per_batch_size / args.group_size) + + if not args.ckpt_interval: + args.ckpt_interval = args.steps_per_epoch + + # lr scheduler + if args.lr_scheduler == 'exponential': + lr = warmup_step_lr(args.lr, + args.lr_epochs, + args.steps_per_epoch, + args.warmup_epochs, + args.max_epoch, + gamma=args.lr_gamma, + ) + elif args.lr_scheduler == 'cosine_annealing': + lr = warmup_cosine_annealing_lr(args.lr, + args.steps_per_epoch, + args.warmup_epochs, + args.max_epoch, + args.T_max, + args.eta_min) + elif args.lr_scheduler == 'cosine_annealing_V2': + lr = warmup_cosine_annealing_lr_V2(args.lr, + args.steps_per_epoch, + args.warmup_epochs, + args.max_epoch, + args.T_max, + args.eta_min) + elif args.lr_scheduler == 'cosine_annealing_sample': + lr = warmup_cosine_annealing_lr_sample(args.lr, + args.steps_per_epoch, + args.warmup_epochs, + args.max_epoch, + args.T_max, + args.eta_min) + else: + raise NotImplementedError(args.lr_scheduler) + + opt = Momentum(params=get_param_groups(network), + learning_rate=Tensor(lr), + momentum=args.momentum, + weight_decay=args.weight_decay, + loss_scale=args.loss_scale) + + network = TrainingWrapper(network, opt) + network.set_train() + + if args.rank_save_ckpt_flag: + # checkpoint save + ckpt_max_num = args.max_epoch * args.steps_per_epoch // args.ckpt_interval + ckpt_config = CheckpointConfig(save_checkpoint_steps=args.ckpt_interval, + keep_checkpoint_max=ckpt_max_num) + ckpt_cb = ModelCheckpoint(config=ckpt_config, + directory=args.outputs_dir, + prefix='{}'.format(args.rank)) + cb_params = _InternalCallbackParam() + cb_params.train_network = network + cb_params.epoch_num = ckpt_max_num + cb_params.cur_epoch_num = 1 + run_context = RunContext(cb_params) + ckpt_cb.begin(run_context) + + old_progress = -1 + t_end = time.time() + data_loader = ds.create_dict_iterator() + + shape_record = ShapeRecord() + for i, data in enumerate(data_loader): + images = data["image"] + input_shape = images.shape[2:4] + args.logger.info('iter[{}], shape{}'.format(i, input_shape[0])) + shape_record.set(input_shape) + + images = Tensor(images) + annos = data["annotation"] + if args.group_size == 1: + batch_y_true_0, batch_y_true_1, batch_y_true_2, batch_gt_box0, batch_gt_box1, batch_gt_box2 = \ + batch_preprocess_true_box(annos, config, input_shape) + else: + batch_y_true_0, batch_y_true_1, batch_y_true_2, batch_gt_box0, batch_gt_box1, batch_gt_box2 = \ + batch_preprocess_true_box_single(annos, config, input_shape) + + batch_y_true_0 = Tensor(batch_y_true_0) + batch_y_true_1 = Tensor(batch_y_true_1) + batch_y_true_2 = Tensor(batch_y_true_2) + batch_gt_box0 = Tensor(batch_gt_box0) + batch_gt_box1 = Tensor(batch_gt_box1) + batch_gt_box2 = Tensor(batch_gt_box2) + + input_shape = Tensor(tuple(input_shape[::-1]), ms.float32) + loss = network(images, batch_y_true_0, batch_y_true_1, batch_y_true_2, batch_gt_box0, batch_gt_box1, + batch_gt_box2, input_shape) + loss_meter.update(loss.asnumpy()) + + if args.rank_save_ckpt_flag: + # ckpt progress + cb_params.cur_step_num = i + 1 # current step number + cb_params.batch_num = i + 2 + ckpt_cb.step_end(run_context) + + if i % args.log_interval == 0: + time_used = time.time() - t_end + epoch = int(i / args.steps_per_epoch) + fps = args.per_batch_size * (i - old_progress) * args.group_size / time_used + if args.rank == 0: + args.logger.info( + 'epoch[{}], iter[{}], {}, {:.2f} imgs/sec, lr:{}'.format(epoch, i, loss_meter, fps, lr[i])) + t_end = time.time() + loss_meter.reset() + old_progress = i + + if (i + 1) % args.steps_per_epoch == 0 and args.rank_save_ckpt_flag: + cb_params.cur_epoch_num += 1 + + if args.need_profiler: + if i == 10: + profiler.analyse() + break + + args.logger.info('==========end training===============') + + +if __name__ == "__main__": + train() diff --git a/model_zoo/yolov3/README.md b/model_zoo/yolov3_resnet18/README.md similarity index 100% rename from model_zoo/yolov3/README.md rename to model_zoo/yolov3_resnet18/README.md diff --git a/model_zoo/yolov3/eval.py b/model_zoo/yolov3_resnet18/eval.py similarity index 100% rename from model_zoo/yolov3/eval.py rename to model_zoo/yolov3_resnet18/eval.py diff --git a/model_zoo/yolov3/scripts/run_distribute_train.sh b/model_zoo/yolov3_resnet18/scripts/run_distribute_train.sh similarity index 100% rename from model_zoo/yolov3/scripts/run_distribute_train.sh rename to model_zoo/yolov3_resnet18/scripts/run_distribute_train.sh diff --git a/model_zoo/yolov3/scripts/run_eval.sh b/model_zoo/yolov3_resnet18/scripts/run_eval.sh similarity index 100% rename from model_zoo/yolov3/scripts/run_eval.sh rename to model_zoo/yolov3_resnet18/scripts/run_eval.sh diff --git a/model_zoo/yolov3/scripts/run_standalone_train.sh b/model_zoo/yolov3_resnet18/scripts/run_standalone_train.sh similarity index 100% rename from model_zoo/yolov3/scripts/run_standalone_train.sh rename to model_zoo/yolov3_resnet18/scripts/run_standalone_train.sh diff --git a/model_zoo/yolov3/src/config.py b/model_zoo/yolov3_resnet18/src/config.py similarity index 100% rename from model_zoo/yolov3/src/config.py rename to model_zoo/yolov3_resnet18/src/config.py diff --git a/model_zoo/yolov3/src/dataset.py b/model_zoo/yolov3_resnet18/src/dataset.py similarity index 100% rename from model_zoo/yolov3/src/dataset.py rename to model_zoo/yolov3_resnet18/src/dataset.py diff --git a/model_zoo/yolov3/src/utils.py b/model_zoo/yolov3_resnet18/src/utils.py similarity index 100% rename from model_zoo/yolov3/src/utils.py rename to model_zoo/yolov3_resnet18/src/utils.py diff --git a/model_zoo/yolov3/src/yolov3.py b/model_zoo/yolov3_resnet18/src/yolov3.py similarity index 100% rename from model_zoo/yolov3/src/yolov3.py rename to model_zoo/yolov3_resnet18/src/yolov3.py diff --git a/model_zoo/yolov3/train.py b/model_zoo/yolov3_resnet18/train.py similarity index 100% rename from model_zoo/yolov3/train.py rename to model_zoo/yolov3_resnet18/train.py From 216ebcb92cc8b7b0ae899fd52ab9cdb9df0af27f Mon Sep 17 00:00:00 2001 From: He Wei Date: Mon, 29 Jun 2020 21:28:58 +0800 Subject: [PATCH 156/254] Enable Lazy allocation for Tensor 1. Tensor data is lazy allocated if no data to be copied; 2. Add prefix to Tensor id to distinguish with other id. --- mindspore/ccsrc/ir/tensor.cc | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/mindspore/ccsrc/ir/tensor.cc b/mindspore/ccsrc/ir/tensor.cc index 673a8da842..dccdbb65b8 100644 --- a/mindspore/ccsrc/ir/tensor.cc +++ b/mindspore/ccsrc/ir/tensor.cc @@ -35,7 +35,7 @@ using Bool = unsigned char; static std::string MakeId() { // Use atomic to make id generator thread safe. static std::atomic last_id{1}; - return std::to_string(last_id.fetch_add(1, std::memory_order_relaxed)); + return "T" + std::to_string(last_id.fetch_add(1, std::memory_order_relaxed)); } static TypeId TypeIdOf(const TypePtr &data_type, TypeId defaultTypeId) { @@ -127,41 +127,47 @@ std::vector CopyData(const std::vector &shape, void *data, size_t data_l template class TensorDataImpl : public TensorData { public: - explicit TensorDataImpl(const std::vector &shape) : shape_(shape), data_(SizeOf(shape)) {} + explicit TensorDataImpl(const std::vector &shape) : ndim_(shape.size()), data_size_(SizeOf(shape)) {} TensorDataImpl(const std::vector &shape, void *data, size_t data_len) - : shape_(shape), data_(CopyData(shape, data, data_len)) {} + : ndim_(shape.size()), data_size_(SizeOf(shape)), data_(CopyData(shape, data, data_len)) {} TensorDataImpl(const std::vector &shape, void *data, TypeId data_type) - : shape_(shape), data_(CopyData(shape, data, data_type)) {} + : ndim_(shape.size()), data_size_(SizeOf(shape)), data_(CopyData(shape, data, data_type)) {} template - TensorDataImpl(const std::vector &shape, InputIt first, InputIt last) : shape_(shape), data_(first, last) {} + TensorDataImpl(const std::vector &shape, InputIt first, InputIt last) + : ndim_(shape.size()), data_size_(SizeOf(shape)), data_(first, last) {} template - TensorDataImpl(const std::vector &shape, Scalar scalar) : shape_(shape), data_({static_cast(scalar)}) {} + TensorDataImpl(const std::vector &shape, Scalar scalar) + : ndim_(shape.size()), data_size_(SizeOf(shape)), data_({static_cast(scalar)}) {} - ssize_t size() const override { return data_.size(); } + ssize_t size() const override { return static_cast(data_size_); } ssize_t itemsize() const override { return static_cast(sizeof(T)); } ssize_t nbytes() const override { return size() * itemsize(); } - ssize_t ndim() const override { return static_cast(shape_.size()); } + ssize_t ndim() const override { return static_cast(ndim_); } void *data() override { static std::vector empty_data(1); - if (data_.empty()) { - // Prevent null pointer for empty data. + if (data_size_ == 0) { + // Prevent null pointer for empty shape. return empty_data.data(); } + if (data_.empty()) { + // Lazy allocation. + data_.resize(data_size_); + } return data_.data(); } bool equals(const TensorData &other) const override { auto ptr = dynamic_cast *>(&other); if (ptr) { - return (ptr == this) || ((shape_ == ptr->shape_) && (data_ == ptr->data_)); + return (ptr == this) || ((ndim_ == ptr->ndim_) && (data_size_ == ptr->data_size_) && (data_ == ptr->data_)); } return false; } @@ -177,7 +183,8 @@ class TensorDataImpl : public TensorData { } private: - std::vector shape_; + size_t ndim_{0}; + size_t data_size_{0}; std::vector data_; }; From 0dc8a42bd17cc4a9ee7263a99cea767fe08e9cc0 Mon Sep 17 00:00:00 2001 From: zhaoting Date: Tue, 30 Jun 2020 11:00:53 +0800 Subject: [PATCH 157/254] fix resnet st group params bug --- .../st/networks/models/resnet50/test_resnet50_imagenet.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/tests/st/networks/models/resnet50/test_resnet50_imagenet.py b/tests/st/networks/models/resnet50/test_resnet50_imagenet.py index 64f4adda99..c88af6bcf7 100644 --- a/tests/st/networks/models/resnet50/test_resnet50_imagenet.py +++ b/tests/st/networks/models/resnet50/test_resnet50_imagenet.py @@ -178,20 +178,18 @@ def train_process(q, device_id, epoch_size, device_num, enable_hccl): net.trainable_params())) no_decayed_params = [param for param in net.trainable_params() if param not in decayed_params] group_params = [{'params': decayed_params, 'weight_decay': config.weight_decay}, - {'params': no_decayed_params}, + {'params': no_decayed_params, 'weight_decay': 0.0}, {'order_params': net.trainable_params()}] if config.use_lars: momentum = nn.Momentum(group_params, lr, config.momentum, - weight_decay=config.weight_decay, loss_scale=config.loss_scale, - use_nesterov=config.use_nesterov) + loss_scale=config.loss_scale, use_nesterov=config.use_nesterov) opt = nn.LARS(momentum, epsilon=config.lars_epsilon, coefficient=config.lars_coefficient, lars_filter=lambda x: 'beta' not in x.name and 'gamma' not in x.name and 'bias' not in x.name) else: opt = nn.Momentum(group_params, lr, config.momentum, - weight_decay=config.weight_decay, loss_scale=config.loss_scale, - use_nesterov=config.use_nesterov) + loss_scale=config.loss_scale, use_nesterov=config.use_nesterov) # model model = Model(net, loss_fn=loss, optimizer=opt, From 68bd5cf6a1f87674b83d0396628cac355d35c1eb Mon Sep 17 00:00:00 2001 From: "wangnan39@huawei.com" Date: Mon, 29 Jun 2020 15:07:48 +0800 Subject: [PATCH 158/254] add cpu sparse optimizer ops with no return --- .../kernel/cpu/sparse_apply_ftrl_cpu_kernel.h | 12 ++ ...sparse_apply_proximal_adagrad_cpu_kernel.h | 13 ++ mindspore/nn/optim/ftrl.py | 3 +- mindspore/nn/optim/proximal_ada_grad.py | 3 +- mindspore/ops/operations/_inner_ops.py | 180 ++++++++++++++++++ mindspore/ops/operations/nn_ops.py | 22 ++- 6 files changed, 221 insertions(+), 12 deletions(-) diff --git a/mindspore/ccsrc/kernel/cpu/sparse_apply_ftrl_cpu_kernel.h b/mindspore/ccsrc/kernel/cpu/sparse_apply_ftrl_cpu_kernel.h index b4e5a48109..9e79dc83c7 100644 --- a/mindspore/ccsrc/kernel/cpu/sparse_apply_ftrl_cpu_kernel.h +++ b/mindspore/ccsrc/kernel/cpu/sparse_apply_ftrl_cpu_kernel.h @@ -53,6 +53,18 @@ MS_REG_CPU_KERNEL(SparseApplyFtrl, .AddOutputAttr(kNumberTypeFloat32) .AddOutputAttr(kNumberTypeFloat32), SparseApplyFtrlCPUKernel); + +MS_REG_CPU_KERNEL(SparseApplyFtrlNoReturn, + KernelAttr() + .AddInputAttr(kNumberTypeFloat32) + .AddInputAttr(kNumberTypeFloat32) + .AddInputAttr(kNumberTypeFloat32) + .AddInputAttr(kNumberTypeFloat32) + .AddInputAttr(kNumberTypeInt32) + .AddOutputAttr(kNumberTypeFloat32) + .AddOutputAttr(kNumberTypeFloat32) + .AddOutputAttr(kNumberTypeFloat32), + SparseApplyFtrlCPUKernel); } // namespace kernel } // namespace mindspore diff --git a/mindspore/ccsrc/kernel/cpu/sparse_apply_proximal_adagrad_cpu_kernel.h b/mindspore/ccsrc/kernel/cpu/sparse_apply_proximal_adagrad_cpu_kernel.h index 00ca5bc693..ff7da7966c 100644 --- a/mindspore/ccsrc/kernel/cpu/sparse_apply_proximal_adagrad_cpu_kernel.h +++ b/mindspore/ccsrc/kernel/cpu/sparse_apply_proximal_adagrad_cpu_kernel.h @@ -51,6 +51,19 @@ MS_REG_CPU_KERNEL(SparseApplyProximalAdagrad, .AddOutputAttr(kNumberTypeFloat32) .AddOutputAttr(kNumberTypeFloat32), SparseApplyProximalAdagradCPUKernel); + +MS_REG_CPU_KERNEL(SparseApplyProximalAdagradNoReturn, + KernelAttr() + .AddInputAttr(kNumberTypeFloat32) + .AddInputAttr(kNumberTypeFloat32) + .AddInputAttr(kNumberTypeFloat32) + .AddInputAttr(kNumberTypeFloat32) + .AddInputAttr(kNumberTypeFloat32) + .AddInputAttr(kNumberTypeFloat32) + .AddInputAttr(kNumberTypeInt32) + .AddOutputAttr(kNumberTypeFloat32) + .AddOutputAttr(kNumberTypeFloat32), + SparseApplyProximalAdagradCPUKernel); } // namespace kernel } // namespace mindspore diff --git a/mindspore/nn/optim/ftrl.py b/mindspore/nn/optim/ftrl.py index b840b89241..b2954430b4 100644 --- a/mindspore/nn/optim/ftrl.py +++ b/mindspore/nn/optim/ftrl.py @@ -16,6 +16,7 @@ from mindspore.ops import functional as F, composite as C, operations as P from mindspore.common import Tensor import mindspore.common.dtype as mstype +from mindspore.ops.operations import _inner_ops as inner from mindspore._checkparam import Validator as validator from mindspore._checkparam import Rel from .optimizer import Optimizer, _apply_decay, _grad_scale @@ -116,7 +117,7 @@ class FTRL(Optimizer): self.decay_tf = tuple((lambda: True)() for x in self.parameters) self.hyper_map = C.HyperMap() self.opt = P.ApplyFtrl(use_locking=use_locking) - self.sparse_opt = P.SparseApplyFtrl(learning_rate, l1, l2, lr_power, use_locking=use_locking) + self.sparse_opt = inner.SparseApplyFtrlNoReturn(learning_rate, l1, l2, lr_power, use_locking=use_locking) def construct(self, grads): params = self.parameters diff --git a/mindspore/nn/optim/proximal_ada_grad.py b/mindspore/nn/optim/proximal_ada_grad.py index 795cf8ab05..75f3994e2a 100644 --- a/mindspore/nn/optim/proximal_ada_grad.py +++ b/mindspore/nn/optim/proximal_ada_grad.py @@ -16,6 +16,7 @@ from mindspore.ops import functional as F, composite as C, operations as P from mindspore.common import Tensor import mindspore.common.dtype as mstype +from mindspore.ops.operations import _inner_ops as inner from mindspore._checkparam import Validator as validator from mindspore._checkparam import Rel from .optimizer import Optimizer @@ -99,7 +100,7 @@ class ProximalAdagrad(Optimizer): self.weight_decay = weight_decay self.hyper_map = C.HyperMap() self.opt = P.ApplyProximalAdagrad(use_locking=use_locking) - self.sparse_opt = P.SparseApplyProximalAdagrad(use_locking=use_locking) + self.sparse_opt = inner.SparseApplyProximalAdagradNoReturn(use_locking=use_locking) def construct(self, grads): params = self.parameters diff --git a/mindspore/ops/operations/_inner_ops.py b/mindspore/ops/operations/_inner_ops.py index 49834fc168..6a88ca674c 100644 --- a/mindspore/ops/operations/_inner_ops.py +++ b/mindspore/ops/operations/_inner_ops.py @@ -18,6 +18,9 @@ from ..._checkparam import Rel from ..._checkparam import Validator as validator from ...common import dtype as mstype +from ..._c_expression import signature_rw as sig_rw +from ..._c_expression import signature_kind as sig_kind +from ..._c_expression import signature_dtype as sig_dtype from ..primitive import PrimitiveWithInfer, prim_attr_register @@ -330,6 +333,183 @@ class EmbeddingLookup(PrimitiveWithInfer): return out +class SparseApplyFtrlNoReturn(PrimitiveWithInfer): + """ + Update relevant entries according to the FTRL-proximal scheme. + + Args: + lr (float): The learning rate value, must be positive. + l1 (float): l1 regularization strength, must be greater than or equal to zero. + l2 (float): l2 regularization strength, must be greater than or equal to zero. + lr_power (float): Learning rate power controls how the learning rate decreases during training, + must be less than or equal to zero. Use fixed learning rate if `lr_power` is zero. + use_locking (bool): Use locks for update operation if True . Default: False. + + Inputs: + - **var** (Parameter): The variable to be updated. The data type must be float32. + - **accum** (Parameter): The accum to be updated, must be same type and shape as `var`. + - **linear** (Parameter): The linear to be updated, must be same type and shape as `var`. + - **grad** (Tensor): A tensor of the same type as `var`, for the gradient. + - **indices** (Tensor): A vector of indices into the first dimension of `var` and `accum`. The shape + of `indices` must be the same as `grad` in first dimension. The type must be int32. + + Outputs: + Tuple of 3 Tensor, this operator will update the input parameters directly, the outputs are useless. + + - **var** (Tensor) - A Tensor with shape (1,). + - **accum** (Tensor) - A Tensor with shape (1,). + - **linear** (Tensor) - A Tensor with shape (1,). + + Examples: + >>> import mindspore + >>> import mindspore.nn as nn + >>> import numpy as np + >>> from mindspore import Parameter + >>> from mindspore import Tensor + >>> from mindspore.ops import operations as P + >>> class SparseApplyFtrlNet(nn.Cell): + >>> def __init__(self): + >>> super(SparseApplyFtrlNet, self).__init__() + >>> self.sparse_apply_ftrl = P.SparseApplyFtrlV2(lr=0.01, l1=0.0, l2=0.0, lr_power=-0.5) + >>> self.var = Parameter(Tensor(np.random.rand(3, 1, 2).astype(np.float32)), name="var") + >>> self.accum = Parameter(Tensor(np.random.rand(3, 1, 2).astype(np.float32)), name="accum") + >>> self.linear = Parameter(Tensor(np.random.rand(3, 1, 2).astype(np.float32)), name="linear") + >>> + >>> def construct(self, grad, indices): + >>> out = self.sparse_apply_ftrl(self.var, self.accum, self.linear, grad, indices) + >>> return out + >>> + >>> net = SparseApplyFtrlNet() + >>> grad = Tensor(np.random.rand(2, 1, 2).astype(np.float32)) + >>> indices = Tensor(np.array([0, 1]).astype(np.int32)) + >>> output = net(grad, indices) + """ + __mindspore_signature__ = ( + ('var', sig_rw.RW_WRITE, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), + ('accum', sig_rw.RW_WRITE, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), + ('linear', sig_rw.RW_WRITE, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), + ('grad', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), + ('indices', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T1) + ) + + @prim_attr_register + def __init__(self, lr, l1, l2, lr_power, use_locking=False): + self.init_prim_io_names(inputs=['var', 'accum', 'linear', 'grad', 'indices'], + outputs=['output']) + validator.check_value_type("lr", lr, [float], self.name) + validator.check_value_type("l1", l1, [float], self.name) + validator.check_value_type("l2", l2, [float], self.name) + validator.check_value_type("lr_power", lr_power, [float], self.name) + self.lr = validator.check_number_range("lr", lr, 0.0, float("inf"), Rel.INC_NEITHER, self.name) + self.l1 = validator.check_number_range("l1", l1, 0.0, float("inf"), Rel.INC_LEFT, self.name) + self.l2 = validator.check_number_range("l2", l2, 0.0, float("inf"), Rel.INC_LEFT, self.name) + self.lr_power = validator.check_number("lr_power", lr_power, 0, Rel.LE, self.name) + self.use_locking = validator.check_value_type("use_locking", use_locking, [bool], self.name) + self.add_prim_attr('primitive_target', 'CPU') + + def infer_shape(self, var_shape, accum_shape, linear_shape, grad_shape, indices_shape): + validator.check('var shape', var_shape, 'accum shape', accum_shape, Rel.EQ, self.name) + validator.check('var shape', var_shape, 'linear shape', linear_shape, Rel.EQ, self.name) + if len(var_shape) > 1: + validator.check('var_shape[1:]', var_shape[1:], 'grad_shape[1:]', grad_shape[1:], Rel.EQ, self.name) + validator.check_integer("indices rank", len(indices_shape), 1, Rel.EQ, self.name) + validator.check('grad_shape[0]', grad_shape[0], 'indices_shape[0]', indices_shape[0], Rel.EQ, self.name) + return [1], [1], [1] + + def infer_dtype(self, var_dtype, accum_dtype, linear_dtype, grad_dtype, indices_dtype): + args = {"var_dtype": var_dtype, "accum_dtype": accum_dtype, + "linear_dtype": linear_dtype, "grad_dtype": grad_dtype} + validator.check_tensor_type_same(args, [mstype.float32], self.name) + validator.check_tensor_type_same({"indices_dtype": indices_dtype}, [mstype.int32], self.name) + return var_dtype, accum_dtype, linear_dtype + + +class SparseApplyProximalAdagradNoReturn(PrimitiveWithInfer): + r""" + Updates relevant entries according to the proximal adagrad algorithm. + + .. math:: + accum += grad * grad + .. math:: + \text{prox_v} = var - lr * grad * \frac{1}{\sqrt{accum}} + .. math:: + var = \frac{sign(\text{prox_v})}{1 + lr * l2} * \max(\left| \text{prox_v} \right| - lr * l1, 0) + + Args: + use_locking (bool): If True, updating of the var and accum tensors will be protected. Default: False. + + Inputs: + - **var** (Parameter) - Variable tensor to be updated. The data type must be float32. + - **accum** (Parameter) - Variable tensor to be updated. Has the same dtype as `var`. + - **lr** (Tensor): The learning rate value. The data type must be float32. + - **l1** (Tensor): l1 regularization strength. The data type must be float32. + - **l2** (Tensor): l2 regularization strength. The data type must be float32. + - **grad** (Tensor) - A tensor of the same type as `var`, for the gradient. The data type must be float32. + - **indices** (Tensor) - A vector of indices into the first dimension of `var` and `accum`. The data type + must be int32. + + Outputs: + Tuple of 2 Tensor, this operator will update the input parameters directly, the outputs are useless. + + - **var** (Tensor) - A Tensor with shape (1,). + - **accum** (Tensor) - A Tensor with shape (1,). + + Examples: + >>> import numpy as np + >>> import mindspore.nn as nn + >>> from mindspore import Tensor, Parameter + >>> from mindspore.ops import operations as P + >>> class Net(nn.Cell): + >>> def __init__(self): + >>> super(Net, self).__init__() + >>> self.sparse_apply_proximal_adagrad = P.SparseApplyProximalAdagradV2() + >>> self.var = Parameter(Tensor(np.random.rand(3, 1, 2).astype(np.float32)), name="var") + >>> self.accum = Parameter(Tensor(np.random.rand(3, 1, 2).astype(np.float32)), name="accum") + >>> self.lr = Tensor(0.01, mstype.float32) + >>> self.l1 = Tensor(0.0, mstype.float32) + >>> self.l2 = Tensor(0.0, mstype.float32) + >>> def construct(self, grad, indices): + >>> out = self.sparse_apply_proximal_adagrad(self.var, self.accum, self.lr, self.l1, + >>> self.l2, grad, indices) + >>> return out + >>> net = Net() + >>> grad = Tensor(np.random.rand(2, 1, 2).astype(np.float32)) + >>> indices = Tensor(np.array([0, 1]).astype(np.int32)) + >>> output = net(grad, indices) + """ + __mindspore_signature__ = ( + ('var', sig_rw.RW_WRITE, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), + ('accum', sig_rw.RW_WRITE, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), + ('lr', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), + ('l1', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), + ('l2', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), + ('grad', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), + ('indices', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T1) + ) + + @prim_attr_register + def __init__(self, use_locking=False): + self.init_prim_io_names(inputs=['var', 'accum', 'lr', 'l1', 'l2', 'grad', 'indices'], + outputs=['output']) + self.use_locking = validator.check_value_type("use_locking", use_locking, [bool], self.name) + self.add_prim_attr('primitive_target', 'CPU') + + def infer_shape(self, var_shape, accum_shape, lr_shape, l1_shape, l2_shape, grad_shape, indices_shape): + validator.check_integer("indices rank", len(indices_shape), 1, Rel.EQ, self.name) + return [1], [1] + + def infer_dtype(self, var_dtype, accum_dtype, lr_dtype, l1_dtype, l2_dtype, grad_dtype, indices_dtype): + args = {'var': var_dtype, 'accum': accum_dtype, 'grad': grad_dtype} + validator.check_tensor_type_same(args, [mstype.float32], self.name) + validator.check_scalar_or_tensor_type_same({"lr": lr_dtype}, [mstype.float32], self.name) + validator.check_scalar_or_tensor_type_same({"l1": l1_dtype}, [mstype.float32], self.name) + validator.check_scalar_or_tensor_type_same({"l2": l2_dtype}, [mstype.float32], self.name) + valid_types = [mstype.int16, mstype.int32, mstype.int64, + mstype.uint16, mstype.uint32, mstype.uint64] + validator.check_tensor_type_same({'indices': indices_dtype}, valid_types, self.name) + return var_dtype, accum_dtype + + class LinSpace(PrimitiveWithInfer): r""" Generates values in an interval. And return the corresponding interpolation accroding to assist. diff --git a/mindspore/ops/operations/nn_ops.py b/mindspore/ops/operations/nn_ops.py index 24b081bdad..fca3104ab4 100644 --- a/mindspore/ops/operations/nn_ops.py +++ b/mindspore/ops/operations/nn_ops.py @@ -2835,11 +2835,11 @@ class SparseApplyAdam(PrimitiveWithInfer): - **indices** (Tensor) - Gradient indices. With int32 data type. Outputs: - Tuple of 3 Tensor, the updated parameters. + Tuple of 3 Tensor, this operator will update the input parameters directly, the outputs are useless. - - **var** (Tensor) - The same shape and data type as `var`. - - **m** (Tensor) - The same shape and data type as `m`. - - **v** (Tensor) - The same shape and data type as `v`. + - **var** (Tensor) - A Tensor with shape (1,). + - **m** (Tensor) - A Tensor with shape (1,). + - **v** (Tensor) - A Tensor with shape (1,). Examples: >>> import numpy as np @@ -2896,6 +2896,7 @@ class SparseApplyAdam(PrimitiveWithInfer): self.init_prim_io_names(inputs=['var', 'm', 'v', 'beta1_power', 'beta2_power', 'lr', 'beta1', 'beta2', 'epsilon', 'grad', 'indices'], outputs=['var', 'm', 'v']) + self.add_prim_attr('primitive_target', 'CPU') def infer_shape(self, var_shape, m_shape, v_shape, beta1_power_shape, beta2_power_shape, lr_shape, beta1_shape, beta2_shape, epsilon_shape, grad_shape, indices_shape): @@ -2907,7 +2908,7 @@ class SparseApplyAdam(PrimitiveWithInfer): raise ValueError(f"For '{self.name}', the shape of updates should be [] or " f"grad_shape = indices_shape + var_shape[1:], but got var_shape: {var_shape}, " f"indices_shape: {indices_shape}, grad_shape: {grad_shape}.") - return var_shape, m_shape, v_shape + return [1], [1], [1] def infer_dtype(self, var_dtype, m_dtype, v_dtype, beta1_power_dtype, beta2_power_dtype, lr_dtype, beta1_dtype, beta2_dtype, epsilon_dtype, grad_dtype, indices_dtype): @@ -2969,11 +2970,11 @@ class SparseApplyLazyAdam(PrimitiveWithInfer): - **indices** (Tensor) - Gradient indices. With int32 data type. Outputs: - Tuple of 3 Tensor, the updated parameters. + Tuple of 3 Tensor, this operator will update the input parameters directly, the outputs are useless. - - **var** (Tensor) - The same shape and data type as `var`. - - **m** (Tensor) - The same shape and data type as `m`. - - **v** (Tensor) - The same shape and data type as `v`. + - **var** (Tensor) - A Tensor with shape (1,). + - **m** (Tensor) - A Tensor with shape (1,). + - **v** (Tensor) - A Tensor with shape (1,). Examples: >>> import numpy as np @@ -3030,6 +3031,7 @@ class SparseApplyLazyAdam(PrimitiveWithInfer): self.init_prim_io_names(inputs=['var', 'm', 'v', 'beta1_power', 'beta2_power', 'lr', 'beta1', 'beta2', 'epsilon', 'grad', 'indices'], outputs=['var', 'm', 'v']) + self.add_prim_attr('primitive_target', 'CPU') def infer_shape(self, var_shape, m_shape, v_shape, beta1_power_shape, beta2_power_shape, lr_shape, beta1_shape, beta2_shape, epsilon_shape, grad_shape, indices_shape): @@ -3041,7 +3043,7 @@ class SparseApplyLazyAdam(PrimitiveWithInfer): raise ValueError(f"For '{self.name}', the shape of updates should be [] or " f"grad_shape = indices_shape + var_shape[1:], but got var_shape: {var_shape}, " f"indices_shape: {indices_shape}, grad_shape: {grad_shape}.") - return var_shape, m_shape, v_shape + return [1], [1], [1] def infer_dtype(self, var_dtype, m_dtype, v_dtype, beta1_power_dtype, beta2_power_dtype, lr_dtype, beta1_dtype, beta2_dtype, epsilon_dtype, grad_dtype, indices_dtype): From 9cfcdce8dc726f8b51f0fe780d7f47e25e386005 Mon Sep 17 00:00:00 2001 From: kingfo Date: Tue, 30 Jun 2020 11:59:36 +0800 Subject: [PATCH 159/254] fix invertPermutation operator in pynative --- mindspore/ops/operations/array_ops.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mindspore/ops/operations/array_ops.py b/mindspore/ops/operations/array_ops.py index f59a3a37e5..532350e84a 100644 --- a/mindspore/ops/operations/array_ops.py +++ b/mindspore/ops/operations/array_ops.py @@ -971,6 +971,7 @@ class InvertPermutation(PrimitiveWithInfer): @prim_attr_register def __init__(self): """init InvertPermutation""" + self.const_value = True def __infer__(self, x): x_shp = x['shape'] From 3398c2d74bb69311ab6ebe75ccf64869638f79c8 Mon Sep 17 00:00:00 2001 From: lichenever Date: Tue, 30 Jun 2020 11:53:12 +0800 Subject: [PATCH 160/254] remove_32byte_limit_for_cpu_gatherv2 --- mindspore/ccsrc/parallel/ops_info/gather_v2_p_info.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mindspore/ccsrc/parallel/ops_info/gather_v2_p_info.cc b/mindspore/ccsrc/parallel/ops_info/gather_v2_p_info.cc index 7a16aeafcb..9fb8df0883 100644 --- a/mindspore/ccsrc/parallel/ops_info/gather_v2_p_info.cc +++ b/mindspore/ccsrc/parallel/ops_info/gather_v2_p_info.cc @@ -79,8 +79,8 @@ Status GatherV2PInfo::CheckStrategy(const StrategyPtr &strategy) { auto param_shape = inputs_shape_.at(0); auto param_strategy = strategy->GetInputDim().at(0); auto slice_shape = param_shape.at(param_shape.size() - 1) / param_strategy.at(param_strategy.size() - 1); - if (slice_shape % 8 != 0 && slice_shape != 1) { - MS_LOG(DEBUG) << name_ << ": Last dim of param slice shape need 32Byte aligned."; + if ((target_ != CPU) && (slice_shape % 8 != 0) && (slice_shape != 1)) { + MS_LOG(ERROR) << name_ << ": Last dim of param slice shape need 32Byte aligned."; return FAILED; } From d1060690b22ca785d6f954a58336f6fb5aaaec55 Mon Sep 17 00:00:00 2001 From: jiangjinsheng Date: Tue, 30 Jun 2020 10:29:39 +0800 Subject: [PATCH 161/254] fixed SpaceToBatchND, Conv2DBackpropFilter --- .../ops/_op_impl/tbe/batch_to_space_nd.py | 4 ++-- .../_op_impl/tbe/conv2d_backprop_filter.py | 2 ++ .../ops/_op_impl/tbe/space_to_batch_nd.py | 4 ++-- mindspore/ops/operations/_grad_ops.py | 1 + mindspore/ops/operations/array_ops.py | 20 +++++++++++++------ 5 files changed, 21 insertions(+), 10 deletions(-) diff --git a/mindspore/ops/_op_impl/tbe/batch_to_space_nd.py b/mindspore/ops/_op_impl/tbe/batch_to_space_nd.py index ad5060e7c1..f942a3836d 100644 --- a/mindspore/ops/_op_impl/tbe/batch_to_space_nd.py +++ b/mindspore/ops/_op_impl/tbe/batch_to_space_nd.py @@ -25,8 +25,8 @@ batch_to_space_nd_op_info = TBERegOp("BatchToSpaceND") \ .partial_flag(True) \ .attr("block_shape", "required", "listInt", "all") \ .attr("crops", "required", "listListInt", "all") \ - .input(0, "x", False, "required", "all") \ - .output(0, "y", False, "required", "all") \ + .input(0, "x", False, "required", "all", reshape_type="NH") \ + .output(0, "y", False, "required", "all", reshape_type="NH") \ .dtype_format(DataType.F16_5HD, DataType.F16_5HD) \ .dtype_format(DataType.F32_5HD, DataType.F32_5HD) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/conv2d_backprop_filter.py b/mindspore/ops/_op_impl/tbe/conv2d_backprop_filter.py index 04b55bb2a3..c309a4f2ab 100644 --- a/mindspore/ops/_op_impl/tbe/conv2d_backprop_filter.py +++ b/mindspore/ops/_op_impl/tbe/conv2d_backprop_filter.py @@ -27,6 +27,8 @@ conv2d_backprop_filter_op_info = TBERegOp("Conv2DBackpropFilter") \ .attr("stride", "required", "listInt", "all") \ .attr("pad_list", "required", "listInt", "all") \ .attr("dilation", "required", "listInt", "all") \ + .attr("groups", "optional", "int", "all") \ + .attr("data_format", "optional", "str", "all") \ .input(0, "out_backprop", False, "required", "all") \ .input(1, "x", False, "required", "all") \ .output(0, "y", False, "required", "all") \ diff --git a/mindspore/ops/_op_impl/tbe/space_to_batch_nd.py b/mindspore/ops/_op_impl/tbe/space_to_batch_nd.py index 3a50b56a24..c1094cb55c 100644 --- a/mindspore/ops/_op_impl/tbe/space_to_batch_nd.py +++ b/mindspore/ops/_op_impl/tbe/space_to_batch_nd.py @@ -25,8 +25,8 @@ space_to_batch_nd_op_info = TBERegOp("SpaceToBatchND") \ .partial_flag(True) \ .attr("block_shape", "required", "listInt", "all") \ .attr("paddings", "required", "listListInt", "all") \ - .input(0, "x", False, "required", "all") \ - .output(0, "y", False, "required", "all") \ + .input(0, "x", False, "required", "all", reshape_type="NH") \ + .output(0, "y", False, "required", "all", reshape_type="NH") \ .dtype_format(DataType.F16_5HD, DataType.F16_5HD) \ .dtype_format(DataType.F32_5HD, DataType.F32_5HD) \ .get_op_info() diff --git a/mindspore/ops/operations/_grad_ops.py b/mindspore/ops/operations/_grad_ops.py index 211c30d143..94ba2f1bd9 100644 --- a/mindspore/ops/operations/_grad_ops.py +++ b/mindspore/ops/operations/_grad_ops.py @@ -237,6 +237,7 @@ class Conv2DBackpropFilter(PrimitiveWithInfer): self.add_prim_attr('stride', self.stride) self.dilation = dilation self.group = group + self.add_prim_attr('groups', group) self.add_prim_attr('data_format', "NCHW") def __infer__(self, doutput, x, w_size): diff --git a/mindspore/ops/operations/array_ops.py b/mindspore/ops/operations/array_ops.py index f59a3a37e5..c85e87b93a 100644 --- a/mindspore/ops/operations/array_ops.py +++ b/mindspore/ops/operations/array_ops.py @@ -2635,16 +2635,20 @@ class SpaceToBatchND(PrimitiveWithInfer): def infer_shape(self, x_shape): x_rank = len(x_shape) + validator.check_integer('x_shape rank', x_rank, 4, Rel.EQ, self.name) out_shape = copy.deepcopy(x_shape) block_shape_prod = 1 - for i in range(x_rank - 2): - padded = out_shape[i + 2] + self.paddings[i][0] + \ + offset = 2 + if x_rank < 4: + offset = 1 + for i in range(len(self.block_shape)): + padded = out_shape[i + offset] + self.paddings[i][0] + \ self.paddings[i][1] if padded % self.block_shape[i] != 0: raise ValueError(f'For \'{self.name}\' padded[{i}] {padded} should be divisible by ' f'block_shape[{i}] {self.block_shape[i]}') - out_shape[i + 2] = padded // self.block_shape[i] + out_shape[i + offset] = padded // self.block_shape[i] block_shape_prod = block_shape_prod * self.block_shape[i] out_shape[0] *= block_shape_prod return out_shape @@ -2715,15 +2719,19 @@ class BatchToSpaceND(PrimitiveWithInfer): def infer_shape(self, x_shape): x_rank = len(x_shape) + validator.check_integer('x_shape rank', x_rank, 4, Rel.EQ, self.name) out_shape = copy.deepcopy(x_shape) block_shape_prod = 1 - for i in range(x_rank - 2): + offset = 2 + if x_rank < 4: + offset = 1 + for i in range(len(self.block_shape)): block_shape_prod = block_shape_prod * self.block_shape[i] - x_block_prod = out_shape[i + 2] * self.block_shape[i] + x_block_prod = out_shape[i + offset] * self.block_shape[i] crops_sum = self.crops[i][0] + self.crops[i][1] validator.check("x block shape prod", x_block_prod, 'crops sum', crops_sum, Rel.GT, self.name) - out_shape[i + 2] = x_block_prod - crops_sum + out_shape[i + offset] = x_block_prod - crops_sum if out_shape[0] % block_shape_prod != 0: raise ValueError(f'For \'{self.name}\' input_x dimension 0 {out_shape[0]} should be divisible by ' From 652093642eddd17ec09ab3db439323318ef6fa6e Mon Sep 17 00:00:00 2001 From: guohongzilong <2713219276@qq.com> Date: Tue, 30 Jun 2020 13:07:31 +0800 Subject: [PATCH 162/254] change order param same as group params --- mindspore/nn/optim/adam.py | 11 ++-- mindspore/nn/optim/lazyadam.py | 15 +++-- mindspore/nn/optim/momentum.py | 11 ++-- mindspore/nn/optim/optimizer.py | 19 +++--- mindspore/nn/optim/rmsprop.py | 11 ++-- mindspore/nn/optim/sgd.py | 11 ++-- .../test_optimizer_with_parameter_groups.py | 63 +++---------------- 7 files changed, 45 insertions(+), 96 deletions(-) diff --git a/mindspore/nn/optim/adam.py b/mindspore/nn/optim/adam.py index f0688a9b47..b73c284aab 100755 --- a/mindspore/nn/optim/adam.py +++ b/mindspore/nn/optim/adam.py @@ -181,8 +181,7 @@ class Adam(Optimizer): - order_params: Optional. If "order_params" in the keys, the value should be the order of parameters and the order will be followed in optimizer. There are no other keys in the `dict` and the parameters which - in the value of 'order_params' but not in any group will use default learning rate and default weight - decay. + in the value of 'order_params' should be in one of group parameters. learning_rate (Union[int, float, Tensor, Iterable]): A value for the learning rate. When the learning_rate is Iterable or a Tensor and the dims of the Tensor is 1, @@ -220,16 +219,14 @@ class Adam(Optimizer): >>> >>> #2) Use parameter groups and set different values >>> conv_params = list(filter(lambda x: 'conv' in x.name, net.trainable_params())) - >>> bias_params = list(filter(lambda x: 'bias' in x.name, net.trainable_params())) + >>> no_conv_params = list(filter(lambda x: 'conv' not in x.name, net.trainable_params())) >>> group_params = [{'params': conv_params, 'weight_decay': 0.01}, - >>> {'params': bias_params, 'lr': 0.01}, + >>> {'params': no_conv_params, 'lr': 0.01}, >>> {'order_params': net.trainable_params()}] >>> opt = nn.Adam(group_params, learning_rate=0.1, weight_decay=0.0) >>> # The conv_params's parameters will use a learning rate of default value 0.1 and a weight decay of 0.01. - >>> # The bias_params's parameters will use a learning rate of 0.01 and a weight decay of default value 0.0. + >>> # The no_conv_params's parameters will use a learning rate of 0.01 and a weight decay of default value 0.0. >>> # The final parameters order in which the optimizer will be followed is the value of 'order_params'. - >>> # The parameters which in the value of 'order_params' but not in any group will use a learning rate - >>> # of default value 0.1 and a weight decay of default value 0.0. >>> >>> loss = nn.SoftmaxCrossEntropyWithLogits() >>> model = Model(net, loss_fn=loss, optimizer=optim) diff --git a/mindspore/nn/optim/lazyadam.py b/mindspore/nn/optim/lazyadam.py index 7d53aad488..4b97d2eb20 100644 --- a/mindspore/nn/optim/lazyadam.py +++ b/mindspore/nn/optim/lazyadam.py @@ -109,6 +109,10 @@ class LazyAdam(Optimizer): - weight_decay: Optional. If "weight_decay" in the keys, the value of corresponding weight decay will be used. If not, the `weight_decay` in the API will be used. + - order_params: Optional. If "order_params" in the keys, the value should be the order of parameters and + the order will be followed in optimizer. There are no other keys in the `dict` and the parameters which + in the value of 'order_params' should be in one of group parameters. + learning_rate (Union[float, Tensor, Iterable]): A value for the learning rate. When the learning_rate is Iterable or a Tensor and the dims of the Tensor is 1, use dynamic learning rate, then the i-th step will @@ -146,12 +150,13 @@ class LazyAdam(Optimizer): >>> #2) Use parameter groups and set different values >>> conv_params = list(filter(lambda x: 'conv' in x.name, net.trainable_params())) >>> no_conv_params = list(filter(lambda x: 'conv' not in x.name, net.trainable_params())) - >>> group_params = [{'params': conv_params, 'weight_decay': 0.01, 'lr': 0.01}, - >>> {'params': no_conv_params}] + >>> group_params = [{'params': conv_params, 'weight_decay': 0.01}, + >>> {'params': no_conv_params, 'lr': 0.01}, + >>> {'order_params': net.trainable_params()}] >>> opt = nn.LazyAdam(group_params, learning_rate=0.1, weight_decay=0.0) - >>> # the conv_params's parameters will use a learning rate of 0.01 and a weight decay of 0.01 - >>> # the no_cov_params's parameters don't set learning and weight decay. So they will use a - >>> # learning rate of 0.1 and a weight decay of 0.0. + >>> # The conv_params's parameters will use a learning rate of default value 0.1 and a weight decay of 0.01. + >>> # The no_conv_params's parameters will use a learning rate of 0.01 and a weight decay of default value 0.0. + >>> # The final parameters order in which the optimizer will be followed is the value of 'order_params'. >>> >>> loss = nn.SoftmaxCrossEntropyWithLogits() >>> model = Model(net, loss_fn=loss, optimizer=optim) diff --git a/mindspore/nn/optim/momentum.py b/mindspore/nn/optim/momentum.py index ebdc5d86bf..1e8ce85570 100755 --- a/mindspore/nn/optim/momentum.py +++ b/mindspore/nn/optim/momentum.py @@ -64,8 +64,7 @@ class Momentum(Optimizer): - order_params: Optional. If "order_params" in the keys, the value should be the order of parameters and the order will be followed in optimizer. There are no other keys in the `dict` and the parameters which - in the value of 'order_params' but not in any group will use default learning rate and default weight - decay. + in the value of 'order_params' should be in one of group parameters. learning_rate (Union[int, float, Tensor, Iterable]): A value for the learning rate. When the learning_rate is Iterable or a Tensor and the dims of the Tensor is 1, @@ -97,16 +96,14 @@ class Momentum(Optimizer): >>> >>> #2) Use parameter groups and set different values >>> conv_params = list(filter(lambda x: 'conv' in x.name, net.trainable_params())) - >>> bias_params = list(filter(lambda x: 'bias' in x.name, net.trainable_params())) + >>> no_conv_params = list(filter(lambda x: 'conv' not in x.name, net.trainable_params())) >>> group_params = [{'params': conv_params, 'weight_decay': 0.01}, - >>> {'params': bias_params, 'lr': 0.01}, + >>> {'params': no_conv_params, 'lr': 0.01}, >>> {'order_params': net.trainable_params()}] >>> opt = nn.Momentum(group_params, learning_rate=0.1, momentum=0.9, weight_decay=0.0) >>> # The conv_params's parameters will use a learning rate of default value 0.1 and a weight decay of 0.01. - >>> # The bias_params's parameters will use a learning rate of 0.01 and a weight decay of default value 0.0. + >>> # The no_conv_params's parameters will use a learning rate of 0.01 and a weight decay of default value 0.0. >>> # The final parameters order in which the optimizer will be followed is the value of 'order_params'. - >>> # The parameters which in the value of 'order_params' but not in any group will use a learning rate - >>> # of default value 0.1 and a weight decay of default value 0.0. >>> >>> loss = nn.SoftmaxCrossEntropyWithLogits() >>> model = Model(net, loss_fn=loss, optimizer=optim, metrics=None) diff --git a/mindspore/nn/optim/optimizer.py b/mindspore/nn/optim/optimizer.py index 5b13d7cfbd..16f252adff 100755 --- a/mindspore/nn/optim/optimizer.py +++ b/mindspore/nn/optim/optimizer.py @@ -77,8 +77,7 @@ class Optimizer(Cell): - order_params: Optional. If "order_params" in the keys, the value should be the order of parameters and the order will be followed in optimizer. There are no other keys in the `dict` and the parameters which - in the value of 'order_params' but not in any group will use default learning rate and default weight - decay. + in the value of 'order_params' should be in one of group parameters. weight_decay (float): A floating point value for the weight decay. It should be equal to or greater than 0. If the type of `weight_decay` input is int, it will be converted to float. Default: 0.0. @@ -351,16 +350,18 @@ class Optimizer(Cell): self.group_weight_decay.append(weight_decay_) if self.is_group_params_ordered: - self._order_and_adjust_group_params(ordered_parameters, learning_rate, weight_decay) + self._order_and_adjust_group_params(ordered_parameters) - def _order_and_adjust_group_params(self, ordered_parameters, learning_rate, weight_decay): + def _order_and_adjust_group_params(self, ordered_parameters): """ - Order group parameter, learning rate and weight decay in group params. And assign the parameters - which in the value of 'order_params' but not in any group to default value. + Order group parameter, learning rate and weight decay in group params. """ - params_length = len(ordered_parameters) - ordered_learning_rate = [Parameter(learning_rate, name="lr_" + param.name) for param in ordered_parameters] - ordered_weight_decay = [weight_decay * self.loss_scale] * params_length + params_length = len(self.group_params) + if len(ordered_parameters) != len(self.group_params): + raise ValueError(f"The value of 'order_params' should be same with all group parameters.") + + ordered_learning_rate = [None] * params_length + ordered_weight_decay = [None] * params_length params_name = [param.name for param in ordered_parameters] for param, lr, wd in zip(self.group_params, self.group_lr, self.group_weight_decay): diff --git a/mindspore/nn/optim/rmsprop.py b/mindspore/nn/optim/rmsprop.py index 05c42fb444..8e8885aff7 100644 --- a/mindspore/nn/optim/rmsprop.py +++ b/mindspore/nn/optim/rmsprop.py @@ -107,8 +107,7 @@ class RMSProp(Optimizer): - order_params: Optional. If "order_params" in the keys, the value should be the order of parameters and the order will be followed in optimizer. There are no other keys in the `dict` and the parameters which - in the value of 'order_params' but not in any group will use default learning rate and default weight - decay. + in the value of 'order_params' should be in one of group parameters. learning_rate (Union[float, Tensor, Iterable]): A value for the learning rate. When the learning_rate is Iterable or a Tensor and the dims of the Tensor is 1, @@ -140,16 +139,14 @@ class RMSProp(Optimizer): >>> >>> #2) Use parameter groups and set different values >>> conv_params = list(filter(lambda x: 'conv' in x.name, net.trainable_params())) - >>> bias_params = list(filter(lambda x: 'bias' in x.name, net.trainable_params())) + >>> no_conv_params = list(filter(lambda x: 'conv' not in x.name, net.trainable_params())) >>> group_params = [{'params': conv_params, 'weight_decay': 0.01}, - >>> {'params': bias_params, 'lr': 0.01}, + >>> {'params': no_conv_params, 'lr': 0.01}, >>> {'order_params': net.trainable_params()}] >>> opt = nn.RMSProp(group_params, learning_rate=0.1, weight_decay=0.0) >>> # The conv_params's parameters will use a learning rate of default value 0.1 and a weight decay of 0.01. - >>> # The bias_params's parameters will use a learning rate of 0.01 and a weight decay of default value 0.0. + >>> # The no_conv_params's parameters will use a learning rate of 0.01 and a weight decay of default value 0.0. >>> # The final parameters order in which the optimizer will be followed is the value of 'order_params'. - >>> # The parameters which in the value of 'order_params' but not in any group will use a learning rate - >>> # of default value 0.1 and a weight decay of default value 0.0. >>> >>> loss = nn.SoftmaxCrossEntropyWithLogits() >>> model = Model(net, loss_fn=loss, optimizer=optim) diff --git a/mindspore/nn/optim/sgd.py b/mindspore/nn/optim/sgd.py index d2680a38e5..382f095627 100755 --- a/mindspore/nn/optim/sgd.py +++ b/mindspore/nn/optim/sgd.py @@ -64,8 +64,7 @@ class SGD(Optimizer): - order_params: Optional. If "order_params" in the keys, the value should be the order of parameters and the order will be followed in optimizer. There are no other keys in the `dict` and the parameters which - in the value of 'order_params' but not in any group will use default learning rate and default weight - decay. + in the value of 'order_params' should be in one of group parameters. learning_rate (Union[float, Tensor, Iterable]): A value for the learning rate. When the learning_rate is Iterable or a Tensor and the dims of the Tensor is 1, @@ -98,16 +97,14 @@ class SGD(Optimizer): >>> >>> #2) Use parameter groups and set different values >>> conv_params = list(filter(lambda x: 'conv' in x.name, net.trainable_params())) - >>> bias_params = list(filter(lambda x: 'bias' in x.name, net.trainable_params())) + >>> no_conv_params = list(filter(lambda x: 'conv' not in x.name, net.trainable_params())) >>> group_params = [{'params': conv_params, 'weight_decay': 0.01}, - >>> {'params': bias_params, 'lr': 0.01}, + >>> {'params': no_conv_params, 'lr': 0.01}, >>> {'order_params': net.trainable_params()}] >>> opt = nn.SGD(group_params, learning_rate=0.1, weight_decay=0.0) >>> # The conv_params's parameters will use a learning rate of default value 0.1 and a weight decay of 0.01. - >>> # The bias_params's parameters will use a learning rate of 0.01 and a weight decay of default value 0.0. + >>> # The no_conv_params's parameters will use a learning rate of 0.01 and a weight decay of default value 0.0. >>> # The final parameters order in which the optimizer will be followed is the value of 'order_params'. - >>> # The parameters which in the value of 'order_params' but not in any group will use a learning rate - >>> # of default value 0.1 and a weight decay of default value 0.0. >>> >>> loss = nn.SoftmaxCrossEntropyWithLogits() >>> model = Model(net, loss_fn=loss, optimizer=optim) diff --git a/tests/ut/python/optimizer/test_optimizer_with_parameter_groups.py b/tests/ut/python/optimizer/test_optimizer_with_parameter_groups.py index 05e58013fa..0aef22284d 100644 --- a/tests/ut/python/optimizer/test_optimizer_with_parameter_groups.py +++ b/tests/ut/python/optimizer/test_optimizer_with_parameter_groups.py @@ -250,8 +250,9 @@ def test_get_lr_parameter_with_order_group(): net = LeNet5() conv_lr = 0.1 conv_params = list(filter(lambda x: 'conv' in x.name, net.trainable_params())) + no_conv_params = list(filter(lambda x: 'conv' not in x.name, net.trainable_params())) group_params = [{'params': conv_params, 'lr': conv_lr}, - {'order_params': net.trainable_params()}] + {'params': no_conv_params}] opt = SGD(group_params) assert opt.is_group_lr is True for param in opt.parameters: @@ -278,65 +279,19 @@ def test_get_lr_parameter_with_no_group(): opt.get_lr_parameter(params_error) -def test_order_params_lr(): - net = LeNet5() - conv_lr = 0.01 - default_lr = 0.1 - conv_params = list(filter(lambda x: 'conv' in x.name, net.trainable_params())) - group_params = [{'params': conv_params, 'lr': conv_lr}, - {'order_params': net.trainable_params()}] - opt = SGD(group_params, learning_rate=default_lr) - assert opt.is_group is True - assert opt.is_group_lr is True - assert opt.is_group_params_ordered is True - for lr, param, order_param in zip(opt.learning_rate, opt.parameters, net.trainable_params()): - if param in conv_params: - assert np.all(lr.data.asnumpy() == Tensor(conv_lr, mstype.float32).asnumpy()) - else: - assert np.all(lr.data.asnumpy() == Tensor(default_lr, mstype.float32).asnumpy()) - - assert param.name == order_param.name - assert lr.name == 'lr_' + param.name - - -def test_order_params_weight_decay(): - net = LeNet5() - conv_weight_decay = 0.01 - default_wd = 0.0 - default_lr = 0.1 - conv_params = list(filter(lambda x: 'conv' in x.name, net.trainable_params())) - group_params = [{'params': conv_params, 'weight_decay': conv_weight_decay}, - {'order_params': net.trainable_params()}] - opt = SGD(group_params, learning_rate=default_lr, weight_decay=default_wd) - assert opt.is_group is True - assert opt.is_group_lr is False - assert opt.is_group_params_ordered is True - assert opt.learning_rate.name == "learning_rate" - assert np.all(opt.learning_rate.data.asnumpy() == Tensor(default_lr, mstype.float32).asnumpy()) - for weight_decay, decay_flags, param, order_param in zip( - opt.weight_decay, opt.decay_flags, opt.parameters, net.trainable_params()): - if param in conv_params: - assert weight_decay == conv_weight_decay - assert decay_flags is True - else: - assert weight_decay == default_wd - assert decay_flags is False - assert param.name == order_param.name - - -def test_order_params_all_1(): +def test_order_params_1(): net = LeNet5() conv_params = list(filter(lambda x: 'conv' in x.name, net.trainable_params())) bias_params = list(filter(lambda x: 'bias' in x.name, net.trainable_params())) group_params = [{'params': conv_params, 'weight_decay': 0.01}, {'params': bias_params, 'lr': 0.01}, - {'order_params': net.trainable_params()}] + {'order_params': bias_params+conv_params}] opt = SGD(group_params, learning_rate=0.1, weight_decay=0.0) assert opt.is_group is True assert opt.is_group_lr is True assert opt.is_group_params_ordered is True for weight_decay, decay_flags, lr, param, order_param in zip( - opt.weight_decay, opt.decay_flags, opt.learning_rate, opt.parameters, net.trainable_params()): + opt.weight_decay, opt.decay_flags, opt.learning_rate, opt.parameters, bias_params+conv_params): if param in conv_params: assert np.all(lr.data.asnumpy() == Tensor(0.1, mstype.float32).asnumpy()) assert weight_decay == 0.01 @@ -354,7 +309,7 @@ def test_order_params_all_1(): assert lr.name == 'lr_' + param.name -def test_order_params_all_2(): +def test_order_params_2(): net = LeNet5() conv_weight_decay = 0.01 fc1_lr = (0.5, 0.4, 0.3) @@ -364,13 +319,13 @@ def test_order_params_all_2(): fc1_params = list(filter(lambda x: 'fc1' in x.name, net.trainable_params())) group_params = [{'params': fc1_params, 'lr': fc1_lr}, {'params': conv_params, 'weight_decay': conv_weight_decay}, - {'order_params': net.trainable_params()}] + {'order_params': fc1_params+conv_params}] opt = SGD(group_params, learning_rate=default_lr, weight_decay=default_wd) assert opt.is_group is True assert opt.is_group_lr is True assert opt.is_group_params_ordered is True for weight_decay, decay_flags, lr, param, order_param in zip( - opt.weight_decay, opt.decay_flags, opt.learning_rate, opt.parameters, net.trainable_params()): + opt.weight_decay, opt.decay_flags, opt.learning_rate, opt.parameters, fc1_params+conv_params): if param in conv_params: assert np.all(lr.data.asnumpy() == Tensor(np.array([default_lr] * 3), mstype.float32).asnumpy()) assert weight_decay == conv_weight_decay @@ -388,7 +343,7 @@ def test_order_params_all_2(): assert lr.name == 'lr_' + param.name -def test_get_order_params_with_not_include(): +def test_get_order_params_with_not_same(): net = LeNet5() conv_weight_decay = 0.8 From 04d01a36f452f66aeeb29bacc716e9d120dced3e Mon Sep 17 00:00:00 2001 From: kswang Date: Tue, 30 Jun 2020 15:17:13 +0800 Subject: [PATCH 163/254] fix mix target device id --- mindspore/ccsrc/vm/backend.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mindspore/ccsrc/vm/backend.cc b/mindspore/ccsrc/vm/backend.cc index 1a27fcb63a..47bc69bbbb 100644 --- a/mindspore/ccsrc/vm/backend.cc +++ b/mindspore/ccsrc/vm/backend.cc @@ -353,7 +353,10 @@ void MsBackend::CreateOtherSession(const std::string &target) { if (other_sess_ == nullptr) { MS_LOG(EXCEPTION) << "Session create failed!, please make sure target device:" << target << " is available."; } - other_sess_->Init(0); + auto context_ptr = MsContext::GetInstance(); + MS_EXCEPTION_IF_NULL(context_ptr); + uint32_t device_id = context_ptr->device_id(); + other_sess_->Init(device_id); other_sess_->RegisterSummaryCallBackFunc(callbacks::SummarySaveCallback); other_device_ = target; } From 4090c1611b5bd4738ab3d58b99c6bfcbeb2a5f9d Mon Sep 17 00:00:00 2001 From: liuwenhao4 Date: Tue, 30 Jun 2020 11:37:32 +0800 Subject: [PATCH 164/254] Fixing type check mistakes of InplaceAdd, InplaceSub and InplaceUpdate vm ops --- mindspore/ops/operations/array_ops.py | 23 ++++----- mindspore/ops/operations/math_ops.py | 74 +++++++++++++-------------- 2 files changed, 46 insertions(+), 51 deletions(-) diff --git a/mindspore/ops/operations/array_ops.py b/mindspore/ops/operations/array_ops.py index 532350e84a..8bac6dfbd3 100644 --- a/mindspore/ops/operations/array_ops.py +++ b/mindspore/ops/operations/array_ops.py @@ -2818,33 +2818,30 @@ class InplaceUpdate(PrimitiveWithInfer): @prim_attr_register def __init__(self, indices): """Init InplaceUpdate""" - self.init_prim_io_names(inputs=['x', 'indices', 'v'], outputs=['y']) + self.init_prim_io_names(inputs=['x', 'v'], outputs=['y']) + self.indices = indices validator.check_value_type("indices", indices, [int, tuple], self.name) if isinstance(indices, int): - self.add_prim_attr('indices', (indices,)) + self.indices = (indices,) for item in self.indices: validator.check_value_type("item of indices", item, [int], self.name) def infer_dtype(self, x_dtype, v_dtype): + args = {'x': x_dtype, 'v': v_dtype} valid_type = [mstype.int32, mstype.float16, mstype.float32] - validator.check_tensor_type_same( - { - "x": x_dtype, - "v": v_dtype - }, valid_type, self.name) - + validator.check_tensor_type_same(args, valid_type, self.name) return x_dtype def infer_shape(self, x_shape, v_shape): validator.check("x", len(x_shape), "v", len(v_shape), Rel.EQ, self.name) - + validator.check("size of indices", len(self.indices), "v's first dimension", v_shape[0], + Rel.EQ, self.name) + for i in self.indices: + if i < 0 or i >= x_shape[0]: + raise ValueError(f'The value of indices must be in [0, {x_shape[0]}), but got {i}.') x_rank = len(x_shape) for idx in range(x_rank)[1:]: validator.check("x dim %d" % idx, x_shape[idx], 'v dim %d' % idx, v_shape[idx], Rel.EQ, self.name) - - validator.check("size of indices", len(self.indices), "v's first dimension", v_shape[0], - Rel.EQ, self.name) - return x_shape diff --git a/mindspore/ops/operations/math_ops.py b/mindspore/ops/operations/math_ops.py index b6aed4d79c..c222f5326a 100644 --- a/mindspore/ops/operations/math_ops.py +++ b/mindspore/ops/operations/math_ops.py @@ -926,32 +926,31 @@ class InplaceAdd(PrimitiveWithInfer): """init InplaceAdd""" self.init_prim_io_names(inputs=['x', 'v'], outputs=['y']) self.indices = indices - - def infer_shape(self, x_shape, v_shape): - validator.check("x", len(x_shape), "v", len(v_shape), Rel.EQ, self.name) - if isinstance(self.indices, int): - validator.check("size of indices", 1, "v's first dimension", v_shape[0], - Rel.EQ, self.name) - if self.indices < 0 or self.indices >= x_shape[0]: - raise ValueError(f'The value of indices must be in [0, {x_shape[0]}), but got {self.indices}.') - else: - validator.check("size of indices", len(self.indices), "v's first dimension", v_shape[0], - Rel.EQ, self.name) - for i in self.indices: - if i < 0 or i >= x_shape[0]: - raise ValueError(f'The value of indices must be in [0, {x_shape[0]}), but got {i}.') - if len(x_shape) > 1: - validator.check("x's ith dimension", x_shape[1:], "v's ith dimension", v_shape[1:], - Rel.EQ, self.name) - return x_shape + validator.check_value_type('indices', indices, [tuple, int], self.name) + if isinstance(indices, int): + self.indices = (indices,) + for item in self.indices: + validator.check_value_type("item of indices", item, [int], self.name) def infer_dtype(self, x_dtype, v_dtype): args = {'x': x_dtype, 'v': v_dtype} valid_type = [mstype.int32, mstype.float16, mstype.float32] validator.check_tensor_type_same(args, valid_type, self.name) - validator.check_value_type('indices', self.indices, [tuple, int], self.name) return x_dtype + def infer_shape(self, x_shape, v_shape): + validator.check("x", len(x_shape), "v", len(v_shape), Rel.EQ, self.name) + validator.check("size of indices", len(self.indices), "v's first dimension", v_shape[0], + Rel.EQ, self.name) + for i in self.indices: + if i < 0 or i >= x_shape[0]: + raise ValueError(f'The value of indices must be in [0, {x_shape[0]}), but got {i}.') + x_rank = len(x_shape) + for idx in range(x_rank)[1:]: + validator.check("x dim %d" % idx, x_shape[idx], 'v dim %d' % idx, v_shape[idx], Rel.EQ, self.name) + + return x_shape + class InplaceSub(PrimitiveWithInfer): """ @@ -985,32 +984,31 @@ class InplaceSub(PrimitiveWithInfer): """init InplaceSub""" self.init_prim_io_names(inputs=['x', 'v'], outputs=['y']) self.indices = indices - - def infer_shape(self, x_shape, v_shape): - validator.check("x", len(x_shape), "v", len(v_shape), Rel.EQ, self.name) - if isinstance(self.indices, int): - validator.check("size of indices", 1, "v's first dimension", v_shape[0], - Rel.EQ, self.name) - if self.indices < 0 or self.indices >= x_shape[0]: - raise ValueError(f'The value of indices must be in [0, {x_shape[0]}), but got {self.indices}.') - else: - validator.check("size of indices", len(self.indices), "v's first dimension", v_shape[0], - Rel.EQ, self.name) - for i in self.indices: - if i < 0 or i >= x_shape[0]: - raise ValueError(f'The value of indices must be in [0, {x_shape[0]}), but got {i}.') - if len(x_shape) > 1: - validator.check("x's ith dimension", x_shape[1:], "v's ith dimension", v_shape[1:], - Rel.EQ, self.name) - return x_shape + validator.check_value_type('indices', indices, [tuple, int], self.name) + if isinstance(indices, int): + self.indices = (indices,) + for item in self.indices: + validator.check_value_type("item of indices", item, [int], self.name) def infer_dtype(self, x_dtype, v_dtype): args = {'x': x_dtype, 'v': v_dtype} valid_type = [mstype.int32, mstype.float16, mstype.float32] validator.check_tensor_type_same(args, valid_type, self.name) - validator.check_value_type('indices', self.indices, [tuple, int], self.name) return x_dtype + def infer_shape(self, x_shape, v_shape): + validator.check("x", len(x_shape), "v", len(v_shape), Rel.EQ, self.name) + validator.check("size of indices", len(self.indices), "v's first dimension", v_shape[0], + Rel.EQ, self.name) + for i in self.indices: + if i < 0 or i >= x_shape[0]: + raise ValueError(f'The value of indices must be in [0, {x_shape[0]}), but got {i}.') + x_rank = len(x_shape) + for idx in range(x_rank)[1:]: + validator.check("x dim %d" % idx, x_shape[idx], 'v dim %d' % idx, v_shape[idx], Rel.EQ, self.name) + + return x_shape + class Sub(_MathBinaryOp): """ From 741154b9b9ea8280d136bc57f1ee3c0af6597048 Mon Sep 17 00:00:00 2001 From: xiefangqi Date: Tue, 30 Jun 2020 15:12:42 +0800 Subject: [PATCH 165/254] fix voc/coco docstring problem --- mindspore/dataset/engine/datasets.py | 74 ++++++++++++++-------------- 1 file changed, 37 insertions(+), 37 deletions(-) diff --git a/mindspore/dataset/engine/datasets.py b/mindspore/dataset/engine/datasets.py index 49cab9002f..1a0757ce9b 100644 --- a/mindspore/dataset/engine/datasets.py +++ b/mindspore/dataset/engine/datasets.py @@ -4036,8 +4036,8 @@ class VOCDataset(MappableDataset): A source dataset for reading and parsing VOC dataset. The generated dataset has two columns : - task='Detection' : ['image', 'annotation']. - task='Segmentation' : ['image', 'target'] + task='Detection' : ['image', 'annotation']; + task='Segmentation' : ['image', 'target']. The shape of both column 'image' and 'target' is [image_size] if decode flag is False, or [H, W, C] otherwise. The type of both tensor 'image' and 'target' is uint8. @@ -4072,20 +4072,20 @@ class VOCDataset(MappableDataset): - False - not allowed - Citation of VOC dataset. + Citation of VOC dataset. .. code-block:: @article{Everingham10, - author = {Everingham, M. and Van~Gool, L. and Williams, C. K. I. and Winn, J. and Zisserman, A.}, - title = {The Pascal Visual Object Classes (VOC) Challenge}, - journal = {International Journal of Computer Vision}, - volume = {88}, - year = {2010}, - number = {2}, - month = {jun}, - pages = {303--338}, - biburl = {http://host.robots.ox.ac.uk/pascal/VOC/pubs/everingham10.html#bibtex}, + author = {Everingham, M. and Van~Gool, L. and Williams, C. K. I. and Winn, J. and Zisserman, A.}, + title = {The Pascal Visual Object Classes (VOC) Challenge}, + journal = {International Journal of Computer Vision}, + volume = {88}, + year = {2010}, + number = {2}, + month = {jun}, + pages = {303--338}, + biburl = {http://host.robots.ox.ac.uk/pascal/VOC/pubs/everingham10.html#bibtex}, howpublished = {http://host.robots.ox.ac.uk/pascal/VOC/voc{year}/index.html}, description = {The PASCAL Visual Object Classes (VOC) challenge is a benchmark in visual object category recognition and detection, providing the vision and machine @@ -4096,8 +4096,8 @@ class VOCDataset(MappableDataset): Args: dataset_dir (str): Path to the root directory that contains the dataset. task (str): Set the task type of reading voc data, now only support "Segmentation" or "Detection" - (default="Segmentation") - mode(str): Set the data list txt file to be readed (default="train") + (default="Segmentation"). + mode (str): Set the data list txt file to be readed (default="train"). class_indexing (dict, optional): A str-to-int mapping from label name to index (default=None, the folder names will be sorted alphabetically and each class will be given a unique index starting from 0). @@ -4116,9 +4116,9 @@ class VOCDataset(MappableDataset): argument should be specified only when num_shards is also specified. Raises: - RuntimeError: If xml of Annotations is a invalid format - RuntimeError: If xml of Annotations loss attribution of "object" - RuntimeError: If xml of Annotations loss attribution of "bndbox" + RuntimeError: If xml of Annotations is a invalid format. + RuntimeError: If xml of Annotations loss attribution of "object". + RuntimeError: If xml of Annotations loss attribution of "bndbox". RuntimeError: If sampler and shuffle are specified at the same time. RuntimeError: If sampler and sharding are specified at the same time. RuntimeError: If num_shards is specified but shard_id is None. @@ -4232,10 +4232,10 @@ class CocoDataset(MappableDataset): """ A source dataset for reading and parsing COCO dataset. - CocoDataset support four kinds of task: - 2017 Train/Val/Test Detection, Keypoints, Stuff, Panoptic. + CocoDataset support four kinds of task: 2017 Train/Val/Test Detection, Keypoints, Stuff, Panoptic. The generated dataset has multi-columns : + - task='Detection', column: [['image', dtype=uint8], ['bbox', dtype=float32], ['category_id', dtype=uint32], ['iscrowd', dtype=uint32]]. - task='Stuff', column: [['image', dtype=uint8], ['segmentation',dtype=float32], ['iscrowd',dtype=uint32]]. @@ -4273,35 +4273,35 @@ class CocoDataset(MappableDataset): - False - not allowed - Citation of Coco dataset. + Citation of Coco dataset. .. code-block:: @article{DBLP:journals/corr/LinMBHPRDZ14, - author = {Tsung{-}Yi Lin and Michael Maire and Serge J. Belongie and - Lubomir D. Bourdev and Ross B. Girshick and James Hays and - Pietro Perona and Deva Ramanan and Piotr Doll{\'{a}}r and C. Lawrence Zitnick}, - title = {Microsoft {COCO:} Common Objects in Context}, - journal = {CoRR}, - volume = {abs/1405.0312}, - year = {2014}, - url = {http://arxiv.org/abs/1405.0312}, + author = {Tsung{-}Yi Lin and Michael Maire and Serge J. Belongie and + Lubomir D. Bourdev and Ross B. Girshick and James Hays and + Pietro Perona and Deva Ramanan and Piotr Doll{\'{a}}r and C. Lawrence Zitnick}, + title = {Microsoft {COCO:} Common Objects in Context}, + journal = {CoRR}, + volume = {abs/1405.0312}, + year = {2014}, + url = {http://arxiv.org/abs/1405.0312}, archivePrefix = {arXiv}, - eprint = {1405.0312}, - timestamp = {Mon, 13 Aug 2018 16:48:13 +0200}, - biburl = {https://dblp.org/rec/journals/corr/LinMBHPRDZ14.bib}, - bibsource = {dblp computer science bibliography, https://dblp.org}, - description = {COCO is a large-scale object detection, segmentation, and captioning dataset. - It contains 91 common object categories with 82 of them having more than 5,000 - labeled instances. In contrast to the popular ImageNet dataset, COCO has fewer - categories but more instances per category.} + eprint = {1405.0312}, + timestamp = {Mon, 13 Aug 2018 16:48:13 +0200}, + biburl = {https://dblp.org/rec/journals/corr/LinMBHPRDZ14.bib}, + bibsource = {dblp computer science bibliography, https://dblp.org}, + description = {COCO is a large-scale object detection, segmentation, and captioning dataset. + It contains 91 common object categories with 82 of them having more than 5,000 + labeled instances. In contrast to the popular ImageNet dataset, COCO has fewer + categories but more instances per category.} } Args: dataset_dir (str): Path to the root directory that contains the dataset. annotation_file (str): Path to the annotation json. task (str): Set the task type of reading coco data, now support 'Detection'/'Stuff'/'Panoptic'/'Keypoint' - (default='Detection') + (default='Detection'). num_samples (int, optional): The number of images to be included in the dataset (default=None, all images). num_parallel_workers (int, optional): Number of workers to read the data From d5255fe311d7282ab3517c73fd07f423a3e79c3a Mon Sep 17 00:00:00 2001 From: zhousiyi Date: Mon, 29 Jun 2020 12:40:48 +0000 Subject: [PATCH 166/254] fix assign used in while --- mindspore/ccsrc/pipeline/pipeline.cc | 2 +- .../static_analysis/abstract_value.cc | 20 ++++++ .../pipeline/static_analysis/abstract_value.h | 1 + .../pipeline/static_analysis/evaluator.cc | 2 + .../python/pipeline/infer/test_net_infer.py | 65 +++++++++++++------ 5 files changed, 69 insertions(+), 21 deletions(-) diff --git a/mindspore/ccsrc/pipeline/pipeline.cc b/mindspore/ccsrc/pipeline/pipeline.cc index d346c980ea..e477a147be 100644 --- a/mindspore/ccsrc/pipeline/pipeline.cc +++ b/mindspore/ccsrc/pipeline/pipeline.cc @@ -314,7 +314,7 @@ std::map> ExecutorPy::FetchI auto weight_name = weight_node->cast()->name(); // find the fakequant from input int count = 0; - int max_depth = 5; + const int max_depth = 5; while (!is_quant_cnode(x)) { if (count >= max_depth) { break; diff --git a/mindspore/ccsrc/pipeline/static_analysis/abstract_value.cc b/mindspore/ccsrc/pipeline/static_analysis/abstract_value.cc index f23c6e31c4..f0cf253408 100644 --- a/mindspore/ccsrc/pipeline/static_analysis/abstract_value.cc +++ b/mindspore/ccsrc/pipeline/static_analysis/abstract_value.cc @@ -429,6 +429,11 @@ AbstractBasePtr AbstractTensor::Join(const AbstractBasePtr &other) { if (other_tensor == nullptr) { MS_LOG(EXCEPTION) << "Join failed as type mismatch, this: " << ToString() << ", other: " << other->ToString(); } + if (*this == *other) { + if (sparse_grad() == other->sparse_grad()) { + return shared_from_base(); + } + } auto element = element_->Join(other_tensor->element_); auto shape = ShapeJoin(this->shape(), other_tensor->shape()); auto ret = std::make_shared(element, shape); @@ -812,6 +817,21 @@ bool AbstractRef::operator==(const AbstractBase &other) const { return false; } +AbstractBasePtr AbstractRef::Join(const AbstractBasePtr &other) { + auto other_ref = other->cast(); + if (other_ref == nullptr) { + MS_LOG(EXCEPTION) << "Join failed as type mismatch, this: " << ToString() << ", other: " << other->ToString(); + } + if (*this == *other) { + return shared_from_base(); + } + auto ref_key = ref_key_->Join(other_ref->ref_key_); + auto ref = ref_->Join(other_ref->ref()); + auto ref_origin = ref_origin_->Join(other_ref->ref_origin_); + + return std::make_shared(ref_key, ref, ref_origin); +} + std::string AbstractRef::ToString() const { std::ostringstream buffer; buffer << type_name() << "(" diff --git a/mindspore/ccsrc/pipeline/static_analysis/abstract_value.h b/mindspore/ccsrc/pipeline/static_analysis/abstract_value.h index f3375d22d6..f45c08e6bc 100644 --- a/mindspore/ccsrc/pipeline/static_analysis/abstract_value.h +++ b/mindspore/ccsrc/pipeline/static_analysis/abstract_value.h @@ -564,6 +564,7 @@ class AbstractRef : public AbstractBase { AbstractBasePtr Broaden() const override { return std::make_shared(ref_key_->Broaden(), ref_->Broaden(), ref_origin_->Broaden()); } + AbstractBasePtr Join(const AbstractBasePtr &other) override; std::size_t hash() const override { return ref_key_->hash() ^ ref_->hash() ^ ref_origin_->hash() ^ (std::hash{}(this->tid()) << 1); } diff --git a/mindspore/ccsrc/pipeline/static_analysis/evaluator.cc b/mindspore/ccsrc/pipeline/static_analysis/evaluator.cc index c9b1ce4f93..34ecfc8980 100644 --- a/mindspore/ccsrc/pipeline/static_analysis/evaluator.cc +++ b/mindspore/ccsrc/pipeline/static_analysis/evaluator.cc @@ -166,6 +166,7 @@ AbstractBasePtrList FuncGraphEvaluator::BroadenUndeterminedArgs(const AbstractBa // If there is loop variant, all arguments need to be broaden to avoid wrong constant propagation. if (!(joined_args_spec_list == args_spec_list)) { func_graph_->set_flag(FUNC_GRAPH_FLAG_IGNORE_VALUES, true); + MS_LOG(DEBUG) << "Set " << func_graph_->ToString() << " with IGNORE_VALUES flag."; } return joined_args_spec_list; } @@ -179,6 +180,7 @@ AbstractBasePtrList FuncGraphEvaluator::BroadenUndeterminedArgs(const AbstractBa if (!(joined_args_spec_list == args_spec_list)) { trace_.push_back(joined_args_spec_list); func_graph_->set_flag(FUNC_GRAPH_FLAG_IGNORE_VALUES, true); + MS_LOG(DEBUG) << "Set " << func_graph_->ToString() << " with IGNORE_VALUES flag."; } MS_LOG(DEBUG) << "Joined eval args: " << ::mindspore::ToString(joined_args_spec_list); return joined_args_spec_list; diff --git a/tests/ut/python/pipeline/infer/test_net_infer.py b/tests/ut/python/pipeline/infer/test_net_infer.py index 003aad827d..6b32a7617d 100644 --- a/tests/ut/python/pipeline/infer/test_net_infer.py +++ b/tests/ut/python/pipeline/infer/test_net_infer.py @@ -16,29 +16,54 @@ import numpy as np import mindspore.nn as nn -from mindspore import Tensor +from mindspore import Tensor, context +from mindspore.common.parameter import Parameter +from mindspore.common.initializer import initializer +import mindspore.ops.operations as op +def test_net_infer(): + """ test_net_infer """ + class Net(nn.Cell): + """ Net definition """ -class Net(nn.Cell): - """ Net definition """ + def __init__(self): + super(Net, self).__init__() + self.conv = nn.Conv2d(3, 64, 3, has_bias=False, weight_init='normal') + self.bn = nn.BatchNorm2d(64) + self.fc = nn.Dense(64, 10) + self.relu = nn.ReLU() + self.flatten = nn.Flatten() - def __init__(self): - super(Net, self).__init__() - self.conv = nn.Conv2d(3, 64, 3, has_bias=False, weight_init='normal') - self.bn = nn.BatchNorm2d(64) - self.fc = nn.Dense(64, 10) - self.relu = nn.ReLU() - self.flatten = nn.Flatten() + def construct(self, x): + x = self.conv(x) + x = self.relu(x) + x = self.flatten(x) + out = self.fc(x) + return out + Tensor(np.random.randint(0, 255, [1, 3, 224, 224])) + Net() - def construct(self, x): - x = self.conv(x) - x = self.relu(x) - x = self.flatten(x) - out = self.fc(x) - return out +def test_assign_in_while(): + context.set_context(mode=context.GRAPH_MODE) + class Net(nn.Cell): + def __init__(self, input_shape): + super().__init__() + self.assign = op.Assign() + self.inputdata = Parameter(initializer(1, input_shape), name="global_step") -def test_net_infer(): - """ test_net_infer """ - Tensor(np.random.randint(0, 255, [1, 3, 224, 224])) - Net() + def construct(self, x, y, z): + out = z + while x < y: + inputdata = self.inputdata + x = x + 1 + out = self.assign(inputdata, z) + return out + + x = Tensor(np.array(1).astype(np.int32)) + y = Tensor(np.array(3).astype(np.int32)) + input_shape = (1024, 512) + z = Tensor(np.random.randn(*input_shape).astype(np.float32)) + net = Net(input_shape) + ret = net(x, y, z) + assert ret == z From 3dc6f6f2d94bfc745c352348eb2a83c1458259e5 Mon Sep 17 00:00:00 2001 From: ougongchang Date: Wed, 24 Jun 2020 14:33:32 +0800 Subject: [PATCH 167/254] add more ut and st for SummaryCollector Has fixed collecting optimizer error when mode is eval --- .../train/callback/_summary_collector.py | 28 +-- tests/st/summary/test_davinci_summary.py | 99 --------- tests/st/summary/test_gpu_summary.py | 89 -------- tests/st/summary/test_summary.py | 194 ++++++++++++++++++ tests/summary_utils.py | 1 + .../train/summary/test_summary_collector.py | 193 +++++++++++++++++ 6 files changed, 402 insertions(+), 202 deletions(-) delete mode 100644 tests/st/summary/test_davinci_summary.py delete mode 100644 tests/st/summary/test_gpu_summary.py create mode 100644 tests/st/summary/test_summary.py diff --git a/mindspore/train/callback/_summary_collector.py b/mindspore/train/callback/_summary_collector.py index cff03ca398..7ef890ec38 100644 --- a/mindspore/train/callback/_summary_collector.py +++ b/mindspore/train/callback/_summary_collector.py @@ -161,7 +161,7 @@ class SummaryCollector(Callback): self._check_custom_lineage_data(custom_lineage_data) self._custom_lineage_data = custom_lineage_data - self._optimizer = None + self._temp_optimizer = None self._has_saved_train_network = False self._has_saved_custom_data = False self._is_parse_loss_success = True @@ -369,15 +369,15 @@ class SummaryCollector(Callback): input_data = getattr(cb_params, 'train_dataset_element', None) if input_data is None: self._collect_specified_data['collect_input_data'] = False - logger.info("There is not a `train_dataset_element` in cb_params.") + logger.info("The 'train_dataset_element' in cb_params is None, maybe there is dataset sink mode.") return if isinstance(input_data, (list, tuple)): input_data = input_data[0] try: self._record.add_value(PluginEnum.IMAGE.value, 'input_data/auto', input_data) - except ValueError as ex: - logger.warning(str(ex)) + except ValueError: + logger.warning('The input data of network are not image, so will not collect by SummaryCollector.') self._collect_specified_data['collect_input_data'] = False return @@ -418,8 +418,8 @@ class SummaryCollector(Callback): try: self._record.add_value(PluginEnum.SCALAR.value, 'loss/auto', loss) - except ValueError as exc: - logger.warning(str(exc)) + except ValueError: + logger.warning("The output of network is not a scalar, so will not collect loss in SummaryCollector.") self._collect_specified_data['collect_metric'] = False def _get_loss(self, cb_params): @@ -438,7 +438,7 @@ class SummaryCollector(Callback): output = cb_params.net_outputs if output is None: - logger.warning("Can not find any output by this network.") + logger.warning("Can not find any output by this network, so will not collect loss in SummaryCollector.") self._is_parse_loss_success = False return None @@ -448,7 +448,7 @@ class SummaryCollector(Callback): # If the output is a list, since the default network returns loss first, # we assume that the first one is loss. loss = output[0] - elif isinstance(output, Tensor) and (not output.shape or output.shape == [1]): + elif isinstance(output, Tensor) and (not output.shape or output.shape == (1,)): loss_numpy = output.asnumpy() loss = float(np.atleast_1d(loss_numpy)[0]) else: @@ -473,15 +473,15 @@ class SummaryCollector(Callback): """ # 'optimizer_failed' means find optimizer failed, so we will not collect data about optimizer. optimizer_failed = 'Failed' - if self._optimizer == optimizer_failed: + if self._temp_optimizer == optimizer_failed: return None - if self._optimizer is not None: - return self._optimizer + if self._temp_optimizer is not None: + return self._temp_optimizer optimizer = cb_params.optimizer if optimizer is None: - network = cb_params.train_network if cb_params.mode == 'train' else cb_params.eval_work + network = cb_params.train_network if cb_params.mode == 'train' else cb_params.eval_network optimizer = self._parse_optimizer_by_network(network) if optimizer is None or not isinstance(optimizer, Optimizer): @@ -489,7 +489,7 @@ class SummaryCollector(Callback): "optimizer, so we will not collect data about optimizer in SummaryCollector.") optimizer = None - self._optimizer = optimizer if optimizer is not None else optimizer_failed + self._temp_optimizer = optimizer if optimizer is not None else optimizer_failed return optimizer @@ -765,7 +765,7 @@ class SummaryCollector(Callback): cb_params (_InternalCallbackParam): Callback parameters. Returns: - Union[Loss_fn, None], a Cell object, if parse failed, will return None. + Union[Cell, None], a Cell object, if parse failed, will return None. """ loss_fn = cb_params.loss_fn if loss_fn is not None: diff --git a/tests/st/summary/test_davinci_summary.py b/tests/st/summary/test_davinci_summary.py deleted file mode 100644 index bc93afe364..0000000000 --- a/tests/st/summary/test_davinci_summary.py +++ /dev/null @@ -1,99 +0,0 @@ -# Copyright 2019 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ -""" test model train """ -import os -import numpy as np -from apply_momentum import ApplyMomentum -import mindspore.context as context -import mindspore.nn as nn -from mindspore.nn import wrap -from mindspore import Tensor, Model -from mindspore.common.api import ms_function -from mindspore.nn.loss import SoftmaxCrossEntropyWithLogits -from mindspore.ops import operations as P -from mindspore.train.summary.summary_record import SummaryRecord - -CUR_DIR = os.getcwd() -SUMMARY_DIR = CUR_DIR + "/test_temp_summary_event_file/" - -context.set_context(device_target="Ascend") - - -class MsWrapper(nn.Cell): - def __init__(self, network): - super(MsWrapper, self).__init__(auto_prefix=False) - self._network = network - - @ms_function - def construct(self, *args): - return self._network(*args) - - -def me_train_tensor(net, input_np, label_np, epoch_size=2): - context.set_context(mode=context.GRAPH_MODE) - loss = SoftmaxCrossEntropyWithLogits(is_grad=False, sparse=True) - opt = ApplyMomentum(Tensor(np.array([0.1])), Tensor(np.array([0.9])), - filter(lambda x: x.requires_grad, net.get_parameters())) - Model(net, loss, opt) - _network = wrap.WithLossCell(net, loss) - _train_net = MsWrapper(wrap.TrainOneStepCell(_network, opt)) - _train_net.set_train() - with SummaryRecord(SUMMARY_DIR, file_suffix="_MS_GRAPH", network=_train_net) as summary_writer: - for epoch in range(0, epoch_size): - print(f"epoch %d" % (epoch)) - output = _train_net(Tensor(input_np), Tensor(label_np)) - summary_writer.record(i) - print("********output***********") - print(output.asnumpy()) - - -def me_infer_tensor(net, input_np): - net.set_train() - net = MsWrapper(net) - output = net(Tensor(input_np)) - return output - - -def test_net(): - class Net(nn.Cell): - def __init__(self, cin, cout): - super(Net, self).__init__() - self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, pad_mode="same") - self.conv = nn.Conv2d(cin, cin, kernel_size=1, stride=1, padding=0, has_bias=False, pad_mode="same") - self.bn = nn.BatchNorm2d(cin, momentum=0.1, eps=0.0001) - self.add = P.TensorAdd() - self.relu = P.ReLU() - self.mean = P.ReduceMean(keep_dims=True) - self.reshape = P.Reshape() - self.dense = nn.Dense(cin, cout) - - def construct(self, input_x): - output = input_x - output = self.maxpool(output) - identity = output - output = self.conv(output) - output = self.bn(output) - output = self.add(output, identity) - output = self.relu(output) - output = self.mean(output, (-2, -1)) - output = self.reshape(output, (32, -1)) - output = self.dense(output) - return output - - net = Net(2048, 1001) - input_np = np.ones([32, 2048, 14, 14]).astype(np.float32) * 0.01 - label_np = np.ones([32]).astype(np.int32) - me_train_tensor(net, input_np, label_np) - # me_infer_tensor(net, input_np) diff --git a/tests/st/summary/test_gpu_summary.py b/tests/st/summary/test_gpu_summary.py deleted file mode 100644 index 9b4095b8d9..0000000000 --- a/tests/st/summary/test_gpu_summary.py +++ /dev/null @@ -1,89 +0,0 @@ -# Copyright 2019 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ -"""Summary gpu st.""" -import os -import random -import tempfile -import shutil - -import numpy as np -import pytest - -import mindspore.context as context -import mindspore.nn as nn -from mindspore.common.tensor import Tensor -from mindspore.ops import operations as P -from mindspore.train.summary.summary_record import SummaryRecord - -context.set_context(mode=context.GRAPH_MODE, device_target="GPU") - - -class SummaryNet(nn.Cell): - """Summary net.""" - def __init__(self, tag_tuple=None, scalar=1): - super(SummaryNet, self).__init__() - self.summary_s = P.ScalarSummary() - self.summary_i = P.ImageSummary() - self.summary_t = P.TensorSummary() - self.histogram_summary = P.HistogramSummary() - self.add = P.TensorAdd() - self.tag_tuple = tag_tuple - self.scalar = scalar - - def construct(self, x, y, image): - """Run summary net.""" - self.summary_i("image", image) - self.summary_s("x1", x) - z = self.add(x, y) - self.summary_t("z1", z) - self.histogram_summary("histogram", z) - return z - - -def train_summary_record(test_writer, steps): - """Train and record summary.""" - net = SummaryNet() - out_me_dict = {} - for i in range(0, steps): - x = Tensor(np.array([1.1 + random.uniform(1, 10)]).astype(np.float32)) - y = Tensor(np.array([1.2 + random.uniform(1, 10)]).astype(np.float32)) - image = Tensor(np.array([[[[1.2]]]]).astype(np.float32)) - out_put = net(x, y, image) - test_writer.record(i) - out_me_dict[i] = out_put.asnumpy() - return out_me_dict - - -class TestGpuSummary: - """Test Gpu summary.""" - summary_dir = tempfile.mkdtemp(suffix='_gpu_summary') - - def setup_method(self): - """Run before method.""" - if not os.path.exists(self.summary_dir): - os.mkdir(self.summary_dir) - - def teardown_method(self): - """Run after method.""" - if os.path.exists(self.summary_dir): - shutil.rmtree(self.summary_dir) - - @pytest.mark.level0 - @pytest.mark.platform_x86_gpu_training - @pytest.mark.env_onecard - def test_summary_step10_summaryrecord1(self): - """Test record 10 step summary.""" - with SummaryRecord(self.summary_dir) as test_writer: - train_summary_record(test_writer, steps=10) diff --git a/tests/st/summary/test_summary.py b/tests/st/summary/test_summary.py new file mode 100644 index 0000000000..b81d15514a --- /dev/null +++ b/tests/st/summary/test_summary.py @@ -0,0 +1,194 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" test model train """ +import os +import re +import tempfile +import shutil + +import pytest + +from mindspore import dataset as ds +from mindspore import nn, Tensor, context +from mindspore.nn.metrics import Accuracy +from mindspore.nn.optim import Momentum +from mindspore.dataset.transforms import c_transforms as C +from mindspore.dataset.transforms.vision import c_transforms as CV +from mindspore.dataset.transforms.vision import Inter +from mindspore.common import dtype as mstype +from mindspore.common.initializer import TruncatedNormal +from mindspore.ops import operations as P +from mindspore.train import Model +from mindspore.train.callback import SummaryCollector + +from tests.summary_utils import SummaryReader + + +def conv(in_channels, out_channels, kernel_size, stride=1, padding=0): + """weight initial for conv layer""" + weight = weight_variable() + return nn.Conv2d(in_channels, out_channels, + kernel_size=kernel_size, stride=stride, padding=padding, + weight_init=weight, has_bias=False, pad_mode="valid") + + +def fc_with_initialize(input_channels, out_channels): + """weight initial for fc layer""" + weight = weight_variable() + bias = weight_variable() + return nn.Dense(input_channels, out_channels, weight, bias) + + +def weight_variable(): + """weight initial""" + return TruncatedNormal(0.02) + + +class LeNet5(nn.Cell): + """Define LeNet5 network.""" + def __init__(self, num_class=10, channel=1): + super(LeNet5, self).__init__() + self.num_class = num_class + self.conv1 = conv(channel, 6, 5) + self.conv2 = conv(6, 16, 5) + self.fc1 = fc_with_initialize(16 * 5 * 5, 120) + self.fc2 = fc_with_initialize(120, 84) + self.fc3 = fc_with_initialize(84, self.num_class) + self.relu = nn.ReLU() + self.max_pool2d = nn.MaxPool2d(kernel_size=2, stride=2) + self.flatten = nn.Flatten() + self.scalar_summary = P.ScalarSummary() + self.image_summary = P.ImageSummary() + self.histogram_summary = P.HistogramSummary() + self.tensor_summary = P.TensorSummary() + self.channel = Tensor(channel) + + def construct(self, data): + """define construct.""" + self.image_summary('image', data) + output = self.conv1(data) + self.histogram_summary('histogram', output) + output = self.relu(output) + self.tensor_summary('tensor', output) + output = self.max_pool2d(output) + output = self.conv2(output) + output = self.relu(output) + output = self.max_pool2d(output) + output = self.flatten(output) + output = self.fc1(output) + output = self.relu(output) + output = self.fc2(output) + output = self.relu(output) + output = self.fc3(output) + self.scalar_summary('scalar', self.channel) + return output + + +def create_dataset(data_path, batch_size=32, repeat_size=1, num_parallel_workers=1): + """create dataset for train or test""" + # define dataset + mnist_ds = ds.MnistDataset(data_path) + + resize_height, resize_width = 32, 32 + rescale = 1.0 / 255.0 + rescale_nml = 1 / 0.3081 + shift_nml = -1 * 0.1307 / 0.3081 + + # define map operations + resize_op = CV.Resize((resize_height, resize_width), interpolation=Inter.LINEAR) # Bilinear mode + rescale_nml_op = CV.Rescale(rescale_nml, shift_nml) + rescale_op = CV.Rescale(rescale, shift=0.0) + hwc2chw_op = CV.HWC2CHW() + type_cast_op = C.TypeCast(mstype.int32) + + # apply map operations on images + mnist_ds = mnist_ds.map(input_columns="label", operations=type_cast_op, num_parallel_workers=num_parallel_workers) + mnist_ds = mnist_ds.map(input_columns="image", operations=resize_op, num_parallel_workers=num_parallel_workers) + mnist_ds = mnist_ds.map(input_columns="image", operations=rescale_op, num_parallel_workers=num_parallel_workers) + mnist_ds = mnist_ds.map(input_columns="image", operations=rescale_nml_op, num_parallel_workers=num_parallel_workers) + mnist_ds = mnist_ds.map(input_columns="image", operations=hwc2chw_op, num_parallel_workers=num_parallel_workers) + + # apply DatasetOps + mnist_ds = mnist_ds.shuffle(buffer_size=10000) # 10000 as in LeNet train script + mnist_ds = mnist_ds.batch(batch_size, drop_remainder=True) + mnist_ds = mnist_ds.repeat(repeat_size) + + return mnist_ds + + +class TestSummary: + """Test summary collector the basic function.""" + base_summary_dir = '' + mnist_path = '/home/workspace/mindspore_dataset/mnist' + + @classmethod + def setup_class(cls): + """Run before test this class.""" + cls.base_summary_dir = tempfile.mkdtemp(suffix='summary') + + @classmethod + def teardown_class(cls): + """Run after test this class.""" + if os.path.exists(cls.base_summary_dir): + shutil.rmtree(cls.base_summary_dir) + + @pytest.mark.level0 + @pytest.mark.platform_x86_ascend_training + @pytest.mark.env_onecard + def test_summary_ascend(self): + """Test summary ascend.""" + context.set_context(mode=context.GRAPH_MODE) + self._run_network() + + def _run_network(self, dataset_sink_mode=True): + lenet = LeNet5() + loss = nn.SoftmaxCrossEntropyWithLogits(is_grad=False, sparse=True, reduction="mean") + optim = Momentum(lenet.trainable_params(), learning_rate=0.1, momentum=0.9) + model = Model(lenet, loss_fn=loss, optimizer=optim, metrics={'acc': Accuracy()}) + summary_dir = tempfile.mkdtemp(dir=self.base_summary_dir) + summary_collector = SummaryCollector(summary_dir=summary_dir, collect_freq=1) + + ds_train = create_dataset(os.path.join(self.mnist_path, "train")) + model.train(1, ds_train, callbacks=[summary_collector], dataset_sink_mode=dataset_sink_mode) + + ds_eval = create_dataset(os.path.join(self.mnist_path, "test")) + model.eval(ds_eval, dataset_sink_mode=dataset_sink_mode, callbacks=[summary_collector]) + + self._check_summary_result(summary_dir) + + @staticmethod + def _check_summary_result(summary_dir): + summary_file_path = '' + for file in os.listdir(summary_dir): + if re.search("_MS", file): + summary_file_path = os.path.join(summary_dir, file) + break + + assert not summary_file_path + + with SummaryReader(summary_file_path) as summary_reader: + tags = set() + + # Read the event that record by SummaryCollector.begin + summary_reader.read_event() + + summary_event = summary_reader.read_event() + for value in summary_event.summary.value: + tags.add(value.tag) + + # There will not record input data when dataset sink mode is True + expected_tags = ['conv1.weight/auto', 'conv2.weight/auto', 'fc1.weight/auto', 'fc1.bias/auto', + 'fc2.weight/auto', 'histogram', 'image', 'scalar', 'tensor'] + assert set(expected_tags) == tags diff --git a/tests/summary_utils.py b/tests/summary_utils.py index 826a3106e5..cc2070f9c6 100644 --- a/tests/summary_utils.py +++ b/tests/summary_utils.py @@ -38,6 +38,7 @@ class SummaryReader: def __init__(self, canonical_file_path, ignore_version_event=True): self._file_path = canonical_file_path self._ignore_version_event = ignore_version_event + self._file_handler = None def __enter__(self): self._file_handler = open(self._file_path, "rb") diff --git a/tests/ut/python/train/summary/test_summary_collector.py b/tests/ut/python/train/summary/test_summary_collector.py index 5e7f8e662c..1390d29bc1 100644 --- a/tests/ut/python/train/summary/test_summary_collector.py +++ b/tests/ut/python/train/summary/test_summary_collector.py @@ -16,9 +16,50 @@ import os import tempfile import shutil +from importlib import import_module +from unittest import mock + +import numpy as np import pytest +from mindspore import Tensor +from mindspore import Parameter from mindspore.train.callback import SummaryCollector +from mindspore.train.callback import _InternalCallbackParam +from mindspore.train.summary.enum import ModeEnum, PluginEnum +from mindspore.train.summary import SummaryRecord +from mindspore.nn import Cell +from mindspore.nn.optim.optimizer import Optimizer +from mindspore.ops.operations import TensorAdd + + +_VALUE_CACHE = list() + + +def add_value(plugin, name, value): + """This function is mock the function in SummaryRecord.""" + global _VALUE_CACHE + _VALUE_CACHE.append((plugin, name, value)) + + +def get_value(): + """Get the value which is added by add_value function.""" + global _VALUE_CACHE + + value = _VALUE_CACHE + _VALUE_CACHE = list() + return value + + +class CustomNet(Cell): + """Define custom netwrok.""" + def __init__(self): + super(CustomNet, self).__init__() + self.add = TensorAdd + self.optimizer = Optimizer(learning_rate=1, parameters=[Parameter(Tensor(1), 'weight')]) + + def construct(self, data): + return data class TestSummaryCollector: @@ -34,6 +75,10 @@ class TestSummaryCollector: if os.path.exists(self.base_summary_dir): shutil.rmtree(self.base_summary_dir) + def teardown_method(self): + """Run after each test function.""" + get_value() + @pytest.mark.parametrize("summary_dir", [1234, None, True, '']) def test_params_with_summary_dir_value_error(self, summary_dir): """Test the exception scenario for summary dir.""" @@ -182,3 +227,151 @@ class TestSummaryCollector: f'bug got {type(param_value).__name__}.' assert expected_msg == str(exc.value) + + def test_check_callback_with_multi_instances(self): + """Use multi SummaryCollector instances to test check_callback function.""" + cb_params = _InternalCallbackParam() + cb_params.list_callback = [ + SummaryCollector(tempfile.mkdtemp(dir=self.base_summary_dir)), + SummaryCollector(tempfile.mkdtemp(dir=self.base_summary_dir)) + ] + with pytest.raises(ValueError) as exc: + SummaryCollector((tempfile.mkdtemp(dir=self.base_summary_dir)))._check_callbacks(cb_params) + assert f"more than one SummaryCollector instance in callback list" in str(exc.value) + + def test_collect_input_data_with_train_dataset_element_none(self): + """Test the param 'train_dataset_element' in cb_params is none.""" + cb_params = _InternalCallbackParam() + cb_params.train_dataset_element = None + summary_collector = SummaryCollector((tempfile.mkdtemp(dir=self.base_summary_dir))) + summary_collector._collect_input_data(cb_params) + assert not summary_collector._collect_specified_data['collect_input_data'] + + @mock.patch.object(SummaryRecord, 'add_value') + def test_collect_input_data_success(self, mock_add_value): + """Mock a image data, and collect image data success.""" + mock_add_value.side_effect = add_value + cb_params = _InternalCallbackParam() + image_data = Tensor(np.random.randint(0, 255, size=(1, 1, 1, 1)).astype(np.uint8)) + cb_params.train_dataset_element = image_data + with SummaryCollector((tempfile.mkdtemp(dir=self.base_summary_dir))) as summary_collector: + summary_collector._collect_input_data(cb_params) + # Note Here need to asssert the result and expected data + + @mock.patch.object(SummaryRecord, 'add_value') + def test_collect_dataset_graph_success(self, mock_add_value): + """Test collect dataset graph.""" + dataset = import_module('mindspore.dataset') + mock_add_value.side_effect = add_value + cb_params = _InternalCallbackParam() + cb_params.train_dataset = dataset.MnistDataset(dataset_dir=tempfile.mkdtemp(dir=self.base_summary_dir)) + cb_params.mode = ModeEnum.TRAIN.value + with SummaryCollector((tempfile.mkdtemp(dir=self.base_summary_dir))) as summary_collector: + summary_collector._collect_dataset_graph(cb_params) + plugin, name, _ = get_value()[0] + assert plugin == 'dataset_graph' + assert name == 'train_dataset' + + @pytest.mark.parametrize("net_output, expected_loss", [ + (1, Tensor(1)), + ([1], Tensor(1)), + ([Tensor(1)], Tensor(1)), + (Tensor([1]), Tensor(1)), + (tuple([1]), Tensor(1)), + (None, None) + ]) + def test_get_loss(self, net_output, expected_loss): + """Test get loss success and failed.""" + cb_params = _InternalCallbackParam() + cb_params.net_outputs = net_output + summary_collector = SummaryCollector((tempfile.mkdtemp(dir=self.base_summary_dir))) + + assert summary_collector._is_parse_loss_success + assert summary_collector._get_loss(cb_params) == expected_loss + + if expected_loss is None: + assert not summary_collector._is_parse_loss_success + + def test_get_optimizer_from_cb_params_success(self): + """Test get optimizer success from cb params.""" + cb_params = _InternalCallbackParam() + cb_params.optimizer = Optimizer(learning_rate=0.1, parameters=[Parameter(Tensor(1), 'weight')]) + summary_collector = SummaryCollector((tempfile.mkdtemp(dir=self.base_summary_dir))) + optimizer = summary_collector._get_optimizer(cb_params) + assert optimizer == cb_params.optimizer + + # Test get optimizer again + assert summary_collector._get_optimizer(cb_params) == cb_params.optimizer + + @pytest.mark.parametrize('mode', [ModeEnum.TRAIN.value, ModeEnum.EVAL.value]) + def test_get_optimizer_from_network(self, mode): + """Get optimizer from train network""" + cb_params = _InternalCallbackParam() + cb_params.optimizer = None + cb_params.mode = mode + if mode == ModeEnum.TRAIN.value: + cb_params.train_network = CustomNet() + else: + cb_params.eval_network = CustomNet() + summary_collector = SummaryCollector((tempfile.mkdtemp(dir=self.base_summary_dir))) + optimizer = summary_collector._get_optimizer(cb_params) + assert isinstance(optimizer, Optimizer) + + def test_get_optimizer_failed(self): + """Test get optimizer failed.""" + class Net(Cell): + """Define net.""" + def __init__(self): + super(Net, self).__init__() + self.add = TensorAdd() + + def construct(self, data): + return data + + cb_params = _InternalCallbackParam() + cb_params.optimizer = None + cb_params.train_network = Net() + cb_params.mode = ModeEnum.TRAIN.value + summary_collector = SummaryCollector((tempfile.mkdtemp(dir=self.base_summary_dir))) + optimizer = summary_collector._get_optimizer(cb_params) + assert optimizer is None + assert summary_collector._temp_optimizer == 'Failed' + + # Test get optimizer again + optimizer = summary_collector._get_optimizer(cb_params) + assert optimizer is None + assert summary_collector._temp_optimizer == 'Failed' + + @pytest.mark.parametrize("histogram_regular, expected_names, expected_values", [ + ( + 'conv1|conv2', + ['conv1.weight1/auto', 'conv2.weight2/auto', 'conv1.bias1/auto'], + [1, 2, 3] + ), + ( + None, + ['conv1.weight1/auto', 'conv2.weight2/auto', 'conv1.bias1/auto', 'conv3.bias/auto', 'conv5.bias/auto'], + [1, 2, 3, 4, 5] + ) + ]) + @mock.patch.object(SummaryRecord, 'add_value') + def test_collect_histogram_from_regular(self, mock_add_value, histogram_regular, expected_names, expected_values): + """Test collect histogram from regular success.""" + mock_add_value.side_effect = add_value + cb_params = _InternalCallbackParam() + parameters = [ + Parameter(Tensor(1), 'conv1.weight1'), + Parameter(Tensor(2), 'conv2.weight2'), + Parameter(Tensor(3), 'conv1.bias1'), + Parameter(Tensor(4), 'conv3.bias'), + Parameter(Tensor(5), 'conv5.bias'), + Parameter(Tensor(6), 'conv6.bias'), + ] + cb_params.optimizer = Optimizer(learning_rate=0.1, parameters=parameters) + with SummaryCollector((tempfile.mkdtemp(dir=self.base_summary_dir))) as summary_collector: + summary_collector._collect_specified_data['histogram_regular'] = histogram_regular + summary_collector._collect_histogram(cb_params) + result = get_value() + assert PluginEnum.HISTOGRAM.value == result[0][0] + assert expected_names == [data[1] for data in result] + assert expected_values == [data[2] for data in result] From e3c4ee75a53d1a7150b0298449e3d70f680c2f43 Mon Sep 17 00:00:00 2001 From: wenchunjiang Date: Tue, 30 Jun 2020 17:03:46 +0800 Subject: [PATCH 168/254] reset call inputs only when graph has been splited --- mindspore/ccsrc/session/ascend_session.cc | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/mindspore/ccsrc/session/ascend_session.cc b/mindspore/ccsrc/session/ascend_session.cc index 288fe0bbb8..05ebcaccf8 100644 --- a/mindspore/ccsrc/session/ascend_session.cc +++ b/mindspore/ccsrc/session/ascend_session.cc @@ -225,7 +225,7 @@ static void BindCallArgsWithParameter(const std::vector ¶meters, // if a call has kernel input, it's a child graph split from ME, so these kernel input should be set into real input of // graph.For example, call input = (prim,graph,kernel1,kernel2),then real_input = [kernel1,kernel2] -static void UpdateRealInput(NotNull graph) { +static void UpdateRealInput(NotNull graph, bool split_flag) { auto call_nodes = graph->FindNodeByPrimitive(prim::kPrimCall); for (auto &call_node : call_nodes) { MS_EXCEPTION_IF_NULL(call_node); @@ -236,7 +236,9 @@ static void UpdateRealInput(NotNull graph) { std::vector(call_node->inputs().begin() + 2, call_node->inputs().end()); std::vector child_inputs = child_graphs[0]->inputs(); BindCallArgsWithParameter(child_inputs, real_args, child_graphs[0].get()); - call_node->set_inputs(std::vector(call_node->inputs().begin(), call_node->inputs().begin() + 2)); + if (split_flag) { + call_node->set_inputs(std::vector(call_node->inputs().begin(), call_node->inputs().begin() + 2)); + } } else if (child_graphs.size() == 2) { auto get_partial_args = [&](size_t input_index) -> std::vector { auto switch_node = call_node->input(1); @@ -248,8 +250,10 @@ static void UpdateRealInput(NotNull graph) { auto partial_cnode = partial->cast(); MS_EXCEPTION_IF_NULL(partial_cnode); auto ret = std::vector(partial_cnode->inputs().begin() + 2, partial_cnode->inputs().end()); - partial_cnode->set_inputs( - std::vector(partial_cnode->inputs().begin(), partial_cnode->inputs().begin() + 2)); + if (split_flag) { + partial_cnode->set_inputs( + std::vector(partial_cnode->inputs().begin(), partial_cnode->inputs().begin() + 2)); + } return ret; }; BindCallArgsWithParameter(child_graphs[0]->inputs(), get_partial_args(2), child_graphs[0].get()); @@ -1678,6 +1682,7 @@ AnfNodePtr AscendSession::BindNewCallToNewGraph(NotNull graph, void AscendSession::SplitGraph(NotNull graph, const std::set &cut_prims) { MS_LOG(INFO) << "Start,graph_id:" << graph->graph_id(); + bool split_flag = false; auto apply_list = GetCNodes(TopoSort(graph->get_return())); // update the root graph child graph order AscendControlParser::UpdateChildGraphOrder(graph); @@ -1710,9 +1715,10 @@ void AscendSession::SplitGraph(NotNull graph, const std::setgraph_id() << "] end"; // recurse to split child graph } From 1089c908a9029441759c33f963762801e8d9a809 Mon Sep 17 00:00:00 2001 From: chenzomi Date: Tue, 30 Jun 2020 16:51:32 +0800 Subject: [PATCH 169/254] cherry-pick r0.5 to master for quantizaiton aware training --- mindspore/ccsrc/utils/checkpoint.proto | 1 - mindspore/nn/cell.py | 9 ++ mindspore/nn/layer/conv.py | 150 +++++++++++++++++- mindspore/nn/layer/quant.py | 74 +++++---- .../_op_impl/_custom_op/batchnorm_fold2.py | 1 - .../_custom_op/batchnorm_fold2_grad.py | 1 - .../_custom_op/batchnorm_fold2_grad_reduce.py | 1 - .../ops/_op_impl/_custom_op/correction_mul.py | 1 - .../_custom_op/correction_mul_grad.py | 2 - .../_custom_op/fake_quant_perchannel.py | 14 +- .../_custom_op/fake_quant_perchannel_grad.py | 14 +- .../_custom_op/minmax_update_perchannel.py | 14 +- mindspore/ops/operations/_quant_ops.py | 41 +++-- mindspore/train/callback/_checkpoint.py | 25 +-- mindspore/train/callback/_loss_monitor.py | 2 +- mindspore/train/quant/quant.py | 43 +++-- mindspore/train/serialization.py | 21 +-- model_zoo/lenet_quant/README.md | 6 +- model_zoo/lenet_quant/eval.py | 2 +- model_zoo/lenet_quant/eval_quant.py | 4 +- model_zoo/lenet_quant/train.py | 5 +- model_zoo/lenet_quant/train_quant.py | 11 +- model_zoo/mobilenetv2/scripts/run_infer.sh | 2 +- model_zoo/mobilenetv2/scripts/run_train.sh | 2 +- model_zoo/mobilenetv3/scripts/run_infer.sh | 2 +- model_zoo/mobilenetv3/scripts/run_train.sh | 2 +- .../train/quant/mobilenetv2_combined.py | 12 +- tests/ut/python/train/quant/test_quant.py | 2 +- 28 files changed, 322 insertions(+), 142 deletions(-) diff --git a/mindspore/ccsrc/utils/checkpoint.proto b/mindspore/ccsrc/utils/checkpoint.proto index 7fca399e2b..31c7cd8004 100644 --- a/mindspore/ccsrc/utils/checkpoint.proto +++ b/mindspore/ccsrc/utils/checkpoint.proto @@ -22,7 +22,6 @@ message Checkpoint { required TensorProto tensor = 2; } repeated Value value = 1; - required string model_type = 2; } diff --git a/mindspore/nn/cell.py b/mindspore/nn/cell.py index 0533546400..cffe00a920 100755 --- a/mindspore/nn/cell.py +++ b/mindspore/nn/cell.py @@ -81,6 +81,7 @@ class Cell: self.enable_hook = False self._bprop_debug = False self._is_run = False + self.cell_type = None @property def is_run(self): @@ -140,6 +141,14 @@ class Cell: for cell_name, cell in cells_name: cell._param_prefix = cell_name + def update_cell_type(self, cell_type): + """ + Update current cell type mainly identify if quantization aware training network. + + After invoked, can set the cell type to 'cell_type'. + """ + self.cell_type = cell_type + @cell_init_args.setter def cell_init_args(self, value): if not isinstance(value, str): diff --git a/mindspore/nn/layer/conv.py b/mindspore/nn/layer/conv.py index b2a0de9cbe..52ec9f2d63 100644 --- a/mindspore/nn/layer/conv.py +++ b/mindspore/nn/layer/conv.py @@ -17,11 +17,12 @@ from mindspore import log as logger from mindspore.ops import operations as P from mindspore.common.parameter import Parameter from mindspore.common.initializer import initializer +from mindspore._checkparam import ParamValidator as validator, Rel from mindspore._checkparam import check_bool, twice, check_int_positive, check_int_non_negative from mindspore._extends import cell_attr_register from ..cell import Cell -__all__ = ['Conv2d', 'Conv2dTranspose'] +__all__ = ['Conv2d', 'Conv2dTranspose', 'DepthwiseConv2d'] class _Conv(Cell): """ @@ -397,3 +398,150 @@ class Conv2dTranspose(_Conv): self.weight, self.bias) return s + + +class DepthwiseConv2d(Cell): + r""" + 2D depthwise convolution layer. + + Applies a 2D depthwise convolution over an input tensor which is typically of shape: + math:`(N, C_{in}, H_{in}, W_{in})`, where :math:`N` is batch size and :math:`C_{in}` is channel number. + For each batch of shape:math:`(C_{in}, H_{in}, W_{in})`, the formula is defined as: + + .. math:: + + out_j = \sum_{i=0}^{C_{in} - 1} ccor(W_{ij}, X_i) + b_j, + + where :math:`ccor` is cross correlation operator, :math:`C_{in}` is the input channel number, :math:`j` ranges + from :math:`0` to :math:`C_{out} - 1`, :math:`W_{ij}` corresponds to :math:`i`-th channel of the :math:`j`-th + filter and :math:`out_{j}` corresponds to the :math:`j`-th channel of the output. :math:`W_{ij}` is a slice + of kernel and it has shape :math:`(\text{ks_h}, \text{ks_w})`, where :math:`\text{ks_h}` and + :math:`\text{ks_w}` are height and width of the convolution kernel. The full kernel has shape + :math:`(C_{out}, C_{in} // \text{group}, \text{ks_h}, \text{ks_w})`, where group is the group number + to split the input in the channel dimension. + + If the 'pad_mode' is set to be "valid", the output height and width will be + :math:`\left \lfloor{1 + \frac{H_{in} + 2 \times \text{padding} - \text{ks_h} - + (\text{ks_h} - 1) \times (\text{dilation} - 1) }{\text{stride}}} \right \rfloor` and + :math:`\left \lfloor{1 + \frac{W_{in} + 2 \times \text{padding} - \text{ks_w} - + (\text{ks_w} - 1) \times (\text{dilation} - 1) }{\text{stride}}} \right \rfloor` respectively. + + The first introduction can be found in paper `Gradient Based Learning Applied to Document Recognition + `_. + + Args: + in_channels (int): The number of input channel :math:`C_{in}`. + out_channels (int): The number of output channel :math:`C_{out}`. + kernel_size (Union[int, tuple[int]]): The data type is int or tuple with 2 integers. Specifies the height + and width of the 2D convolution window. Single int means the value if for both height and width of + the kernel. A tuple of 2 ints means the first value is for the height and the other is for the + width of the kernel. + stride (Union[int, tuple[int]]): The distance of kernel moving, an int number that represents + the height and width of movement are both strides, or a tuple of two int numbers that + represent height and width of movement respectively. Default: 1. + pad_mode (str): Specifies padding mode. The optional values are + "same", "valid", "pad". Default: "same". + + - same: Adopts the way of completion. Output height and width will be the same as the input. + Total number of padding will be calculated for horizontal and vertical + direction and evenly distributed to top and bottom, left and right if possible. Otherwise, the + last extra padding will be done from the bottom and the right side. If this mode is set, `padding` + must be 0. + + - valid: Adopts the way of discarding. The possibly largest height and width of output will be return + without padding. Extra pixels will be discarded. If this mode is set, `padding` + must be 0. + + - pad: Implicit paddings on both sides of the input. The number of `padding` will be padded to the input + Tensor borders. `padding` should be greater than or equal to 0. + + padding (int): Implicit paddings on both sides of the input. Default: 0. + dilation (Union[int, tuple[int]]): The data type is int or tuple with 2 integers. Specifies the dilation rate + to use for dilated convolution. If set to be :math:`k > 1`, there will + be :math:`k - 1` pixels skipped for each sampling location. Its value should + be greater or equal to 1 and bounded by the height and width of the + input. Default: 1. + group (int): Split filter into groups, `in_ channels` and `out_channels` should be + divisible by the number of groups. Default: 1. + has_bias (bool): Specifies whether the layer uses a bias vector. Default: False. + weight_init (Union[Tensor, str, Initializer, numbers.Number]): Initializer for the convolution kernel. + It can be a Tensor, a string, an Initializer or a numbers.Number. When a string is specified, + values from 'TruncatedNormal', 'Normal', 'Uniform', 'HeUniform' and 'XavierUniform' distributions as well + as constant 'One' and 'Zero' distributions are possible. Alias 'xavier_uniform', 'he_uniform', 'ones' + and 'zeros' are acceptable. Uppercase and lowercase are both acceptable. Refer to the values of + Initializer for more details. Default: 'normal'. + bias_init (Union[Tensor, str, Initializer, numbers.Number]): Initializer for the bias vector. Possible + Initializer and string are the same as 'weight_init'. Refer to the values of + Initializer for more details. Default: 'zeros'. + + Inputs: + - **input** (Tensor) - Tensor of shape :math:`(N, C_{in}, H_{in}, W_{in})`. + + Outputs: + Tensor of shape :math:`(N, C_{out}, H_{out}, W_{out})`. + + Examples: + >>> net = nn.DepthwiseConv2d(120, 240, 4, has_bias=False, weight_init='normal') + >>> input = Tensor(np.ones([1, 120, 1024, 640]), mindspore.float32) + >>> net(input).shape + (1, 240, 1024, 640) + """ + def __init__(self, + in_channels, + out_channels, + kernel_size, + stride=1, + pad_mode='same', + padding=0, + dilation=1, + group=1, + has_bias=False, + weight_init='normal', + bias_init='zeros'): + super(DepthwiseConv2d, self).__init__() + self.kernel_size = twice(kernel_size) + self.stride = twice(stride) + self.dilation = twice(dilation) + self.in_channels = check_int_positive(in_channels) + self.out_channels = check_int_positive(out_channels) + validator.check_integer('group', group, in_channels, Rel.EQ) + validator.check_integer('group', group, out_channels, Rel.EQ) + validator.check_integer('group', group, 1, Rel.GE) + self.pad_mode = pad_mode + self.padding = padding + self.dilation = dilation + self.group = group + self.has_bias = has_bias + self.conv = P.DepthwiseConv2dNative(channel_multiplier=1, + kernel_size=self.kernel_size, + pad_mode=self.pad_mode, + pad=self.padding, + stride=self.stride, + dilation=self.dilation) + self.bias_add = P.BiasAdd() + weight_shape = [1, in_channels, *self.kernel_size] + self.weight = Parameter(initializer(weight_init, weight_shape), name='weight') + if check_bool(has_bias): + self.bias = Parameter(initializer(bias_init, [out_channels]), name='bias') + else: + if bias_init != 'zeros': + logger.warning("value of `has_bias` is False, value of `bias_init` will be ignore.") + self.bias = None + + def construct(self, x): + out = self.conv(x, self.weight) + if self.has_bias: + out = self.bias_add(out, self.bias) + return out + + def extend_repr(self): + s = 'input_channels={}, output_channels={}, kernel_size={}, stride={}, ' \ + 'pad_mode={}, padding={}, dilation={}, group={},' \ + 'has_bias={}, weight_init={}, bias_init={}'.format( + self.in_channels, self.out_channels, self.kernel_size, self.stride, + self.pad_mode, self.padding, self.dilation, self.group, + self.has_bias, self.weight_init, self.bias_init) + + if self.has_bias: + s += ', bias={}'.format(self.bias) + return s diff --git a/mindspore/nn/layer/quant.py b/mindspore/nn/layer/quant.py index 14731c6262..225d37bf84 100644 --- a/mindspore/nn/layer/quant.py +++ b/mindspore/nn/layer/quant.py @@ -16,6 +16,7 @@ from functools import partial import numpy as np + import mindspore.common.dtype as mstype from mindspore.ops import operations as P from mindspore.ops import functional as F @@ -23,10 +24,9 @@ from mindspore.common.parameter import Parameter from mindspore.common.initializer import initializer from mindspore.common.tensor import Tensor from mindspore._checkparam import check_int_positive, check_bool, twice -from mindspore._checkparam import Validator as validator, Rel -from mindspore.nn.cell import Cell -from mindspore.nn.layer.activation import get_activation +from mindspore._checkparam import Rel import mindspore.context as context + from .normalization import BatchNorm2d from .activation import get_activation from ..cell import Cell @@ -82,7 +82,7 @@ class Conv2dBnAct(Cell): bias_init (Union[Tensor, str, Initializer, numbers.Number]): Initializer for the bias vector. Possible Initializer and string are the same as 'weight_init'. Refer to the values of Initializer for more details. Default: 'zeros'. - batchnorm (bool): Specifies to used batchnorm or not. Default: None. + has_bn (bool): Specifies to used batchnorm or not. Default: False. activation (string): Specifies activation type. The optional values are as following: 'softmax', 'logsoftmax', 'relu', 'relu6', 'tanh', 'gelu', 'sigmoid', 'prelu', 'leakyrelu', 'hswish', 'hsigmoid'. Default: None. @@ -94,7 +94,7 @@ class Conv2dBnAct(Cell): Tensor of shape :math:`(N, C_{out}, H_{out}, W_{out})`. Examples: - >>> net = Conv2dBnAct(120, 240, 4, batchnorm=True, activation='ReLU') + >>> net = Conv2dBnAct(120, 240, 4, has_bn=True, activation='ReLU') >>> input = Tensor(np.ones([1, 120, 1024, 640]), mindspore.float32) >>> net(input).shape (1, 240, 1024, 640) @@ -112,28 +112,39 @@ class Conv2dBnAct(Cell): has_bias=False, weight_init='normal', bias_init='zeros', - batchnorm=None, + has_bn=False, activation=None): super(Conv2dBnAct, self).__init__() - self.conv = conv.Conv2d( - in_channels, - out_channels, - kernel_size, - stride, - pad_mode, - padding, - dilation, - group, - has_bias, - weight_init, - bias_init) - self.has_bn = batchnorm is not None + + if context.get_context('device_target') == "Ascend" and group > 1: + self.conv = conv.DepthwiseConv2d(in_channels, + out_channels, + kernel_size=kernel_size, + stride=stride, + pad_mode=pad_mode, + padding=padding, + dilation=dilation, + group=group, + has_bias=has_bias, + weight_init=weight_init, + bias_init=bias_init) + else: + self.conv = conv.Conv2d(in_channels, + out_channels, + kernel_size=kernel_size, + stride=stride, + pad_mode=pad_mode, + padding=padding, + dilation=dilation, + group=group, + has_bias=has_bias, + weight_init=weight_init, + bias_init=bias_init) + + self.has_bn = validator.check_bool("has_bn", has_bn) self.has_act = activation is not None - self.batchnorm = batchnorm - if batchnorm is True: + if has_bn: self.batchnorm = BatchNorm2d(out_channels) - elif batchnorm is not None: - validator.check_isinstance('batchnorm', batchnorm, (BatchNorm2d,)) self.activation = get_activation(activation) def construct(self, x): @@ -160,7 +171,7 @@ class DenseBnAct(Cell): same as input x. The values of str refer to the function `initializer`. Default: 'zeros'. has_bias (bool): Specifies whether the layer uses a bias vector. Default: True. activation (str): Regularizer function applied to the output of the layer, eg. 'relu'. Default: None. - batchnorm (bool): Specifies to used batchnorm or not. Default: None. + has_bn (bool): Specifies to used batchnorm or not. Default: False. activation (string): Specifies activation type. The optional values are as following: 'softmax', 'logsoftmax', 'relu', 'relu6', 'tanh', 'gelu', 'sigmoid', 'prelu', 'leakyrelu', 'hswish', 'hsigmoid'. Default: None. @@ -183,7 +194,7 @@ class DenseBnAct(Cell): weight_init='normal', bias_init='zeros', has_bias=True, - batchnorm=None, + has_bn=False, activation=None): super(DenseBnAct, self).__init__() self.dense = basic.Dense( @@ -192,12 +203,10 @@ class DenseBnAct(Cell): weight_init, bias_init, has_bias) - self.has_bn = batchnorm is not None + self.has_bn = validator.check_bool("has_bn", has_bn) self.has_act = activation is not None - if batchnorm is True: + if has_bn: self.batchnorm = BatchNorm2d(out_channels) - elif batchnorm is not None: - validator.check_isinstance('batchnorm', batchnorm, (BatchNorm2d,)) self.activation = get_activation(activation) def construct(self, x): @@ -312,6 +321,10 @@ class FakeQuantWithMinMax(Cell): quant_delay=0): """init FakeQuantWithMinMax layer""" super(FakeQuantWithMinMax, self).__init__() + validator.check_type("min_init", min_init, [int, float]) + validator.check_type("max_init", max_init, [int, float]) + validator.check("min_init", min_init, "max_init", max_init, rel=Rel.LT) + validator.check_integer('quant_delay', quant_delay, 0, Rel.GE) self.min_init = min_init self.max_init = max_init self.num_bits = num_bits @@ -1183,12 +1196,13 @@ class QuantBlock(Cell): self.has_bias = bias is None self.activation = activation self.has_act = activation is None + self.bias_add = P.BiasAdd() def construct(self, x): x = self.quant(x) x = self.core_op(x, self.weight) if self.has_bias: - output = self.bias_add(output, self.bias) + x = self.bias_add(x, self.bias) if self.has_act: x = self.activation(x) x = self.dequant(x, self.dequant_scale) diff --git a/mindspore/ops/_op_impl/_custom_op/batchnorm_fold2.py b/mindspore/ops/_op_impl/_custom_op/batchnorm_fold2.py index 7e98517057..9daab5a75f 100644 --- a/mindspore/ops/_op_impl/_custom_op/batchnorm_fold2.py +++ b/mindspore/ops/_op_impl/_custom_op/batchnorm_fold2.py @@ -30,7 +30,6 @@ batchnorm_fold2_op_info = TBERegOp("BatchNormFold2_D") \ .compute_cost(10) \ .kernel_name("batchnorm_fold2") \ .partial_flag(True) \ - .op_pattern("formatAgnostic") \ .input(0, "x", None, "required", None) \ .input(1, "beta", None, "required", None) \ .input(2, "gamma", None, "required", None) \ diff --git a/mindspore/ops/_op_impl/_custom_op/batchnorm_fold2_grad.py b/mindspore/ops/_op_impl/_custom_op/batchnorm_fold2_grad.py index 824da62d19..9994a88f30 100644 --- a/mindspore/ops/_op_impl/_custom_op/batchnorm_fold2_grad.py +++ b/mindspore/ops/_op_impl/_custom_op/batchnorm_fold2_grad.py @@ -30,7 +30,6 @@ batchnorm_fold2_grad_op_info = TBERegOp("BatchNormFold2GradD") \ .compute_cost(10) \ .kernel_name("batchnorm_fold2_grad") \ .partial_flag(True) \ - .op_pattern("formatAgnostic") \ .input(0, "dout", None, "required", None) \ .input(1, "dout_reduce", None, "required", None) \ .input(2, "dout_x_reduce", None, "required", None) \ diff --git a/mindspore/ops/_op_impl/_custom_op/batchnorm_fold2_grad_reduce.py b/mindspore/ops/_op_impl/_custom_op/batchnorm_fold2_grad_reduce.py index 7806c6834e..92b91ff712 100644 --- a/mindspore/ops/_op_impl/_custom_op/batchnorm_fold2_grad_reduce.py +++ b/mindspore/ops/_op_impl/_custom_op/batchnorm_fold2_grad_reduce.py @@ -31,7 +31,6 @@ batchnorm_fold2_grad_reduce_op_info = TBERegOp("BatchNormFold2GradReduce") \ .compute_cost(10) \ .kernel_name("batchnorm_fold2_grad_reduce") \ .partial_flag(True) \ - .op_pattern("formatAgnostic") \ .input(0, "dout", None, "required", None) \ .input(1, "x", None, "required", None) \ .output(0, "dout_reduce", True, "required", "all") \ diff --git a/mindspore/ops/_op_impl/_custom_op/correction_mul.py b/mindspore/ops/_op_impl/_custom_op/correction_mul.py index ce92d2bbc5..49cd35cc11 100644 --- a/mindspore/ops/_op_impl/_custom_op/correction_mul.py +++ b/mindspore/ops/_op_impl/_custom_op/correction_mul.py @@ -30,7 +30,6 @@ correction_mul_op_info = TBERegOp("CorrectionMul") \ .compute_cost(10) \ .kernel_name("correction_mul") \ .partial_flag(True) \ - .op_pattern("formatAgnostic") \ .attr("channel_axis", "optional", "int", "all") \ .input(0, "x", None, "required", None) \ .input(1, "batch_std", None, "required", None) \ diff --git a/mindspore/ops/_op_impl/_custom_op/correction_mul_grad.py b/mindspore/ops/_op_impl/_custom_op/correction_mul_grad.py index da3a634454..6c11ce6855 100644 --- a/mindspore/ops/_op_impl/_custom_op/correction_mul_grad.py +++ b/mindspore/ops/_op_impl/_custom_op/correction_mul_grad.py @@ -30,7 +30,6 @@ correction_mul_grad_op_info = TBERegOp("CorrectionMulGrad") \ .compute_cost(10) \ .kernel_name("correction_mul_grad") \ .partial_flag(True) \ - .op_pattern("formatAgnostic") \ .attr("channel_axis", "optional", "int", "all") \ .input(0, "dout", None, "required", None) \ .input(1, "x", None, "required", None) \ @@ -128,7 +127,6 @@ correction_mul_grad_reduce_op_info = TBERegOp("CorrectionMulGradReduce") \ .compute_cost(10) \ .kernel_name("correction_mul_grad_reduce") \ .partial_flag(True) \ - .op_pattern("formatAgnostic") \ .attr("channel_axis", "optional", "int", "all") \ .input(0, "dout", None, "required", None) \ .output(0, "d_batch_std", True, "required", "all") \ diff --git a/mindspore/ops/_op_impl/_custom_op/fake_quant_perchannel.py b/mindspore/ops/_op_impl/_custom_op/fake_quant_perchannel.py index f6c133c808..dae2d7058d 100644 --- a/mindspore/ops/_op_impl/_custom_op/fake_quant_perchannel.py +++ b/mindspore/ops/_op_impl/_custom_op/fake_quant_perchannel.py @@ -99,11 +99,15 @@ def fake_quant_perchannel(x, min_val, max_val, y, min_dtype = min_val.get("dtype") max_shape = max_val.get("ori_shape") max_dtype = max_val.get("dtype") - + # for Dense weight quant, 2d[co,ci] -> 4d[1,co,ci,1], channel_axis_ need change to 1. + if channel_axis == 0 and x_shape_[0] != min_shape[0] and x_shape_[1] == min_shape[0]: + channel_axis_ = 1 + else: + channel_axis_ = channel_axis util.check_kernel_name(kernel_name) util.check_shape_rule(x_shape) - util.check_shape_rule(min_shape, 1, 1, x_shape_[channel_axis]) - util.check_shape_rule(max_shape, 1, 1, x_shape_[channel_axis]) + util.check_shape_rule(min_shape, 1, 1, x_shape_[channel_axis_]) + util.check_shape_rule(max_shape, 1, 1, x_shape_[channel_axis_]) util.check_tensor_shape_size(x_shape) util.check_tensor_shape_size(min_shape) util.check_tensor_shape_size(max_shape) @@ -126,8 +130,8 @@ def fake_quant_perchannel(x, min_val, max_val, y, quant_min = quant_min + 1 shape_c = [1] * len(x_shape) - shape_c[channel_axis] = min_val.get("ori_shape")[0] - if x_format == "NC1HWC0" and channel_axis == 1: + shape_c[channel_axis_] = min_val.get("ori_shape")[0] + if x_format == "NC1HWC0" and channel_axis_ == 1: shape_c = min_val.get("shape") input_data = tvm.placeholder(x_shape, name="x", dtype=x_dtype) min_data = tvm.placeholder(shape_c, name="min_val", dtype=x_dtype) diff --git a/mindspore/ops/_op_impl/_custom_op/fake_quant_perchannel_grad.py b/mindspore/ops/_op_impl/_custom_op/fake_quant_perchannel_grad.py index 4e9053fcb1..795aab52a3 100644 --- a/mindspore/ops/_op_impl/_custom_op/fake_quant_perchannel_grad.py +++ b/mindspore/ops/_op_impl/_custom_op/fake_quant_perchannel_grad.py @@ -124,11 +124,15 @@ def fake_quant_perchannel_grad(dout, x, min_val, max_val, dx, min_dtype = min_val.get("dtype") max_shape = max_val.get("ori_shape") max_dtype = max_val.get("dtype") - + # for Dense weight quant, 2d[co,ci] -> 4d[1,co,ci,1], channel_axis_ need change to 1. + if channel_axis == 0 and x_shape_[0] != min_shape[0] and x_shape_[1] == min_shape[0]: + channel_axis_ = 1 + else: + channel_axis_ = channel_axis util.check_kernel_name(kernel_name) util.check_shape_rule(x_shape) - util.check_shape_rule(min_shape, 1, 1, x_shape_[channel_axis]) - util.check_shape_rule(max_shape, 1, 1, x_shape_[channel_axis]) + util.check_shape_rule(min_shape, 1, 1, x_shape_[channel_axis_]) + util.check_shape_rule(max_shape, 1, 1, x_shape_[channel_axis_]) util.check_tensor_shape_size(x_shape) util.check_tensor_shape_size(min_shape) util.check_tensor_shape_size(max_shape) @@ -151,8 +155,8 @@ def fake_quant_perchannel_grad(dout, x, min_val, max_val, dx, quant_min = quant_min + 1 shape_c = [1] * len(x_shape) - shape_c[channel_axis] = min_val.get("ori_shape")[0] - if x_format == "NC1HWC0" and channel_axis == 1: + shape_c[channel_axis_] = min_val.get("ori_shape")[0] + if x_format == "NC1HWC0" and channel_axis_ == 1: shape_c = min_val.get("shape") dout_data = tvm.placeholder(x_shape, name="dout", dtype=x_dtype) input_data = tvm.placeholder(x_shape, name="x", dtype=x_dtype) diff --git a/mindspore/ops/_op_impl/_custom_op/minmax_update_perchannel.py b/mindspore/ops/_op_impl/_custom_op/minmax_update_perchannel.py index 1ff63464c3..f29fc53325 100644 --- a/mindspore/ops/_op_impl/_custom_op/minmax_update_perchannel.py +++ b/mindspore/ops/_op_impl/_custom_op/minmax_update_perchannel.py @@ -88,11 +88,15 @@ def minmax_update_perchannel(x, min_val, max_val, min_up, max_up, min_dtype = min_val.get("dtype") max_shape = max_val.get("ori_shape") max_dtype = max_val.get("dtype") - + # for Dense weight quant, 2d[co,ci] -> 4d[1,co,ci,1], channel_axis_ need change to 1. + if channel_axis == 0 and x_shape[0] != min_shape[0] and x_shape[1] == min_shape[0]: + channel_axis_ = 1 + else: + channel_axis_ = channel_axis util.check_kernel_name(kernel_name) util.check_shape_rule(x_shape) - util.check_shape_rule(min_shape, 1, 1, x_shape[channel_axis]) - util.check_shape_rule(max_shape, 1, 1, x_shape[channel_axis]) + util.check_shape_rule(min_shape, 1, 1, x_shape[channel_axis_]) + util.check_shape_rule(max_shape, 1, 1, x_shape[channel_axis_]) util.check_tensor_shape_size(x_shape) util.check_tensor_shape_size(min_shape) util.check_tensor_shape_size(max_shape) @@ -105,7 +109,7 @@ def minmax_update_perchannel(x, min_val, max_val, min_up, max_up, util.check_dtype_rule(min_dtype, check_list) util.check_dtype_rule(max_dtype, check_list) - if channel_axis == 0: + if channel_axis_ == 0: shape_c = min_val.get("ori_shape") else: shape_c = [min_val.get("shape")[1], min_val.get("shape")[-1]] @@ -113,7 +117,7 @@ def minmax_update_perchannel(x, min_val, max_val, min_up, max_up, min_data = tvm.placeholder(shape_c, name="min_val", dtype=x_dtype) max_data = tvm.placeholder(shape_c, name="max_val", dtype=x_dtype) res_list = minmax_update_perchannel_compute(input_data, min_data, max_data, - ema, ema_decay, channel_axis) + ema, ema_decay, channel_axis_) with tvm.target.cce(): sch = generic.auto_schedule(res_list) diff --git a/mindspore/ops/operations/_quant_ops.py b/mindspore/ops/operations/_quant_ops.py index 42c2406906..1f4de03d3c 100644 --- a/mindspore/ops/operations/_quant_ops.py +++ b/mindspore/ops/operations/_quant_ops.py @@ -106,7 +106,7 @@ class MinMaxUpdatePerChannel(PrimitiveWithInfer): Args: ema (bool): Use EMA algorithm update value min and max. Default: False. ema_decay (int) : EMA algorithm decay parameter. Default: 0.999. - channel_axis (int): Channel asis for per channel compute. Default: 1. + channel_axis (int): Quantization by channel axis. Ascend backend only supports 0 or 1. Default: 1. Inputs: - **x** (Tensor) : float32 Tensor representing the shape of the output tensor. @@ -123,11 +123,13 @@ class MinMaxUpdatePerChannel(PrimitiveWithInfer): >>> output_tensor = MinMaxUpdatePerChannel(num_bits=8)(x, min, max) """ support_quant_bit = [4, 7, 8] + ascend_support_x_rank = [2, 4] @prim_attr_register def __init__(self, ema=False, ema_decay=0.999, channel_axis=1): """init FakeQuantPerChannelUpdate OP for Ascend""" - if context.get_context('device_target') == "Ascend": + self.is_ascend = context.get_context('device_target') == "Ascend" + if self.is_ascend: from mindspore.ops._op_impl._custom_op import minmax_update_perchannel if ema and not ema_decay: raise ValueError( @@ -136,13 +138,18 @@ class MinMaxUpdatePerChannel(PrimitiveWithInfer): self.ema = validator.check_value_type('ema', ema, (bool,), self.name) self.ema_decay = validator.check_number_range( 'ema_decay', ema_decay, 0, 1, Rel.INC_BOTH, self.name) - self.channel_axis = validator.check_integer( - 'channel axis', channel_axis, 0, Rel.GE, self.name) + if self.is_ascend: + self.channel_axis = validator.check_int_range('channel_axis', channel_axis, 0, 1, Rel.INC_BOTH, self.name) + else: + self.channel_axis = validator.check_integer('channel_axis', channel_axis, 0, Rel.GE, self.name) self.init_prim_io_names( inputs=['x', 'min', 'max'], outputs=['min_up', 'max_up']) def infer_shape(self, x_shape, min_shape, max_shape): - validator.check_integer("x rank", len(x_shape), 1, Rel.GT, self.name) + if self.is_ascend and len(x_shape) not in self.ascend_support_x_rank: + raise ValueError(f"For '{self.name}' x rank should be in '{self.ascend_support_x_rank}'") + if not self.is_ascend: + validator.check_integer("x rank", len(x_shape), 1, Rel.GE, self.name) validator.check("min shape", min_shape, "max shape", max_shape, Rel.EQ, self.name) validator.check_integer("min shape", len( @@ -221,8 +228,8 @@ class FakeQuantPerLayer(PrimitiveWithInfer): 'ema_decay', ema_decay, 0, 1, Rel.INC_BOTH, self.name) self.num_bits = validator.check_integer( 'num_bits', num_bits, 0, Rel.GT, self.name) - self.quant_delay = validator.check_value_type( - 'quant_delay', quant_delay, (int,), self.name) + self.quant_delay = validator.check_integer( + 'quant_delay', quant_delay, 0, Rel.GE, self.name) self.init_prim_io_names(inputs=['x', 'min', 'max'], outputs=['out']) @@ -314,6 +321,7 @@ class FakeQuantPerChannel(PrimitiveWithInfer): symmetric (bool): Quantization algorithm use symmetric or not. Default: False. narrow_range (bool): Quantization algorithm use narrow range or not. Default: False. training (bool): Training the network or not. Default: True. + channel_axis (int): Quantization by channel axis. Ascend backend only supports 0 or 1. Default: 1. Inputs: - **x** (Tensor) : 4-D float32 Tensor representing the shape of the output tensor. @@ -331,6 +339,7 @@ class FakeQuantPerChannel(PrimitiveWithInfer): >>> result = fake_quant(input_x, _min, _max) """ support_quant_bit = [4, 7, 8] + ascend_support_x_rank = [2, 4] @prim_attr_register def __init__(self, @@ -343,7 +352,8 @@ class FakeQuantPerChannel(PrimitiveWithInfer): training=True, channel_axis=1): """init FakeQuantPerChannel OP""" - if context.get_context('device_target') == "Ascend": + self.is_ascend = context.get_context('device_target') == "Ascend" + if self.is_ascend: from mindspore.ops._op_impl._custom_op import fake_quant_perchannel if num_bits not in self.support_quant_bit: raise ValueError( @@ -363,14 +373,19 @@ class FakeQuantPerChannel(PrimitiveWithInfer): 'ema_decay', ema_decay, 0, 1, Rel.INC_BOTH, self.name) self.num_bits = validator.check_integer( 'num_bits', num_bits, 0, Rel.GT, self.name) - self.quant_delay = validator.check_value_type( - 'quant_delay', quant_delay, (int,), self.name) - self.channel_axis = validator.check_integer( - 'channel_axis', channel_axis, 0, Rel.GE, self.name) + self.quant_delay = validator.check_integer( + 'quant_delay', quant_delay, 0, Rel.GE, self.name) + if self.is_ascend: + self.channel_axis = validator.check_int_range('channel_axis', channel_axis, 0, 1, Rel.INC_BOTH, self.name) + else: + self.channel_axis = validator.check_integer('channel_axis', channel_axis, 0, Rel.GE, self.name) self.init_prim_io_names(inputs=['x', 'min', 'max'], outputs=['out']) def infer_shape(self, x_shape, min_shape, max_shape): - validator.check_integer("x rank", len(x_shape), 1, Rel.GE, self.name) + if self.is_ascend and len(x_shape) not in self.ascend_support_x_rank: + raise ValueError(f"For '{self.name}' x rank should be in '{self.ascend_support_x_rank}'") + if not self.is_ascend: + validator.check_integer("x rank", len(x_shape), 1, Rel.GE, self.name) validator.check("min shape", min_shape, "max shape", max_shape, Rel.EQ, self.name) validator.check_integer( "min shape", min_shape[0], x_shape[self.channel_axis], Rel.EQ, self.name) diff --git a/mindspore/train/callback/_checkpoint.py b/mindspore/train/callback/_checkpoint.py index 4e686c414f..e0048ad713 100644 --- a/mindspore/train/callback/_checkpoint.py +++ b/mindspore/train/callback/_checkpoint.py @@ -21,7 +21,7 @@ import time import mindspore.context as context from mindspore import log as logger -from mindspore._checkparam import check_bool, check_string, check_int_non_negative +from mindspore._checkparam import check_bool, check_int_non_negative from mindspore.train._utils import _make_directory from mindspore.train.serialization import _exec_save_checkpoint, _save_graph from ._callback import Callback, set_cur_net @@ -86,7 +86,6 @@ class CheckpointConfig: Can't be used with keep_checkpoint_max at the same time. integrated_save (bool): Whether to intergrated save in automatic model parallel scene. Default: True. Integrated save function is only supported in automatic parallel scene, not supported in manual parallel. - model_type (str): Model type in `normal`, `fusion` or `quant`. Default: "normal". Raises: ValueError: If the input_param is None or 0. @@ -101,8 +100,7 @@ class CheckpointConfig: save_checkpoint_seconds=0, keep_checkpoint_max=5, keep_checkpoint_per_n_minutes=0, - integrated_save=True, - model_type="normal"): + integrated_save=True): if not save_checkpoint_steps and not save_checkpoint_seconds and \ not keep_checkpoint_max and not keep_checkpoint_per_n_minutes: @@ -116,8 +114,6 @@ class CheckpointConfig: keep_checkpoint_max = check_int_non_negative(keep_checkpoint_max) if keep_checkpoint_per_n_minutes: keep_checkpoint_per_n_minutes = check_int_non_negative(keep_checkpoint_per_n_minutes) - if model_type: - model_type = check_string(model_type, ["normal", "fusion", "quant"]) self._save_checkpoint_steps = save_checkpoint_steps self._save_checkpoint_seconds = save_checkpoint_seconds @@ -132,7 +128,6 @@ class CheckpointConfig: if not self._keep_checkpoint_per_n_minutes or self._keep_checkpoint_per_n_minutes == 0: self._keep_checkpoint_max = 1 - self._model_type = model_type self._integrated_save = check_bool(integrated_save) @property @@ -160,18 +155,12 @@ class CheckpointConfig: """Get the value of _integrated_save.""" return self._integrated_save - @property - def model_type(self): - """Get the value of model_type.""" - return self._model_type - def get_checkpoint_policy(self): """Get the policy of checkpoint.""" checkpoint_policy = {'save_checkpoint_steps': self._save_checkpoint_steps, 'save_checkpoint_seconds': self._save_checkpoint_seconds, 'keep_checkpoint_max': self._keep_checkpoint_max, - 'keep_checkpoint_per_n_minutes': self._keep_checkpoint_per_n_minutes, - 'model_type': self._model_type} + 'keep_checkpoint_per_n_minutes': self._keep_checkpoint_per_n_minutes} return checkpoint_policy @@ -236,7 +225,7 @@ class ModelCheckpoint(Callback): graph_file_name = os.path.join(self._directory, self._prefix + '-graph.meta') _save_graph(cb_params.train_network, graph_file_name) self._graph_saved = True - self._save_ckpt(cb_params, self._config.model_type) + self._save_ckpt(cb_params) def end(self, run_context): """ @@ -247,7 +236,7 @@ class ModelCheckpoint(Callback): """ cb_params = run_context.original_args() _to_save_last_ckpt = True - self._save_ckpt(cb_params, self._config.model_type, _to_save_last_ckpt) + self._save_ckpt(cb_params, _to_save_last_ckpt) from mindspore.parallel._cell_wrapper import destroy_allgather_cell destroy_allgather_cell() @@ -266,7 +255,7 @@ class ModelCheckpoint(Callback): return False - def _save_ckpt(self, cb_params, model_type, force_to_save=False): + def _save_ckpt(self, cb_params, force_to_save=False): """Save checkpoint files.""" if cb_params.cur_step_num == self._last_triggered_step: return @@ -302,7 +291,7 @@ class ModelCheckpoint(Callback): set_cur_net(cb_params.train_network) cb_params.train_network.exec_checkpoint_graph() - _exec_save_checkpoint(cb_params.train_network, gen_file, model_type, self._config.integrated_save) + _exec_save_checkpoint(cb_params.train_network, gen_file, self._config.integrated_save) if os.path.exists(gen_file): shutil.move(gen_file, cur_file) diff --git a/mindspore/train/callback/_loss_monitor.py b/mindspore/train/callback/_loss_monitor.py index 3f93c6314d..bdd2220441 100644 --- a/mindspore/train/callback/_loss_monitor.py +++ b/mindspore/train/callback/_loss_monitor.py @@ -86,7 +86,7 @@ class LossMonitor(Callback): if self._per_print_times != 0 and cb_params.cur_step_num % self._per_print_times == 0: print("Epoch: [{:3d}/{:3d}], step: [{:5d}/{:5d}], " - "loss: [{:5.4f}/{:5.4f}], time: [{:5.4f}]".format( + "loss: [{:5.4f}], avg los: [{:5.4f}], time: [{:5.4f}]".format( cb_params.cur_epoch_num, cb_params.epoch_num, cur_step_in_epoch, int(cb_params.batch_num), step_loss, np.mean(self.losses), diff --git a/mindspore/train/quant/quant.py b/mindspore/train/quant/quant.py index cb4cb39e66..3709c171e5 100644 --- a/mindspore/train/quant/quant.py +++ b/mindspore/train/quant/quant.py @@ -42,15 +42,14 @@ _ACTIVATION_MAP = {nn.ReLU: quant.ReLUQuant, class _AddFakeQuantInput(nn.Cell): """ - Add FakeQuant at input and output of the Network. Only support one input and one output case. + Add FakeQuant OP at input of the network. Only support one input case. """ def __init__(self, network, quant_delay=0): super(_AddFakeQuantInput, self).__init__(auto_prefix=False) + self.fake_quant_input = quant.FakeQuantWithMinMax(min_init=-6, max_init=6, quant_delay=quant_delay, ema=True) + self.fake_quant_input.update_parameters_name('fake_quant_input.') self.network = network - self.fake_quant_input = quant.FakeQuantWithMinMax( - min_init=-6, max_init=6, quant_delay=quant_delay, ema=True) - self.fake_quant_input.update_parameters_name('fake_quant_input') def construct(self, data): data = self.fake_quant_input(data) @@ -60,7 +59,7 @@ class _AddFakeQuantInput(nn.Cell): class _AddFakeQuantAfterSubCell(nn.Cell): """ - Add FakeQuant after of the sub Cell. + Add FakeQuant OP after of the sub Cell. """ def __init__(self, subcell, **kwargs): @@ -115,11 +114,12 @@ class ConvertToQuantNetwork: self.network.update_cell_prefix() network = self._convert_subcells2quant(self.network) network = _AddFakeQuantInput(network) + self.network.update_cell_type("quant") return network def _convert_subcells2quant(self, network): """ - convet sub cell to quant cell + convert sub cell like `Conv2dBnAct` and `DenseBnAct` to quant cell """ cells = network.name_cells() change = False @@ -138,13 +138,13 @@ class ConvertToQuantNetwork: if isinstance(network, nn.SequentialCell) and change: network.cell_list = list(network.cells()) - # tensoradd to tensoradd quant + # add FakeQuant OP after OP in while list add_list = [] for name in network.__dict__: if name[0] == '_': continue attr = network.__dict__[name] - if isinstance(attr, ops.Primitive) and attr.name in ConvertToQuantNetwork.__quant_op_name__: + if isinstance(attr, ops.Primitive) and attr.name in self.__quant_op_name__: add_list.append((name, attr)) for name, prim_op in add_list: prefix = name @@ -164,11 +164,11 @@ class ConvertToQuantNetwork: def _convert_conv(self, subcell): """ - convet conv cell to quant cell + convert Conv2d cell to quant cell """ conv_inner = subcell.conv - bn_inner = subcell.batchnorm - if subcell.batchnorm is not None and self.bn_fold: + if subcell.has_bn and self.bn_fold: + bn_inner = subcell.batchnorm conv_inner = quant.Conv2dBatchNormQuant(conv_inner.in_channels, conv_inner.out_channels, kernel_size=conv_inner.kernel_size, @@ -178,7 +178,7 @@ class ConvertToQuantNetwork: dilation=conv_inner.dilation, group=conv_inner.group, eps=bn_inner.eps, - momentum=bn_inner.momentum, + momentum=1 - bn_inner.momentum, quant_delay=self.weight_qdelay, freeze_bn=self.freeze_bn, per_channel=self.weight_channel, @@ -186,6 +186,11 @@ class ConvertToQuantNetwork: fake=True, symmetric=self.weight_symmetric, narrow_range=self.weight_range) + # change original network BatchNormal OP parameters to quant network + conv_inner.gamma = subcell.batchnorm.gamma + conv_inner.beta = subcell.batchnorm.beta + conv_inner.moving_mean = subcell.batchnorm.moving_mean + conv_inner.moving_variance = subcell.batchnorm.moving_variance del subcell.batchnorm subcell.batchnorm = None subcell.has_bn = False @@ -204,6 +209,10 @@ class ConvertToQuantNetwork: num_bits=self.weight_bits, symmetric=self.weight_symmetric, narrow_range=self.weight_range) + # change original network Conv2D OP parameters to quant network + conv_inner.weight = subcell.conv.weight + if subcell.conv.has_bias: + conv_inner.bias = subcell.conv.bias subcell.conv = conv_inner if subcell.has_act and subcell.activation is not None: subcell.activation = self._convert_activation(subcell.activation) @@ -230,6 +239,10 @@ class ConvertToQuantNetwork: per_channel=self.weight_channel, symmetric=self.weight_symmetric, narrow_range=self.weight_range) + # change original network Dense OP parameters to quant network + dense_inner.weight = subcell.dense.weight + if subcell.dense.has_bias: + dense_inner.bias = subcell.dense.bias subcell.dense = dense_inner if subcell.has_act and subcell.activation is not None: subcell.activation = self._convert_activation(subcell.activation) @@ -247,12 +260,12 @@ class ConvertToQuantNetwork: act_class = activation.__class__ if act_class not in _ACTIVATION_MAP: raise ValueError( - "Unsupported activation in auto Quant: ", act_class) + "Unsupported activation in auto quant: ", act_class) return _ACTIVATION_MAP[act_class](num_bits=self.act_bits, quant_delay=self.act_qdelay, per_channel=self.act_channel, - symmetric=self.weight_symmetric, - narrow_range=self.weight_range) + symmetric=self.act_symmetric, + narrow_range=self.act_range) class ExportQuantNetworkDeploy: diff --git a/mindspore/train/serialization.py b/mindspore/train/serialization.py index ff1b8c3122..fc135b18a9 100644 --- a/mindspore/train/serialization.py +++ b/mindspore/train/serialization.py @@ -40,8 +40,6 @@ tensor_to_np_type = {"Int8": np.int8, "Uint8": np.uint8, "Int16": np.int16, "Uin "Int32": np.int32, "Uint32": np.uint32, "Int64": np.int64, "Uint64": np.uint64, "Float16": np.float16, "Float32": np.float32, "Float64": np.float64, "Bool": np.bool_} -ModelType = ["normal", "fusion", "quant"] - def _special_process_par(par, new_par): """ @@ -103,7 +101,7 @@ def _update_param(param, new_param): param.set_parameter_data(type(param.data)(new_param.data)) -def save_checkpoint(parameter_list, ckpt_file_name, model_type="normal"): +def save_checkpoint(parameter_list, ckpt_file_name): """ Saves checkpoint info to a specified file. @@ -111,14 +109,12 @@ def save_checkpoint(parameter_list, ckpt_file_name, model_type="normal"): parameter_list (list): Parameters list, each element is a dict like {"name":xx, "type":xx, "shape":xx, "data":xx}. ckpt_file_name (str): Checkpoint file name. - model_type (str): The name of model type. Default: "normal". Raises: RuntimeError: Failed to save the Checkpoint file. """ logger.info("Execute save checkpoint process.") checkpoint_list = Checkpoint() - checkpoint_list.model_type = model_type try: for param in parameter_list: @@ -147,13 +143,12 @@ def save_checkpoint(parameter_list, ckpt_file_name, model_type="normal"): logger.info("Save checkpoint process finish.") -def load_checkpoint(ckpt_file_name, model_type="normal", net=None): +def load_checkpoint(ckpt_file_name, net=None): """ Loads checkpoint info from a specified file. Args: ckpt_file_name (str): Checkpoint file name. - model_type (str): The name of model type in `normal`, `fusion` or `quant`. Default: "normal". net (Cell): Cell network. Default: None Returns: @@ -165,9 +160,6 @@ def load_checkpoint(ckpt_file_name, model_type="normal", net=None): if not isinstance(ckpt_file_name, str): raise ValueError("The ckpt_file_name must be string.") - if model_type not in ModelType: - raise ValueError(f"The model_type is not in {ModelType}.") - if not os.path.exists(ckpt_file_name) or ckpt_file_name[-5:] != ".ckpt": raise ValueError("Please input the correct checkpoint file name.") @@ -186,10 +178,6 @@ def load_checkpoint(ckpt_file_name, model_type="normal", net=None): raise ValueError(e.__str__()) parameter_dict = {} - if checkpoint_list.model_type: - if model_type != checkpoint_list.model_type: - raise KeyError("Checkpoint file model type({}) is not equal to input model type({}).".format( - checkpoint_list.model_type, model_type)) try: for element in checkpoint_list.value: data = element.tensor.tensor_content @@ -314,14 +302,13 @@ def _save_graph(network, file_name): os.chmod(file_name, stat.S_IWUSR | stat.S_IRUSR) -def _exec_save_checkpoint(train_network, ckpt_file_name, model_type="normal", integrated_save=True): +def _exec_save_checkpoint(train_network, ckpt_file_name, integrated_save=True): """ Saves checkpoint for 'ms' backend. Args: train_network (Network): The train network for training. ckpt_file_name (str): The name of checkpoint file. - model_type (str): The name of model type in `normal`, `fusion` or `quant`. Default: "normal". integrated_save (bool): Whether to integrated save in automatic model parallel scene. """ @@ -346,7 +333,7 @@ def _exec_save_checkpoint(train_network, ckpt_file_name, model_type="normal", in each_param["data"] = param_data param_list.append(each_param) - save_checkpoint(param_list, ckpt_file_name, model_type) + save_checkpoint(param_list, ckpt_file_name) def _get_merged_param_data(net, param_name, param_data): diff --git a/model_zoo/lenet_quant/README.md b/model_zoo/lenet_quant/README.md index 2f949f6d76..2fd3e129a2 100644 --- a/model_zoo/lenet_quant/README.md +++ b/model_zoo/lenet_quant/README.md @@ -33,7 +33,7 @@ Then you will get the following display ```bash >>> Found existing installation: mindspore-ascend >>> Uninstalling mindspore-ascend: ->>> Successfully uninstalled mindspore-ascend. +>>> Successfully uninstalled mindspore-ascend. ``` ### Prepare Dataset @@ -186,7 +186,7 @@ model = Model(network, net_loss, net_opt, metrics={"Accuracy": Accuracy()}) ### train quantization aware model -Also, you can just run this command instread. +Also, you can just run this command instead. ```python python train_quant.py --data_path MNIST_Data --device_target Ascend --ckpt_path checkpoint_lenet.ckpt @@ -235,7 +235,7 @@ The top1 accuracy would display on shell. Here are some optional parameters: ```bash ---device_target {Ascend,GPU,CPU} +--device_target {Ascend,GPU} device where the code will be implemented (default: Ascend) --data_path DATA_PATH path where the dataset is saved diff --git a/model_zoo/lenet_quant/eval.py b/model_zoo/lenet_quant/eval.py index d94e77279f..c0293ae1f7 100644 --- a/model_zoo/lenet_quant/eval.py +++ b/model_zoo/lenet_quant/eval.py @@ -31,7 +31,7 @@ from src.lenet_fusion import LeNet5 as LeNet5Fusion parser = argparse.ArgumentParser(description='MindSpore MNIST Example') parser.add_argument('--device_target', type=str, default="Ascend", - choices=['Ascend', 'GPU', 'CPU'], + choices=['Ascend', 'GPU'], help='device where the code will be implemented (default: Ascend)') parser.add_argument('--data_path', type=str, default="./MNIST_Data", help='path where the dataset is saved') diff --git a/model_zoo/lenet_quant/eval_quant.py b/model_zoo/lenet_quant/eval_quant.py index 2c2477123f..bc9b62121d 100644 --- a/model_zoo/lenet_quant/eval_quant.py +++ b/model_zoo/lenet_quant/eval_quant.py @@ -32,7 +32,7 @@ from src.lenet_fusion import LeNet5 as LeNet5Fusion parser = argparse.ArgumentParser(description='MindSpore MNIST Example') parser.add_argument('--device_target', type=str, default="Ascend", - choices=['Ascend', 'GPU', 'CPU'], + choices=['Ascend', 'GPU'], help='device where the code will be implemented (default: Ascend)') parser.add_argument('--data_path', type=str, default="./MNIST_Data", help='path where the dataset is saved') @@ -61,7 +61,7 @@ if __name__ == "__main__": model = Model(network, net_loss, net_opt, metrics={"Accuracy": Accuracy()}) # load quantization aware network checkpoint - param_dict = load_checkpoint(args.ckpt_path, model_type="quant") + param_dict = load_checkpoint(args.ckpt_path) load_param_into_net(network, param_dict) print("============== Starting Testing ==============") diff --git a/model_zoo/lenet_quant/train.py b/model_zoo/lenet_quant/train.py index b6040776ef..a34b6d5ed6 100644 --- a/model_zoo/lenet_quant/train.py +++ b/model_zoo/lenet_quant/train.py @@ -31,7 +31,7 @@ from src.lenet_fusion import LeNet5 as LeNet5Fusion parser = argparse.ArgumentParser(description='MindSpore MNIST Example') parser.add_argument('--device_target', type=str, default="Ascend", - choices=['Ascend', 'GPU', 'CPU'], + choices=['Ascend', 'GPU'], help='device where the code will be implemented (default: Ascend)') parser.add_argument('--data_path', type=str, default="./MNIST_Data", help='path where the dataset is saved') @@ -56,8 +56,7 @@ if __name__ == "__main__": # call back and monitor time_cb = TimeMonitor(data_size=ds_train.get_dataset_size()) config_ckpt = CheckpointConfig(save_checkpoint_steps=cfg.epoch_size * step_size, - keep_checkpoint_max=cfg.keep_checkpoint_max, - model_type=network.type) + keep_checkpoint_max=cfg.keep_checkpoint_max) ckpt_callback = ModelCheckpoint(prefix="checkpoint_lenet", config=config_ckpt) # define model diff --git a/model_zoo/lenet_quant/train_quant.py b/model_zoo/lenet_quant/train_quant.py index eb1f783a7c..ba54e63d80 100644 --- a/model_zoo/lenet_quant/train_quant.py +++ b/model_zoo/lenet_quant/train_quant.py @@ -33,7 +33,7 @@ from src.lenet_fusion import LeNet5 as LeNet5Fusion parser = argparse.ArgumentParser(description='MindSpore MNIST Example') parser.add_argument('--device_target', type=str, default="Ascend", - choices=['Ascend', 'GPU', 'CPU'], + choices=['Ascend', 'GPU'], help='device where the code will be implemented (default: Ascend)') parser.add_argument('--data_path', type=str, default="./MNIST_Data", help='path where the dataset is saved') @@ -50,11 +50,13 @@ if __name__ == "__main__": # define fusion network network = LeNet5Fusion(cfg.num_classes) + + # convert fusion network to quantization aware network + network = quant.convert_quant_network(network, quant_delay=0, bn_fold=False, freeze_bn=10000) + # load quantization aware network checkpoint param_dict = load_checkpoint(args.ckpt_path, network.type) load_param_into_net(network, param_dict) - # convert fusion network to quantization aware network - network = quant.convert_quant_network(network, quant_delay=0, bn_fold=False, freeze_bn=10000) # define network loss net_loss = nn.SoftmaxCrossEntropyWithLogits(is_grad=False, sparse=True, reduction="mean") @@ -64,8 +66,7 @@ if __name__ == "__main__": # call back and monitor time_cb = TimeMonitor(data_size=ds_train.get_dataset_size()) config_ckpt = CheckpointConfig(save_checkpoint_steps=cfg.epoch_size * step_size, - keep_checkpoint_max=cfg.keep_checkpoint_max, - model_type="quant") + keep_checkpoint_max=cfg.keep_checkpoint_max) ckpt_callback = ModelCheckpoint(prefix="checkpoint_lenet", config=config_ckpt) # define model diff --git a/model_zoo/mobilenetv2/scripts/run_infer.sh b/model_zoo/mobilenetv2/scripts/run_infer.sh index e200e600bf..7385a221d4 100644 --- a/model_zoo/mobilenetv2/scripts/run_infer.sh +++ b/model_zoo/mobilenetv2/scripts/run_infer.sh @@ -40,7 +40,7 @@ export PYTHONPATH=${BASEPATH}:$PYTHONPATH export DEVICE_ID=0 export RANK_ID=0 export RANK_SIZE=1 -if [ -d "eval" ]; +if [ -d "../eval" ]; then rm -rf ../eval fi diff --git a/model_zoo/mobilenetv2/scripts/run_train.sh b/model_zoo/mobilenetv2/scripts/run_train.sh index fc013d474c..3414aa7528 100644 --- a/model_zoo/mobilenetv2/scripts/run_train.sh +++ b/model_zoo/mobilenetv2/scripts/run_train.sh @@ -62,7 +62,7 @@ run_gpu() BASEPATH=$(cd "`dirname $0`" || exit; pwd) export PYTHONPATH=${BASEPATH}:$PYTHONPATH - if [ -d "train" ]; + if [ -d "../train" ]; then rm -rf ../train fi diff --git a/model_zoo/mobilenetv3/scripts/run_infer.sh b/model_zoo/mobilenetv3/scripts/run_infer.sh index e200e600bf..7385a221d4 100644 --- a/model_zoo/mobilenetv3/scripts/run_infer.sh +++ b/model_zoo/mobilenetv3/scripts/run_infer.sh @@ -40,7 +40,7 @@ export PYTHONPATH=${BASEPATH}:$PYTHONPATH export DEVICE_ID=0 export RANK_ID=0 export RANK_SIZE=1 -if [ -d "eval" ]; +if [ -d "../eval" ]; then rm -rf ../eval fi diff --git a/model_zoo/mobilenetv3/scripts/run_train.sh b/model_zoo/mobilenetv3/scripts/run_train.sh index 78b79b235f..47dabffe01 100644 --- a/model_zoo/mobilenetv3/scripts/run_train.sh +++ b/model_zoo/mobilenetv3/scripts/run_train.sh @@ -60,7 +60,7 @@ run_gpu() BASEPATH=$(cd "`dirname $0`" || exit; pwd) export PYTHONPATH=${BASEPATH}:$PYTHONPATH - if [ -d "train" ]; + if [ -d "../train" ]; then rm -rf ../train fi diff --git a/tests/ut/python/train/quant/mobilenetv2_combined.py b/tests/ut/python/train/quant/mobilenetv2_combined.py index b0cbafb29a..51916192d8 100644 --- a/tests/ut/python/train/quant/mobilenetv2_combined.py +++ b/tests/ut/python/train/quant/mobilenetv2_combined.py @@ -31,7 +31,7 @@ def _conv_bn(in_channel, out_channel, kernel_size=ksize, stride=stride, - batchnorm=True)]) + has_bn=True)]) class InvertedResidual(nn.Cell): @@ -49,25 +49,25 @@ class InvertedResidual(nn.Cell): 3, stride, group=hidden_dim, - batchnorm=True, + has_bn=True, activation='relu6'), nn.Conv2dBnAct(hidden_dim, oup, 1, 1, - batchnorm=True) + has_bn=True) ]) else: self.conv = nn.SequentialCell([ nn.Conv2dBnAct(inp, hidden_dim, 1, 1, - batchnorm=True, + has_bn=True, activation='relu6'), nn.Conv2dBnAct(hidden_dim, hidden_dim, 3, stride, group=hidden_dim, - batchnorm=True, + has_bn=True, activation='relu6'), nn.Conv2dBnAct(hidden_dim, oup, 1, 1, - batchnorm=True) + has_bn=True) ]) self.add = P.TensorAdd() diff --git a/tests/ut/python/train/quant/test_quant.py b/tests/ut/python/train/quant/test_quant.py index 54563d86eb..1a21bc2c02 100644 --- a/tests/ut/python/train/quant/test_quant.py +++ b/tests/ut/python/train/quant/test_quant.py @@ -42,7 +42,7 @@ class LeNet5(nn.Cell): def __init__(self, num_class=10): super(LeNet5, self).__init__() self.num_class = num_class - self.conv1 = nn.Conv2dBnAct(1, 6, kernel_size=5, batchnorm=True, activation='relu6', pad_mode="valid") + self.conv1 = nn.Conv2dBnAct(1, 6, kernel_size=5, has_bn=True, activation='relu6', pad_mode="valid") self.conv2 = nn.Conv2dBnAct(6, 16, kernel_size=5, activation='relu', pad_mode="valid") self.fc1 = nn.DenseBnAct(16 * 5 * 5, 120, activation='relu') self.fc2 = nn.DenseBnAct(120, 84, activation='relu') From d6635bbbe2e15c18713649badfb5474a3778f9f2 Mon Sep 17 00:00:00 2001 From: panyifeng Date: Mon, 8 Jun 2020 19:49:56 +0800 Subject: [PATCH 170/254] Add IndexedSlices --- mindspore/_extends/parse/resources.py | 4 + mindspore/ccsrc/debug/dump_proto.cc | 4 + mindspore/ccsrc/ir/dtype.cc | 84 +++++ mindspore/ccsrc/ir/dtype.h | 53 +++- mindspore/ccsrc/ir/dtype/type.cc | 4 + mindspore/ccsrc/ir/dtype/type.h | 9 +- mindspore/ccsrc/ir/dtype/type_id.h | 2 + mindspore/ccsrc/ir/dtype_extends.cc | 58 ++++ .../operator/composite/multitype_funcgraph.cc | 50 ++- .../operator/composite/multitype_funcgraph.h | 1 + mindspore/ccsrc/operator/ops.cc | 7 + mindspore/ccsrc/operator/ops.h | 7 + mindspore/ccsrc/operator/prim_others.cc | 76 +++++ mindspore/ccsrc/optimizer/clean.cc | 3 +- mindspore/ccsrc/optimizer/irpass.cc | 6 + mindspore/ccsrc/optimizer/irpass.h | 3 + .../irpass/indexed_slices_eliminate.h | 75 +++++ mindspore/ccsrc/pipeline/action.cc | 3 + mindspore/ccsrc/pipeline/init.cc | 4 +- mindspore/ccsrc/pipeline/pass.cc | 1 + mindspore/ccsrc/pipeline/resource.cc | 293 +++++++++--------- .../static_analysis/abstract_value.cc | 97 +++++- .../pipeline/static_analysis/abstract_value.h | 79 +++-- .../pipeline/static_analysis/evaluator.h | 14 + .../static_analysis/param_validator.h | 1 + .../ccsrc/pipeline/static_analysis/prim.cc | 49 +++ .../ccsrc/pipeline/static_analysis/prim.h | 11 + .../static_analysis/static_analysis.cc | 4 + mindspore/ccsrc/pipeline/validator.cc | 4 +- mindspore/ccsrc/utils/context/ms_context.cc | 1 + mindspore/ccsrc/utils/context/ms_context.h | 4 + mindspore/common/__init__.py | 4 +- mindspore/common/parameter.py | 16 +- mindspore/common/tensor.py | 7 +- mindspore/context.py | 12 +- mindspore/ops/functional.py | 8 + mindspore/ops/operations/array_ops.py | 2 +- tests/ut/cpp/optimizer/lib_test.cc | 13 + .../gtest_input/optimizer/opt_test.py | 35 +++ tests/ut/python/ir/test_indexed_slices.py | 290 +++++++++++++++++ .../nn/optim/test_adam_with_tuple_grad.py | 2 +- 41 files changed, 1208 insertions(+), 192 deletions(-) create mode 100644 mindspore/ccsrc/optimizer/irpass/indexed_slices_eliminate.h create mode 100644 tests/ut/python/ir/test_indexed_slices.py diff --git a/mindspore/_extends/parse/resources.py b/mindspore/_extends/parse/resources.py index eb89c965df..3af574caf9 100644 --- a/mindspore/_extends/parse/resources.py +++ b/mindspore/_extends/parse/resources.py @@ -17,6 +17,7 @@ """Resources for ast tree parse.""" import ast import math +from mindspore import IndexedSlices from mindspore.ops.composite import multitype_ops from mindspore.ops import functional as F, composite as C from . import standard_method as M @@ -135,4 +136,7 @@ convert_object_map = { math.sin: NO_IMPLEMENT, math.cos: NO_IMPLEMENT, math.tan: NO_IMPLEMENT, + + # user defined + IndexedSlices: F.make_indexed_slices, } diff --git a/mindspore/ccsrc/debug/dump_proto.cc b/mindspore/ccsrc/debug/dump_proto.cc index ab2ce1322a..99440537c7 100644 --- a/mindspore/ccsrc/debug/dump_proto.cc +++ b/mindspore/ccsrc/debug/dump_proto.cc @@ -120,6 +120,10 @@ void ProtoExporter::SetNodeOutputType(const TypePtr &type, const BaseShapePtr &s type_proto->mutable_tensor_type()->mutable_shape()->add_dim()->set_size(elem); } } + } else if (type->isa()) { + // Do Nothing + } else if (type->isa()) { + // Do Nothing } else if (type->isa()) { TuplePtr tuple_type = dyn_cast(type); type_proto->set_data_type(irpb::DT_TUPLE); diff --git a/mindspore/ccsrc/ir/dtype.cc b/mindspore/ccsrc/ir/dtype.cc index 5e049c0623..71a78bdcf6 100644 --- a/mindspore/ccsrc/ir/dtype.cc +++ b/mindspore/ccsrc/ir/dtype.cc @@ -94,6 +94,48 @@ bool Slice::operator==(const Type &other) const { std::string Slice::DumpText() const { return ToString(); } +TypePtr UndeterminedType::DeepCopy() const { + MS_EXCEPTION_IF_NULL(element_type_); + if (IsGeneric()) { + return std::make_shared(); + } + return std::make_shared(element_type_->DeepCopy()); +} + +std::string UndeterminedType::ToReprString() const { + if (element_type_ == nullptr) { + return "Undetermined"; + } + return "Undetermined[" + element_type_->ToReprString() + "]"; +} + +std::string UndeterminedType::ToString() const { + if (element_type_ == nullptr) { + return "Undetermined"; + } + return "Undetermined[" + element_type_->ToString() + "]"; +} + +std::string UndeterminedType::DumpText() const { + if (element_type_ == nullptr) { + return "Undetermined"; + } + return "Undetermined[" + element_type_->DumpText() + "]"; +} + +bool UndeterminedType::operator==(const Type &other) const { + if (!IsSameObjectType(*this, other)) { + return false; + } + auto other_elem_type = static_cast(other).element_type_; + if (element_type_ == nullptr && other_elem_type == nullptr) { + return true; + } else if (element_type_ == nullptr || other_elem_type == nullptr) { + return false; + } + return *element_type_ == *other_elem_type; +} + TypePtr TensorType::DeepCopy() const { MS_EXCEPTION_IF_NULL(element_type_); if (IsGeneric()) { @@ -137,6 +179,48 @@ bool TensorType::operator==(const Type &other) const { return *element_type_ == *other_elem_type; } +TypePtr IndexedSlicesType::DeepCopy() const { + MS_EXCEPTION_IF_NULL(element_type_); + if (IsGeneric()) { + return std::make_shared(); + } + return std::make_shared(element_type_->DeepCopy()); +} + +std::string IndexedSlicesType::ToReprString() const { + if (element_type_ == nullptr) { + return "IndexedSlices"; + } + return "IndexedSlices[" + element_type_->ToReprString() + "]"; +} + +std::string IndexedSlicesType::ToString() const { + if (element_type_ == nullptr) { + return "IndexedSlices"; + } + return "IndexedSlices[" + element_type_->ToString() + "]"; +} + +std::string IndexedSlicesType::DumpText() const { + if (element_type_ == nullptr) { + return "IndexedSlices"; + } + return "IndexedSlices[" + element_type_->DumpText() + "]"; +} + +bool IndexedSlicesType::operator==(const Type &other) const { + if (!IsSameObjectType(*this, other)) { + return false; + } + auto other_elem_type = static_cast(other).element_type_; + if (element_type_ == nullptr && other_elem_type == nullptr) { + return true; + } else if (element_type_ == nullptr || other_elem_type == nullptr) { + return false; + } + return *element_type_ == *other_elem_type; +} + Function::Function() : Object(kObjectTypeFunction) { args_ = std::vector(); retval_ = nullptr; diff --git a/mindspore/ccsrc/ir/dtype.h b/mindspore/ccsrc/ir/dtype.h index 9659a27e36..f10c56e659 100644 --- a/mindspore/ccsrc/ir/dtype.h +++ b/mindspore/ccsrc/ir/dtype.h @@ -108,10 +108,34 @@ class Slice : public Object { }; using SlicePtr = std::shared_ptr; +class UndeterminedType : public Object { + public: + UndeterminedType() : Object(kObjectTypeUndeterminedType) {} + explicit UndeterminedType(const TypePtr &ele) + : Object(kObjectTypeUndeterminedType, kMetaTypeObject, false), element_type_(ele) {} + ~UndeterminedType() override = default; + MS_DECLARE_PARENT(UndeterminedType, Object) + + TypeId generic_type_id() const override { return kObjectTypeUndeterminedType; } + const TypePtr element() const { return element_type_; } + void set_element(const TypePtr &element_type) { element_type_ = element_type; } + + TypePtr DeepCopy() const override; + std::string ToString() const override; + std::string ToReprString() const override; + std::string DumpText() const override; + bool operator==(const Type &other) const override; + + protected: + TypePtr element_type_; +}; +using MetaTensorTypePtr = std::shared_ptr; + class TensorType : public Object { public: - TensorType() : Object(kObjectTypeTensorType) {} - explicit TensorType(const TypePtr &ele) : Object(kObjectTypeTensorType, false), element_type_(ele) {} + TensorType() : Object(kObjectTypeTensorType, kObjectTypeUndeterminedType) {} + explicit TensorType(const TypePtr &ele) + : Object(kObjectTypeTensorType, kObjectTypeUndeterminedType, false), element_type_(ele) {} ~TensorType() override = default; MS_DECLARE_PARENT(TensorType, Object) @@ -130,6 +154,29 @@ class TensorType : public Object { }; using TensorTypePtr = std::shared_ptr; +class IndexedSlicesType : public Object { + public: + IndexedSlicesType() : Object(kObjectTypeIndexedSlicesType, kObjectTypeUndeterminedType) {} + explicit IndexedSlicesType(const TypePtr &ele) + : Object(kObjectTypeIndexedSlicesType, kObjectTypeUndeterminedType, false), element_type_(ele) {} + ~IndexedSlicesType() override = default; + MS_DECLARE_PARENT(IndexedSlicesType, Object) + + TypeId generic_type_id() const override { return kObjectTypeIndexedSlicesType; } + const TypePtr element() const { return element_type_; } + void set_element(const TypePtr &element_type) { element_type_ = element_type; } + + TypePtr DeepCopy() const override; + std::string ToString() const override; + std::string ToReprString() const override; + std::string DumpText() const override; + bool operator==(const Type &other) const override; + + private: + TypePtr element_type_; +}; +using IndexedSlicesTypePtr = std::shared_ptr; + class Function : public Object { public: Function(); @@ -255,6 +302,8 @@ TypePtr StringToType(const std::string &type_name); // Judge whether x is predicate or is a subclass of predicate. bool IsIdentidityOrSubclass(TypePtr const &x, TypePtr const &base_type); +bool IsParentOrChildrenType(TypePtr const &x, TypePtr const &base_type); + // Whether t1 is identity or a subclass of t2. bool IsSubType(TypePtr const &t1, TypePtr const &t2 = nullptr); diff --git a/mindspore/ccsrc/ir/dtype/type.cc b/mindspore/ccsrc/ir/dtype/type.cc index 5395b59617..754876a366 100644 --- a/mindspore/ccsrc/ir/dtype/type.cc +++ b/mindspore/ccsrc/ir/dtype/type.cc @@ -115,6 +115,10 @@ const char *ObjectIdLabel(const TypeId &v) { return "kObjectTypeKeyword"; case kObjectTypeTensorType: return "kObjectTypeTensorType"; + case kObjectTypeIndexedSlicesType: + return "kObjectTypeIndexedSlicesType"; + case kObjectTypeUndeterminedType: + return "kObjectTypeUndeterminedType"; case kObjectTypeDictionary: return "kObjectTypeDictionary"; case kObjectTypeClass: diff --git a/mindspore/ccsrc/ir/dtype/type.h b/mindspore/ccsrc/ir/dtype/type.h index bfe39af43c..cba0d17fce 100644 --- a/mindspore/ccsrc/ir/dtype/type.h +++ b/mindspore/ccsrc/ir/dtype/type.h @@ -67,6 +67,7 @@ class Type : public Value { virtual bool equal(const TypePtr other) const { return *this == *other; } virtual TypeId object_type() const { return kTypeUnknown; } + virtual TypeId parent_type() const { return kTypeUnknown; } virtual TypeId number_type() const { return kTypeUnknown; } virtual TypePtr DeepCopy() const = 0; virtual TypePtr Clone() const { return DeepCopy(); } @@ -97,13 +98,16 @@ using TypePtrList = std::vector; // class Object : public Type { public: - Object() : Type(kMetaTypeObject), object_type_(kMetaTypeObject) {} + Object() : Type(kMetaTypeObject), object_type_(kMetaTypeObject), parent_type_(kMetaTypeObject) {} explicit Object(const TypeId object_type, bool is_generic = true) - : Type(kMetaTypeObject, is_generic), object_type_(object_type) {} + : Type(kMetaTypeObject, is_generic), object_type_(object_type), parent_type_(kMetaTypeObject) {} + explicit Object(const TypeId object_type, const TypeId parent_type, bool is_generic = true) + : Type(kMetaTypeObject, is_generic), object_type_(object_type), parent_type_(parent_type) {} ~Object() override = default; MS_DECLARE_PARENT(Object, Type) TypeId object_type() const override { return object_type_; } + TypeId parent_type() const override { return parent_type_; } TypeId type_id() const override { return object_type_; } TypeId generic_type_id() const override { return kMetaTypeObject; } bool equal(const TypePtr other) const override; @@ -114,6 +118,7 @@ class Object : public Type { private: const TypeId object_type_; + const TypeId parent_type_; }; std::ostream &operator<<(std::ostream &os, const TypePtrList &types); diff --git a/mindspore/ccsrc/ir/dtype/type_id.h b/mindspore/ccsrc/ir/dtype/type_id.h index 17862ad798..a711779e91 100644 --- a/mindspore/ccsrc/ir/dtype/type_id.h +++ b/mindspore/ccsrc/ir/dtype/type_id.h @@ -50,6 +50,8 @@ enum TypeId : int { kObjectTypeSlice, kObjectTypeKeyword, kObjectTypeTensorType, + kObjectTypeIndexedSlicesType, + kObjectTypeUndeterminedType, kObjectTypeClass, kObjectTypeDictionary, kObjectTypeFunction, diff --git a/mindspore/ccsrc/ir/dtype_extends.cc b/mindspore/ccsrc/ir/dtype_extends.cc index e7af812922..732872cb4f 100644 --- a/mindspore/ccsrc/ir/dtype_extends.cc +++ b/mindspore/ccsrc/ir/dtype_extends.cc @@ -192,6 +192,40 @@ TypePtr TensorStrToType(const std::string &type_name) { return type; } +TypePtr IndexedSlicesStrToType(const std::string &type_name) { + if (type_name == "IndexedSlices") { + return std::make_shared(); + } + auto start = type_name.find_first_of('[') + 1; + auto end = type_name.find_last_of(']'); + if (start >= type_name.size()) { + return nullptr; + } + auto element_str = type_name.substr(start, end - start); + auto element_type = StringToType(element_str); + if (element_type == nullptr) { + return nullptr; + } + return std::make_shared(element_type); +} + +TypePtr UndeterminedStrToType(const std::string &type_name) { + if (type_name == "Undetermined") { + return std::make_shared(); + } + auto start = type_name.find_first_of('[') + 1; + auto end = type_name.find_last_of(']'); + if (start >= type_name.size()) { + return nullptr; + } + auto element_str = type_name.substr(start, end - start); + auto element_type = StringToType(element_str); + if (element_type == nullptr) { + return nullptr; + } + return std::make_shared(element_type); +} + TypePtr ListStrToType(const std::string &type_name) { TypePtr type = nullptr; if (type_name == "List") { @@ -313,6 +347,10 @@ TypePtr StringToType(const std::string &type_name) { type = StringToNumberType(type_name, "Float"); } else if (type_name.compare(0, strlen("Tensor"), "Tensor") == 0) { type = TensorStrToType(type_name); + } else if (type_name.compare(0, strlen("Undetermined"), "Undetermined") == 0) { + type = UndeterminedStrToType(type_name); + } else if (type_name.compare(0, strlen("IndexedSlices"), "IndexedSlices") == 0) { + type = IndexedSlicesStrToType(type_name); } else if (type_name.compare(0, strlen("List"), "List") == 0) { type = ListStrToType(type_name); } else if (type_name.compare(0, strlen("Tuple"), "Tuple") == 0) { @@ -340,6 +378,20 @@ TypePtr StringToType(const std::string &type_name) { return type; } +bool IsParentOrChildrenType(TypePtr const &x, TypePtr const &base_type) { + if (x == nullptr || base_type == nullptr) { + MS_LOG(ERROR) << "Type is nullptr."; + return false; + } + if (base_type->type_id() == kTypeUnknown || x->type_id() == kTypeUnknown) { + return false; + } + if (base_type->type_id() == x->parent_type() || x->type_id() == base_type->parent_type()) { + return true; + } + return false; +} + bool IsIdentidityOrSubclass(TypePtr const &x, TypePtr const &base_type) { if (x == nullptr || base_type == nullptr) { MS_LOG(ERROR) << "Type is nullptr."; @@ -481,6 +533,10 @@ REGISTER_PYBIND_DEFINE( TensorType data(TypeIdToType(TypeId(static_cast(t[0].cast())))); return data; })); + (void)py::class_>(m_sub, "IndexedSlicesType") + .def(py::init()); + (void)py::class_>(m_sub, "UndeterminedType") + .def(py::init()); (void)py::class_>(m_sub, "Function") .def(py::init()) .def(py::init, TypePtr>(), py::arg("args"), py::arg("retval")); @@ -501,6 +557,8 @@ const TypePtr kTypeExternal = std::make_shared(); const TypePtr kTypeEnv = std::make_shared(); const TypePtr kTypeType = std::make_shared(); const TypePtr kTensorType = std::make_shared(); +const TypePtr kIndexedSlicesType = std::make_shared(); +const TypePtr kUndeterminedType = std::make_shared(); const TypePtr kString = std::make_shared(); const TypePtr kList = std::make_shared(); const TypePtr kTuple = std::make_shared(); diff --git a/mindspore/ccsrc/operator/composite/multitype_funcgraph.cc b/mindspore/ccsrc/operator/composite/multitype_funcgraph.cc index 88b3134508..de6526f642 100644 --- a/mindspore/ccsrc/operator/composite/multitype_funcgraph.cc +++ b/mindspore/ccsrc/operator/composite/multitype_funcgraph.cc @@ -93,15 +93,17 @@ static TypePtr UnwrapRef(const TypePtr &type) { } return type; } -FuncGraphPtr MultitypeFuncGraph::GenerateFromTypes(const TypePtrList &types) { - bool find_fn = false; - py::function py_fn; + +// Return Exact match if exists, else return non ambiguous sub class match +// Return py::none() if matching is ambiguous +const py::function MultitypeFuncGraph::SignMatch(const TypePtrList &types) { + // Exact match for (auto &item : fn_cache_py_) { TypePtrList sign = item.first; if (sign.size() != types.size()) { continue; } - bool match = true; + auto match = true; for (size_t i = 0; i < sign.size(); ++i) { if (!IsIdentidityOrSubclass(UnwrapRef(types[i]), sign[i])) { match = false; @@ -111,13 +113,45 @@ FuncGraphPtr MultitypeFuncGraph::GenerateFromTypes(const TypePtrList &types) { if (!match) { continue; } - find_fn = true; - py_fn = item.second; - break; + return item.second; } + // Try best match + py::function py_fn_subclass; + size_t subclass_match_cnt = 0; + for (auto &item : fn_cache_py_) { + TypePtrList sign = item.first; + if (sign.size() != types.size()) { + continue; + } + auto match = true; + for (size_t i = 0; i < sign.size(); ++i) { + if (!IsIdentidityOrSubclass(UnwrapRef(types[i]), sign[i]) && + !IsParentOrChildrenType(UnwrapRef(types[i]), sign[i])) { + match = false; + break; + } + } + if (!match) { + continue; + } + py_fn_subclass = item.second; + subclass_match_cnt++; + } + if (subclass_match_cnt > 1) { + MS_LOG(EXCEPTION) << "There are more than one prototypes for overload function match by subclass"; + } + if (subclass_match_cnt == 1) { + MS_LOG(DEBUG) << "Found one subclass match"; + return py_fn_subclass; + } + return py::none(); +} + +FuncGraphPtr MultitypeFuncGraph::GenerateFromTypes(const TypePtrList &types) { + auto py_fn = SignMatch(types); std::ostringstream buffer; buffer << types; - if (find_fn) { + if (py_fn != py::none()) { FuncGraphPtr func_graph = parse::ParsePythonCode(py_fn); if (func_graph == nullptr) { MS_LOG(EXCEPTION) << "Fail to parse overload function " << buffer.str(); diff --git a/mindspore/ccsrc/operator/composite/multitype_funcgraph.h b/mindspore/ccsrc/operator/composite/multitype_funcgraph.h index feb38f17ba..ababf21883 100644 --- a/mindspore/ccsrc/operator/composite/multitype_funcgraph.h +++ b/mindspore/ccsrc/operator/composite/multitype_funcgraph.h @@ -54,6 +54,7 @@ class MultitypeFuncGraph : public MetaFuncGraph { } private: + const py::function SignMatch(const TypePtrList &types); std::unordered_map fn_cache_; std::unordered_map fn_cache_py_; }; diff --git a/mindspore/ccsrc/operator/ops.cc b/mindspore/ccsrc/operator/ops.cc index 88001bf63f..b682847ed7 100755 --- a/mindspore/ccsrc/operator/ops.cc +++ b/mindspore/ccsrc/operator/ops.cc @@ -277,5 +277,12 @@ const PrimitivePtr kPrimImageSummary = std::make_shared("ImageSummary const PrimitivePtr kPrimTensorSummary = std::make_shared("TensorSummary"); const PrimitivePtr kPrimHistogramSummary = std::make_shared("HistogramSummary"); const PrimitivePtr kPrimDebug = std::make_shared("Debug"); + +// IndexedSlices +const PrimitivePtr kPrimMakeIndexedSlices = std::make_shared("MakeIndexedSlices"); +const PrimitivePtr kPrimIndexedSlicesGetValues = std::make_shared("IndexedSlicesGetValues"); +const PrimitivePtr kPrimIndexedSlicesGetIndices = std::make_shared("IndexedSlicesGetIndices"); +const PrimitivePtr kPrimIndexedSlicesGetDenseShape = std::make_shared("IndexedSlicesGetDenseShape"); +const PrimitivePtr kPrimIsIndexedSlices = std::make_shared("IsIndexedSlices"); } // namespace prim } // namespace mindspore diff --git a/mindspore/ccsrc/operator/ops.h b/mindspore/ccsrc/operator/ops.h index efa6683468..f778013896 100755 --- a/mindspore/ccsrc/operator/ops.h +++ b/mindspore/ccsrc/operator/ops.h @@ -287,6 +287,13 @@ extern const PrimitivePtr kPrimMirror; extern const PrimitivePtr kPrimVirtualDiv; extern const PrimitivePtr kPrimVirtualDataset; +// IndexedSlices +extern const PrimitivePtr kPrimMakeIndexedSlices; +extern const PrimitivePtr kPrimIndexedSlicesGetValues; +extern const PrimitivePtr kPrimIndexedSlicesGetIndices; +extern const PrimitivePtr kPrimIndexedSlicesGetDenseShape; +extern const PrimitivePtr kPrimIsIndexedSlices; + class DoSignaturePrimitive : public Primitive { public: explicit DoSignaturePrimitive(const std::string &name, const ValuePtr &function) diff --git a/mindspore/ccsrc/operator/prim_others.cc b/mindspore/ccsrc/operator/prim_others.cc index 9350e9aa3b..ff9ec712bb 100644 --- a/mindspore/ccsrc/operator/prim_others.cc +++ b/mindspore/ccsrc/operator/prim_others.cc @@ -24,6 +24,7 @@ #include "pipeline/static_analysis/prim.h" #include "pipeline/static_analysis/utils.h" #include "utils/symbolic.h" +#include "utils/context/ms_context.h" namespace mindspore { namespace abstract { @@ -173,6 +174,13 @@ AbstractBasePtr InferImplEnvGetItem(const AnalysisEnginePtr &, const PrimitivePt return std::make_shared(sparse_list); } + auto context = MsContext::GetInstance(); + MS_EXCEPTION_IF_NULL(context); + bool enable_sparse_flag = context->enable_sparse_flag(); + if (enable_sparse_flag && key->has_indexed_slices_grad() && dflt->isa()) { + auto dflt_tensor = dflt->cast(); + return std::make_shared(dflt_tensor->element()->Clone(), dflt_tensor->shape()->Clone()); + } if (!key->GetValueTrack()->isa()) { return dflt; } @@ -236,6 +244,7 @@ AbstractBasePtr InferImplMakeRef(const AnalysisEnginePtr &, const PrimitivePtr & } auto ret = std::make_shared(args_spec_list[0], args_spec_list[1], args_spec_list[2]); ret->set_sparse_grad(args_spec_list[2]->sparse_grad()); + ret->set_has_indexed_slices_grad(args_spec_list[2]->has_indexed_slices_grad()); return ret; } @@ -437,5 +446,72 @@ AbstractBasePtr InferImplControlDepend(const AnalysisEnginePtr &, const Primitiv } return std::make_shared(kAnyValue, kBool); } + +AbstractBasePtr InferImplMakeIndexedSlices(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list) { + // Inputs: two tensors and a tuple. + const std::string op_name = primitive->name(); + CheckArgsSize(op_name, args_spec_list, 3); + auto indices = CheckArg(op_name, args_spec_list, 0); + auto values = CheckArg(op_name, args_spec_list, 1); + auto dense_shape = CheckArg(op_name, args_spec_list, 2); + + auto dense_shape_value = dense_shape->BuildValue()->cast(); + MS_EXCEPTION_IF_NULL(dense_shape_value); + auto shp = dense_shape_value->value(); + std::vector dense_shape_vec; + (void)std::transform(std::begin(shp), std::end(shp), std::back_inserter(dense_shape_vec), + [](const ValuePtr &e) -> int { + auto elem = GetValue(e); + return elem; + }); + auto ret = std::make_shared(values->element()->BuildType(), dense_shape_vec); + ret->set_indices(indices); + ret->set_values(values); + ret->set_dense_shape(dense_shape); + return ret; +} + +AbstractBasePtr InferImplIndexedSlicesGetValues(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list) { + // Inputs: two tensors and a tuple. + const std::string op_name = primitive->name(); + CheckArgsSize(op_name, args_spec_list, 1); + auto indexed_slices = CheckArg(op_name, args_spec_list, 0); + MS_EXCEPTION_IF_NULL(indexed_slices->values()); + return indexed_slices->values(); +} + +AbstractBasePtr InferImplIndexedSlicesGetIndices(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list) { + // Inputs: two tensors and a tuple. + const std::string op_name = primitive->name(); + CheckArgsSize(op_name, args_spec_list, 1); + auto indexed_slices = CheckArg(op_name, args_spec_list, 0); + MS_EXCEPTION_IF_NULL(indexed_slices->indices()); + return indexed_slices->indices(); +} + +AbstractBasePtr InferImplIndexedSlicesGetDenseShape(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list) { + // Inputs: two tensors and a tuple. + const std::string op_name = primitive->name(); + CheckArgsSize(op_name, args_spec_list, 1); + auto indexed_slices = CheckArg(op_name, args_spec_list, 0); + MS_EXCEPTION_IF_NULL(indexed_slices->dense_shape()); + return indexed_slices->dense_shape(); +} + +AbstractBasePtr InferImplIsIndexedSlices(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list) { + const std::string op_name = primitive->name(); + CheckArgsSize(op_name, args_spec_list, 1); + bool ret = false; + if (args_spec_list[0]->isa()) { + ret = true; + } + MS_LOG(DEBUG) << "IsIndexedSlices result: " << ret << ", input: " << args_spec_list[0]->ToString(); + return std::make_shared(ret); +} } // namespace abstract } // namespace mindspore diff --git a/mindspore/ccsrc/optimizer/clean.cc b/mindspore/ccsrc/optimizer/clean.cc index 6a54597282..bb52273568 100644 --- a/mindspore/ccsrc/optimizer/clean.cc +++ b/mindspore/ccsrc/optimizer/clean.cc @@ -36,6 +36,7 @@ using mindspore::abstract::AbstractJTagged; using mindspore::abstract::AbstractList; using mindspore::abstract::AbstractScalar; using mindspore::abstract::AbstractTuple; +using mindspore::abstract::AbstractUndetermined; static AbstractBasePtr Reabs(const AbstractBasePtr &t) { if (t == nullptr) { @@ -78,7 +79,7 @@ AnfNodePtr ConvertGetAttrToTupleGetItem(const CNodePtr &node) { MS_EXCEPTION_IF_NULL(cons); auto dt = data->abstract(); - if (dt == nullptr) { + if (dt == nullptr || dt->BuildType()->type_id() == kObjectTypeUndeterminedType) { return nullptr; } diff --git a/mindspore/ccsrc/optimizer/irpass.cc b/mindspore/ccsrc/optimizer/irpass.cc index 3e8cfea37f..166151751f 100644 --- a/mindspore/ccsrc/optimizer/irpass.cc +++ b/mindspore/ccsrc/optimizer/irpass.cc @@ -42,6 +42,7 @@ #include "optimizer/irpass/tile_eliminate.h" #include "optimizer/irpass/transpose_eliminate.h" #include "optimizer/opt.h" +#include "optimizer/irpass/indexed_slices_eliminate.h" namespace mindspore { namespace opt { @@ -153,6 +154,11 @@ OptimizeIRPassLib::OptimizeIRPassLib() { // Mark interface fusion mark_interface_fusion_ = MakeSubstitution(std::make_shared(), "mark_interface_fusion", prim::kPrimSelect); + + // IndexedSlices Eliminate + indexed_slices_eliminate_ = MakeSubstitution( + std::make_shared(), "indexed_slices_eliminate", + {prim::kPrimIndexedSlicesGetIndices, prim::kPrimIndexedSlicesGetValues, prim::kPrimIndexedSlicesGetDenseShape}); } ResolveIRPassLib::ResolveIRPassLib() { diff --git a/mindspore/ccsrc/optimizer/irpass.h b/mindspore/ccsrc/optimizer/irpass.h index fa4d1e4cae..782eae6124 100644 --- a/mindspore/ccsrc/optimizer/irpass.h +++ b/mindspore/ccsrc/optimizer/irpass.h @@ -104,6 +104,9 @@ class OptimizeIRPassLib { // Fusion SubstitutionPtr mark_interface_fusion_; + + // IndexedSlices Eliminate + SubstitutionPtr indexed_slices_eliminate_; }; // the collection of irpass for resolve action diff --git a/mindspore/ccsrc/optimizer/irpass/indexed_slices_eliminate.h b/mindspore/ccsrc/optimizer/irpass/indexed_slices_eliminate.h new file mode 100644 index 0000000000..630d567549 --- /dev/null +++ b/mindspore/ccsrc/optimizer/irpass/indexed_slices_eliminate.h @@ -0,0 +1,75 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_OPTIMIZER_IRPASS_INDEXED_SLICES_ELIMINATE_H_ +#define MINDSPORE_CCSRC_OPTIMIZER_IRPASS_INDEXED_SLICES_ELIMINATE_H_ + +#include +#include + +#include "optimizer/irpass.h" +#include "optimizer/optimizer.h" +#include "ir/visitor.h" +#include "operator/ops.h" + +namespace mindspore { +namespace opt { +namespace irpass { +// {prim::kPrimIndexedSlicesGetIndices, {prim::kPrimMakeIndexedSlices, Xs}} +// {prim::kPrimIndexedSlicesGetValues, {prim::kPrimMakeIndexedSlices, Xs}} +// {prim::kPrimIndexedSlicesGetDenseShape, {prim::kPrimMakeIndexedSlices, Xs}} +class IndexedSlicesEliminater : public AnfVisitor { + public: + AnfNodePtr operator()(const OptimizerPtr &, const AnfNodePtr &node) override { + Reset(); + AnfVisitor::Match(prim::kPrimIndexedSlicesGetIndices, {IsCNode})(node); + + if (is_match_) { + return tuple_->input(1); + } + AnfVisitor::Match(prim::kPrimIndexedSlicesGetValues, {IsCNode})(node); + + if (is_match_) { + return tuple_->input(2); + } + AnfVisitor::Match(prim::kPrimIndexedSlicesGetDenseShape, {IsCNode})(node); + + if (is_match_) { + return tuple_->input(3); + } + return nullptr; + } + + void Visit(const CNodePtr &cnode) override { + if (IsPrimitiveCNode(cnode, prim::kPrimMakeIndexedSlices)) { + tuple_ = cnode; + is_match_ = true; + } + } + + void Reset() { + tuple_ = nullptr; + is_match_ = false; + } + + private: + bool is_match_{false}; + CNodePtr tuple_{nullptr}; +}; +} // namespace irpass +} // namespace opt +} // namespace mindspore +#endif // MINDSPORE_CCSRC_OPTIMIZER_IRPASS_INDEXED_SLICES_ELIMINATE_H_ diff --git a/mindspore/ccsrc/pipeline/action.cc b/mindspore/ccsrc/pipeline/action.cc index 7d56551ff0..c76053d241 100644 --- a/mindspore/ccsrc/pipeline/action.cc +++ b/mindspore/ccsrc/pipeline/action.cc @@ -232,6 +232,9 @@ bool AbstractSpecializeAction(const ResourcePtr &res) { auto sparse_grad = py::cast(parse::python_adapter::GetPyObjAttr(param_value->value(), "sparse_grad")); ptr->set_sparse_grad(sparse_grad); + auto has_indexed_slices_grad = + py::cast(parse::python_adapter::GetPyObjAttr(param_value->value(), "has_indexed_slices_grad")); + ptr->set_has_indexed_slices_grad(has_indexed_slices_grad); parallel::ParallelParameterContextRestoreInNoTraining(func_graph, param_node, ptr); args_spec.push_back(ptr); diff --git a/mindspore/ccsrc/pipeline/init.cc b/mindspore/ccsrc/pipeline/init.cc index dc309808d9..f28be181dd 100644 --- a/mindspore/ccsrc/pipeline/init.cc +++ b/mindspore/ccsrc/pipeline/init.cc @@ -154,7 +154,9 @@ PYBIND11_MODULE(_c_expression, m) { .def("set_print_file_path", &mindspore::MsContext::set_print_file_path, "Set path to print.") .def("set_enable_graph_kernel", &mindspore::MsContext::set_enable_graph_kernel, "Set the GraphKernel switch to on or off.") - .def("get_enable_graph_kernel", &mindspore::MsContext::enable_graph_kernel, "Get the value of GraphKernel switch."); + .def("get_enable_graph_kernel", &mindspore::MsContext::enable_graph_kernel, "Get the value of GraphKernel switch.") + .def("get_enable_sparse_flag", &mindspore::MsContext::enable_sparse_flag, "Get whether to enable sparse.") + .def("set_enable_sparse_flag", &mindspore::MsContext::set_enable_sparse_flag, "Set whether to enable sparse."); (void)py::class_>(m, "MpiConfig") .def_static("get_instance", &mindspore::MpiConfig::GetInstance, "Get mpi config instance.") diff --git a/mindspore/ccsrc/pipeline/pass.cc b/mindspore/ccsrc/pipeline/pass.cc index 9876c0280a..f6cfd6362c 100644 --- a/mindspore/ccsrc/pipeline/pass.cc +++ b/mindspore/ccsrc/pipeline/pass.cc @@ -156,6 +156,7 @@ OptPassGroupMap GetOptPassesB(const opt::irpass::OptimizeIRPassLib &irpass) { irpass.replace_refkey_by_param_, irpass.make_ref_eliminate_, irpass.get_ref_param_eliminate_, + irpass.indexed_slices_eliminate_, }); OptPassGroupMap map({ {"b_1", b_1}, diff --git a/mindspore/ccsrc/pipeline/resource.cc b/mindspore/ccsrc/pipeline/resource.cc index 50ccef2f44..faf1f2015d 100644 --- a/mindspore/ccsrc/pipeline/resource.cc +++ b/mindspore/ccsrc/pipeline/resource.cc @@ -33,148 +33,157 @@ namespace mindspore { namespace pipeline { MethodMap &GetMethodMap() { - static MethodMap method_map = {{kObjectTypeString, - { - {"__bool__", std::string("str_bool")} // C.str_bool - }}, - {kMetaTypeNone, - { - {"__bool__", std::string("none_bool")} // C.none_bool - }}, - {kNumberTypeBool, - { - {"__and__", prim::kPrimBoolAnd}, // P.bool_and - {"__or__", prim::kPrimBoolOr}, // P.bool_or - {"__eq__", prim::kPrimBoolEq}, // P.bool_eq - {"__ne__", std::string("bool_ne")}, // C.bool_ne - {"__bool__", prim::kPrimIdentity} // P.identity - }}, - {kNumberTypeInt, - { - {"__add__", prim::kPrimScalarAdd}, // P.scalar_add - {"__sub__", prim::kPrimScalarSub}, // P.scalar_sub - {"__mul__", prim::kPrimScalarMul}, // P.scalar_mul - {"__floordiv__", std::string("int_floordiv")}, // C.int_floordiv - {"__truediv__", std::string("int_truediv")}, // C.int_truediv - {"__mod__", prim::kPrimScalarMod}, // P.scalar_mod - {"__pow__", prim::kPrimScalarPow}, // P.scalar_pow - {"__floor__", prim::kPrimIdentity}, // P.identity - {"__trunc__", prim::kPrimIdentity}, // P.identity - {"__pos__", prim::kPrimScalarUadd}, // P.scalar_uadd - {"__neg__", prim::kPrimScalarUsub}, // P.scalar_usub - {"__eq__", prim::kPrimScalarEq}, // P.scalar_eq - {"__ne__", prim::kPrimScalarNe}, // P.scalar_ne - {"__lt__", prim::kPrimScalarLt}, // P.scalar_lt - {"__gt__", prim::kPrimScalarGt}, // P.scalar_gt - {"__le__", prim::kPrimScalarLe}, // P.scalar_le - {"__ge__", prim::kPrimScalarGe}, // P.scalar_ge - {"__bool__", std::string("int_bool")}, // C.int_bool - {"__ms_to_array__", prim::kPrimScalarToArray}, // P.scalar_to_array - }}, - {kNumberTypeUInt, - { - {"__add__", prim::kPrimScalarAdd}, // P.scalar_add, - {"__sub__", prim::kPrimScalarSub}, // P.scalar_sub, - {"__mul__", prim::kPrimScalarMul}, // P.scalar_mul, - {"__floordiv__", prim::kPrimScalarDiv}, // P.scalar_div, - {"__truediv__", std::string("int_truediv")}, // C.int_truediv - {"__mod__", prim::kPrimScalarMod}, // P.scalar_mod, - {"__pow__", prim::kPrimScalarPow}, // P.scalar_pow, - {"__floor__", prim::kPrimIdentity}, // P.identity, - {"__trunc__", prim::kPrimIdentity}, // P.identity, - {"__pos__", prim::kPrimScalarUadd}, // P.scalar_uadd, - {"__neg__", prim::kPrimScalarUsub}, // P.scalar_usub, - {"__eq__", prim::kPrimScalarEq}, // P.scalar_eq, - {"__ne__", prim::kPrimScalarNe}, // P.scalar_ne, - {"__lt__", prim::kPrimScalarLt}, // P.scalar_lt, - {"__gt__", prim::kPrimScalarGt}, // P.scalar_gt, - {"__le__", prim::kPrimScalarLe}, // P.scalar_le, - {"__ge__", prim::kPrimScalarGe}, // P.scalar_ge, - {"__bool__", std::string("int_bool")}, // C.int_bool - {"__ms_to_array__", prim::kPrimScalarToArray}, // P.scalar_to_array, - }}, - {kNumberTypeFloat, - { - {"__add__", prim::kPrimScalarAdd}, // P.scalar_add, - {"__sub__", prim::kPrimScalarSub}, // P.scalar_sub, - {"__mul__", prim::kPrimScalarMul}, // P.scalar_mul, - {"__floordiv__", std::string("float_floordiv")}, // C.float_floordiv - {"__truediv__", prim::kPrimScalarDiv}, // P.scalar_div, - {"__mod__", prim::kPrimScalarMod}, // P.scalar_mod, - {"__pow__", prim::kPrimScalarPow}, // P.scalar_pow, - {"__floor__", prim::kPrimScalarFloor}, // P.scalar_floor, - {"__trunc__", prim::kPrimScalarTrunc}, // P.scalar_trunc, - {"__pos__", prim::kPrimScalarUadd}, // P.scalar_uadd, - {"__neg__", prim::kPrimScalarUsub}, // P.scalar_usub, - {"__eq__", prim::kPrimScalarEq}, // P.scalar_eq, - {"__ne__", prim::kPrimScalarNe}, // P.scalar_ne, - {"__lt__", prim::kPrimScalarLt}, // P.scalar_lt, - {"__gt__", prim::kPrimScalarGt}, // P.scalar_gt, - {"__le__", prim::kPrimScalarLe}, // P.scalar_le, - {"__ge__", prim::kPrimScalarGe}, // P.scalar_ge, - {"__bool__", std::string("float_bool")}, // C.float_bool - {"__ms_to_array__", prim::kPrimScalarToArray}, // P.scalar_to_array, - }}, - {kObjectTypeTuple, - { - {"__len__", prim::kPrimTupleLen}, // P.tuple_len, - {"__getitem__", prim::kPrimTupleGetItem}, // P.tuple_getitem, - {"__setitem__", prim::kPrimTupleSetItem}, // P.tuple_setitem, - {"__ms_iter__", prim::kPrimIdentity}, // P.identity, - {"__ms_next__", std::string("tuple_next")}, // C.tuple_next, - {"__ms_hasnext__", std::string("tuple_hasnext")}, // C.tuple_hasnext - {"__bool__", std::string("tuple_bool")} // C.tuple_bool - }}, - {kObjectTypeList, - { - {"__len__", prim::kPrimListLen}, // P.list_len, - {"__getitem__", prim::kPrimListGetItem}, // P.list_getitem, - {"__setitem__", prim::kPrimListSetItem}, // P.list_setitem, - {"__ms_iter__", prim::kPrimIdentity}, // P.identity - {"__ms_next__", std::string("list_next")}, // C.list_next - {"append", std::string("list_append")}, // C.list_next - {"__bool__", std::string("list_bool")}, // C.list_bool - {"__ms_hasnext__", std::string("list_hasnext")}, - }}, - {kObjectTypeDictionary, - { - {"__len__", prim::kPrimDictLen}, // P.dict_len - {"__getitem__", prim::kPrimDictGetItem}, // P.dict_getitem - {"__setitem__", prim::kPrimDictSetItem}, // P.dict_setitem, - {"__bool__", std::string("dict_bool")} // C.dict_bool - }}, - {kObjectTypeTensorType, - { - {"__add__", std::string("add")}, // C.add - {"__sub__", std::string("sub")}, // C.sub - {"__mul__", std::string("mul")}, // C.mul - {"__truediv__", std::string("truediv")}, // C.truediv - {"__floordiv__", std::string("floordiv")}, // C.floordiv - {"__mod__", std::string("mod")}, // C.mod - {"__pow__", std::string("pow_")}, // C.pow - {"__floor__", std::string("array_floor")}, // C.array_floor - {"__trunc__", std::string("array_trunc")}, // C.array_trunc - {"__pos__", std::string("array_uadd")}, // C.array_uadd - {"__neg__", std::string("array_usub")}, // C.array_usub - {"__eq__", std::string("eq")}, // C.eq - {"__ne__", std::string("ne")}, // C.ne - {"__lt__", std::string("lt")}, // C.lt - {"__gt__", std::string("gt")}, // C.gt - {"__le__", std::string("le")}, // C.le - {"__ge__", std::string("ge")}, // C.ge - {"__matmul__", prim::kPrimDot}, // P.dot, - {"__len__", prim::kPrimArrayLen}, // P.array_len, - {"__getitem__", prim::kPrimArrayGetItem}, // P.array_getitem, - {"__setitem__", prim::kPrimArraySetItem}, // P.array_setitem, - {"__ms_iter__", std::string("array_iter")}, // C.array_iter - {"__ms_to_array__", prim::kPrimIdentity}, // P.identity, - {"item", prim::kPrimArrayToScalar}, // P.array_to_scalar, - {"transpose", std::string("transpose")}, // P.transpose - {"__bool__", std::string("tensor_bool")}, // C.tensor_bool - }}, - {kObjectTypeJTagged, {}}, - {kObjectTypeSymbolicKeyType, {}}, - {kObjectTypeEnvType, {}}}; + static MethodMap method_map = { + {kObjectTypeString, + { + {"__bool__", std::string("str_bool")} // C.str_bool + }}, + {kMetaTypeNone, + { + {"__bool__", std::string("none_bool")} // C.none_bool + }}, + {kNumberTypeBool, + { + {"__and__", prim::kPrimBoolAnd}, // P.bool_and + {"__or__", prim::kPrimBoolOr}, // P.bool_or + {"__eq__", prim::kPrimBoolEq}, // P.bool_eq + {"__ne__", std::string("bool_ne")}, // C.bool_ne + {"__bool__", prim::kPrimIdentity} // P.identity + }}, + {kNumberTypeInt, + { + {"__add__", prim::kPrimScalarAdd}, // P.scalar_add + {"__sub__", prim::kPrimScalarSub}, // P.scalar_sub + {"__mul__", prim::kPrimScalarMul}, // P.scalar_mul + {"__floordiv__", std::string("int_floordiv")}, // C.int_floordiv + {"__truediv__", std::string("int_truediv")}, // C.int_truediv + {"__mod__", prim::kPrimScalarMod}, // P.scalar_mod + {"__pow__", prim::kPrimScalarPow}, // P.scalar_pow + {"__floor__", prim::kPrimIdentity}, // P.identity + {"__trunc__", prim::kPrimIdentity}, // P.identity + {"__pos__", prim::kPrimScalarUadd}, // P.scalar_uadd + {"__neg__", prim::kPrimScalarUsub}, // P.scalar_usub + {"__eq__", prim::kPrimScalarEq}, // P.scalar_eq + {"__ne__", prim::kPrimScalarNe}, // P.scalar_ne + {"__lt__", prim::kPrimScalarLt}, // P.scalar_lt + {"__gt__", prim::kPrimScalarGt}, // P.scalar_gt + {"__le__", prim::kPrimScalarLe}, // P.scalar_le + {"__ge__", prim::kPrimScalarGe}, // P.scalar_ge + {"__bool__", std::string("int_bool")}, // C.int_bool + {"__ms_to_array__", prim::kPrimScalarToArray}, // P.scalar_to_array + }}, + {kNumberTypeUInt, + { + {"__add__", prim::kPrimScalarAdd}, // P.scalar_add, + {"__sub__", prim::kPrimScalarSub}, // P.scalar_sub, + {"__mul__", prim::kPrimScalarMul}, // P.scalar_mul, + {"__floordiv__", prim::kPrimScalarDiv}, // P.scalar_div, + {"__truediv__", std::string("int_truediv")}, // C.int_truediv + {"__mod__", prim::kPrimScalarMod}, // P.scalar_mod, + {"__pow__", prim::kPrimScalarPow}, // P.scalar_pow, + {"__floor__", prim::kPrimIdentity}, // P.identity, + {"__trunc__", prim::kPrimIdentity}, // P.identity, + {"__pos__", prim::kPrimScalarUadd}, // P.scalar_uadd, + {"__neg__", prim::kPrimScalarUsub}, // P.scalar_usub, + {"__eq__", prim::kPrimScalarEq}, // P.scalar_eq, + {"__ne__", prim::kPrimScalarNe}, // P.scalar_ne, + {"__lt__", prim::kPrimScalarLt}, // P.scalar_lt, + {"__gt__", prim::kPrimScalarGt}, // P.scalar_gt, + {"__le__", prim::kPrimScalarLe}, // P.scalar_le, + {"__ge__", prim::kPrimScalarGe}, // P.scalar_ge, + {"__bool__", std::string("int_bool")}, // C.int_bool + {"__ms_to_array__", prim::kPrimScalarToArray}, // P.scalar_to_array, + }}, + {kNumberTypeFloat, + { + {"__add__", prim::kPrimScalarAdd}, // P.scalar_add, + {"__sub__", prim::kPrimScalarSub}, // P.scalar_sub, + {"__mul__", prim::kPrimScalarMul}, // P.scalar_mul, + {"__floordiv__", std::string("float_floordiv")}, // C.float_floordiv + {"__truediv__", prim::kPrimScalarDiv}, // P.scalar_div, + {"__mod__", prim::kPrimScalarMod}, // P.scalar_mod, + {"__pow__", prim::kPrimScalarPow}, // P.scalar_pow, + {"__floor__", prim::kPrimScalarFloor}, // P.scalar_floor, + {"__trunc__", prim::kPrimScalarTrunc}, // P.scalar_trunc, + {"__pos__", prim::kPrimScalarUadd}, // P.scalar_uadd, + {"__neg__", prim::kPrimScalarUsub}, // P.scalar_usub, + {"__eq__", prim::kPrimScalarEq}, // P.scalar_eq, + {"__ne__", prim::kPrimScalarNe}, // P.scalar_ne, + {"__lt__", prim::kPrimScalarLt}, // P.scalar_lt, + {"__gt__", prim::kPrimScalarGt}, // P.scalar_gt, + {"__le__", prim::kPrimScalarLe}, // P.scalar_le, + {"__ge__", prim::kPrimScalarGe}, // P.scalar_ge, + {"__bool__", std::string("float_bool")}, // C.float_bool + {"__ms_to_array__", prim::kPrimScalarToArray}, // P.scalar_to_array, + }}, + {kObjectTypeTuple, + { + {"__len__", prim::kPrimTupleLen}, // P.tuple_len, + {"__getitem__", prim::kPrimTupleGetItem}, // P.tuple_getitem, + {"__setitem__", prim::kPrimTupleSetItem}, // P.tuple_setitem, + {"__ms_iter__", prim::kPrimIdentity}, // P.identity, + {"__ms_next__", std::string("tuple_next")}, // C.tuple_next, + {"__ms_hasnext__", std::string("tuple_hasnext")}, // C.tuple_hasnext + {"__bool__", std::string("tuple_bool")} // C.tuple_bool + }}, + {kObjectTypeList, + { + {"__len__", prim::kPrimListLen}, // P.list_len, + {"__getitem__", prim::kPrimListGetItem}, // P.list_getitem, + {"__setitem__", prim::kPrimListSetItem}, // P.list_setitem, + {"__ms_iter__", prim::kPrimIdentity}, // P.identity + {"__ms_next__", std::string("list_next")}, // C.list_next + {"append", std::string("list_append")}, // C.list_next + {"__bool__", std::string("list_bool")}, // C.list_bool + {"__ms_hasnext__", std::string("list_hasnext")}, + }}, + {kObjectTypeDictionary, + { + {"__len__", prim::kPrimDictLen}, // P.dict_len + {"__getitem__", prim::kPrimDictGetItem}, // P.dict_getitem + {"__setitem__", prim::kPrimDictSetItem}, // P.dict_setitem, + {"__bool__", std::string("dict_bool")} // C.dict_bool + }}, + {kObjectTypeTensorType, + { + {"__add__", std::string("add")}, // C.add + {"__sub__", std::string("sub")}, // C.sub + {"__mul__", std::string("mul")}, // C.mul + {"__truediv__", std::string("truediv")}, // C.truediv + {"__floordiv__", std::string("floordiv")}, // C.floordiv + {"__mod__", std::string("mod")}, // C.mod + {"__pow__", std::string("pow_")}, // C.pow + {"__floor__", std::string("array_floor")}, // C.array_floor + {"__trunc__", std::string("array_trunc")}, // C.array_trunc + {"__pos__", std::string("array_uadd")}, // C.array_uadd + {"__neg__", std::string("array_usub")}, // C.array_usub + {"__eq__", std::string("eq")}, // C.eq + {"__ne__", std::string("ne")}, // C.ne + {"__lt__", std::string("lt")}, // C.lt + {"__gt__", std::string("gt")}, // C.gt + {"__le__", std::string("le")}, // C.le + {"__ge__", std::string("ge")}, // C.ge + {"__matmul__", prim::kPrimDot}, // P.dot, + {"__len__", prim::kPrimArrayLen}, // P.array_len, + {"__getitem__", prim::kPrimArrayGetItem}, // P.array_getitem, + {"__setitem__", prim::kPrimArraySetItem}, // P.array_setitem, + {"__ms_iter__", std::string("array_iter")}, // C.array_iter + {"__ms_to_array__", prim::kPrimIdentity}, // P.identity, + {"item", prim::kPrimArrayToScalar}, // P.array_to_scalar, + {"transpose", std::string("transpose")}, // P.transpose + {"__bool__", std::string("tensor_bool")}, // C.tensor_bool + {"is_indexed_slices", prim::kPrimIsIndexedSlices}, // F.is_indexed_slices + }}, + {kObjectTypeIndexedSlicesType, + { + {"is_indexed_slices", prim::kPrimIsIndexedSlices}, // F.is_indexed_slices + {"values", prim::kPrimIndexedSlicesGetValues}, // F.indexed_slices_get_values + {"indices", prim::kPrimIndexedSlicesGetIndices}, // F.indexed_slices_get_indices + {"dense_shape", prim::kPrimIndexedSlicesGetDenseShape}, // F.indexed_slices_get_dense_shape + }}, + {kObjectTypeJTagged, {}}, + {kObjectTypeSymbolicKeyType, {}}, + {kObjectTypeEnvType, {}}}; return method_map; } diff --git a/mindspore/ccsrc/pipeline/static_analysis/abstract_value.cc b/mindspore/ccsrc/pipeline/static_analysis/abstract_value.cc index f23c6e31c4..86bfecf14b 100644 --- a/mindspore/ccsrc/pipeline/static_analysis/abstract_value.cc +++ b/mindspore/ccsrc/pipeline/static_analysis/abstract_value.cc @@ -30,6 +30,10 @@ bool AbstractBase::operator==(const AbstractBase &other) const { if (tid() != other.tid()) { return false; } + if (BuildType()->type_id() == kObjectTypeUndeterminedType && + other.BuildType()->type_id() == kObjectTypeUndeterminedType) { + return true; + } if (value_ == nullptr || other.value_ == nullptr) { MS_LOG(EXCEPTION) << "If value_ is nullptr, AbstractBase::operator== should not be called. this: " << this->ToString() << ", other: " << other.ToString(); @@ -65,7 +69,7 @@ std::string AbstractBase::ToString() const { MS_EXCEPTION_IF_NULL(shape_); buffer << type_name() << "(" << "Type: " << type_->ToString() << " Value: " << value << " Shape: " << shape_->ToString() - << " sparse_grad: " << sparse_grad_ << ")"; + << " sparse_grad: " << sparse_grad_ << " has_indexed_slices_grad: " << has_indexed_slices_grad_ << ")"; return buffer.str(); } @@ -76,6 +80,7 @@ AbstractBasePtr AbstractScalar::Join(const AbstractBasePtr &other) { if (*this == *other) { auto ret = shared_from_base(); ret->set_sparse_grad(sparse_grad()); + ret->set_has_indexed_slices_grad(has_indexed_slices_grad()); return ret; } auto value_self = GetValueTrack(); @@ -85,10 +90,12 @@ AbstractBasePtr AbstractScalar::Join(const AbstractBasePtr &other) { if (res_value == value_self) { auto ret = shared_from_base(); ret->set_sparse_grad(sparse_grad()); + ret->set_has_indexed_slices_grad(has_indexed_slices_grad()); return ret; } auto ret = std::make_shared(res_value, res_type); ret->set_sparse_grad(sparse_grad()); + ret->set_has_indexed_slices_grad(has_indexed_slices_grad()); return ret; } @@ -409,6 +416,14 @@ std::size_t AbstractSlice::hash() const { return hash_combine({tid(), start_->hash(), stop_->hash(), step_->hash()}); } +ShapePtr AbstractUndetermined::shape() const { + auto shp = dyn_cast(GetShapeTrack()); + if (shp == nullptr) { + MS_LOG(EXCEPTION) << "Tensor should have a shape."; + } + return shp; +} + TypePtr AbstractTensor::BuildType() const { MS_EXCEPTION_IF_NULL(element_); TypePtr element_type = element_->BuildType(); @@ -425,6 +440,13 @@ BaseShapePtr AbstractTensor::BuildShape() const { } AbstractBasePtr AbstractTensor::Join(const AbstractBasePtr &other) { + if (other->BuildType()->type_id() == kObjectTypeUndeterminedType) { + auto other_tensor = dyn_cast(other); + auto element = element_->Join(other_tensor->element()); + auto shape = ShapeJoin(this->shape(), other_tensor->shape()); + auto ret = std::make_shared(element, shape); + return ret; + } auto other_tensor = dyn_cast(other); if (other_tensor == nullptr) { MS_LOG(EXCEPTION) << "Join failed as type mismatch, this: " << ToString() << ", other: " << other->ToString(); @@ -433,6 +455,7 @@ AbstractBasePtr AbstractTensor::Join(const AbstractBasePtr &other) { auto shape = ShapeJoin(this->shape(), other_tensor->shape()); auto ret = std::make_shared(element, shape); ret->set_sparse_grad(sparse_grad()); + ret->set_has_indexed_slices_grad(has_indexed_slices_grad()); return ret; } @@ -474,6 +497,7 @@ AbstractBasePtr AbstractTensor::Clone() const { clone->set_shape(shp->Clone()); clone->set_value(GetValueTrack()); clone->set_sparse_grad(sparse_grad()); + clone->set_has_indexed_slices_grad(has_indexed_slices_grad()); return clone; } @@ -484,6 +508,7 @@ AbstractBasePtr AbstractTensor::Broaden() const { broaden->set_shape(shp->Clone()); broaden->set_value(kAnyValue); broaden->set_sparse_grad(sparse_grad()); + broaden->set_has_indexed_slices_grad(has_indexed_slices_grad()); return broaden; } @@ -495,17 +520,10 @@ AbstractBasePtr AbstractTensor::BroadenWithShape() const { broaden->set_shape(shp); broaden->set_value(kAnyValue); broaden->set_sparse_grad(sparse_grad()); + broaden->set_has_indexed_slices_grad(has_indexed_slices_grad()); return broaden; } -ShapePtr AbstractTensor::shape() const { - auto shp = dyn_cast(GetShapeTrack()); - if (shp == nullptr) { - MS_LOG(EXCEPTION) << "Tensor should have a shape."; - } - return shp; -} - std::string AbstractTensor::ToString() const { std::ostringstream buffer; BaseShapePtr shape_track = GetShapeTrack(); @@ -516,7 +534,7 @@ std::string AbstractTensor::ToString() const { buffer << type_name() << "(" << "shape: " << shape_track->ToString() << ", element: " << element_->ToString() << ", value_ptr: " << value_track << ", value: " << value_track->ToString() << " sparse_grad " << sparse_grad() - << ")"; + << " has_indexed_slices_grad " << has_indexed_slices_grad() << ")"; return buffer.str(); } @@ -1019,5 +1037,64 @@ std::size_t AbstractBasePtrListHasher::operator()(const AbstractBasePtrList &arg bool AbstractBasePtrListEqual::operator()(const AbstractBasePtrList &lhs, const AbstractBasePtrList &rhs) const { return AbstractBasePtrListDeepEqual(lhs, rhs); } + +// IndexedSlices +TypePtr AbstractIndexedSlices::BuildType() const { + MS_EXCEPTION_IF_NULL(element()); + TypePtr element_type = element()->BuildType(); + return std::make_shared(element_type); +} + +AbstractBasePtr AbstractIndexedSlices::Clone() const { + MS_EXCEPTION_IF_NULL(element()); + auto clone = std::make_shared(element()->Clone()); + ShapePtr shp = shape(); + clone->set_shape(shp->Clone()); + clone->set_value(GetValueTrack()); + clone->set_indices(indices_->Clone()->cast()); + clone->set_values(values_->Clone()->cast()); + clone->set_dense_shape(dense_shape_->Clone()->cast()); + return clone; +} + +AbstractBasePtr AbstractIndexedSlices::Broaden() const { + MS_EXCEPTION_IF_NULL(element()); + auto broaden = std::make_shared(element()->Broaden()); + auto shp = shape(); + broaden->set_shape(shp->Clone()); + broaden->set_value(kAnyValue); + broaden->set_indices(indices_->Clone()->cast()); + broaden->set_values(values_->Clone()->cast()); + broaden->set_dense_shape(dense_shape_->Clone()->cast()); + return broaden; +} + +AbstractBasePtr AbstractIndexedSlices::BroadenWithShape() const { + MS_EXCEPTION_IF_NULL(element()); + auto broaden = std::make_shared(element()->Broaden()); + auto shp = shape()->Clone(); + shp->Broaden(); + broaden->set_shape(shp); + broaden->set_value(kAnyValue); + broaden->set_indices(indices_->Clone()->cast()); + broaden->set_values(values_->Clone()->cast()); + broaden->set_dense_shape(dense_shape_->Clone()->cast()); + return broaden; +} + +std::string AbstractIndexedSlices::ToString() const { + std::ostringstream buffer; + BaseShapePtr shape_track = GetShapeTrack(); + MS_EXCEPTION_IF_NULL(shape_track); + MS_EXCEPTION_IF_NULL(element()); + auto value_track = GetValueTrack(); + MS_EXCEPTION_IF_NULL(value_track); + buffer << type_name() << "(" + << "shape: " << shape_track->ToString() << ", element: " << element()->ToString() + << ", value_ptr: " << value_track << ", value: " << value_track->ToString() << ")" + << ", indices: " << indices_->ToString() << ", values" << values_->ToString() + << ", dense_shape: " << dense_shape_->ToString(); + return buffer.str(); +} } // namespace abstract } // namespace mindspore diff --git a/mindspore/ccsrc/pipeline/static_analysis/abstract_value.h b/mindspore/ccsrc/pipeline/static_analysis/abstract_value.h index 5b54c749b6..a5b4acff45 100644 --- a/mindspore/ccsrc/pipeline/static_analysis/abstract_value.h +++ b/mindspore/ccsrc/pipeline/static_analysis/abstract_value.h @@ -44,7 +44,7 @@ class AbstractBase : public Base { public: explicit AbstractBase(const ValuePtr &value = nullptr, const TypePtr &type = kAnyType, const BaseShapePtr &shape = kNoShape) - : value_(value), type_(type), shape_(shape), sparse_grad_("") {} + : value_(value), type_(type), shape_(shape), sparse_grad_(""), has_indexed_slices_grad_(false) {} ~AbstractBase() override = default; MS_DECLARE_PARENT(AbstractBase, Base) @@ -54,12 +54,16 @@ class AbstractBase : public Base { virtual bool operator==(const AbstractBase &other) const; void set_value(const ValuePtr &value) { value_ = value; } void set_sparse_grad(const std::string &sparse_grad) { sparse_grad_ = sparse_grad; } + void set_has_indexed_slices_grad(const bool &has_indexed_slices_grad) { + has_indexed_slices_grad_ = has_indexed_slices_grad; + } void set_type(const TypePtr &type) { type_ = type; } void set_shape(const BaseShapePtr &shape) { shape_ = shape; } void set_value_desc(const std::string &desc) { value_desc_ = desc; } const std::string &value_desc() const { return value_desc_; } ValuePtr GetValueTrack() const { return value_; } const std::string &sparse_grad() const { return sparse_grad_; } + const bool &has_indexed_slices_grad() const { return has_indexed_slices_grad_; } TypePtr GetTypeTrack() const { return type_; } BaseShapePtr GetShapeTrack() const { return shape_; } @@ -88,6 +92,7 @@ class AbstractBase : public Base { BaseShapePtr shape_; std::string value_desc_; // store initial value description for error report std::string sparse_grad_; + bool has_indexed_slices_grad_; }; class AbstractScalar : public AbstractBase { @@ -231,35 +236,49 @@ class AbstractKeywordArg : public AbstractBase { }; using AbstractKeywordArgPtr = std::shared_ptr; -class AbstractTensor : public AbstractBase { +class AbstractUndetermined : public AbstractBase { public: + // shape and type are all unknown + AbstractUndetermined() : AbstractBase(kAnyValue) {} // only element_ and value, shape track are valid member, type track are unknown. - explicit AbstractTensor(const AbstractBasePtr &element, const BaseShapePtr &shape = std::make_shared()) + explicit AbstractUndetermined(const AbstractBasePtr &element, const BaseShapePtr &shape = std::make_shared()) : AbstractBase(kAnyValue), element_(element) { if (element == nullptr) { MS_LOG(EXCEPTION) << "element is nullptr"; } - if (element->isa()) { + if (element->isa()) { MS_LOG(EXCEPTION) << "element type error"; } set_shape(shape); } - AbstractTensor(const TypePtr &element_type, const std::vector &shape) + AbstractUndetermined(const TypePtr &element_type, const std::vector &shape) : AbstractBase(kAnyValue), element_(std::make_shared(kAnyValue, element_type)) { if (element_type == nullptr) { MS_LOG(EXCEPTION) << "element_type is nullptr"; } set_shape(std::make_shared(shape)); } - explicit AbstractTensor(const tensor::TensorPtr &tensor) - : AbstractBase(tensor), element_(std::make_shared(kAnyValue, tensor->Dtype())) { - if (tensor == nullptr) { - MS_LOG(EXCEPTION) << "tensor is nullptr"; - } - set_shape(std::make_shared(tensor->shape())); - } + ~AbstractUndetermined() override = default; + MS_DECLARE_PARENT(AbstractUndetermined, AbstractBase) + TypePtr BuildType() const override { return std::make_shared(); } + AbstractBasePtr Clone() const override { return std::make_shared(); } + const AbstractBasePtr element() const { return element_; } + ShapePtr shape() const; + + protected: + AbstractBasePtr element_; +}; + +class AbstractTensor : public AbstractUndetermined { + public: + // only element_ and value, shape track are valid member, type track are unknown. + explicit AbstractTensor(const AbstractBasePtr &element, const BaseShapePtr &shape = std::make_shared()) + : AbstractUndetermined(element, shape) {} + AbstractTensor(const TypePtr &element_type, const std::vector &shape) + : AbstractUndetermined(element_type, shape) {} + explicit AbstractTensor(const tensor::TensorPtr &tensor) : AbstractUndetermined(tensor->Dtype(), tensor->shape()) {} ~AbstractTensor() override = default; - MS_DECLARE_PARENT(AbstractTensor, AbstractBase) + MS_DECLARE_PARENT(AbstractTensor, AbstractUndetermined) TypePtr BuildType() const override; BaseShapePtr BuildShape() const override; @@ -271,9 +290,7 @@ class AbstractTensor : public AbstractBase { bool operator==(const AbstractTensor &other) const; bool operator==(const AbstractBase &other) const override; - ShapePtr shape() const; std::string ToString() const override; - const AbstractBasePtr element() const { return element_; } std::size_t hash() const override { auto value = GetValueTrack(); auto hash_sum = hash_combine(tid(), element_->hash()); @@ -285,9 +302,6 @@ class AbstractTensor : public AbstractBase { } return hash_sum; } - - private: - AbstractBasePtr element_; }; using AbstractTensorPtr = std::shared_ptr; using AbstractTensorPtrList = std::vector; @@ -585,6 +599,35 @@ struct AbstractBasePtrListEqual { std::size_t AbstractBasePtrListHash(const AbstractBasePtrList &args_spec_list); bool AbstractBasePtrListDeepEqual(const AbstractBasePtrList &lhs, const AbstractBasePtrList &rhs); + +// IndexedSlices +class AbstractIndexedSlices : public AbstractUndetermined { + public: + explicit AbstractIndexedSlices(const AbstractBasePtr &element, const BaseShapePtr &shape = std::make_shared()) + : AbstractUndetermined(element, shape) {} + AbstractIndexedSlices(const TypePtr &element_type, const std::vector &shape) + : AbstractUndetermined(element_type, shape) {} + ~AbstractIndexedSlices() override = default; + MS_DECLARE_PARENT(AbstractIndexedSlices, AbstractUndetermined) + + const AbstractTensorPtr indices() const { return indices_; } + const AbstractTensorPtr values() const { return values_; } + const AbstractTuplePtr dense_shape() const { return dense_shape_; } + void set_indices(const AbstractTensorPtr &indices) { indices_ = indices; } + void set_values(const AbstractTensorPtr &values) { values_ = values; } + void set_dense_shape(const AbstractTuplePtr &dense_shape) { dense_shape_ = dense_shape; } + TypePtr BuildType() const override; + AbstractBasePtr Clone() const override; + AbstractBasePtr Broaden() const override; + AbstractBasePtr BroadenWithShape() const; + + std::string ToString() const override; + + private: + AbstractTensorPtr indices_; + AbstractTensorPtr values_; + AbstractTuplePtr dense_shape_; +}; } // namespace abstract } // namespace mindspore #endif // PIPELINE_STATIC_ANALYSIS_ABSTRACT_VALUE_H_ diff --git a/mindspore/ccsrc/pipeline/static_analysis/evaluator.h b/mindspore/ccsrc/pipeline/static_analysis/evaluator.h index c7a004ac44..f6430eda84 100644 --- a/mindspore/ccsrc/pipeline/static_analysis/evaluator.h +++ b/mindspore/ccsrc/pipeline/static_analysis/evaluator.h @@ -58,6 +58,20 @@ class Evaluator : public Base { return args_spec_list; } + virtual EvalResultPtr AbstractEval(const AbstractBasePtrList &args_spec_list) { + auto is_abstract = std::any_of(args_spec_list.begin(), args_spec_list.end(), [](auto &arg) { + if (arg->BuildType()->type_id() == kObjectTypeUndeterminedType) { + return true; + } + return false; + }); + if (is_abstract) { + MS_LOG(DEBUG) << "Eval " << identifier_ << " return abstract result"; + return std::make_shared(std::make_shared(), std::make_shared()); + } + return nullptr; + } + std::string ToString() const override { return identifier_; } virtual AnfNodePtr bound_node() const { return bound_node_.lock(); } diff --git a/mindspore/ccsrc/pipeline/static_analysis/param_validator.h b/mindspore/ccsrc/pipeline/static_analysis/param_validator.h index ecb9529a58..2f5729aa73 100644 --- a/mindspore/ccsrc/pipeline/static_analysis/param_validator.h +++ b/mindspore/ccsrc/pipeline/static_analysis/param_validator.h @@ -66,6 +66,7 @@ ABSTRACT_REPORT_NAME_TRAITS(Function) ABSTRACT_REPORT_NAME_TRAITS(Type) ABSTRACT_REPORT_NAME_TRAITS(KeywordArg) ABSTRACT_REPORT_NAME_TRAITS(Class) +ABSTRACT_REPORT_NAME_TRAITS(IndexedSlices) template std::shared_ptr CheckArg(const std::string &op, const AbstractBasePtrList &args_spec_list, size_t index) { diff --git a/mindspore/ccsrc/pipeline/static_analysis/prim.cc b/mindspore/ccsrc/pipeline/static_analysis/prim.cc index bf1f319ae2..99dc085989 100644 --- a/mindspore/ccsrc/pipeline/static_analysis/prim.cc +++ b/mindspore/ccsrc/pipeline/static_analysis/prim.cc @@ -36,6 +36,7 @@ #include "pipeline/parse/resolve.h" #include "ir/tensor.h" #include "utils/convert_utils.h" +#include "utils/context/ms_context.h" #include "pipeline/parse/data_converter.h" #include "pipeline/static_analysis/param_validator.h" #include "common/utils.h" @@ -132,6 +133,12 @@ PrimitiveEvalImplMap &GetPrimitiveToEvalImplMap() { {prim::kPrimControlDepend, {InferImplControlDepend, true}}, // Debug {prim::kPrimDebug, {InferImplDebug, true}}, + // IndexedSlices + {prim::kPrimMakeIndexedSlices, {InferImplMakeIndexedSlices, true}}, + {prim::kPrimIndexedSlicesGetValues, {InferImplIndexedSlicesGetValues, true}}, + {prim::kPrimIndexedSlicesGetIndices, {InferImplIndexedSlicesGetIndices, true}}, + {prim::kPrimIndexedSlicesGetDenseShape, {InferImplIndexedSlicesGetDenseShape, true}}, + {prim::kPrimIsIndexedSlices, {InferImplIsIndexedSlices, true}}, }; return prim_eval_implement_map; } @@ -139,6 +146,16 @@ PrimitiveEvalImplMap &GetPrimitiveToEvalImplMap() { using mindspore::parse::PyObjectWrapper; EvalResultPtr StandardPrimEvaluator::EvalPrim(const AnalysisEnginePtr &engine, const AbstractBasePtrList &args) { + auto context = MsContext::GetInstance(); + MS_EXCEPTION_IF_NULL(context); + bool enable_sparse_flag = context->enable_sparse_flag(); + if (enable_sparse_flag && prim_ != prim::kPrimMakeTuple && prim_ != prim::kPrimSwitch) { + auto ret_abstract = AbstractEval(args); + if (ret_abstract != nullptr) { + MS_LOG(DEBUG) << "StandardPrimEvaluator eval Undetermined"; + return ret_abstract; + } + } prim_->BeginRecordAddAttr(); AbstractBasePtr abs_base = eval_impl_(engine, prim_, args); prim_->EndRecordAddAttr(); @@ -485,6 +502,16 @@ AbstractBasePtr PyInferRes2Abstract(const PrimitivePyPtr &prim_py, const py::dic } // end anonymous namespace EvalResultPtr PythonPrimEvaluator::EvalPrim(const AnalysisEnginePtr &, const AbstractBasePtrList &args) { + auto context = MsContext::GetInstance(); + MS_EXCEPTION_IF_NULL(context); + bool enable_sparse_flag = context->enable_sparse_flag(); + if (enable_sparse_flag) { + auto ret_abstract = AbstractEval(args); + if (ret_abstract != nullptr) { + MS_LOG(DEBUG) << "PythonPrimEvaluator eval Undetermined"; + return ret_abstract; + } + } MS_LOG(DEBUG) << "Eval for:" << prim_py_->ToString(); const auto &iter = cache_->find(args); @@ -512,6 +539,16 @@ EvalResultPtr PythonPrimEvaluator::EvalPrim(const AnalysisEnginePtr &, const Abs } EvalResultPtr UniformPrimEvaluator::EvalPrim(const AnalysisEnginePtr &, const AbstractBasePtrList &args) { + auto context = MsContext::GetInstance(); + MS_EXCEPTION_IF_NULL(context); + bool enable_sparse_flag = context->enable_sparse_flag(); + if (enable_sparse_flag) { + auto ret_abstract = AbstractEval(args); + if (ret_abstract != nullptr) { + MS_LOG(DEBUG) << "UniformPrimEvaluator eval Undetermined"; + return ret_abstract; + } + } // if func_desc_.retval type is super class of parameter type, then make the retval type as parameter type. if (nargs_ != args.size()) { MS_LOG(ERROR) << "UniformPrimEvaluator expect " << nargs_ << " args, but got " << args.size() << " inputs"; @@ -871,6 +908,7 @@ class RefToEmbedEvaluator : public SymbolicPrimEvaluator { auto ref_value = ref_abs->ref(); MS_EXCEPTION_IF_NULL(ref_value); ret->set_sparse_grad(ref_value->sparse_grad()); + ret->set_has_indexed_slices_grad(ref_value->has_indexed_slices_grad()); return std::make_shared(ret, std::make_shared()); } @@ -886,6 +924,7 @@ class RefToEmbedEvaluator : public SymbolicPrimEvaluator { std::shared_ptr key = std::make_shared(node, x); std::shared_ptr abs_scalar = std::make_shared(key, type); abs_scalar->set_sparse_grad(x->sparse_grad()); + abs_scalar->set_has_indexed_slices_grad(x->has_indexed_slices_grad()); return std::make_shared(abs_scalar, std::make_shared()); } }; @@ -897,6 +936,16 @@ class GetAttrEvaluator : public TransitionPrimEvaluator { MS_DECLARE_PARENT(GetAttrEvaluator, TransitionPrimEvaluator); EvalResultPtr EvalPrim(const AnalysisEnginePtr &engine, const AbstractBasePtrList &args_spec_list, const ConfigPtr &in_conf0, const AnfNodeConfigPtr &out_conf) override { + auto context = MsContext::GetInstance(); + MS_EXCEPTION_IF_NULL(context); + bool enable_sparse_flag = context->enable_sparse_flag(); + if (enable_sparse_flag) { + auto ret_abstract = AbstractEval(args_spec_list); + if (ret_abstract != nullptr) { + MS_LOG(DEBUG) << "GetAttrEvaluator eval Undetermined"; + return ret_abstract; + } + } // Inputs: data, item if (args_spec_list.size() != 2) { MS_LOG(EXCEPTION) << "Expected args_spec_list size = 2, but has size:" << args_spec_list.size(); diff --git a/mindspore/ccsrc/pipeline/static_analysis/prim.h b/mindspore/ccsrc/pipeline/static_analysis/prim.h index 5954179aa5..1346dba2a2 100644 --- a/mindspore/ccsrc/pipeline/static_analysis/prim.h +++ b/mindspore/ccsrc/pipeline/static_analysis/prim.h @@ -350,6 +350,17 @@ AbstractBasePtr InferImplControlDepend(const AnalysisEnginePtr &, const Primitiv AbstractBasePtr InferImplDebug(const AnalysisEnginePtr &, const PrimitivePtr &primitive, const AbstractBasePtrList &args_spec_list); void InitUndeterminedFromEnv(const std::string &sparse_shape_types); + +AbstractBasePtr InferImplMakeIndexedSlices(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list); +AbstractBasePtr InferImplIndexedSlicesGetValues(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list); +AbstractBasePtr InferImplIndexedSlicesGetIndices(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list); +AbstractBasePtr InferImplIndexedSlicesGetDenseShape(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list); +AbstractBasePtr InferImplIsIndexedSlices(const AnalysisEnginePtr &, const PrimitivePtr &primitive, + const AbstractBasePtrList &args_spec_list); } // namespace abstract } // namespace mindspore diff --git a/mindspore/ccsrc/pipeline/static_analysis/static_analysis.cc b/mindspore/ccsrc/pipeline/static_analysis/static_analysis.cc index 9da148d2a7..5416576680 100644 --- a/mindspore/ccsrc/pipeline/static_analysis/static_analysis.cc +++ b/mindspore/ccsrc/pipeline/static_analysis/static_analysis.cc @@ -228,6 +228,10 @@ EvalResultPtr AnalysisEngine::EvalCNode(const CNodePtr &cnode, const AnfNodeConf MS_LOG(EXCEPTION) << "func_conf.GetEvaluatedValue() return null, func_conf: " << func_conf->ToString() << " NodeInfo: " << trace::GetDebugInfo(cnode->debug_info()); } + if (maybe_func->BuildType()->type_id() == kObjectTypeUndeterminedType) { + MS_LOG(DEBUG) << "EvalCNode eval Undetermined"; + return std::make_shared(maybe_func->Clone(), std::make_shared()); + } AbstractFunctionPtr func = dyn_cast(maybe_func); if (func == nullptr) { MS_LOG(EXCEPTION) << "func_conf.GetEvaluatedValue() return not AbstractFunction: " << maybe_func->ToString() diff --git a/mindspore/ccsrc/pipeline/validator.cc b/mindspore/ccsrc/pipeline/validator.cc index 4866d43b93..bbca3c8721 100644 --- a/mindspore/ccsrc/pipeline/validator.cc +++ b/mindspore/ccsrc/pipeline/validator.cc @@ -32,6 +32,7 @@ using mindspore::abstract::AbstractBase; using mindspore::abstract::AbstractClass; using mindspore::abstract::AbstractError; using mindspore::abstract::AbstractFunction; +using mindspore::abstract::AbstractIndexedSlices; using mindspore::abstract::AbstractJTagged; using mindspore::abstract::AbstractList; using mindspore::abstract::AbstractScalar; @@ -93,7 +94,8 @@ void ValidateAbstract(const AnfNodePtr &node) { } if (ptrBase->isa() || ptrBase->isa() || ptrBase->isa() || - ptrBase->isa() || ptrBase->isa() || ptrBase->isa()) { + ptrBase->isa() || ptrBase->isa() || ptrBase->isa() || + ptrBase->isa()) { return; } diff --git a/mindspore/ccsrc/utils/context/ms_context.cc b/mindspore/ccsrc/utils/context/ms_context.cc index d385ec7a3f..3d367b90e2 100644 --- a/mindspore/ccsrc/utils/context/ms_context.cc +++ b/mindspore/ccsrc/utils/context/ms_context.cc @@ -89,6 +89,7 @@ MsContext::MsContext(const std::string &policy, const std::string &target) { max_device_memory_ = kDefaultMaxDeviceMemory; print_file_path_ = ""; enable_graph_kernel_ = false; + enable_sparse_flag_ = false; } std::shared_ptr MsContext::GetInstance() { diff --git a/mindspore/ccsrc/utils/context/ms_context.h b/mindspore/ccsrc/utils/context/ms_context.h index 9afe1fa5aa..3bca16f8ee 100644 --- a/mindspore/ccsrc/utils/context/ms_context.h +++ b/mindspore/ccsrc/utils/context/ms_context.h @@ -161,6 +161,9 @@ class MsContext { void set_enable_graph_kernel(bool enable_graph_kernel) { enable_graph_kernel_ = enable_graph_kernel; } bool enable_graph_kernel() const { return enable_graph_kernel_; } + bool enable_sparse_flag() const { return enable_sparse_flag_; } + void set_enable_sparse_flag(bool enable_sparse_flag) { enable_sparse_flag_ = enable_sparse_flag; } + private: MsContext(const std::string &backend_policy, const std::string &target); void GetGeOptions(std::map *ge_options) const; @@ -204,6 +207,7 @@ class MsContext { float max_device_memory_; std::string print_file_path_; bool enable_graph_kernel_; + bool enable_sparse_flag_; }; } // namespace mindspore diff --git a/mindspore/common/__init__.py b/mindspore/common/__init__.py index ead8aee556..c896805d75 100644 --- a/mindspore/common/__init__.py +++ b/mindspore/common/__init__.py @@ -17,10 +17,10 @@ from . import dtype from .api import ms_function from .dtype import * from .parameter import Parameter, ParameterTuple -from .tensor import MetaTensor, Tensor +from .tensor import MetaTensor, Tensor, IndexedSlices __all__ = [ - "MetaTensor", "Tensor", # tensor + "MetaTensor", "Tensor", "IndexedSlices", # tensor 'ms_function', # api 'Parameter', 'ParameterTuple', # parameter "dtype" diff --git a/mindspore/common/parameter.py b/mindspore/common/parameter.py index 773f6a99a6..571cc9cb40 100644 --- a/mindspore/common/parameter.py +++ b/mindspore/common/parameter.py @@ -52,13 +52,16 @@ class Parameter: layerwise_parallel (bool): A kind of model parallel mode. When layerwise_parallel is true in paralle mode, broadcast and gradients communication would not be applied on parameters. Default: False. sparse_grad (str): Set if the parameter's gradient is sparse. Default: empty. + has_indexed_slices (bool): Set if the parameter's gradient is indexed_slices. Default: false. """ - def __init__(self, default_input, name, requires_grad=True, layerwise_parallel=False, sparse_grad=""): + def __init__(self, default_input, name, requires_grad=True, layerwise_parallel=False, + sparse_grad="", has_indexed_slices_grad=False): self.set_parameter_data(default_input) self.name = name self.requires_grad = requires_grad self.layerwise_parallel = layerwise_parallel self.sparse_grad = sparse_grad + self.has_indexed_slices_grad = has_indexed_slices_grad self._is_init = False self._sliced = False self.clone_info = _CloneInfo() @@ -186,6 +189,17 @@ class Parameter: raise TypeError("`sparse_grad` parameter must be str type") self._sparse_grad = value + @property + def has_indexed_slices_grad(self): + """Return whether the parameter's gradient is indexed_slices.""" + return self._has_indexed_slices_grad + + @has_indexed_slices_grad.setter + def has_indexed_slices_grad(self, value=False): + if not isinstance(value, bool): + raise TypeError("`has_indexed_slices_grad` parameter must be bool type") + self._has_indexed_slices_grad = value + @property def data(self): return self.default_input diff --git a/mindspore/common/tensor.py b/mindspore/common/tensor.py index 92c600520f..4bb845af55 100644 --- a/mindspore/common/tensor.py +++ b/mindspore/common/tensor.py @@ -21,7 +21,7 @@ from .._checkparam import check_type, check_typename from . import dtype as mstype from ._register_for_tensor import tensor_operator_registry -__all__ = ['Tensor', 'MetaTensor'] +__all__ = ['Tensor', 'MetaTensor', 'IndexedSlices'] np_types = (np.int8, np.int16, np.int32, np.int64, np.uint8, np.uint16, np.uint32, np.uint64, np.float16, np.float32, np.float64, np.bool_) @@ -214,3 +214,8 @@ class Tensor(Tensor_): raise TypeError("init_flag must be bool.") self.set_init_flag(value) self._init_flag = value + + +class IndexedSlices: + def __init__(self, indices, values, dense_shape): + raise NotImplementedError diff --git a/mindspore/context.py b/mindspore/context.py index 070544c529..b5be6c3213 100644 --- a/mindspore/context.py +++ b/mindspore/context.py @@ -355,6 +355,14 @@ class _Context: def check_bprop(self, check_bprop_flag): self._context_handle.set_check_bprop_flag(check_bprop_flag) + @property + def enable_sparse(self): + return self._context_handle.get_enable_sparse_flag() + + @enable_sparse.setter + def enable_sparse(self, enable_sparse_flag): + self._context_handle.set_enable_sparse_flag(enable_sparse_flag) + @property def max_device_memory(self): return self._context_handle.get_max_device_memory() @@ -510,7 +518,8 @@ def reset_auto_parallel_context(): save_graphs_path=str, save_ms_model=bool, save_ms_model_path=str, enable_dump=bool, save_dump_path=str, enable_reduce_precision=bool, variable_memory_max_size=str, enable_profiling=bool, profiling_options=str, enable_auto_mixed_precision=bool, - enable_graph_kernel=bool, check_bprop=bool, max_device_memory=str, print_file_path=str) + enable_graph_kernel=bool, check_bprop=bool, max_device_memory=str, print_file_path=str, + enable_sparse=bool) def set_context(**kwargs): """ Sets context for running environment. @@ -567,6 +576,7 @@ def set_context(**kwargs): The format is "xxGB". Default: "1024GB". print_file_path (str): The path of print data to save. If this parameter is set, print data is saved to a file by default, and turn off printing to the screen. + enable_sparse (bool): Whether to enable sparse feature. Default: False. Raises: ValueError: If input key is not an attribute in context. diff --git a/mindspore/ops/functional.py b/mindspore/ops/functional.py index 840c4e745e..a5c3165ab1 100644 --- a/mindspore/ops/functional.py +++ b/mindspore/ops/functional.py @@ -153,6 +153,14 @@ shape_mul = Primitive("shape_mul") # a primitive to compare between tuple. stop_gradient = Primitive("stop_gradient") + +make_indexed_slices = Primitive('MakeIndexedSlices') +indexed_slices_get_values = Primitive('IndexedSlicesGetValues') +indexed_slices_get_indices = Primitive('IndexedSlicesGetIndices') +indexed_slices_get_dense_shape = Primitive('IndexedSlicesGetDenseShape') +is_indexed_slices = Primitive('IsIndexedSlices') + + tensor_operator_registry.register('__add__', tensor_add) tensor_operator_registry.register('__sub__', tensor_sub) tensor_operator_registry.register('__mul__', tensor_mul) diff --git a/mindspore/ops/operations/array_ops.py b/mindspore/ops/operations/array_ops.py index 8bac6dfbd3..f568c86b4c 100644 --- a/mindspore/ops/operations/array_ops.py +++ b/mindspore/ops/operations/array_ops.py @@ -564,7 +564,7 @@ class SparseGatherV2(GatherV2): >>> input_params = Tensor(np.array([[1, 2, 7, 42], [3, 4, 54, 22], [2, 2, 55, 3]]), mindspore.float32) >>> input_indices = Tensor(np.array([1, 2]), mindspore.int32) >>> axis = 1 - >>> out = P.GatherV2()(input_params, input_indices, axis) + >>> out = P.SparseGatherV2()(input_params, input_indices, axis) """ diff --git a/tests/ut/cpp/optimizer/lib_test.cc b/tests/ut/cpp/optimizer/lib_test.cc index ebbcdf6f7c..bc8561f171 100644 --- a/tests/ut/cpp/optimizer/lib_test.cc +++ b/tests/ut/cpp/optimizer/lib_test.cc @@ -603,5 +603,18 @@ TEST_F(TestOptLib, test_adjust_allreduce_mul_add) { ASSERT_TRUE(CheckOpt(before2l, after2, patterns)); ASSERT_TRUE(CheckOpt(before2r, after2, patterns)); } + +TEST_F(TestOptLib, test_indexed_slices) { + FuncGraphPtr before_get_indices = getPyFun.CallAndParseRet("test_indexed_slices", "before_get_indices"); + FuncGraphPtr after_get_indices = getPyFun.CallAndParseRet("test_indexed_slices", "after_get_indices"); + FuncGraphPtr before_get_values = getPyFun.CallAndParseRet("test_indexed_slices", "before_get_values"); + FuncGraphPtr after_get_values = getPyFun.CallAndParseRet("test_indexed_slices", "after_get_values"); + FuncGraphPtr before_get_dense_shape = getPyFun.CallAndParseRet("test_indexed_slices", "before_get_dense_shape"); + FuncGraphPtr after_get_dense_shape = getPyFun.CallAndParseRet("test_indexed_slices", "after_get_dense_shape"); + auto patterns = std::vector({irpass.indexed_slices_eliminate_}); + ASSERT_TRUE(CheckOpt(before_get_indices, after_get_indices, patterns)); + ASSERT_TRUE(CheckOpt(before_get_values, after_get_values, patterns)); + ASSERT_TRUE(CheckOpt(before_get_dense_shape, after_get_dense_shape, patterns)); +} } // namespace opt } // namespace mindspore diff --git a/tests/ut/cpp/python_input/gtest_input/optimizer/opt_test.py b/tests/ut/cpp/python_input/gtest_input/optimizer/opt_test.py index af8cab902c..22e2535819 100644 --- a/tests/ut/cpp/python_input/gtest_input/optimizer/opt_test.py +++ b/tests/ut/cpp/python_input/gtest_input/optimizer/opt_test.py @@ -1130,3 +1130,38 @@ def test_adjust_allreduce_mul_add(tag): return Mul(AllReduce(AddN((Mul(z, z), x))), y) return fns[tag] + + +def test_indexed_slices(tag): + """ test_add_zero """ + fns = FnDict() + make_indexed_slices = Primitive('MakeIndexedSlices') + indexed_slices_get_values = Primitive('IndexedSlicesGetValues') + indexed_slices_get_indices = Primitive('IndexedSlicesGetIndices') + indexed_slices_get_dense_shape = Primitive('IndexedSlicesGetDenseShape') + + @fns + def before_get_indices(x, y, z): + return indexed_slices_get_indices(make_indexed_slices(x, y, z)) + + @fns + def after_get_indices(x, y, z): + return x + + @fns + def before_get_values(x, y, z): + return indexed_slices_get_values(make_indexed_slices(x, y, z)) + + @fns + def after_get_values(x, y, z): + return y + + @fns + def before_get_dense_shape(x, y, z): + return indexed_slices_get_dense_shape(make_indexed_slices(x, y, z)) + + @fns + def after_get_dense_shape(x, y, z): + return z + + return fns[tag] diff --git a/tests/ut/python/ir/test_indexed_slices.py b/tests/ut/python/ir/test_indexed_slices.py new file mode 100644 index 0000000000..8690183090 --- /dev/null +++ b/tests/ut/python/ir/test_indexed_slices.py @@ -0,0 +1,290 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +@File : test_indexed_slices.py +@Author: +@Date : 2020-06-08 +@Desc : test mindspore indexed_slices's operation +""" +import numpy as np + +import mindspore as ms +import mindspore.nn as nn +from mindspore.ops import composite as C +from mindspore.ops import functional as F +from mindspore.ops import operations as P +from mindspore.ops.composite.multitype_ops.zeros_like_impl import zeros_like +from mindspore.ops.primitive import constexpr +from mindspore.ops._grad.grad_base import bprop_getters +from mindspore import Tensor, IndexedSlices, context +from mindspore.common.parameter import Parameter, ParameterTuple +from mindspore.common import dtype as mstype +from mindspore._checkparam import Validator as validator +from mindspore._checkparam import Rel +from mindspore.nn import Optimizer +from mindspore.nn import TrainOneStepCell, WithLossCell + +reduce_sum = P.ReduceSum() +unsorted_segment_sum = P.UnsortedSegmentSum() +transpose = P.Transpose() +shape_op = P.Shape() +reshape = P.Reshape() +size_op = P.Size() +invert_permutation = P.InvertPermutation() +logical_and = P.LogicalAnd() +context.set_context(mode=context.GRAPH_MODE, enable_sparse=True) + +@constexpr +def _generate_shape_index(out_shape, indices_shape, axis): + out_rank = len(out_shape) + ind_rank = len(indices_shape) + if axis < 0: + axis += out_rank - ind_rank + 1 + perm_part1 = tuple(range(axis, axis + ind_rank)) + index = tuple(range(out_rank)) + perm = perm_part1 + index[:axis] + index[axis + ind_rank:] + return perm + +@constexpr +def _generate_inverse_index(x_shape, axis): + x_rank = len(x_shape) + index = tuple(range(x_rank)) + if axis < 0: + axis += x_rank + perm = index[1:1 + axis] + (0,) + index[1 + axis:] + return perm + +class MySparseGatherV2(P.GatherV2): + """ + For test + """ + +@bprop_getters.register(MySparseGatherV2) +def get_bprop_sparse_gather_v2(self): + """Generate bprop for MySparseGatherV2""" + + def bprop(x, indices, axis, out, dout): + x_shp = shape_op(x) + if axis == 0: + indices_size = (size_op(indices),) + x_tail_shp = x_shp[1:] + values_shape = indices_size + x_tail_shp + values = reshape(dout, values_shape) + indices = reshape(indices, indices_size) + return IndexedSlices(indices, values, x_shp), zeros_like(indices), zeros_like(axis) + if F.rank(dout) == 0: + dout = P.ExpandDims()(dout, -1) + if F.rank(indices) == 0: + indices = P.ExpandDims()(indices, -1) + out_shp = shape_op(dout) + ind_shp = shape_op(indices) + # Example: out_shape:(3,2,3) axis 1 -> (1,0,2) + perm_1 = _generate_shape_index(out_shp, ind_shp, axis) + values_transpose = transpose(dout, perm_1) + params_grad = unsorted_segment_sum(values_transpose, indices, shape_op(x)[axis]) + # Example: out_shape:(3,2,3) axis 2 -> (1,2,0) + perm_2 = _generate_inverse_index(x_shp, axis) + params_grad = transpose(params_grad, perm_2) + return params_grad, zeros_like(indices), zeros_like(axis) + + return bprop + +adam_opt_for_map = C.MultitypeFuncGraph("adam_opt_for_map") +@adam_opt_for_map.register("Tensor", "Tensor", "Tensor", "Tensor", "Tensor", + "Tensor", "Tensor", "Tensor", "Undetermined", "Bool") +def _update_run_op_for_map(beta1, beta2, eps, lr, weight_decay_tensor, param, m, v, gradient, decay_flag): + if gradient.is_indexed_slices(): + return gradient.values() + op_mul = P.Mul() + op_square = P.Square() + op_sqrt = P.Sqrt() + op_cast = P.Cast() + op_reshape = P.Reshape() + op_shape = P.Shape() + + param_fp32 = op_cast(param, mstype.float32) + m_fp32 = op_cast(m, mstype.float32) + v_fp32 = op_cast(v, mstype.float32) + gradient_fp32 = op_cast(gradient, mstype.float32) + + next_m = op_mul(beta1, m_fp32) + op_mul(op_cast(F.tuple_to_array((1.0,)), mstype.float32) - beta1, gradient_fp32) + + next_v = op_mul(beta2, v_fp32) + op_mul(op_cast(F.tuple_to_array((1.0,)), mstype.float32) + - beta2, op_square(gradient_fp32)) + + update = next_m / (op_sqrt(next_v) + eps) + if decay_flag: + update = update + op_mul(weight_decay_tensor, param_fp32) + + update_with_lr = op_mul(lr, update) + next_param = param_fp32 - op_reshape(update_with_lr, op_shape(param_fp32)) + + next_v = F.depend(next_v, F.assign(param, next_param)) + next_v = F.depend(next_v, F.assign(m, next_m)) + next_v = F.depend(next_v, F.assign(v, next_v)) + return next_v + + +def _check_param_value(beta1, beta2, eps, weight_decay, prim_name): + """Check the type of inputs.""" + validator.check_value_type("beta1", beta1, [float], prim_name) + validator.check_value_type("beta2", beta2, [float], prim_name) + validator.check_value_type("eps", eps, [float], prim_name) + validator.check_value_type("weight_dacay", weight_decay, [float], prim_name) + validator.check_number_range("beta1", beta1, 0.0, 1.0, Rel.INC_NEITHER, prim_name) + validator.check_number_range("beta2", beta2, 0.0, 1.0, Rel.INC_NEITHER, prim_name) + validator.check_number_range("eps", eps, 0.0, float("inf"), Rel.INC_NEITHER, prim_name) + validator.check_number_range("weight_decay", weight_decay, 0.0, float("inf"), Rel.INC_LEFT, prim_name) + + +class AdamWeightDecaySparse(Optimizer): + def __init__(self, params, learning_rate=1e-3, beta1=0.9, beta2=0.999, eps=1e-6, weight_decay=0.0, + decay_filter=lambda x: 'beta' not in x.name and 'gamma' not in x.name): + super(AdamWeightDecaySparse, self).__init__(learning_rate, params) + if self.is_group: + raise RuntimeError(f"The {self.cls_name} optimizer cannot support group setting.") + _check_param_value(beta1, beta2, eps, weight_decay, self.cls_name) + self.beta1 = Tensor(np.array([beta1]).astype(np.float32)) + self.beta2 = Tensor(np.array([beta2]).astype(np.float32)) + self.eps = Tensor(np.array([eps]).astype(np.float32)) + self.weight_decay_tensor = Tensor(np.array([weight_decay]).astype(np.float32)) + + self.params = self.parameters + self.moments1 = self.params.clone(prefix="adam_m", init='zeros') + self.moments2 = self.params.clone(prefix="adam_v", init='zeros') + self.decay_flag = tuple(decay_filter(x) for x in self.params) + self.map = C.Map() + + def construct(self, gradients): + lr = self.get_lr() + updated_velocity = self.map(F.partial(adam_opt_for_map, self.beta1, self.beta2, self.eps, lr, + self.weight_decay_tensor), + self.params, self.moments1, self.moments2, gradients, self.decay_flag) + return updated_velocity + + +def test_indexed_slices_make_indexed_slices(): + class MakeIndexedSlices(nn.Cell): + def __init__(self): + super(MakeIndexedSlices, self).__init__() + self.dense_shape = (3, 4) + def construct(self, indices, values): + ret = (IndexedSlices(indices, values, self.dense_shape),) + return ret[0].is_indexed_slices() + indices = Tensor([[0, 0], [1, 2]]) + values = Tensor([1, 2], dtype=ms.float32) + MakeIndexedSlices()(indices, values) + + +def test_indexed_slices_attr(): + class IndexedSlicesGetAttr(nn.Cell): + def __init__(self): + super(IndexedSlicesGetAttr, self).__init__() + self.dense_shape = (3, 4) + def construct(self, indices, values): + x = IndexedSlices(indices, values, self.dense_shape) + return x.values(), x.indices(), x.dense_shape() + indices = Tensor([[0, 0], [1, 2]]) + values = Tensor([1, 2], dtype=ms.float32) + IndexedSlicesGetAttr()(indices, values) + + +def test_indexed_slices_sparse_gatherv2_grad_all(): + grad_all = C.GradOperation('get_all', get_all=True) + class GradWrap(nn.Cell): + def __init__(self, network): + super(GradWrap, self).__init__() + self.network = network + def construct(self, x, y): + grad = grad_all(self.network)(x, y) + return grad, grad[0].is_indexed_slices(), grad[1].is_indexed_slices() + class SparseGatherV2(nn.Cell): + def __init__(self): + super(SparseGatherV2, self).__init__() + self.sparse_gatherv2 = MySparseGatherV2() + self.axis = 0 + def construct(self, params, indices): + return self.sparse_gatherv2(params, indices, self.axis) + params = Tensor(np.ones([3, 1, 2]).astype(np.int32)) + indices = Tensor(np.array([0, 1]).astype(np.int32)) + GradWrap(SparseGatherV2())(params, indices) + + +def test_indexed_slices_sparse_gatherv2_grad_with_pram(): + grad_by_list = C.GradOperation('get_by_list', get_by_list=True) + class GradWrap(nn.Cell): + def __init__(self, network): + super(GradWrap, self).__init__() + self.network = network + self.weights = ParameterTuple(filter(lambda x: x.requires_grad, network.get_parameters())) + def construct(self, x): + weights = self.weights + grad = grad_by_list(self.network, weights)(x) + x = grad[0] + return x.is_indexed_slices(), x.values(), x.indices(), x.dense_shape() + class SparseGatherV2(nn.Cell): + def __init__(self): + super(SparseGatherV2, self).__init__() + self.sparse_gatherv2 = MySparseGatherV2() + self.axis = 0 + self.params = Parameter(Tensor(np.ones([3, 1, 2]).astype(np.int32)), + name="params", has_indexed_slices_grad=True) + def construct(self, indices): + return self.sparse_gatherv2(self.params, indices, self.axis) + indices = Tensor(np.array([0, 1]).astype(np.int32)) + network = GradWrap(SparseGatherV2()) + network(indices) + + +def test_indexed_slices_is_indexed_slices(): + class MakeIndexedSlices(nn.Cell): + def __init__(self): + super(MakeIndexedSlices, self).__init__() + self.dense_shape = (3, 4) + def construct(self, indices, values): + indexed_slices = IndexedSlices(indices, values, self.dense_shape) + ret = indexed_slices.is_indexed_slices() + return ret + indices = Tensor([[0, 0], [1, 2]]) + values = Tensor([1, 2], dtype=ms.float32) + MakeIndexedSlices()(indices, values) + + +def test_indexed_slices_env_get(): + class Loss(nn.Cell): + def __init__(self): + super(Loss, self).__init__() + def construct(self, base, target): + return base + class NetWithSparseGatherV2(nn.Cell): + def __init__(self): + super(NetWithSparseGatherV2, self).__init__() + self.w1 = Parameter(Tensor(np.ones([3, 1, 2]).astype(np.float32)), name="w1", has_indexed_slices_grad=True) + self.w2 = Parameter(Tensor(np.ones([2, 1, 2]).astype(np.float32)), name="w2") + self.gatherv2 = MySparseGatherV2() + self.axis = 0 + def construct(self, indices): + return self.gatherv2(self.w1, indices, self.axis) * self.w2 + + inputs = Tensor(np.array([0, 1]).astype(np.int32)) + label = Tensor(np.zeros([2, 1, 2]).astype(np.float32)) + net = NetWithSparseGatherV2() + net.set_train() + loss = Loss() + optimizer = AdamWeightDecaySparse(net.trainable_params()) + + net_with_loss = WithLossCell(net, loss) + train_network = TrainOneStepCell(net_with_loss, optimizer) + train_network(inputs, label) diff --git a/tests/ut/python/nn/optim/test_adam_with_tuple_grad.py b/tests/ut/python/nn/optim/test_adam_with_tuple_grad.py index 5222f920ba..7f9f341a93 100644 --- a/tests/ut/python/nn/optim/test_adam_with_tuple_grad.py +++ b/tests/ut/python/nn/optim/test_adam_with_tuple_grad.py @@ -155,7 +155,7 @@ def test_AdamWeightDecaySparse(): def __init__(self): super(NetWithSparseGatherV2, self).__init__() self.w1 = Parameter(Tensor(np.ones([3, 1, 2]).astype(np.float32)), name="w1", sparse_grad="sparse_key_w1") - self.w2 = Parameter(Tensor(np.ones([2, 1, 2]).astype(np.float32)), name="w2", sparse_grad="sparse_key_w2") + self.w2 = Parameter(Tensor(np.ones([2, 1, 2]).astype(np.float32)), name="w2") self.gatherv2 = P.SparseGatherV2() self.axis = 0 def construct(self, indices): From 53757d5f284646c0e4f149dcf7e4f92baedf99e8 Mon Sep 17 00:00:00 2001 From: jonyguo Date: Tue, 30 Jun 2020 17:58:59 +0800 Subject: [PATCH 171/254] update release notes --- RELEASE.md | 1 + 1 file changed, 1 insertion(+) diff --git a/RELEASE.md b/RELEASE.md index eb390d3624..4b829152a2 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -44,6 +44,7 @@ * Bert, Move Bert from `example` to `model_zoo`, optimize network for better performance. ([!1902](https://gitee.com/mindspore/mindspore/pulls/1902)) * VGG16, Move VGG16 from `example` to `model_zoo`, optimize network for better accuracy. ([!2645](https://gitee.com/mindspore/mindspore/pulls/2645)) * Alexnet, modify parameter setting to improve accuracy ([!1364](https://gitee.com/mindspore/mindspore/pulls/2370)) + * Wide&Deep, Move Wide&Deep from `example` to `model_zoo`, optimize network for better performance. ([!2221](https://gitee.com/mindspore/mindspore/pulls/2221)) * Python API * Fix bug in auto cast([!1766](https://gitee.com/mindspore/mindspore/pulls/1766)) * Fix bug of register_backward_hook([!2148](https://gitee.com/mindspore/mindspore/pulls/2148)) From f49b8ad6d2bd8ecc889aa4f36f117376002e7362 Mon Sep 17 00:00:00 2001 From: VectorSL Date: Tue, 30 Jun 2020 15:12:14 +0800 Subject: [PATCH 172/254] update argmaxwithvalue --- .../gpu/arrays/argmaxwithvalue_gpu_kernel.h | 2 +- .../gpu/cuda_impl/argmaxwithvalue_impl.cu | 46 +++++++++---------- .../gpu/cuda_impl/argmaxwithvalue_impl.cuh | 4 +- .../kernel/gpu/cuda_impl/unary_op_impl.cu | 7 +++ 4 files changed, 32 insertions(+), 27 deletions(-) diff --git a/mindspore/ccsrc/kernel/gpu/arrays/argmaxwithvalue_gpu_kernel.h b/mindspore/ccsrc/kernel/gpu/arrays/argmaxwithvalue_gpu_kernel.h index fb7796b022..304f0ab161 100644 --- a/mindspore/ccsrc/kernel/gpu/arrays/argmaxwithvalue_gpu_kernel.h +++ b/mindspore/ccsrc/kernel/gpu/arrays/argmaxwithvalue_gpu_kernel.h @@ -38,7 +38,7 @@ class ArgmaxWithValueGpuKernel : public GpuKernel { T *input = GetDeviceAddress(inputs, 0); T *output = GetDeviceAddress(outputs, 1); S *index = GetDeviceAddress(outputs, 0); - CalArgmaxWithValue(input_size_ / sizeof(T), input, bound_, outerSize_, innerSize_, index, output, + CalArgmaxWithValue(input, bound_, outerSize_, innerSize_, index, output, reinterpret_cast(stream_ptr)); return true; } diff --git a/mindspore/ccsrc/kernel/gpu/cuda_impl/argmaxwithvalue_impl.cu b/mindspore/ccsrc/kernel/gpu/cuda_impl/argmaxwithvalue_impl.cu index a0687a2768..3313fc6853 100644 --- a/mindspore/ccsrc/kernel/gpu/cuda_impl/argmaxwithvalue_impl.cu +++ b/mindspore/ccsrc/kernel/gpu/cuda_impl/argmaxwithvalue_impl.cu @@ -18,41 +18,39 @@ #include "device/gpu/cuda_common.h" #include "include/cuda_fp16.h" template -__global__ void ArgmaxWithValue(size_t size, const T* input, const int bound, int outerSize, int innerSize, - S* index, T* output) { - for (size_t pos = blockIdx.x * blockDim.x + threadIdx.x; pos < (size); pos += blockDim.x * gridDim.x) { - for (int i = 0; i < outerSize; i++) { - int inputOutterOffset = i * innerSize * bound; - int outputOutterOffset = i * innerSize; - for (int j = 0; j < innerSize; j++) { - auto outputInnerOffset = outputOutterOffset + j; - S idx = 0; - T maxData = input[j + inputOutterOffset]; - for (S c = 0; c < bound; c++) { - int offset = j + c * innerSize; - auto inputData = input[inputOutterOffset + offset]; - idx = inputData > maxData ? c : idx; - maxData = inputData > maxData ? inputData : maxData; - } - output[outputInnerOffset] = maxData; - index[outputInnerOffset] = idx; +__global__ void ArgmaxWithValue(const T* input, const int bound, int outerSize, int innerSize, S* index, + T* output) { + for (size_t pos = blockIdx.x * blockDim.x + threadIdx.x; pos < (outerSize); pos += blockDim.x * gridDim.x) { + int inputOutterOffset = pos * innerSize * bound; + int outputOutterOffset = pos * innerSize; + for (int j = 0; j < innerSize; j++) { + auto outputInnerOffset = outputOutterOffset + j; + S idx = 0; + T maxData = input[j + inputOutterOffset]; + for (S c = 0; c < bound; c++) { + int offset = j + c * innerSize; + auto inputData = input[inputOutterOffset + offset]; + idx = inputData > maxData ? c : idx; + maxData = inputData > maxData ? inputData : maxData; + } + output[outputInnerOffset] = maxData; + index[outputInnerOffset] = idx; } - } } return; } template -void CalArgmaxWithValue(size_t size, const T* input, const int bound_, const int outerSize_, const int innerSize_, +void CalArgmaxWithValue(const T* input, const int bound_, const int outerSize_, const int innerSize_, S* index, T* output, cudaStream_t cuda_stream) { - ArgmaxWithValue<<>>(size, input, bound_, outerSize_, innerSize_, - index, output); + ArgmaxWithValue<<>>(input, bound_, outerSize_, innerSize_, + index, output); return; } -template void CalArgmaxWithValue(size_t size, const float* input, const int bound_, const int outerSize_, +template void CalArgmaxWithValue(const float* input, const int bound_, const int outerSize_, const int innerSize_, int* index, float* output, cudaStream_t cuda_stream); -template void CalArgmaxWithValue(size_t size, const half* input, const int bound_, const int outerSize_, +template void CalArgmaxWithValue(const half* input, const int bound_, const int outerSize_, const int innerSize_, int* index, half* output, cudaStream_t cuda_stream); diff --git a/mindspore/ccsrc/kernel/gpu/cuda_impl/argmaxwithvalue_impl.cuh b/mindspore/ccsrc/kernel/gpu/cuda_impl/argmaxwithvalue_impl.cuh index 0d4f4b62a3..67c061a966 100644 --- a/mindspore/ccsrc/kernel/gpu/cuda_impl/argmaxwithvalue_impl.cuh +++ b/mindspore/ccsrc/kernel/gpu/cuda_impl/argmaxwithvalue_impl.cuh @@ -17,6 +17,6 @@ #ifndef MINDSPORE_CCSRC_KERNEL_GPU_CUDA_IMP_ARGMAXWITHVALUE_H_ #define MINDSPORE_CCSRC_KERNEL_GPU_CUDA_IMP_ARGMAXWITHVALUE_H_ template -void CalArgmaxWithValue(size_t size, const T* input, const int bound_, const int outerSize_, const int innerSize_, - S* index, T* output, cudaStream_t cuda_stream); +void CalArgmaxWithValue(const T *input, const int bound_, const int outerSize_, const int innerSize_, S *index, + T *output, cudaStream_t cuda_stream); #endif // MINDSPORE_CCSRC_KERNEL_GPU_CUDA_IMP_ARGMAXWITHVALUE_H_ diff --git a/mindspore/ccsrc/kernel/gpu/cuda_impl/unary_op_impl.cu b/mindspore/ccsrc/kernel/gpu/cuda_impl/unary_op_impl.cu index 4a8af83aa4..09b347e3d5 100755 --- a/mindspore/ccsrc/kernel/gpu/cuda_impl/unary_op_impl.cu +++ b/mindspore/ccsrc/kernel/gpu/cuda_impl/unary_op_impl.cu @@ -36,6 +36,13 @@ __global__ void LogarithmKernel(T *input, T *output, size_t count) { } return; } +template <> +__global__ void LogarithmKernel(half *input, half *output, size_t count) { + for (size_t i = blockIdx.x * blockDim.x + threadIdx.x; i < (count); i += blockDim.x * gridDim.x) { + output[i] = hlog(input[i]); + } + return; +} template __global__ void NegativeKernel(T *input, T *output, size_t count) { T neg_one = -1; From bf8e7f8131af476a118e1aa85aca3dfcbc3ed882 Mon Sep 17 00:00:00 2001 From: Jesse Lee Date: Tue, 30 Jun 2020 07:07:48 -0400 Subject: [PATCH 173/254] Gcc 9 fix --- mindspore/ccsrc/dataset/util/queue.h | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/mindspore/ccsrc/dataset/util/queue.h b/mindspore/ccsrc/dataset/util/queue.h index 2d55ded736..7fca93d944 100644 --- a/mindspore/ccsrc/dataset/util/queue.h +++ b/mindspore/ccsrc/dataset/util/queue.h @@ -71,7 +71,7 @@ class Queue { arr_(nullptr), head_(0), tail_(0), - my_name_(std::move(Services::GetUniqueID())), + my_name_(Services::GetUniqueID()), alloc_(Services::GetInstance().GetServiceMemPool()) { Init(); MS_LOG(DEBUG) << "Create Q with uuid " << my_name_ << " of size " << sz_ << "."; @@ -154,7 +154,16 @@ class Queue { uint32_t k = head_++ % sz_; *p = std::move(arr_[k]); if (std::is_destructible::value) { + // std::move above only changes arr_[k] from rvalue to lvalue. + // The real implementation of move constructor depends on T. + // It may be compiler generated or user defined. But either case + // the result of arr_[k] is still a valid object of type T, and + // we will not keep any extra copy in the queue. arr_[k].~T(); + // For gcc 9, an extra fix is needed here to clear the memory content + // of arr_[k] because this slot can be reused by another Add which can + // do another std::move. We have seen SEGV here in this case. + std::allocator_traits>::construct(alloc_, &(arr_[k])); } full_cv_.NotifyAll(); _lock.unlock(); From 2a31e52bf14031458583ca596ef98be662ce70e7 Mon Sep 17 00:00:00 2001 From: yujianfeng Date: Tue, 30 Jun 2020 19:25:34 +0800 Subject: [PATCH 174/254] Adjust the thread num to compute optimizer in cpu --- mindspore/ccsrc/kernel/common_utils.cc | 7 ++++--- mindspore/ccsrc/kernel/common_utils.h | 2 +- mindspore/ccsrc/kernel/cpu/sparse_apply_adam_cpu_kernel.cc | 7 +++---- mindspore/ccsrc/kernel/cpu/sparse_apply_ftrl_cpu_kernel.cc | 3 +-- .../ccsrc/kernel/cpu/sparse_apply_lazy_adam_cpu_kernel.cc | 3 +-- .../kernel/cpu/sparse_apply_proximal_adagrad_cpu_kernel.cc | 3 +-- 6 files changed, 11 insertions(+), 14 deletions(-) diff --git a/mindspore/ccsrc/kernel/common_utils.cc b/mindspore/ccsrc/kernel/common_utils.cc index 012fecbadb..ab4f59e549 100644 --- a/mindspore/ccsrc/kernel/common_utils.cc +++ b/mindspore/ccsrc/kernel/common_utils.cc @@ -876,12 +876,13 @@ bool IsWeightBoundary(const AnfNodePtr &node) { return false; } -void MultiThreadCompute(const MultiThreadComputeFunc &func, MultiThreadComputeParams *params, size_t thread_num, +void MultiThreadCompute(const MultiThreadComputeFunc &func, MultiThreadComputeParams *params, size_t total_compute_size) { + const size_t kThreadNum = 24; std::vector threads; - threads.reserve(thread_num); + threads.reserve(kThreadNum); size_t start = 0; - size_t once_compute_size = (total_compute_size + thread_num - 1) / thread_num; + size_t once_compute_size = (total_compute_size + kThreadNum - 1) / kThreadNum; while (start < total_compute_size) { size_t end = (start + once_compute_size) > total_compute_size ? total_compute_size : (start + once_compute_size); threads.emplace_back(std::thread(func, params, start, end)); diff --git a/mindspore/ccsrc/kernel/common_utils.h b/mindspore/ccsrc/kernel/common_utils.h index e25421c57d..e9d72848f6 100644 --- a/mindspore/ccsrc/kernel/common_utils.h +++ b/mindspore/ccsrc/kernel/common_utils.h @@ -128,7 +128,7 @@ void GetValidKernelNodes(const FuncGraphPtr &func_graph, std::vector bool GetInputTensorValue(const AnfNodePtr &anf_node, size_t input_idx, nlohmann::json *const node_json); void GetGraphRealOutput(const FuncGraphPtr &func_graph, std::vector> *node_list); bool IsWeightBoundary(const AnfNodePtr &node); -void MultiThreadCompute(const MultiThreadComputeFunc &func, MultiThreadComputeParams *params, size_t thread_num, +void MultiThreadCompute(const MultiThreadComputeFunc &func, MultiThreadComputeParams *params, size_t total_compute_size); } // namespace kernel } // namespace mindspore diff --git a/mindspore/ccsrc/kernel/cpu/sparse_apply_adam_cpu_kernel.cc b/mindspore/ccsrc/kernel/cpu/sparse_apply_adam_cpu_kernel.cc index 7405ec02ec..ef3db78275 100644 --- a/mindspore/ccsrc/kernel/cpu/sparse_apply_adam_cpu_kernel.cc +++ b/mindspore/ccsrc/kernel/cpu/sparse_apply_adam_cpu_kernel.cc @@ -155,15 +155,14 @@ bool SparseApplyAdamCPUKernel::Launch(const std::vector &inp input_params.v_ = v; input_params.beta1_ = beta1; input_params.beta2_ = beta2; - const size_t kThreadNum = 16; - MultiThreadCompute(ComputeMomentum, &input_params, kThreadNum, total_dim_size); + MultiThreadCompute(ComputeMomentum, &input_params, total_dim_size); input_params.m_t_ = m_t; input_params.use_nesterov_ = use_nesterov_; input_params.sparse_grad_ = unique_sparse_grad; input_params.var_first_dim_size_ = var_first_dim_size_; input_params.var_outer_dim_size_ = var_outer_dim_size_; - MultiThreadCompute(ComputeAdam, &input_params, kThreadNum, unique_sparse_grad.indices_size_); + MultiThreadCompute(ComputeAdam, &input_params, unique_sparse_grad.indices_size_); if (use_nesterov_) { input_params.m_ = input_params.m_t_; @@ -171,7 +170,7 @@ bool SparseApplyAdamCPUKernel::Launch(const std::vector &inp input_params.var_ = var; input_params.lr_ = lr; input_params.epsilon_ = epsilon; - MultiThreadCompute(ComputeWeight, &input_params, kThreadNum, total_dim_size); + MultiThreadCompute(ComputeWeight, &input_params, total_dim_size); return true; } } // namespace kernel diff --git a/mindspore/ccsrc/kernel/cpu/sparse_apply_ftrl_cpu_kernel.cc b/mindspore/ccsrc/kernel/cpu/sparse_apply_ftrl_cpu_kernel.cc index af014022d1..0537e746f3 100644 --- a/mindspore/ccsrc/kernel/cpu/sparse_apply_ftrl_cpu_kernel.cc +++ b/mindspore/ccsrc/kernel/cpu/sparse_apply_ftrl_cpu_kernel.cc @@ -145,8 +145,7 @@ bool SparseApplyFtrlCPUKernel::Launch(const std::vector &inp input_params.sparse_grad_ = unique_sparse_grad; input_params.var_first_dim_size_ = var_first_dim_size_; input_params.var_outer_dim_size_ = var_outer_dim_size_; - const size_t kThreadNum = 16; - MultiThreadCompute(ComputeFtrl, &input_params, kThreadNum, unique_sparse_grad.indices_size_); + MultiThreadCompute(ComputeFtrl, &input_params, unique_sparse_grad.indices_size_); return true; } } // namespace kernel diff --git a/mindspore/ccsrc/kernel/cpu/sparse_apply_lazy_adam_cpu_kernel.cc b/mindspore/ccsrc/kernel/cpu/sparse_apply_lazy_adam_cpu_kernel.cc index 2460dc0f27..16cb901b04 100644 --- a/mindspore/ccsrc/kernel/cpu/sparse_apply_lazy_adam_cpu_kernel.cc +++ b/mindspore/ccsrc/kernel/cpu/sparse_apply_lazy_adam_cpu_kernel.cc @@ -139,8 +139,7 @@ bool SparseApplyLazyAdamCPUKernel::Launch(const std::vector input_params.sparse_grad_ = unique_sparse_grad; input_params.var_first_dim_size_ = var_first_dim_size_; input_params.var_outer_dim_size_ = var_outer_dim_size_; - const size_t kThreadNum = 16; - MultiThreadCompute(ComputeLazyAdam, &input_params, kThreadNum, unique_sparse_grad.indices_size_); + MultiThreadCompute(ComputeLazyAdam, &input_params, unique_sparse_grad.indices_size_); return true; } } // namespace kernel diff --git a/mindspore/ccsrc/kernel/cpu/sparse_apply_proximal_adagrad_cpu_kernel.cc b/mindspore/ccsrc/kernel/cpu/sparse_apply_proximal_adagrad_cpu_kernel.cc index 64cb65764f..6069fb708e 100644 --- a/mindspore/ccsrc/kernel/cpu/sparse_apply_proximal_adagrad_cpu_kernel.cc +++ b/mindspore/ccsrc/kernel/cpu/sparse_apply_proximal_adagrad_cpu_kernel.cc @@ -132,8 +132,7 @@ bool SparseApplyProximalAdagradCPUKernel::Launch(const std::vector Date: Tue, 30 Jun 2020 19:37:10 +0800 Subject: [PATCH 175/254] update serving/ms_service.proto. add copyright --- serving/ms_service.proto | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/serving/ms_service.proto b/serving/ms_service.proto index 6b03896b76..f11bc235e7 100644 --- a/serving/ms_service.proto +++ b/serving/ms_service.proto @@ -1,3 +1,19 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + // ms_service.proto syntax = "proto3"; From cf8ea2cb665d31af970f3b4f00f5a63a921a1dc9 Mon Sep 17 00:00:00 2001 From: tronzhang <6517937+tronzhang@user.noreply.gitee.com> Date: Tue, 30 Jun 2020 15:13:31 +0800 Subject: [PATCH 176/254] clean review bot warning --- .../ccsrc/kernel/akg/akg_kernel_build.cc | 1 - mindspore/ccsrc/kernel/akg/akg_kernel_build.h | 1 - .../akg/ascend/akg_ascend_kernel_build.cc | 325 ++++++++++-------- .../akg/ascend/akg_ascend_kernel_build.h | 4 + .../akg/ascend/akg_ascend_kernel_mod.cc | 2 +- 5 files changed, 186 insertions(+), 147 deletions(-) diff --git a/mindspore/ccsrc/kernel/akg/akg_kernel_build.cc b/mindspore/ccsrc/kernel/akg/akg_kernel_build.cc index 1f88bbb89a..6bd1e7747c 100644 --- a/mindspore/ccsrc/kernel/akg/akg_kernel_build.cc +++ b/mindspore/ccsrc/kernel/akg/akg_kernel_build.cc @@ -618,6 +618,5 @@ size_t AkgKernelBuild::GetOutputTensorIdxInc() { size_t idx = output_tensor_idx_++; return idx; } - } // namespace kernel } // namespace mindspore diff --git a/mindspore/ccsrc/kernel/akg/akg_kernel_build.h b/mindspore/ccsrc/kernel/akg/akg_kernel_build.h index d32bd48ce6..15fa03f45b 100644 --- a/mindspore/ccsrc/kernel/akg/akg_kernel_build.h +++ b/mindspore/ccsrc/kernel/akg/akg_kernel_build.h @@ -70,7 +70,6 @@ void SetTensorName(const std::string &tag, const std::string &new_name, const st nlohmann::json *const node_json); std::string GetTensorName(const nlohmann::json &node_json, const std::string &tag, const std::pair &position); - } // namespace kernel } // namespace mindspore diff --git a/mindspore/ccsrc/kernel/akg/ascend/akg_ascend_kernel_build.cc b/mindspore/ccsrc/kernel/akg/ascend/akg_ascend_kernel_build.cc index 454b8052ab..7200a91ac0 100644 --- a/mindspore/ccsrc/kernel/akg/ascend/akg_ascend_kernel_build.cc +++ b/mindspore/ccsrc/kernel/akg/ascend/akg_ascend_kernel_build.cc @@ -35,7 +35,6 @@ namespace mindspore { namespace kernel { - constexpr int32_t PARALLEL_ARGS_SIZE = 3; constexpr int32_t PROCESS_NUM = 16; constexpr int32_t TIME_OUT = 300; @@ -48,76 +47,9 @@ constexpr auto kOutputDesc = "output_desc"; constexpr auto kTensorName = "tensor_name"; constexpr auto kCompileAkgKernelParallelFunc = "compile_akg_kernel_parallel"; constexpr auto kMultiProcModule = "mindspore._extends.parallel_compile.akg_compiler.multi_process_compiler"; - -bool AkgAscendKernelBuilder::CollectJson(const AnfNodePtr &anf_node) { - MS_EXCEPTION_IF_NULL(anf_node); - std::string op_name = AnfAlgo::GetCNodeName(anf_node); - MS_LOG(INFO) << "AKG start compile, op[" << op_name << "], device[" << AkgKernelBuild::GetProcessor(anf_node) << "]"; - auto it = kAkgKernelAttrsProcessMap.find(op_name); - if (it != kAkgKernelAttrsProcessMap.end()) { - it->second(anf_node); - } - MS_LOG(INFO) << "Akg start compile, op[" << op_name << "], device[" << AkgKernelBuild::GetProcessor(anf_node) << "]"; - nlohmann::json node_json; - if (!GenerateSingleKernelJson(anf_node, op_name, &node_json)) { - MS_LOG(ERROR) << "Op[" << op_name << "] create single kernel json failed."; - } - - kernel_json_ = node_json.dump(); - - if (!GetIOSize(node_json, &input_size_list_, &output_size_list_)) { - MS_LOG(ERROR) << "Cal mem size failed."; - return false; - } - - return true; -} - -bool AkgAscendKernelBuilder::CollectFusedJson(const std::vector &anf_nodes, - const std::vector &input_list, - const std::vector &output_list) { - if (anf_nodes.empty() || input_list.empty()) { - MS_LOG(ERROR) << "Invalid input size, anf_nodes [" << anf_nodes.size() << "], input_list [" << input_list.size() - << "]."; - return false; - } - MS_LOG(INFO) << "anf_nodes [" << output_list.size() << "], input_list [" << anf_nodes.size() << "], output_list [" - << input_list.size() << "]."; - - std::map node_json_map; - - for (auto const &anf_node : anf_nodes) { - MS_EXCEPTION_IF_NULL(anf_node); - std::string op_name = AnfAlgo::GetCNodeName(anf_node); - if (!AnfAlgo::IsRealKernel(anf_node)) { - MS_LOG(ERROR) << "Invalid anf node to build [" << anf_node->fullname_with_scope() << "]."; - return false; - } - auto it = kAkgKernelAttrsProcessMap.find(op_name); - if (it != kAkgKernelAttrsProcessMap.end()) { - it->second(anf_node); - } - - nlohmann::json node_json; - if (!GenerateSingleKernelJson(anf_node, op_name, &node_json)) { - MS_LOG(ERROR) << "Op [" << op_name << "] create single kernel json failed."; - return false; - } - // No need for composite op. - node_json.erase("id"); - node_json.erase("op"); - node_json.erase("composite"); - - auto primitive = AnfAlgo::GetCNodePrimitive(anf_node); - MS_EXCEPTION_IF_NULL(primitive); - - if (primitive->GetAttr("fusion") != nullptr) { - node_json["fusion"] = primitive->GetAttr("fusion")->ToString(); - } - - node_json_map[anf_node] = node_json; - } - +namespace { +void UpdateTensorNameInJson(const std::vector &anf_nodes, + std::map *node_json_map) { for (auto const &anf_node : anf_nodes) { std::vector dyn_input_sizes; auto primitive = AnfAlgo::GetCNodePrimitive(anf_node); @@ -134,11 +66,11 @@ bool AkgAscendKernelBuilder::CollectFusedJson(const std::vector &anf size_t input_tensor_num = is_dynamic_input ? IntToSize(dyn_input_sizes[i]) : 1; for (size_t j = 0; j < input_tensor_num; ++j) { auto tmp_input = GetKernelInput(anf_node, real_input_index); - std::string tensor_name = GetTensorName(node_json_map[anf_node], kInputDesc, std::make_pair(i, j)); - if (node_json_map.find(tmp_input.first) != node_json_map.end()) { + std::string tensor_name = GetTensorName((*node_json_map)[anf_node], kInputDesc, std::make_pair(i, j)); + if (node_json_map->find(tmp_input.first) != node_json_map->end()) { std::string new_tensor_name = - GetTensorName(node_json_map[tmp_input.first], kOutputDesc, std::make_pair(0, tmp_input.second)); - SetTensorName(kInputDesc, new_tensor_name, std::make_pair(i, j), &(node_json_map[anf_node])); + GetTensorName((*node_json_map)[tmp_input.first], kOutputDesc, std::make_pair(0, tmp_input.second)); + SetTensorName(kInputDesc, new_tensor_name, std::make_pair(i, j), &((*node_json_map)[anf_node])); MS_LOG(DEBUG) << "Update [" << real_input_index << "] input [" << tensor_name << "] of [" << anf_node->fullname_with_scope() << "] to [" << tmp_input.second << "] output [" << new_tensor_name << "] of [" << tmp_input.first->fullname_with_scope() << "]."; @@ -150,13 +82,10 @@ bool AkgAscendKernelBuilder::CollectFusedJson(const std::vector &anf } } } +} - nlohmann::json fused_node_json; - std::vector node_json_desc; - std::transform(anf_nodes.begin(), anf_nodes.end(), std::back_inserter(node_json_desc), - [&node_json_map](const AnfNodePtr &anf_node) { return node_json_map[anf_node]; }); - fused_node_json[kOpDesc] = node_json_desc; - +nlohmann::json GetInputsJson(const std::vector &anf_nodes, const std::vector &input_list, + std::map *node_json_map) { nlohmann::json inputs_json; auto input_index = GetInputIndex(anf_nodes, input_list); for (size_t i = 0; i < input_index.size(); ++i) { @@ -164,13 +93,18 @@ bool AkgAscendKernelBuilder::CollectFusedJson(const std::vector &anf auto type_id = AnfAlgo::GetInputDeviceDataType(tmp_input.first, tmp_input.second.first); std::string dtype = TypeId2String(type_id); nlohmann::json input_desc_json; - input_desc_json[kTensorName] = GetTensorName(node_json_map[tmp_input.first], kInputDesc, tmp_input.second); + input_desc_json[kTensorName] = GetTensorName((*node_json_map)[tmp_input.first], kInputDesc, tmp_input.second); input_desc_json[kDataType] = dtype; input_desc_json[kShape] = AnfAlgo::GetInputDeviceShape(tmp_input.first, tmp_input.second.first); inputs_json.emplace_back(std::vector{input_desc_json}); } - fused_node_json[kInputDesc] = inputs_json; + return inputs_json; +} + +nlohmann::json GetOutputsJson(const std::vector &anf_nodes, const std::vector &input_list, + const std::vector &output_list, const nlohmann::json &inputs_json, + std::map *node_json_map) { nlohmann::json outputs_json; auto output_index = GetOutputIndex(anf_nodes, input_list, output_list); for (size_t i = 0; i < output_index.size(); ++i) { @@ -188,7 +122,7 @@ bool AkgAscendKernelBuilder::CollectFusedJson(const std::vector &anf auto type_id = AnfAlgo::GetOutputDeviceDataType(tmp_output.first, tmp_output.second); std::string dtype = TypeId2String(type_id); output_desc_json[kTensorName] = - GetTensorName(node_json_map[tmp_output.first], kOutputDesc, std::make_pair(0, tmp_output.second)); + GetTensorName((*node_json_map)[tmp_output.first], kOutputDesc, std::make_pair(0, tmp_output.second)); output_desc_json[kDataType] = dtype; auto output_shape = AnfAlgo::GetOutputDeviceShape(tmp_output.first, tmp_output.second); if (output_shape.empty()) { @@ -198,7 +132,166 @@ bool AkgAscendKernelBuilder::CollectFusedJson(const std::vector &anf } outputs_json.emplace_back(output_desc_json); } - fused_node_json[kOutputDesc] = outputs_json; + + return outputs_json; +} + +std::pair, std::vector>> PreProcessJsonForBuild( + const std::vector> &build_args) { + // Remove cached nodes, gether unique nodes, and collect repeated nodes which need postprecess. + std::vector jsons; + std::vector> repeat_nodes; + std::unordered_set json_name_set; + for (const auto &[builder, anf_node] : build_args) { + MS_EXCEPTION_IF_NULL(anf_node); + auto json_name = builder.json_name(); + MS_LOG(DEBUG) << "Akg start compile op: " << json_name; + auto cached_kernel_pack = tbe::TbeUtils::SearchCache(json_name, AkgKernelBuild::GetProcessor(anf_node)); + if (cached_kernel_pack != nullptr) { + MS_LOG(DEBUG) << "Use cached kernel, json_name_[" << json_name << "], fullname_with_scope[" + << anf_node->fullname_with_scope() << "]."; + auto kernel_mod_ptr = std::make_shared(cached_kernel_pack); + kernel_mod_ptr->SetInputSizeList(builder.input_size_list()); + kernel_mod_ptr->SetOutputSizeList(builder.output_size_list()); + AnfAlgo::SetKernelMod(kernel_mod_ptr, anf_node.get()); + continue; + } + + if (json_name_set.count(json_name) != 0) { + repeat_nodes.push_back({builder, anf_node}); + continue; + } + json_name_set.insert(json_name); + auto node_json = builder.kernel_json(); + kernel::SaveJsonInfo(json_name, node_json); + jsons.push_back(node_json); + } + + return std::make_pair(jsons, repeat_nodes); +} + +bool PostProcessAfterCompile(const std::vector> &build_args, + const std::vector> &repeat_nodes) { + for (const auto &[builder, anf_node] : build_args) { + auto json_name = builder.json_name(); + auto new_kernel_pack = tbe::TbeUtils::InsertCache(json_name, AkgKernelBuild::GetProcessor(anf_node)); + if (new_kernel_pack == nullptr) { + MS_LOG(ERROR) << "Insert to cache failed, json_name_[" << json_name << "], fullname_with_scope[" + << anf_node->fullname_with_scope() << "]."; + return false; + } + auto kernel_mod_ptr = std::make_shared(new_kernel_pack); + kernel_mod_ptr->SetInputSizeList(builder.input_size_list()); + kernel_mod_ptr->SetOutputSizeList(builder.output_size_list()); + AnfAlgo::SetKernelMod(kernel_mod_ptr, anf_node.get()); + MS_LOG(DEBUG) << "Akg compile " << json_name << " kernel and insert cache successfully!"; + } + + for (const auto &[builder, anf_node] : repeat_nodes) { + auto node_json = builder.kernel_json(); + auto json_name = builder.json_name(); + auto cached_kernel_pack = tbe::TbeUtils::SearchCache(json_name, AkgKernelBuild::GetProcessor(anf_node)); + if (cached_kernel_pack == nullptr) { + return false; + } + MS_LOG(INFO) << "Use just compiled kernel, json_name_[" << json_name << "], fullname_with_scope[" + << anf_node->fullname_with_scope() << "]."; + auto kernel_mod_ptr = std::make_shared(cached_kernel_pack); + kernel_mod_ptr->SetInputSizeList(builder.input_size_list()); + kernel_mod_ptr->SetOutputSizeList(builder.output_size_list()); + AnfAlgo::SetKernelMod(kernel_mod_ptr, anf_node.get()); + } + + return true; +} +} // namespace + +bool AkgAscendKernelBuilder::CollectJson(const AnfNodePtr &anf_node) { + MS_EXCEPTION_IF_NULL(anf_node); + std::string op_name = AnfAlgo::GetCNodeName(anf_node); + MS_LOG(INFO) << "AKG start compile, op[" << op_name << "], device[" << AkgKernelBuild::GetProcessor(anf_node) << "]"; + auto it = kAkgKernelAttrsProcessMap.find(op_name); + if (it != kAkgKernelAttrsProcessMap.end()) { + it->second(anf_node); + } + MS_LOG(INFO) << "Akg start compile, op[" << op_name << "], device[" << AkgKernelBuild::GetProcessor(anf_node) << "]"; + nlohmann::json node_json; + if (!GenerateSingleKernelJson(anf_node, op_name, &node_json)) { + MS_LOG(ERROR) << "Op[" << op_name << "] create single kernel json failed."; + } + + kernel_json_ = node_json.dump(); + + if (!GetIOSize(node_json, &input_size_list_, &output_size_list_)) { + MS_LOG(ERROR) << "Cal mem size failed."; + return false; + } + + return true; +} + +bool AkgAscendKernelBuilder::GenJsonAndPreprocess4Fused(const std::vector &anf_nodes, + std::map *node_json_map) { + for (auto const &anf_node : anf_nodes) { + MS_EXCEPTION_IF_NULL(anf_node); + std::string op_name = AnfAlgo::GetCNodeName(anf_node); + if (!AnfAlgo::IsRealKernel(anf_node)) { + MS_LOG(ERROR) << "Invalid anf node to build [" << anf_node->fullname_with_scope() << "]."; + return false; + } + auto it = kAkgKernelAttrsProcessMap.find(op_name); + if (it != kAkgKernelAttrsProcessMap.end()) { + it->second(anf_node); + } + + nlohmann::json node_json; + if (!GenerateSingleKernelJson(anf_node, op_name, &node_json)) { + MS_LOG(ERROR) << "Op [" << op_name << "] create single kernel json failed."; + return false; + } + // No need for composite op. + node_json.erase("id"); + node_json.erase("op"); + node_json.erase("composite"); + + auto primitive = AnfAlgo::GetCNodePrimitive(anf_node); + MS_EXCEPTION_IF_NULL(primitive); + + if (primitive->GetAttr("fusion") != nullptr) { + node_json["fusion"] = primitive->GetAttr("fusion")->ToString(); + } + + (*node_json_map)[anf_node] = node_json; + } + return true; +} + +bool AkgAscendKernelBuilder::CollectFusedJson(const std::vector &anf_nodes, + const std::vector &input_list, + const std::vector &output_list) { + if (anf_nodes.empty() || input_list.empty()) { + MS_LOG(ERROR) << "Invalid input size, anf_nodes [" << anf_nodes.size() << "], input_list [" << input_list.size() + << "]."; + return false; + } + MS_LOG(INFO) << "anf_nodes [" << output_list.size() << "], input_list [" << anf_nodes.size() << "], output_list [" + << input_list.size() << "]."; + + std::map node_json_map; + if (!GenJsonAndPreprocess4Fused(anf_nodes, &node_json_map)) { + return false; + } + + UpdateTensorNameInJson(anf_nodes, &node_json_map); + + nlohmann::json fused_node_json; + std::vector node_json_desc; + std::transform(anf_nodes.begin(), anf_nodes.end(), std::back_inserter(node_json_desc), + [&node_json_map](const AnfNodePtr &anf_node) { return node_json_map[anf_node]; }); + fused_node_json[kOpDesc] = node_json_desc; + fused_node_json[kInputDesc] = GetInputsJson(anf_nodes, input_list, &node_json_map); + fused_node_json[kOutputDesc] = + GetOutputsJson(anf_nodes, input_list, output_list, fused_node_json[kInputDesc], &node_json_map); size_t hash_id = std::hash()(fused_node_json.dump()); json_name_ = "Fused_"; @@ -243,36 +336,7 @@ void GenParallelCompileFuncArgs(const std::vector &kernel_jsons, Py } bool AkgOpParallelBuild(const std::vector> &build_args) { - // Remove cached nodes, gether unique nodes, and collect repeated nodes which need postprecess. - std::vector jsons; - std::unordered_set json_name_set; - std::vector> repeat_nodes; - for (const auto &[builder, anf_node] : build_args) { - MS_EXCEPTION_IF_NULL(anf_node); - auto json_name = builder.json_name(); - MS_LOG(DEBUG) << "Akg start compile op: " << json_name; - auto cached_kernel_pack = tbe::TbeUtils::SearchCache(json_name, AkgKernelBuild::GetProcessor(anf_node)); - if (cached_kernel_pack != nullptr) { - MS_LOG(DEBUG) << "Use cached kernel, json_name_[" << json_name << "], fullname_with_scope[" - << anf_node->fullname_with_scope() << "]."; - auto kernel_mod_ptr = std::make_shared(cached_kernel_pack); - kernel_mod_ptr->SetInputSizeList(builder.input_size_list()); - kernel_mod_ptr->SetOutputSizeList(builder.output_size_list()); - AnfAlgo::SetKernelMod(kernel_mod_ptr, anf_node.get()); - continue; - } - - if (json_name_set.count(json_name) != 0) { - repeat_nodes.push_back({builder, anf_node}); - continue; - } - json_name_set.insert(json_name); - auto node_json = builder.kernel_json(); - kernel::SaveJsonInfo(json_name, node_json); - jsons.push_back(node_json); - } - - // No nodes need to be compiled! + auto [jsons, repeat_nodes] = PreProcessJsonForBuild(build_args); if (jsons.empty()) { return true; } @@ -307,34 +371,8 @@ bool AkgOpParallelBuild(const std::vectorfullname_with_scope() << "]."; - return false; - } - auto kernel_mod_ptr = std::make_shared(new_kernel_pack); - kernel_mod_ptr->SetInputSizeList(builder.input_size_list()); - kernel_mod_ptr->SetOutputSizeList(builder.output_size_list()); - AnfAlgo::SetKernelMod(kernel_mod_ptr, anf_node.get()); - MS_LOG(DEBUG) << "Akg compile " << json_name << " kernel and insert cache successfully!"; - } - - // Handle repeated nodes. - for (const auto &[builder, anf_node] : repeat_nodes) { - auto node_json = builder.kernel_json(); - auto json_name = builder.json_name(); - auto cached_kernel_pack = tbe::TbeUtils::SearchCache(json_name, AkgKernelBuild::GetProcessor(anf_node)); - if (cached_kernel_pack == nullptr) return false; - MS_LOG(INFO) << "Use just compiled kernel, json_name_[" << json_name << "], fullname_with_scope[" - << anf_node->fullname_with_scope() << "]."; - auto kernel_mod_ptr = std::make_shared(cached_kernel_pack); - kernel_mod_ptr->SetInputSizeList(builder.input_size_list()); - kernel_mod_ptr->SetOutputSizeList(builder.output_size_list()); - AnfAlgo::SetKernelMod(kernel_mod_ptr, anf_node.get()); + if (!PostProcessAfterCompile(build_args, repeat_nodes)) { + return false; } return true; @@ -380,6 +418,5 @@ bool AkgAscendKernelParallelBuild(const std::vector &anf_nodes) { return AkgOpParallelBuild(json_and_node); } - } // namespace kernel } // namespace mindspore diff --git a/mindspore/ccsrc/kernel/akg/ascend/akg_ascend_kernel_build.h b/mindspore/ccsrc/kernel/akg/ascend/akg_ascend_kernel_build.h index 619b583fde..01752911ed 100644 --- a/mindspore/ccsrc/kernel/akg/ascend/akg_ascend_kernel_build.h +++ b/mindspore/ccsrc/kernel/akg/ascend/akg_ascend_kernel_build.h @@ -20,6 +20,7 @@ #include #include #include +#include #include "ir/anf.h" #include "kernel/kernel.h" #include "kernel/akg/akg_kernel_build.h" @@ -40,6 +41,9 @@ class AkgAscendKernelBuilder : public AkgKernelBuild { const std::vector &output_size_list() const { return output_size_list_; } private: + bool GenJsonAndPreprocess4Fused(const std::vector &anf_nodes, + std::map *node_json_map); + std::string kernel_json_; std::vector input_size_list_; std::vector output_size_list_; diff --git a/mindspore/ccsrc/kernel/akg/ascend/akg_ascend_kernel_mod.cc b/mindspore/ccsrc/kernel/akg/ascend/akg_ascend_kernel_mod.cc index bad6de64aa..69fc82aad3 100644 --- a/mindspore/ccsrc/kernel/akg/ascend/akg_ascend_kernel_mod.cc +++ b/mindspore/ccsrc/kernel/akg/ascend/akg_ascend_kernel_mod.cc @@ -55,7 +55,7 @@ const std::vector &AkgKernelMod::GetWorkspaceSizeList() const { return w bool AkgKernelMod::Launch(const std::vector &inputs, const std::vector &, const std::vector &outputs, void *stream_ptr) { - if (stream_ptr == 0) { + if (stream_ptr == nullptr) { MS_LOG(ERROR) << "stream_ptr should not be nullptr."; return false; } From 365c901ee08f3a5564ee0b9d9feda7e4397b7c00 Mon Sep 17 00:00:00 2001 From: yanghaitao Date: Tue, 30 Jun 2020 20:57:53 +0800 Subject: [PATCH 177/254] get default value if num_parallel_worker is None --- mindspore/dataset/engine/serializer_deserializer.py | 4 +++- tests/ut/python/dataset/test_config.py | 5 ++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/mindspore/dataset/engine/serializer_deserializer.py b/mindspore/dataset/engine/serializer_deserializer.py index 833f660f16..9d3339e26d 100644 --- a/mindspore/dataset/engine/serializer_deserializer.py +++ b/mindspore/dataset/engine/serializer_deserializer.py @@ -22,7 +22,7 @@ import sys from mindspore import log as logger from . import datasets as de from ..transforms.vision.utils import Inter, Border - +from ..core.configuration import config def serialize(dataset, json_filepath=None): """ @@ -164,6 +164,8 @@ def traverse(node): node_repr[k] = v.to_json() elif k in set(['schema', 'dataset_files', 'dataset_dir', 'schema_file_path']): expand_path(node_repr, k, v) + elif k == "num_parallel_workers" and v is None: + node_repr[k] = config.get_num_parallel_workers() else: node_repr[k] = v diff --git a/tests/ut/python/dataset/test_config.py b/tests/ut/python/dataset/test_config.py index 874ce5db54..259f42d948 100644 --- a/tests/ut/python/dataset/test_config.py +++ b/tests/ut/python/dataset/test_config.py @@ -84,12 +84,11 @@ def test_pipeline(): num_parallel_workers_original = ds.config.get_num_parallel_workers() data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, shuffle=False) - ds.config.set_num_parallel_workers(2) data1 = data1.map(input_columns=["image"], operations=[c_vision.Decode(True)]) ds.serialize(data1, "testpipeline.json") - data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, shuffle=False) - ds.config.set_num_parallel_workers(4) + data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, num_parallel_workers=num_parallel_workers_original, + shuffle=False) data2 = data2.map(input_columns=["image"], operations=[c_vision.Decode(True)]) ds.serialize(data2, "testpipeline2.json") From cd868aea52d6bd984d53266f2fb61eaae4a9fb9e Mon Sep 17 00:00:00 2001 From: ougongchang Date: Tue, 30 Jun 2020 19:44:28 +0800 Subject: [PATCH 178/254] fix get loss error and NoneType error cause by _proceesor_specified_data fix get loss error when it not a scalar and fix process specified data failed when the action is False, and collect_specified_data parameter is not None --- .../train/callback/_summary_collector.py | 15 +++++---- .../train/summary/test_summary_collector.py | 32 ++++++++++++++++--- 2 files changed, 36 insertions(+), 11 deletions(-) diff --git a/mindspore/train/callback/_summary_collector.py b/mindspore/train/callback/_summary_collector.py index a94eb2fc55..c77a41fedc 100644 --- a/mindspore/train/callback/_summary_collector.py +++ b/mindspore/train/callback/_summary_collector.py @@ -239,7 +239,8 @@ class SummaryCollector(Callback): unexpected_params = set(specified_data) - set(self._DEFAULT_SPECIFIED_DATA) if unexpected_params: - raise ValueError(f'For `collect_specified_data` the keys {unexpected_params} are unsupported.') + raise ValueError(f'For `collect_specified_data` the keys {unexpected_params} are unsupported, ' + f'expect the follow keys: {list(self._DEFAULT_SPECIFIED_DATA.keys())}') if 'histogram_regular' in specified_data: check_value_type('histogram_regular', specified_data.get('histogram_regular'), (str, type(None))) @@ -250,7 +251,8 @@ class SummaryCollector(Callback): check_value_type(item, specified_data.get(item), bool) if action: - result = dict(self._DEFAULT_SPECIFIED_DATA).update(specified_data) + result = dict(self._DEFAULT_SPECIFIED_DATA) + result.update(specified_data) else: result = specified_data return result @@ -444,15 +446,12 @@ class SummaryCollector(Callback): self._is_parse_loss_success = False return None - if isinstance(output, (int, float)): + if isinstance(output, (int, float, Tensor)): loss = output - elif isinstance(output, (list, tuple)): + elif isinstance(output, (list, tuple)) and output: # If the output is a list, since the default network returns loss first, # we assume that the first one is loss. loss = output[0] - elif isinstance(output, Tensor) and (not output.shape or output.shape == (1,)): - loss_numpy = output.asnumpy() - loss = float(np.atleast_1d(loss_numpy)[0]) else: logger.warning("The output type could not be identified, so no loss was recorded in SummaryCollector.") self._is_parse_loss_success = False @@ -461,6 +460,8 @@ class SummaryCollector(Callback): if not isinstance(loss, Tensor): loss = Tensor(loss) + precision = 4 + loss = Tensor(round(np.mean(loss.asnumpy()), precision)) return loss def _get_optimizer(self, cb_params): diff --git a/tests/ut/python/train/summary/test_summary_collector.py b/tests/ut/python/train/summary/test_summary_collector.py index 1390d29bc1..31552e44bd 100644 --- a/tests/ut/python/train/summary/test_summary_collector.py +++ b/tests/ut/python/train/summary/test_summary_collector.py @@ -50,6 +50,9 @@ def get_value(): _VALUE_CACHE = list() return value +_SPECIFIED_DATA = SummaryCollector._DEFAULT_SPECIFIED_DATA +_SPECIFIED_DATA['collect_metric'] = False + class CustomNet(Cell): """Define custom netwrok.""" @@ -190,8 +193,8 @@ class TestSummaryCollector: data = {'unexpected_key': True} with pytest.raises(ValueError) as exc: SummaryCollector(summary_dir, collect_specified_data=data) - expected_msg = f"For `collect_specified_data` the keys {set(data)} are unsupported." - assert expected_msg == str(exc.value) + expected_msg = f"For `collect_specified_data` the keys {set(data)} are unsupported" + assert expected_msg in str(exc.value) @pytest.mark.parametrize("custom_lineage_data", [ 123, @@ -273,12 +276,16 @@ class TestSummaryCollector: assert name == 'train_dataset' @pytest.mark.parametrize("net_output, expected_loss", [ + (None, None), (1, Tensor(1)), + (1.5, Tensor(1.5)), + (Tensor(1), Tensor(1)), ([1], Tensor(1)), ([Tensor(1)], Tensor(1)), - (Tensor([1]), Tensor(1)), + ({}, None), + (Tensor([[1, 2], [3, 4]]), Tensor(2.5)), + ([Tensor([[3, 4, 3]]), Tensor([3, 4])], Tensor(3.33333)), (tuple([1]), Tensor(1)), - (None, None) ]) def test_get_loss(self, net_output, expected_loss): """Test get loss success and failed.""" @@ -375,3 +382,20 @@ class TestSummaryCollector: assert PluginEnum.HISTOGRAM.value == result[0][0] assert expected_names == [data[1] for data in result] assert expected_values == [data[2] for data in result] + + @pytest.mark.parametrize("specified_data, action, expected_result", [ + (None, True, SummaryCollector._DEFAULT_SPECIFIED_DATA), + (None, False, {}), + ({}, True, SummaryCollector._DEFAULT_SPECIFIED_DATA), + ({}, False, {}), + ({'collect_metric': False}, True, _SPECIFIED_DATA), + ({'collect_metric': True}, False, {'collect_metric': True}) + ]) + def test_process_specified_data(self, specified_data, action, expected_result): + """Test process specified data.""" + summary_dir = tempfile.mkdtemp(dir=self.base_summary_dir) + summary_collector = SummaryCollector(summary_dir, + collect_specified_data=specified_data, + keep_default_action=action) + + assert summary_collector._collect_specified_data == expected_result From 7a8ee4725b025d824e3fc3c75da105bc1e192010 Mon Sep 17 00:00:00 2001 From: yoonlee666 Date: Tue, 30 Jun 2020 21:18:11 +0800 Subject: [PATCH 179/254] add bert support for glue --- model_zoo/bert/README.md | 4 +- model_zoo/bert/evaluation.py | 140 ++++++++++++++++++++++++++++++----- model_zoo/bert/finetune.py | 36 +++++---- model_zoo/bert/src/utils.py | 61 +++++++++++++-- 4 files changed, 201 insertions(+), 40 deletions(-) diff --git a/model_zoo/bert/README.md b/model_zoo/bert/README.md index 1fa755e72f..3ed2bf6783 100644 --- a/model_zoo/bert/README.md +++ b/model_zoo/bert/README.md @@ -89,7 +89,7 @@ config.py: optimizer optimizer used in the network: AdamWerigtDecayDynamicLR | Lamb | Momentum, default is "Lamb" finetune_config.py: - task task type: NER | SQUAD | OTHERS + task task type: SeqLabeling | Regression | Classification | COLA | SQUAD num_labels number of labels to do classification data_file dataset file to load: PATH, default is "/your/path/train.tfrecord" schema_file dataset schema file to load: PATH, default is "/your/path/schema.json" @@ -101,7 +101,7 @@ finetune_config.py: optimizer optimizer used in fine-tune network: AdamWeigtDecayDynamicLR | Lamb | Momentum, default is "Lamb" evaluation_config.py: - task task type: NER | SQUAD | OTHERS + task task type: SeqLabeling | Regression | Classification | COLA num_labels number of labels to do classsification data_file dataset file to load: PATH, default is "/your/path/evaluation.tfrecord" schema_file dataset schema file to load: PATH, default is "/your/path/schema.json" diff --git a/model_zoo/bert/evaluation.py b/model_zoo/bert/evaluation.py index 4877b60cef..4e8b2a3aea 100644 --- a/model_zoo/bert/evaluation.py +++ b/model_zoo/bert/evaluation.py @@ -19,6 +19,7 @@ Bert evaluation script. import os import argparse +import math import numpy as np import mindspore.common.dtype as mstype from mindspore import context @@ -29,19 +30,24 @@ import mindspore.dataset.transforms.c_transforms as C from mindspore.train.model import Model from mindspore.train.serialization import load_checkpoint, load_param_into_net from src.evaluation_config import cfg, bert_net_cfg -from src.utils import BertNER, BertCLS +from src.utils import BertNER, BertCLS, BertReg from src.CRF import postprocess from src.cluener_evaluation import submit from src.finetune_config import tag_to_index + class Accuracy(): - ''' + """ calculate accuracy - ''' + """ def __init__(self): self.acc_num = 0 self.total_num = 0 + def update(self, logits, labels): + """ + Update accuracy + """ labels = labels.asnumpy() labels = np.reshape(labels, -1) logits = logits.asnumpy() @@ -50,18 +56,20 @@ class Accuracy(): self.total_num += len(labels) print("=========================accuracy is ", self.acc_num / self.total_num) + class F1(): - ''' + """ calculate F1 score - ''' + """ def __init__(self): self.TP = 0 self.FP = 0 self.FN = 0 + def update(self, logits, labels): - ''' + """ update F1 score - ''' + """ labels = labels.asnumpy() labels = np.reshape(labels, -1) if cfg.use_crf: @@ -80,10 +88,76 @@ class F1(): self.FP += np.sum(pos_eva&(~pos_label)) self.FN += np.sum((~pos_eva)&pos_label) + +class MCC(): + """ + Calculate Matthews Correlation Coefficient. + """ + def __init__(self): + self.TP = 0 + self.FP = 0 + self.FN = 0 + self.TN = 0 + + def update(self, logits, labels): + """ + Update MCC score + """ + labels = labels.asnumpy() + labels = np.reshape(labels, -1) + labels = labels.astype(np.bool) + logits = logits.asnumpy() + logit_id = np.argmax(logits, axis=-1) + logit_id = np.reshape(logit_id, -1) + logit_id = logit_id.astype(np.bool) + ornot = logit_id ^ labels + + self.TP += (~ornot & labels).sum() + self.FP += (ornot & ~labels).sum() + self.FN += (ornot & labels).sum() + self.TN += (~ornot & ~labels).sum() + + +class Spearman_Correlation(): + """ + calculate Spearman Correlation coefficient + """ + def __init__(self): + self.label = [] + self.logit = [] + + def update(self, logits, labels): + """ + Update Spearman Correlation + """ + labels = labels.asnumpy() + labels = np.reshape(labels, -1) + logits = logits.asnumpy() + logits = np.reshape(logits, -1) + self.label.append(labels) + self.logit.append(logits) + + def cal(self): + """ + Calculate Spearman Correlation + """ + label = np.concatenate(self.label) + logit = np.concatenate(self.logit) + sort_label = label.argsort()[::-1] + sort_logit = logit.argsort()[::-1] + n = len(label) + d_acc = 0 + for i in range(n): + d = np.where(sort_label == i)[0] - np.where(sort_logit == i)[0] + d_acc += d**2 + ps = 1 - 6*d_acc/n/(n**2-1) + return ps + + def get_dataset(batch_size=1, repeat_count=1, distribute_file=''): - ''' + """ get dataset - ''' + """ _ = distribute_file ds = de.TFRecordDataset([cfg.data_file], cfg.schema_file, columns_list=["input_ids", "input_mask", @@ -92,7 +166,11 @@ def get_dataset(batch_size=1, repeat_count=1, distribute_file=''): ds = ds.map(input_columns="segment_ids", operations=type_cast_op) ds = ds.map(input_columns="input_mask", operations=type_cast_op) ds = ds.map(input_columns="input_ids", operations=type_cast_op) - ds = ds.map(input_columns="label_ids", operations=type_cast_op) + if cfg.task == "Regression": + type_cast_op_float = C.TypeCast(mstype.float32) + ds = ds.map(input_columns="label_ids", operations=type_cast_op_float) + else: + ds = ds.map(input_columns="label_ids", operations=type_cast_op) ds = ds.repeat(repeat_count) # apply shuffle operation @@ -103,10 +181,11 @@ def get_dataset(batch_size=1, repeat_count=1, distribute_file=''): ds = ds.batch(batch_size, drop_remainder=True) return ds + def bert_predict(Evaluation): - ''' + """ prediction function - ''' + """ target = args_opt.device_target if target == "Ascend": devid = int(os.getenv('DEVICE_ID')) @@ -131,15 +210,33 @@ def bert_predict(Evaluation): return model, dataset def test_eval(): - ''' + """ evaluation function - ''' - task_type = BertNER if cfg.task == "NER" else BertCLS + """ + if cfg.task == "SeqLabeling": + task_type = BertNER + elif cfg.task == "Regression": + task_type = BertReg + elif cfg.task == "Classification": + task_type = BertCLS + elif cfg.task == "COLA": + task_type = BertCLS + else: + raise ValueError("Task not supported.") model, dataset = bert_predict(task_type) + if cfg.clue_benchmark: submit(model, cfg.data_file, bert_net_cfg.seq_length) else: - callback = F1() if cfg.task == "NER" else Accuracy() + if cfg.task == "SeqLabeling": + callback = F1() + elif cfg.task == "COLA": + callback = MCC() + elif cfg.task == "Regression": + callback = Spearman_Correlation() + else: + callback = Accuracy() + columns_list = ["input_ids", "input_mask", "segment_ids", "label_ids"] for data in dataset.create_dict_iterator(): input_data = [] @@ -149,10 +246,19 @@ def test_eval(): logits = model.predict(input_ids, input_mask, token_type_id, label_ids) callback.update(logits, label_ids) print("==============================================================") - if cfg.task == "NER": + if cfg.task == "SeqLabeling": print("Precision {:.6f} ".format(callback.TP / (callback.TP + callback.FP))) print("Recall {:.6f} ".format(callback.TP / (callback.TP + callback.FN))) print("F1 {:.6f} ".format(2*callback.TP / (2*callback.TP + callback.FP + callback.FN))) + elif cfg.task == "COLA": + TP = callback.TP + TN = callback.TN + FP = callback.FP + FN = callback.FN + mcc = (TP*TN-FP*FN)/math.sqrt((TP+FP)*(TP+FN)*(TN+FP)*(TN+FN)) + print("MCC: {:.6f}".format(mcc)) + elif cfg.task == "Regression": + print("Spearman Correlation is {:.6f}".format(callback.cal()[0])) else: print("acc_num {} , total_num {}, accuracy {:.6f}".format(callback.acc_num, callback.total_num, callback.acc_num / callback.total_num)) diff --git a/model_zoo/bert/finetune.py b/model_zoo/bert/finetune.py index df16e3c91d..eb1880b9cc 100644 --- a/model_zoo/bert/finetune.py +++ b/model_zoo/bert/finetune.py @@ -13,13 +13,13 @@ # limitations under the License. # ============================================================================ -''' +""" Bert finetune script. -''' +""" import os import argparse -from src.utils import BertFinetuneCell, BertCLS, BertNER, BertSquad, BertSquadCell +from src.utils import BertFinetuneCell, BertCLS, BertNER, BertSquad, BertSquadCell, BertReg from src.finetune_config import cfg, bert_net_cfg, tag_to_index import mindspore.common.dtype as mstype from mindspore import context @@ -34,14 +34,14 @@ from mindspore.train.callback import CheckpointConfig, ModelCheckpoint from mindspore.train.serialization import load_checkpoint, load_param_into_net class LossCallBack(Callback): - ''' + """ Monitor the loss in training. If the loss is NAN or INF, terminate training. Note: If per_print_times is 0, do not print loss. Args: per_print_times (int): Print loss every times. Default: 1. - ''' + """ def __init__(self, per_print_times=1): super(LossCallBack, self).__init__() if not isinstance(per_print_times, int) or per_print_times < 0: @@ -56,16 +56,20 @@ class LossCallBack(Callback): f.write("\n") def get_dataset(batch_size=1, repeat_count=1, distribute_file=''): - ''' + """ get dataset - ''' + """ ds = de.TFRecordDataset([cfg.data_file], cfg.schema_file, columns_list=["input_ids", "input_mask", "segment_ids", "label_ids"]) type_cast_op = C.TypeCast(mstype.int32) ds = ds.map(input_columns="segment_ids", operations=type_cast_op) ds = ds.map(input_columns="input_mask", operations=type_cast_op) ds = ds.map(input_columns="input_ids", operations=type_cast_op) - ds = ds.map(input_columns="label_ids", operations=type_cast_op) + if cfg.task == "Regression": + type_cast_op_float = C.TypeCast(mstype.float32) + ds = ds.map(input_columns="label_ids", operations=type_cast_op_float) + else: + ds = ds.map(input_columns="label_ids", operations=type_cast_op) ds = ds.repeat(repeat_count) # apply shuffle operation @@ -77,9 +81,9 @@ def get_dataset(batch_size=1, repeat_count=1, distribute_file=''): return ds def get_squad_dataset(batch_size=1, repeat_count=1, distribute_file=''): - ''' + """ get SQuAD dataset - ''' + """ ds = de.TFRecordDataset([cfg.data_file], cfg.schema_file, columns_list=["input_ids", "input_mask", "segment_ids", "start_positions", "end_positions", "unique_ids", "is_impossible"]) @@ -97,9 +101,9 @@ def get_squad_dataset(batch_size=1, repeat_count=1, distribute_file=''): return ds def test_train(): - ''' + """ finetune function - ''' + """ target = args_opt.device_target if target == "Ascend": devid = int(os.getenv('DEVICE_ID')) @@ -113,7 +117,7 @@ def test_train(): raise Exception("Target error, GPU or Ascend is supported.") #BertCLSTrain for classification #BertNERTrain for sequence labeling - if cfg.task == 'NER': + if cfg.task == 'SeqLabeling': if cfg.use_crf: netwithloss = BertNER(bert_net_cfg, True, num_labels=len(tag_to_index), use_crf=True, tag_to_index=tag_to_index, dropout_prob=0.1) @@ -121,8 +125,12 @@ def test_train(): netwithloss = BertNER(bert_net_cfg, True, num_labels=cfg.num_labels, dropout_prob=0.1) elif cfg.task == 'SQUAD': netwithloss = BertSquad(bert_net_cfg, True, 2, dropout_prob=0.1) - else: + elif cfg.task == 'Regression': + netwithloss = BertReg(bert_net_cfg, True, num_labels=cfg.num_labels, dropout_prob=0.1) + elif cfg.task == 'Classification': netwithloss = BertCLS(bert_net_cfg, True, num_labels=cfg.num_labels, dropout_prob=0.1) + else: + raise Exception("Target error, GPU or Ascend is supported.") if cfg.task == 'SQUAD': dataset = get_squad_dataset(bert_net_cfg.batch_size, cfg.epoch_num) else: diff --git a/model_zoo/bert/src/utils.py b/model_zoo/bert/src/utils.py index 9b5383877b..ec5651b205 100644 --- a/model_zoo/bert/src/utils.py +++ b/model_zoo/bert/src/utils.py @@ -13,9 +13,9 @@ # limitations under the License. # ============================================================================ -''' +""" Functional Cells used in Bert finetune and evaluation. -''' +""" import mindspore.nn as nn from mindspore.common.initializer import TruncatedNormal @@ -245,6 +245,32 @@ class BertSquadCell(nn.Cell): ret = (loss, cond) return F.depend(ret, succ) + +class BertRegressionModel(nn.Cell): + """ + Bert finetune model for regression task + """ + def __init__(self, config, is_training, num_labels=2, dropout_prob=0.0, use_one_hot_embeddings=False): + super(BertRegressionModel, self).__init__() + self.bert = BertModel(config, is_training, use_one_hot_embeddings) + self.cast = P.Cast() + self.weight_init = TruncatedNormal(config.initializer_range) + self.log_softmax = P.LogSoftmax(axis=-1) + self.dtype = config.dtype + self.num_labels = num_labels + self.dropout = nn.Dropout(1 - dropout_prob) + self.dense_1 = nn.Dense(config.hidden_size, 1, weight_init=self.weight_init, + has_bias=True).to_float(mstype.float16) + + def construct(self, input_ids, input_mask, token_type_id): + _, pooled_output, _ = self.bert(input_ids, token_type_id, input_mask) + cls = self.cast(pooled_output, self.dtype) + cls = self.dropout(cls) + logits = self.dense_1(cls) + logits = self.cast(logits, self.dtype) + return logits + + class BertCLSModel(nn.Cell): """ This class is responsible for classification task evaluation, i.e. XNLI(num_labels=3), @@ -274,9 +300,9 @@ class BertCLSModel(nn.Cell): return log_probs class BertSquadModel(nn.Cell): - ''' - This class is responsible for SQuAD - ''' + """ + Bert finetune model for SQuAD v1.1 task + """ def __init__(self, config, is_training, num_labels=2, dropout_prob=0.0, use_one_hot_embeddings=False): super(BertSquadModel, self).__init__() self.bert = BertModel(config, is_training, use_one_hot_embeddings) @@ -401,9 +427,9 @@ class BertNER(nn.Cell): return loss class BertSquad(nn.Cell): - ''' + """ Train interface for SQuAD finetuning task. - ''' + """ def __init__(self, config, is_training, num_labels=2, dropout_prob=0.0, use_one_hot_embeddings=False): super(BertSquad, self).__init__() self.bert = BertSquadModel(config, is_training, num_labels, dropout_prob, use_one_hot_embeddings) @@ -432,3 +458,24 @@ class BertSquad(nn.Cell): end_logits = self.squeeze(logits[:, :, 1:2]) total_loss = (unique_id, start_logits, end_logits) return total_loss + + +class BertReg(nn.Cell): + """ + Bert finetune model with loss for regression task + """ + def __init__(self, config, is_training, num_labels=2, dropout_prob=0.0, use_one_hot_embeddings=False): + super(BertReg, self).__init__() + self.bert = BertRegressionModel(config, is_training, num_labels, dropout_prob, use_one_hot_embeddings) + self.loss = nn.MSELoss() + self.is_training = is_training + self.sigmoid = P.Sigmoid() + self.cast = P.Cast() + self.mul = P.Mul() + def construct(self, input_ids, input_mask, token_type_id, labels): + logits = self.bert(input_ids, input_mask, token_type_id) + if self.is_training: + loss = self.loss(logits, labels) + else: + loss = logits + return loss From a0a863f20c8bd95f23822afd60b61f5bca5f477f Mon Sep 17 00:00:00 2001 From: Jesse Lee Date: Fri, 26 Jun 2020 15:06:23 -0400 Subject: [PATCH 180/254] Stage 2 porting --- .../dataset/engine/datasetops/concat_op.cc | 2 +- .../dataset/engine/datasetops/dataset_op.cc | 66 +++++++++++++++++-- .../dataset/engine/datasetops/dataset_op.h | 23 ++++++- .../ccsrc/dataset/engine/datasetops/map_op.cc | 2 +- .../dataset/engine/datasetops/parallel_op.cc | 4 +- .../dataset/engine/datasetops/parallel_op.h | 3 +- .../dataset/engine/datasetops/pipeline_op.cc | 3 +- .../dataset/engine/datasetops/pipeline_op.h | 3 +- .../dataset/engine/datasetops/repeat_op.cc | 6 +- .../engine/datasetops/source/celeba_op.cc | 3 +- .../engine/datasetops/source/celeba_op.h | 1 - .../engine/datasetops/source/cifar_op.cc | 3 +- .../engine/datasetops/source/cifar_op.h | 1 - .../datasetops/source/image_folder_op.cc | 3 +- .../datasetops/source/image_folder_op.h | 1 - .../engine/datasetops/source/manifest_op.cc | 3 +- .../engine/datasetops/source/manifest_op.h | 1 - .../engine/datasetops/source/mnist_op.cc | 3 +- .../engine/datasetops/source/mnist_op.h | 1 - .../datasetops/source/random_data_op.cc | 25 +++++-- .../engine/datasetops/source/random_data_op.h | 20 +++++- .../source/sampler/distributed_sampler.cc | 9 ++- .../datasetops/source/sampler/pk_sampler.cc | 8 +++ .../datasetops/source/sampler/pk_sampler.h | 5 ++ .../source/sampler/python_sampler.cc | 9 +++ .../source/sampler/python_sampler.h | 5 ++ .../source/sampler/random_sampler.cc | 9 ++- .../datasetops/source/sampler/sampler.cc | 9 +-- .../source/sampler/sequential_sampler.cc | 11 +++- .../source/sampler/sequential_sampler.h | 3 + .../source/sampler/subset_random_sampler.cc | 9 +++ .../source/sampler/subset_random_sampler.h | 5 ++ .../source/sampler/weighted_random_sampler.cc | 9 +++ .../source/sampler/weighted_random_sampler.h | 5 ++ .../engine/datasetops/source/text_file_op.cc | 13 ++-- .../engine/datasetops/source/text_file_op.h | 13 +++- .../engine/datasetops/source/tf_reader_op.cc | 53 ++++++++++++--- .../engine/datasetops/source/tf_reader_op.h | 18 ++++- .../engine/datasetops/source/voc_op.cc | 3 +- .../dataset/engine/datasetops/source/voc_op.h | 1 - .../dataset/engine/datasetops/take_op.cc | 2 +- .../ccsrc/dataset/engine/execution_tree.cc | 39 +++++++---- .../ccsrc/dataset/engine/execution_tree.h | 35 ++++++---- 43 files changed, 351 insertions(+), 99 deletions(-) diff --git a/mindspore/ccsrc/dataset/engine/datasetops/concat_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/concat_op.cc index c5aac523d2..4bada31e7e 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/concat_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/concat_op.cc @@ -128,7 +128,7 @@ Status ConcatOp::Verify(int32_t id, const std::unique_ptr &buf) { Status ConcatOp::PrepareNodePostAction() { RETURN_IF_NOT_OK(PipelineOp::PrepareNodePostAction()); - tree_->AddToRepeatStack(shared_from_this()); + tree_->AddToEOEOpStack(shared_from_this()); return Status::OK(); } diff --git a/mindspore/ccsrc/dataset/engine/datasetops/dataset_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/dataset_op.cc index 727c543958..91ed7fbc5f 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/dataset_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/dataset_op.cc @@ -18,23 +18,26 @@ #include #include #include +#include #include #include #include #include "dataset/engine/execution_tree.h" #include "dataset/engine/datasetops/device_queue_op.h" +#include "dataset/engine/datasetops/source/sampler/sampler.h" #include "dataset/engine/data_buffer.h" #include "dataset/engine/db_connector.h" #include "dataset/engine/opt/pass.h" - +#include "utils/system/crc32c.h" #include "utils/log_adapter.h" namespace mindspore { namespace dataset { // Constructor -DatasetOp::DatasetOp(int32_t op_connector_size) +DatasetOp::DatasetOp(int32_t op_connector_size, std::shared_ptr sampler) : oc_queue_size_(op_connector_size), + sampler_(sampler), operator_id_(kInvalidOperatorId), tree_(nullptr), state_(OpState::kDeOpIdle), @@ -150,6 +153,9 @@ void DatasetOp::Print(std::ostream &out, bool show_all) const { } out << "\nConnector queue size : " << oc_queue_size_ << "\nOperator control flags : 0x" << std::hex << std::setw(8) << std::setfill('0') << op_ctrl_flags_ << std::dec << std::setfill(' '); + if (sampler_) { + sampler_->Print(out, show_all); + } } } @@ -222,11 +228,10 @@ Status DatasetOp::PrepareNodePreAction() { Status DatasetOp::PrepareNodePostAction() { // If this op does not have any children and it is in a repeat path of the tree... if (child_.empty() && BitTest(op_ctrl_flags_, kDeOpRepeated)) { - // push ourselves onto the tree repeat stack. Later, the repeat operator + // push ourselves onto the eoe operator stack. Later, a repeat/epoch ctrl operator // above us will consume them. - tree_->AddToRepeatStack(shared_from_this()); + tree_->AddToEOEOpStack(shared_from_this()); } - // Creating Connector object for each op. // The consumer of the root node is assumed to be one thread. // If multiple threads are consuming from the root node, they will get the ordered data in round robin fashion. @@ -289,5 +294,56 @@ Status DatasetOp::Accept(NodePass *p, bool *modified) { // This method will only be called if its derived class does not implement one. return p->RunOnNode(shared_from_this(), modified); } + +// A helper function with some common code that leaf nodes can use during +// prepare phase for checking if they need to assign a sampler to the cache. +// @return - Status +Status DatasetOp::SaveSamplerForCache(bool random_access_op) { + // If we are a descendant under a cache op and we have a sampler, then save this sampler + // to a stack so that the cache can pick it up during it's processing above us. + if (sampler_) { + if (BitTest(tree_->PrepareFlags(), ExecutionTree::kDePrepCache)) { + // use move semantic to set our sampler_ to null after the move. This is okay because a sampler is + // useless to a random data op. It was only being used as a temporary holding until the cache can + // be created + tree_->AddToSamplerStack(sampler_); + MS_LOG(INFO) << "Preparing a leaf op: passing sampler up the tree for Cache handling."; + } else if (!random_access_op) { + // A sampler exists, but we are not in a caching tree and we are not a random access mappable leaf. + // This is an error because that type of leaf does not use sampling unless there's a cache to hook it into. + return Status( + StatusCode::kUnexpectedError, __LINE__, __FILE__, + "Non-mappable leaf op has a sampler, but it only supports sampling if there is a cache after it in the tree"); + } + } + + if (!random_access_op) { + // Since we don't truly need the sampler for this non-mappable dataset and it's been saved for the cache + // we can remove it now from the base. + sampler_.reset(); + } + + return Status::OK(); +} +uint32_t DatasetOp::GenerateCRC(const std::shared_ptr &op) { + std::stringstream ss; + op->tree_->Print(ss, op); + std::string ss_str = ss.str(); + + // Filter out the Operator control flags field when generating the check sum + ss_str = std::regex_replace(ss_str, std::regex("Operator control flags.*\n"), ""); + + // Filter out the Device id field to allow cache sharing for a distributed run of the same pipeline + ss_str = std::regex_replace(ss_str, std::regex("Device id.*\n"), ""); + ss_str = std::regex_replace(ss_str, std::regex("device_id.*\n"), ""); + + // The Cache crc and Server cache id field is different when creating new cache_client and re-using the same + // cache_client later. So we filter out these two fields to allow cache sharing. + ss_str = std::regex_replace(ss_str, std::regex("Cache crc.*\n"), ""); + ss_str = std::regex_replace(ss_str, std::regex("Server cache id.*\n"), ""); + + uint32_t cache_crc = system::Crc32c::GetMaskCrc32cValue(ss_str.c_str(), ss_str.length()); + return cache_crc; +} } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/dataset_op.h b/mindspore/ccsrc/dataset/engine/datasetops/dataset_op.h index 2516fdbee1..254cd411c5 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/dataset_op.h +++ b/mindspore/ccsrc/dataset/engine/datasetops/dataset_op.h @@ -34,6 +34,8 @@ class DataBuffer; class NodePass; +class Sampler; + // The base class DatasetOp is the main tree node. It is an abstract class, so // the actual implementation of the operators will be derived from here. class DatasetOp : public std::enable_shared_from_this { @@ -55,7 +57,8 @@ class DatasetOp : public std::enable_shared_from_this { // Constructor // @param op_connector_size - The size for the output connector of this operator. - explicit DatasetOp(int32_t op_connector_size); + // @param sampler - The sampler for the op + explicit DatasetOp(int32_t op_connector_size, std::shared_ptr sampler); // Destructor virtual ~DatasetOp() { tree_ = nullptr; } @@ -204,6 +207,10 @@ class DatasetOp : public std::enable_shared_from_this { // @return Sets the control flags void set_control_flag(uint64_t flag) { BitSet(&op_ctrl_flags_, flag); } + // Setter function + // @return Sets the control flags + void ClearControlFlag(uint64_t flag) { BitClear(&op_ctrl_flags_, flag); } + // Register the internal worker connectors. No op unless it is a parallel op // @return Status virtual Status RegisterWorkerConnectors() { return Status::OK(); } @@ -271,6 +278,13 @@ class DatasetOp : public std::enable_shared_from_this { // @return Pointer to the ExecutionTree the current op belongs to, no ownership ExecutionTree *Tree() { return tree_; } + // Getter for the sampler + // @return Shared pointer to the sampler (may return nullptr) + std::shared_ptr sampler() { return sampler_; } + + // Computes a CRC value for the operator + static uint32_t GenerateCRC(const std::shared_ptr &op); + protected: // Adds a parent operator to this operator // @notes External callers do not have access to this function. @@ -289,8 +303,15 @@ class DatasetOp : public std::enable_shared_from_this { // @return - Status virtual Status ComputeColMap(); + // A helper function with some common code that leaf nodes can use during + // prepare phase for checking if they need to assign a sampler to the cache. + // @param random_access_op - indicate if this is a mappable random access leaf or not + // @return - Status + Status SaveSamplerForCache(bool random_access_op); + std::vector> child_; // Child nodes std::vector parent_; // Parent nodes. No ownership + std::shared_ptr sampler_; // Some leaf ops might have a sampler int32_t oc_queue_size_; // Capacity for each out_connector_ int32_t operator_id_; // Generated id for the node ExecutionTree *tree_; // Back pointer to our tree. diff --git a/mindspore/ccsrc/dataset/engine/datasetops/map_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/map_op.cc index fcb2e357e8..020f40d268 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/map_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/map_op.cc @@ -100,7 +100,7 @@ void MapOp::Print(std::ostream &out, bool show_all) const { } out << "\n TensorOps:"; for (size_t i = 0; i < tfuncs_.size(); i++) { - out << " " << tfuncs_[i]; + out << " " << *(tfuncs_[i].get()); } out << "\n\n"; } diff --git a/mindspore/ccsrc/dataset/engine/datasetops/parallel_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/parallel_op.cc index c0a8d95f15..244861a6c8 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/parallel_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/parallel_op.cc @@ -26,8 +26,8 @@ namespace mindspore { namespace dataset { // Constructor -ParallelOp::ParallelOp(int32_t num_workers, int32_t op_connector_size) - : DatasetOp(op_connector_size), +ParallelOp::ParallelOp(int32_t num_workers, int32_t op_connector_size, std::shared_ptr sampler) + : DatasetOp(op_connector_size, sampler), num_workers_(num_workers), num_producers_(num_workers), worker_connector_size_(1), diff --git a/mindspore/ccsrc/dataset/engine/datasetops/parallel_op.h b/mindspore/ccsrc/dataset/engine/datasetops/parallel_op.h index 142ec78360..f59d4bfc53 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/parallel_op.h +++ b/mindspore/ccsrc/dataset/engine/datasetops/parallel_op.h @@ -38,7 +38,8 @@ class ParallelOp : public DatasetOp { // Constructor // @param num_workers // @param op_connector_size - size of the output connector for this operator - ParallelOp(int32_t num_workers, int32_t op_connector_size); + // @param sampler - The sampler for the op + ParallelOp(int32_t num_workers, int32_t op_connector_size, std::shared_ptr sampler = nullptr); // Destructor ~ParallelOp() = default; diff --git a/mindspore/ccsrc/dataset/engine/datasetops/pipeline_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/pipeline_op.cc index 46eded8ea1..1d017a4d3e 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/pipeline_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/pipeline_op.cc @@ -20,7 +20,8 @@ namespace mindspore { namespace dataset { // Constructor -PipelineOp::PipelineOp(int32_t op_connector_size) : DatasetOp(op_connector_size) {} +PipelineOp::PipelineOp(int32_t op_connector_size, std::shared_ptr sampler) + : DatasetOp(op_connector_size, sampler) {} // A print method typically used for debugging void PipelineOp::Print(std::ostream &out, bool show_all) const { diff --git a/mindspore/ccsrc/dataset/engine/datasetops/pipeline_op.h b/mindspore/ccsrc/dataset/engine/datasetops/pipeline_op.h index a14279032d..cb3c76813b 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/pipeline_op.h +++ b/mindspore/ccsrc/dataset/engine/datasetops/pipeline_op.h @@ -32,7 +32,8 @@ class PipelineOp : public DatasetOp { // Constructor // @param op_connector_size - size of the output connector // @return Builder setter method returns reference to the builder. - explicit PipelineOp(int32_t op_connector_size); + // @param sampler - The sampler for the op + explicit PipelineOp(int32_t op_connector_size, std::shared_ptr sampler = nullptr); // Destructor ~PipelineOp() = default; diff --git a/mindspore/ccsrc/dataset/engine/datasetops/repeat_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/repeat_op.cc index 66e2177636..86903e540a 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/repeat_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/repeat_op.cc @@ -82,14 +82,14 @@ void RepeatOp::Print(std::ostream &out, bool show_all) const { Status RepeatOp::PrepareNodePostAction() { // Run any common code from super class first before adding our own specific logic RETURN_IF_NOT_OK(PipelineOp::PrepareNodePostAction()); - std::shared_ptr leaf_op = tree_->PopFromRepeatStack(); + std::shared_ptr leaf_op = tree_->PopFromEOEOpStack(); while (leaf_op != nullptr) { // Track the leaf operators that are under this repeat op. eoe_ops_.push_back(leaf_op); - leaf_op = tree_->PopFromRepeatStack(); + leaf_op = tree_->PopFromEOEOpStack(); } // Push ourselves to the stack in case one of our ascendants is repeat too. - tree_->AddToRepeatStack(shared_from_this()); + tree_->AddToEOEOpStack(shared_from_this()); return Status::OK(); } diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/celeba_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/celeba_op.cc index 7889362555..c7a4269a39 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/celeba_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/celeba_op.cc @@ -70,13 +70,12 @@ Status CelebAOp::Builder::SanityCheck() { CelebAOp::CelebAOp(int32_t num_workers, int32_t rows_per_buffer, const std::string &dir, int32_t queue_size, bool decode, const std::string &dataset_type, const std::set &exts, std::unique_ptr schema, std::shared_ptr sampler) - : ParallelOp(num_workers, queue_size), + : ParallelOp(num_workers, queue_size, std::move(sampler)), rows_per_buffer_(rows_per_buffer), folder_path_(dir), decode_(decode), extensions_(exts), data_schema_(std::move(schema)), - sampler_(std::move(sampler)), num_rows_in_attr_file_(0), dataset_type_(dataset_type) { attr_info_queue_ = std::make_unique>>(queue_size); diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/celeba_op.h b/mindspore/ccsrc/dataset/engine/datasetops/source/celeba_op.h index f8a49dabb2..a6fa495a14 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/celeba_op.h +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/celeba_op.h @@ -221,7 +221,6 @@ class CelebAOp : public ParallelOp, RandomAccessOp { bool decode_; std::set extensions_; // extensions allowed std::unique_ptr data_schema_; - std::shared_ptr sampler_; std::unique_ptr>> attr_info_queue_; int64_t num_rows_in_attr_file_; // rows number specified in attr file QueueList> io_block_queues_; diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/cifar_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/cifar_op.cc index e7c418b146..8dd615a8c1 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/cifar_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/cifar_op.cc @@ -79,12 +79,11 @@ Status CifarOp::Builder::SanityCheck() { CifarOp::CifarOp(CifarType type, int32_t num_works, int32_t rows_per_buf, const std::string &file_dir, int32_t queue_size, std::unique_ptr data_schema, std::shared_ptr sampler) - : ParallelOp(num_works, queue_size), + : ParallelOp(num_works, queue_size, std::move(sampler)), cifar_type_(type), rows_per_buffer_(rows_per_buf), folder_path_(file_dir), data_schema_(std::move(data_schema)), - sampler_(std::move(sampler)), row_cnt_(0), buf_cnt_(0) { constexpr uint64_t kUtilQueueSize = 512; diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/cifar_op.h b/mindspore/ccsrc/dataset/engine/datasetops/source/cifar_op.h index 21ed80ceab..917b23db94 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/cifar_op.h +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/cifar_op.h @@ -216,7 +216,6 @@ class CifarOp : public ParallelOp, public RandomAccessOp { int32_t rows_per_buffer_; std::string folder_path_; std::unique_ptr data_schema_; - std::shared_ptr sampler_; int64_t row_cnt_; int64_t buf_cnt_; diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/image_folder_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/image_folder_op.cc index c28ed2d3ab..cb17158bff 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/image_folder_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/image_folder_op.cc @@ -65,7 +65,7 @@ ImageFolderOp::ImageFolderOp(int32_t num_wkrs, int32_t rows_per_buffer, std::str bool recursive, bool do_decode, const std::set &exts, const std::map &map, std::unique_ptr data_schema, std::shared_ptr sampler) - : ParallelOp(num_wkrs, queue_size), + : ParallelOp(num_wkrs, queue_size, std::move(sampler)), rows_per_buffer_(rows_per_buffer), folder_path_(file_dir), recursive_(recursive), @@ -73,7 +73,6 @@ ImageFolderOp::ImageFolderOp(int32_t num_wkrs, int32_t rows_per_buffer, std::str extensions_(exts), class_index_(map), data_schema_(std::move(data_schema)), - sampler_(std::move(sampler)), row_cnt_(0), buf_cnt_(0), sampler_ind_(0), diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/image_folder_op.h b/mindspore/ccsrc/dataset/engine/datasetops/source/image_folder_op.h index 06f39deee0..6629fd6092 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/image_folder_op.h +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/image_folder_op.h @@ -259,7 +259,6 @@ class ImageFolderOp : public ParallelOp, public RandomAccessOp { std::set extensions_; // extensions allowed std::map class_index_; std::unique_ptr data_schema_; - std::shared_ptr sampler_; int64_t row_cnt_; int64_t buf_cnt_; int64_t sampler_ind_; diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/manifest_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/manifest_op.cc index e26bb7de65..e65da8707b 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/manifest_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/manifest_op.cc @@ -64,7 +64,7 @@ Status ManifestOp::Builder::SanityCheck() { ManifestOp::ManifestOp(int32_t num_works, int32_t rows_per_buffer, std::string file, int32_t queue_size, bool decode, const std::map &class_index, std::unique_ptr data_schema, std::shared_ptr sampler, std::string usage) - : ParallelOp(num_works, queue_size), + : ParallelOp(num_works, queue_size, std::move(sampler)), rows_per_buffer_(rows_per_buffer), io_block_pushed_(0), row_cnt_(0), @@ -72,7 +72,6 @@ ManifestOp::ManifestOp(int32_t num_works, int32_t rows_per_buffer, std::string f data_schema_(std::move(data_schema)), file_(file), class_index_(class_index), - sampler_(std::move(sampler)), decode_(decode), usage_(usage), buf_cnt_(0) { diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/manifest_op.h b/mindspore/ccsrc/dataset/engine/datasetops/source/manifest_op.h index 1bdf683084..c180ea581d 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/manifest_op.h +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/manifest_op.h @@ -230,7 +230,6 @@ class ManifestOp : public ParallelOp, public RandomAccessOp { std::unique_ptr data_schema_; std::string file_; // file that store the information of images std::map class_index_; - std::shared_ptr sampler_; bool decode_; std::string usage_; int64_t buf_cnt_; diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/mnist_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/mnist_op.cc index 67e7757da5..e98f8ae8c1 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/mnist_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/mnist_op.cc @@ -66,12 +66,11 @@ Status MnistOp::Builder::SanityCheck() { MnistOp::MnistOp(int32_t num_workers, int32_t rows_per_buffer, std::string folder_path, int32_t queue_size, std::unique_ptr data_schema, std::shared_ptr sampler) - : ParallelOp(num_workers, queue_size), + : ParallelOp(num_workers, queue_size, std::move(sampler)), buf_cnt_(0), row_cnt_(0), folder_path_(folder_path), rows_per_buffer_(rows_per_buffer), - sampler_(std::move(sampler)), data_schema_(std::move(data_schema)) { io_block_queues_.Init(num_workers, queue_size); } diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/mnist_op.h b/mindspore/ccsrc/dataset/engine/datasetops/source/mnist_op.h index c22ee24acd..9bd6276a11 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/mnist_op.h +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/mnist_op.h @@ -235,7 +235,6 @@ class MnistOp : public ParallelOp, public RandomAccessOp { WaitPost wp_; std::string folder_path_; // directory of image folder int32_t rows_per_buffer_; - std::shared_ptr sampler_; std::unique_ptr data_schema_; std::vector image_label_pairs_; std::vector image_names_; diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/random_data_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/random_data_op.cc index afd7ebcc55..3a865d8d69 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/random_data_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/random_data_op.cc @@ -21,6 +21,7 @@ #include "dataset/core/config_manager.h" #include "dataset/util/random.h" #include "dataset/util/wait_post.h" +#include "dataset/engine/datasetops/source/sampler/sequential_sampler.h" namespace mindspore { namespace dataset { @@ -30,7 +31,8 @@ RandomDataOp::Builder::Builder() builder_num_workers_(0), builder_op_connector_size_(0), builder_rows_per_buffer_(0), - builder_total_rows_(0) { + builder_total_rows_(0), + builder_sampler_(nullptr) { // Some arguments to the RandomDataOp have a default argument that is taken from the config. // The user may override these defaults by using the builder set methods. std::shared_ptr cfg = GlobalContext::config_manager(); @@ -43,8 +45,9 @@ RandomDataOp::Builder::Builder() Status RandomDataOp::Builder::Build(std::shared_ptr *out_op) { RETURN_IF_NOT_OK(SanityCheck()); - *out_op = std::make_shared(builder_num_workers_, builder_op_connector_size_, builder_rows_per_buffer_, - builder_total_rows_, std::move(builder_data_schema_)); + *out_op = + std::make_shared(builder_num_workers_, builder_op_connector_size_, builder_rows_per_buffer_, + builder_total_rows_, std::move(builder_data_schema_), std::move(builder_sampler_)); // If the user did not provide a schema, then we will ask the op to generate a pseudo-random // schema. @@ -66,8 +69,8 @@ Status RandomDataOp::Builder::SanityCheck() const { // Constructor for RandomDataOp RandomDataOp::RandomDataOp(int32_t num_workers, int32_t op_connector_size, int64_t rows_per_buffer, int64_t total_rows, - std::unique_ptr data_schema) - : ParallelOp(num_workers, op_connector_size), + std::unique_ptr data_schema, std::shared_ptr sampler) + : ParallelOp(num_workers, op_connector_size, std::move(sampler)), buffer_id_(0), rows_per_buffer_(rows_per_buffer), total_rows_(total_rows), @@ -124,7 +127,7 @@ Status RandomDataOp::GenerateSchema() { // For each column: // - choose a datatype // - generate a shape that randomly chooses the number of dimensions and the dimension values. - DataType::Type newType = static_cast(GenRandomInt(0, DataType::NUM_OF_TYPES - 2)); + DataType::Type newType = static_cast(GenRandomInt(1, DataType::NUM_OF_TYPES - 2)); int32_t rank = GenRandomInt(1, kMaxRank); std::vector dims; for (int32_t d = 0; d < rank; d++) { @@ -412,5 +415,15 @@ Status RandomDataOp::ComputeColMap() { } return Status::OK(); } + +// During tree prepare phase, operators may have specific post-operations to perform depending on +// their role. +Status RandomDataOp::PrepareNodePostAction() { + // Run common code from super class before adding RandomDataOp specific handling + RETURN_IF_NOT_OK(ParallelOp::PrepareNodePostAction()); + // Specific handling for this op, we need to do cache op work to assign the sampler to the cache. + RETURN_IF_NOT_OK(DatasetOp::SaveSamplerForCache(false)); + return Status::OK(); +} } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/random_data_op.h b/mindspore/ccsrc/dataset/engine/datasetops/source/random_data_op.h index 020c9a6e09..b2af27dda3 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/random_data_op.h +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/random_data_op.h @@ -42,7 +42,7 @@ class RandomDataOp : public ParallelOp { // Some constants to provide limits to random generation. static constexpr int32_t kMaxNumColumns = 4; static constexpr int32_t kMaxRank = 4; - static constexpr int32_t kMaxDimValue = 2048; + static constexpr int32_t kMaxDimValue = 32; static constexpr int32_t kMaxTotalRows = 1024; // A nested builder class to aid in the construction of a RandomDataOp @@ -117,6 +117,14 @@ class RandomDataOp : public ParallelOp { return *this; } + // Setter method + // @param std::shared_ptr sampler + // @return Builder setter method returns reference to the builder. + Builder &SetSampler(std::shared_ptr sampler) { + builder_sampler_ = std::move(sampler); + return *this; + } + private: /** * Check if the required parameters are set by the builder. @@ -125,6 +133,7 @@ class RandomDataOp : public ParallelOp { Status SanityCheck() const; std::unique_ptr builder_data_schema_; + std::shared_ptr builder_sampler_; int32_t builder_num_workers_; int32_t builder_op_connector_size_; int64_t builder_rows_per_buffer_; @@ -139,10 +148,11 @@ class RandomDataOp : public ParallelOp { * @param rows_per_buffer - The number of rows in each DataBuffer * @param data_schema - A user-provided schema * @param total_rows - The total number of rows in the dataset + * @param sampler - allow a sampler. Only valid if a cache exists in ascendent tree nodes * @return Builder - The modified builder by reference */ RandomDataOp(int32_t num_workers, int32_t op_connector_size, int64_t rows_per_buffer, int64_t total_rows, - std::unique_ptr data_schema); + std::unique_ptr data_schema, std::shared_ptr sampler); /** * Destructor @@ -193,6 +203,12 @@ class RandomDataOp : public ParallelOp { // @return Name of the current Op std::string Name() const override { return "RandomDataOp"; } + // During tree prepare phase, operators may have specific post-operations to perform depending on + // their role. + // @notes Derived versions of this function should always call it's superclass version first + // before providing their own implementations. + Status PrepareNodePostAction() override; + private: /** * The entry point code for when workers are launched diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/distributed_sampler.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/distributed_sampler.cc index 226647df14..9f4a9cf55c 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/distributed_sampler.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/distributed_sampler.cc @@ -107,12 +107,11 @@ Status DistributedSampler::ResetSampler() { } void DistributedSampler::Print(std::ostream &out, bool show_all) const { - out << "(sampler): DistributedSampler\n"; + out << "\nSampler: DistributedSampler"; if (show_all) { - out << "seed_: " << seed_ << '\n'; - out << "device_id_: " << device_id_ << '\n'; - out << "num_devices_: " << num_devices_ << '\n'; - out << "shuffle_: " << shuffle_ << '\n'; + Sampler::Print(out, show_all); + out << "\nseed: " << seed_ << "\ndevice_id: " << device_id_ << "\nnum_devices: " << num_devices_ + << "\nshuffle: " << shuffle_; } } diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/pk_sampler.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/pk_sampler.cc index 92a880d599..cd2cadb9ff 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/pk_sampler.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/pk_sampler.cc @@ -113,5 +113,13 @@ Status PKSampler::HandshakeRandomAccessOp(const RandomAccessOp *op) { return Status::OK(); } +void PKSampler::Print(std::ostream &out, bool show_all) const { + out << "\nSampler: PKSampler"; + if (show_all) { + // Call the super class for displaying any common detailed info + Sampler::Print(out, show_all); + // Then add our own info if any + } +} } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/pk_sampler.h b/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/pk_sampler.h index 7b1423326a..cde8a75b5b 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/pk_sampler.h +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/pk_sampler.h @@ -56,6 +56,11 @@ class PKSampler : public Sampler { // NOT YET FINISHED // @return - The error code return Status ResetSampler() override; + // Printer for debugging purposes. + // @param out - output stream to write to + // @param show_all - bool to show detailed vs summary + void Print(std::ostream &out, bool show_all) const override; + private: bool shuffle_; uint32_t seed_; diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/python_sampler.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/python_sampler.cc index af4aa20bb2..d204c55ce9 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/python_sampler.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/python_sampler.cc @@ -103,5 +103,14 @@ Status PythonSampler::ResetSampler() { return Status::OK(); } + +void PythonSampler::Print(std::ostream &out, bool show_all) const { + out << "\nSampler: PythonSampler"; + if (show_all) { + // Call the super class for displaying any common detailed info + Sampler::Print(out, show_all); + // Then add our own info if any + } +} } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/python_sampler.h b/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/python_sampler.h index 49ff12878d..7d653b2087 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/python_sampler.h +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/python_sampler.h @@ -50,6 +50,11 @@ class PythonSampler : public Sampler { // @return - The error code return Status GetNextSample(std::unique_ptr *out_buffer) override; + // Printer for debugging purposes. + // @param out - output stream to write to + // @param show_all - bool to show detailed vs summary + void Print(std::ostream &out, bool show_all) const override; + private: bool need_to_reset_; // Whether Reset() should be called before calling GetNextBuffer() diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/random_sampler.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/random_sampler.cc index b3dfaad7f7..db0a96ea3a 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/random_sampler.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/random_sampler.cc @@ -113,13 +113,12 @@ Status RandomSampler::ResetSampler() { } void RandomSampler::Print(std::ostream &out, bool show_all) const { - out << "(sampler): RandomSampler\n"; - + out << "\nSampler: RandomSampler"; if (show_all) { - out << "num_samples_: " << num_samples_ << '\n'; - out << "next_id_: " << next_id_ << '\n'; + // Call the super class for displaying any common detailed info + Sampler::Print(out, show_all); + // Then add our own info if any } } - } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/sampler.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/sampler.cc index b3c595870f..1584166dc3 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/sampler.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/sampler.cc @@ -80,11 +80,12 @@ Status Sampler::CreateSamplerTensor(std::shared_ptr *sample_ids, int64_t } void Sampler::Print(std::ostream &out, bool show_all) const { - out << "(sampler): base\n"; - + // Sampler printing is usually only called in the show_all mode. + // Derived classes will display the name, then call back to this base + // for common info. + // No-op in the summary mode. if (show_all) { - out << "num_rows_: " << num_rows_ << '\n'; - out << "num_samples_: " << num_samples_ << '\n'; + out << "\nnum_rows_: " << num_rows_ << "\nnum_samples_: " << num_samples_; } } diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/sequential_sampler.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/sequential_sampler.cc index f0ff6a2c02..28598da55f 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/sequential_sampler.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/sequential_sampler.cc @@ -89,7 +89,14 @@ Status SequentialSampler::ResetSampler() { return Status::OK(); } -void SequentialSampler::Print(std::ostream &out, bool show_all) const { out << "(sampler): SequentialSampler\n"; } - +void SequentialSampler::Print(std::ostream &out, bool show_all) const { + out << "\nSampler: SequentialSampler"; + if (show_all) { + // Call the super class for displaying any common detailed info + Sampler::Print(out, show_all); + // Then add our own info + out << "\nStart index: " << start_index_; + } +} } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/sequential_sampler.h b/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/sequential_sampler.h index 2cb7a9ff8d..06f084fb7a 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/sequential_sampler.h +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/sequential_sampler.h @@ -49,6 +49,9 @@ class SequentialSampler : public Sampler { // @return - The error code return Status GetNextSample(std::unique_ptr *out_buffer) override; + // Printer for debugging purposes. + // @param out - output stream to write to + // @param show_all - bool to show detailed vs summary void Print(std::ostream &out, bool show_all) const override; private: diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/subset_random_sampler.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/subset_random_sampler.cc index 54491889fc..08a623ed1b 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/subset_random_sampler.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/subset_random_sampler.cc @@ -119,5 +119,14 @@ Status SubsetRandomSampler::GetNextSample(std::unique_ptr *out_buffe return Status::OK(); } + +void SubsetRandomSampler::Print(std::ostream &out, bool show_all) const { + out << "\nSampler: SubsetRandomSampler"; + if (show_all) { + // Call the super class for displaying any common detailed info + Sampler::Print(out, show_all); + // Then add our own info if any + } +} } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/subset_random_sampler.h b/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/subset_random_sampler.h index 980ffe578a..ffc7cb17bc 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/subset_random_sampler.h +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/subset_random_sampler.h @@ -51,6 +51,11 @@ class SubsetRandomSampler : public Sampler { // @note the sample ids (int64_t) will be placed in one Tensor and be placed into pBuffer. Status GetNextSample(std::unique_ptr *out_buffer) override; + // Printer for debugging purposes. + // @param out - output stream to write to + // @param show_all - bool to show detailed vs summary + void Print(std::ostream &out, bool show_all) const override; + private: // A list of indices (already randomized in constructor). std::vector indices_; diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/weighted_random_sampler.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/weighted_random_sampler.cc index 759af99352..6bf3d2d85e 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/weighted_random_sampler.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/weighted_random_sampler.cc @@ -156,5 +156,14 @@ Status WeightedRandomSampler::GetNextSample(std::unique_ptr *out_buf return Status::OK(); } + +void WeightedRandomSampler::Print(std::ostream &out, bool show_all) const { + out << "\nSampler: WeightedRandomSampler"; + if (show_all) { + // Call the super class for displaying any common detailed info + Sampler::Print(out, show_all); + // Then add our own info if any + } +} } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/weighted_random_sampler.h b/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/weighted_random_sampler.h index 257501250d..1fbe29ed80 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/weighted_random_sampler.h +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/sampler/weighted_random_sampler.h @@ -53,6 +53,11 @@ class WeightedRandomSampler : public Sampler { // @note the sample ids (int64_t) will be placed in one Tensor and be placed into pBuffer. Status GetNextSample(std::unique_ptr *out_buffer) override; + // Printer for debugging purposes. + // @param out - output stream to write to + // @param show_all - bool to show detailed vs summary + void Print(std::ostream &out, bool show_all) const override; + private: // A list of weights for each sample. std::vector weights_; diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/text_file_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/text_file_op.cc index fbba73de21..818b5ab3f4 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/text_file_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/text_file_op.cc @@ -33,7 +33,11 @@ namespace mindspore { namespace dataset { TextFileOp::Builder::Builder() - : builder_device_id_(0), builder_num_devices_(1), builder_total_rows_(0), builder_shuffle_files_(false) { + : builder_device_id_(0), + builder_num_devices_(1), + builder_total_rows_(0), + builder_shuffle_files_(false), + builder_sampler_(nullptr) { std::shared_ptr config_manager = GlobalContext::config_manager(); builder_num_workers_ = config_manager->num_parallel_workers(); builder_op_connector_size_ = config_manager->op_connector_size(); @@ -64,7 +68,7 @@ Status TextFileOp::Builder::Build(std::shared_ptr *op) { std::shared_ptr text_file_op = std::make_shared( builder_num_workers_, builder_rows_per_buffer_, builder_total_rows_, builder_worker_connector_size_, std::move(builder_schema_), builder_text_files_list_, builder_op_connector_size_, builder_shuffle_files_, - builder_num_devices_, builder_device_id_); + builder_num_devices_, builder_device_id_, std::move(builder_sampler_)); RETURN_IF_NOT_OK(text_file_op->Init()); *op = std::move(text_file_op); @@ -73,8 +77,9 @@ Status TextFileOp::Builder::Build(std::shared_ptr *op) { TextFileOp::TextFileOp(int32_t num_workers, int64_t rows_per_buffer, int64_t total_rows, int32_t worker_connector_size, std::unique_ptr schema, std::vector text_files_list, - int32_t op_connector_size, bool shuffle_files, int32_t num_device, int32_t device_id) - : ParallelOp(num_workers, op_connector_size), + int32_t op_connector_size, bool shuffle_files, int32_t num_device, int32_t device_id, + std::shared_ptr sampler) + : ParallelOp(num_workers, op_connector_size, std::move(sampler)), device_id_(device_id), num_devices_(num_device), rows_per_buffer_(rows_per_buffer), diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/text_file_op.h b/mindspore/ccsrc/dataset/engine/datasetops/source/text_file_op.h index 5379263979..5b787d4dad 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/text_file_op.h +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/text_file_op.h @@ -20,6 +20,7 @@ #include #include #include +#include #include #include "dataset/util/status.h" @@ -112,6 +113,14 @@ class TextFileOp : public ParallelOp { return *this; } + // Setter method + // @param std::shared_ptr sampler + // @return Builder setter method returns reference to the builder. + Builder &SetSampler(std::shared_ptr sampler) { + builder_sampler_ = std::move(sampler); + return *this; + } + private: int32_t builder_device_id_; int32_t builder_num_devices_; @@ -123,6 +132,7 @@ class TextFileOp : public ParallelOp { std::vector builder_text_files_list_; bool builder_shuffle_files_; std::unique_ptr builder_schema_; + std::shared_ptr builder_sampler_; }; // Constructor of TextFileOp @@ -136,9 +146,10 @@ class TextFileOp : public ParallelOp { // @param columns_to_load - the names of the columns to load data from. // @param shuffle_files - whether or not to shuffle the files before reading data. // @param equal_rows_per_shard - whether or not to get equal rows for each process. + // @param sampler - allow a sampler. Only valid if a cache exists in ascendent tree nodes TextFileOp(int32_t num_workers, int64_t rows_per_buffer, int64_t total_rows, int32_t worker_connector_size, std::unique_ptr, std::vector text_files_list, int32_t op_connector_size, - bool shuffle_files, int32_t num_devices, int32_t device_id); + bool shuffle_files, int32_t num_devices, int32_t device_id, std::shared_ptr sampler); // Default destructor ~TextFileOp() = default; diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/tf_reader_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/tf_reader_op.cc index b05fa54978..a2b04bcc01 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/tf_reader_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/tf_reader_op.cc @@ -48,7 +48,11 @@ namespace mindspore { namespace dataset { TFReaderOp::Builder::Builder() - : builder_device_id_(0), builder_num_devices_(1), builder_total_rows_(0), builder_equal_rows_per_shard_(false) { + : builder_device_id_(0), + builder_num_devices_(1), + builder_total_rows_(0), + builder_equal_rows_per_shard_(false), + builder_sampler_(nullptr) { std::shared_ptr config_manager = GlobalContext::config_manager(); builder_num_workers_ = config_manager->num_parallel_workers(); builder_worker_connector_size_ = config_manager->worker_connector_size(); @@ -87,11 +91,6 @@ Status TFReaderOp::Builder::ValidateInputs() const { err_msg += "Number of parallel workers is smaller or equal to 0\n"; } - if (!builder_equal_rows_per_shard_ && - builder_dataset_files_list_.size() < static_cast(builder_num_devices_)) { - err_msg += "Not enough tfrecord files provided\n"; - } - if (builder_device_id_ >= builder_num_devices_ || builder_num_devices_ < 1) { err_msg += "Wrong sharding configs\n"; } @@ -125,7 +124,8 @@ Status TFReaderOp::Builder::Build(std::shared_ptr *out_tf_reader_op) std::shared_ptr new_tf_reader_op = std::make_shared( builder_num_workers_, builder_worker_connector_size_, builder_rows_per_buffer_, builder_total_rows_, builder_dataset_files_list_, std::move(builder_data_schema_), builder_op_connector_size_, builder_columns_to_load_, - builder_shuffle_files_, builder_num_devices_, builder_device_id_, builder_equal_rows_per_shard_); + builder_shuffle_files_, builder_num_devices_, builder_device_id_, builder_equal_rows_per_shard_, + std::move(builder_sampler_)); RETURN_IF_NOT_OK(new_tf_reader_op->Init()); *out_tf_reader_op = std::move(new_tf_reader_op); @@ -136,8 +136,8 @@ TFReaderOp::TFReaderOp(int32_t num_workers, int32_t worker_connector_size, int64 int64_t total_num_rows, std::vector dataset_files_list, std::unique_ptr data_schema, int32_t op_connector_size, std::vector columns_to_load, bool shuffle_files, int32_t num_device, - int32_t device_id, bool equal_rows_per_shard) - : ParallelOp(num_workers, op_connector_size), + int32_t device_id, bool equal_rows_per_shard, std::shared_ptr sampler) + : ParallelOp(num_workers, op_connector_size, std::move(sampler)), device_id_(device_id), num_devices_(num_device), rows_per_buffer_(rows_per_buffer), @@ -1018,5 +1018,40 @@ Status TFReaderOp::ComputeColMap() { } return Status::OK(); } + +// During tree prepare phase, operators may have specific post-operations to perform depending on +// their role. +Status TFReaderOp::PrepareNodePostAction() { + // Run common code from super class before adding TFReaderOp specific handling + RETURN_IF_NOT_OK(ParallelOp::PrepareNodePostAction()); + + // Specific handling for this op, we need to do cache op work so assign the sampler to the cache + // TF is a special case because it can support file-based sharding/shuffling, or, if there + // is a cache, then it can also do row-based sampler using the sampler on the cache. + // Thus, pass true for random access op flag when saving the sampler. This is a special case, + // since usually a non-mappable dataset would pass false here. + RETURN_IF_NOT_OK(DatasetOp::SaveSamplerForCache(true)); + + // Now that the sampler has been saved for the cache, we need to adjust the TFReaderOp to turn it into + // a simpler producer of all data (no shuffling or sharding or anything) + if (BitTest(tree_->PrepareFlags(), ExecutionTree::kDePrepCache)) { + device_id_ = 0; + num_devices_ = 1; + total_rows_ = 0; + shuffle_files_ = false; + equal_rows_per_shard_ = false; + sampler_.reset(); // Normally SaveSampler code did this for us, but we passed in true above (See comment) + } else { + // This sanity check had been delayed until now in the prepare loop. + // If we are not in a cache path, then we can validate the the file-based sharding config. + // If we are in a cache path, there is no file-based sharding so the check is not correct in that + // situation. + if (!equal_rows_per_shard_ && dataset_files_list_.size() < static_cast(num_devices_)) { + RETURN_STATUS_UNEXPECTED("Not enough tfrecord files provided\n"); + } + } + + return Status::OK(); +} } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/tf_reader_op.h b/mindspore/ccsrc/dataset/engine/datasetops/source/tf_reader_op.h index 9d2e38ec6b..9226c4c6c5 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/tf_reader_op.h +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/tf_reader_op.h @@ -153,8 +153,17 @@ class TFReaderOp : public ParallelOp { return *this; } + // Setter method + // @param std::shared_ptr sampler + // @return Builder setter method returns reference to the builder. + Builder &SetSampler(std::shared_ptr sampler) { + builder_sampler_ = std::move(sampler); + return *this; + } + private: std::unique_ptr builder_data_schema_; + std::shared_ptr builder_sampler_; int32_t builder_device_id_; int32_t builder_num_devices_; int32_t builder_num_workers_; @@ -180,10 +189,11 @@ class TFReaderOp : public ParallelOp { // @param columns_to_load - the names of the columns to load data from. // @param shuffle_files - whether or not to shuffle the files before reading data. // @param equal_rows_per_shard - whether or not to get equal rows for each process. + // @param sampler - allow a sampler. Only valid if a cache exists in ascendent tree nodes TFReaderOp(int32_t num_workers, int32_t worker_connector_size, int64_t rows_per_buffer, int64_t total_num_rows, std::vector dataset_files_list, std::unique_ptr data_schema, int32_t op_connector_size, std::vector columns_to_load, bool shuffle_files, - int32_t num_devices, int32_t device_id, bool equal_rows_per_shard); + int32_t num_devices, int32_t device_id, bool equal_rows_per_shard, std::shared_ptr sampler); // Default destructor ~TFReaderOp() = default; @@ -236,6 +246,12 @@ class TFReaderOp : public ParallelOp { // @return Vector of the input file names std::vector FileNames() { return dataset_files_list_; } + // During tree prepare phase, operators may have specific post-operations to perform depending on + // their role. + // @notes Derived versions of this function should always call it's superclass version first + // before providing their own implementations. + Status PrepareNodePostAction() override; + private: // The entry point for when workers are launched. // @param worker_id - the id of the worker that is executing this function. diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/voc_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/voc_op.cc index 5d9f0ee92c..958aa65b06 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/voc_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/voc_op.cc @@ -88,7 +88,7 @@ Status VOCOp::Builder::SanityCheck() { VOCOp::VOCOp(const TaskType &task_type, const std::string &task_mode, const std::string &folder_path, const std::map &class_index, int32_t num_workers, int32_t rows_per_buffer, int32_t queue_size, bool decode, std::unique_ptr data_schema, std::shared_ptr sampler) - : ParallelOp(num_workers, queue_size), + : ParallelOp(num_workers, queue_size, std::move(sampler)), decode_(decode), row_cnt_(0), buf_cnt_(0), @@ -97,7 +97,6 @@ VOCOp::VOCOp(const TaskType &task_type, const std::string &task_mode, const std: folder_path_(folder_path), class_index_(class_index), rows_per_buffer_(rows_per_buffer), - sampler_(std::move(sampler)), data_schema_(std::move(data_schema)) { io_block_queues_.Init(num_workers_, queue_size); } diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/voc_op.h b/mindspore/ccsrc/dataset/engine/datasetops/source/voc_op.h index a0f5eba4d6..89875341ca 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/voc_op.h +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/voc_op.h @@ -274,7 +274,6 @@ class VOCOp : public ParallelOp, public RandomAccessOp { TaskType task_type_; std::string task_mode_; int32_t rows_per_buffer_; - std::shared_ptr sampler_; std::unique_ptr data_schema_; WaitPost wp_; diff --git a/mindspore/ccsrc/dataset/engine/datasetops/take_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/take_op.cc index 05c224ee2e..259ae8e62b 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/take_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/take_op.cc @@ -129,7 +129,7 @@ Status TakeOp::FillBuffer(std::unique_ptr *buffer, std::unique_ptrAddToRepeatStack(shared_from_this()); + tree_->AddToEOEOpStack(shared_from_this()); return Status::OK(); } diff --git a/mindspore/ccsrc/dataset/engine/execution_tree.cc b/mindspore/ccsrc/dataset/engine/execution_tree.cc index 8dd622912b..2f88ee1795 100644 --- a/mindspore/ccsrc/dataset/engine/execution_tree.cc +++ b/mindspore/ccsrc/dataset/engine/execution_tree.cc @@ -88,13 +88,13 @@ Status ExecutionTree::AssignRoot(const std::shared_ptr &op) { } // A print method typically used for debugging -void ExecutionTree::Print(std::ostream &out) const { +void ExecutionTree::Print(std::ostream &out, const std::shared_ptr &op) const { out << "Execution tree summary:\n" << "-----------------------\n"; - this->PrintNode(out, root_, "", true, false); + this->PrintNode(out, op == nullptr ? root_ : op, "", true, false); out << "\nExecution tree operator details:\n" << "--------------------------------\n"; - this->PrintNode(out, root_, "", true, true); + this->PrintNode(out, op == nullptr ? root_ : op, "", true, true); } // A helper functions for doing the recursive printing @@ -269,27 +269,40 @@ Status ExecutionTree::PrepareNode(const std::shared_ptr &dataset_op) RETURN_IF_NOT_OK(this->PrepareNode(i)); } - // Then clear the flags from this op now that we have prepared it. - BitClear(&prepare_flags_, op_prep_flags); - // No more children, now we execute any prepare actions before going back up the // the tree on recursive function RETURN_IF_NOT_OK(dataset_op->PrepareNodePostAction()); + // Then clear the flags from this op now that we have prepared it. + BitClear(&prepare_flags_, op_prep_flags); + return Status::OK(); } -// Adds an operator to the repeat stack during prepare phase. -void ExecutionTree::AddToRepeatStack(std::shared_ptr dataset_op) { repeat_stack_.push(dataset_op); } +// Adds an operator to the eoe operator stack during prepare phase. +void ExecutionTree::AddToEOEOpStack(std::shared_ptr dataset_op) { eoe_stack_.push(dataset_op); } -// Pops an operator from the repeat stack during prepare phase. -std::shared_ptr ExecutionTree::PopFromRepeatStack() { +// Pops an operator from the eoe operator stack during prepare phase. +std::shared_ptr ExecutionTree::PopFromEOEOpStack() { std::shared_ptr top_op = nullptr; - if (!repeat_stack_.empty()) { - top_op = repeat_stack_.top(); - repeat_stack_.pop(); + if (!eoe_stack_.empty()) { + top_op = eoe_stack_.top(); + eoe_stack_.pop(); } return top_op; } + +// Adds a sampler to the sampler stack during prepare phase. +void ExecutionTree::AddToSamplerStack(std::shared_ptr sampler) { sampler_stack_.push(sampler); } + +// Pops an operator from the sampler stack during prepare phase. +std::shared_ptr ExecutionTree::PopFromSamplerStack() { + std::shared_ptr top_sampler = nullptr; + if (!sampler_stack_.empty()) { + top_sampler = sampler_stack_.top(); + sampler_stack_.pop(); + } + return top_sampler; +} } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/execution_tree.h b/mindspore/ccsrc/dataset/engine/execution_tree.h index b0391bf77b..5ebfa539ad 100644 --- a/mindspore/ccsrc/dataset/engine/execution_tree.h +++ b/mindspore/ccsrc/dataset/engine/execution_tree.h @@ -37,7 +37,8 @@ class ExecutionTree { // Prepare flags used during tree prepare phase enum PrepareFlags { kDePrepNone = 0, - kDePrepRepeat = 1 // Processing a repeat operation + kDePrepRepeat = 1, // Processing a repeat operation + kDePrepCache = 2 // Processing a cache operation }; // State flags for the lifecycle of the tree @@ -118,9 +119,9 @@ class ExecutionTree { // @return Status - The error code return Status Launch(); - // A print method typically used for debugging - // @param out - The output stream to write output to - void Print(std::ostream &out) const; + /// A print method typically used for debugging + /// \param out - The output stream to write output to + void Print(std::ostream &out, const std::shared_ptr &op = nullptr) const; // Returns an iterator positioned at the start // @return Iterator - The iterator @@ -199,14 +200,23 @@ class ExecutionTree { // @return Status - The error code return Status PrepareNode(const std::shared_ptr &dataset_op); - // Adds an operator to the repeat stack during prepare phase. - // @param op - The dataset op to work add to repeat stack - // @return Status - The error code return - void AddToRepeatStack(std::shared_ptr dataset_op); + /// Adds an operator to the eoe operator stack during prepare phase. + /// \param op - The dataset op to work add to eoe stack + /// \return Status - The error code return + void AddToEOEOpStack(std::shared_ptr dataset_op); + + /// Pops an operator from the eoe operator stack during prepare phase. + /// \return shared_ptr to the popped operator + std::shared_ptr PopFromEOEOpStack(); + + /// Adds a sampler to the sampler stack during prepare phase. + /// \param samplerop - The dataset op to work add to eoe stack + /// \return Status - The error code return + void AddToSamplerStack(std::shared_ptr sampler); - // Pops an operator from the repeat stack during prepare phase. - // @return shared_ptr to the popped operator - std::shared_ptr PopFromRepeatStack(); + /// Pops an operator from the sampler stack during prepare phase. + /// \return shared_ptr to the popped operator + std::shared_ptr PopFromSamplerStack(); // Return the pointer to the TaskGroup // @return raw pointer to the TaskGroup @@ -236,9 +246,10 @@ class ExecutionTree { int32_t id_count_; // Counter for generating operator id's uint32_t prepare_flags_; // Flags used during tree prepare TreeState tree_state_; // Tracking the current tree state - std::stack> repeat_stack_; // A stack used during prepare phase std::unique_ptr perf_monitor_; // Performance Monitor std::unique_ptr profiling_manager_; // Profiling manager + std::stack> eoe_stack_; // A stack used during prepare phase + std::stack> sampler_stack_; // A stack used during prepare phase }; } // namespace dataset } // namespace mindspore From 48d3bc4686909600b853a7cad705dc22c2f0658f Mon Sep 17 00:00:00 2001 From: yuchaojie Date: Mon, 29 Jun 2020 20:26:16 +0800 Subject: [PATCH 181/254] modify tokenization for transformer model --- model_zoo/Transformer/create_data.py | 12 +- .../Transformer/scripts/replace-quote.perl | 15 ++ model_zoo/Transformer/src/tokenization.py | 225 ++++++++---------- 3 files changed, 116 insertions(+), 136 deletions(-) diff --git a/model_zoo/Transformer/create_data.py b/model_zoo/Transformer/create_data.py index af941623cb..fc81d3c232 100644 --- a/model_zoo/Transformer/create_data.py +++ b/model_zoo/Transformer/create_data.py @@ -37,13 +37,13 @@ class SampleInstance(): def __str__(self): s = "" s += "source sos tokens: %s\n" % (" ".join( - [tokenization.printable_text(x) for x in self.source_sos_tokens])) + [tokenization.convert_to_printable(x) for x in self.source_sos_tokens])) s += "source eos tokens: %s\n" % (" ".join( - [tokenization.printable_text(x) for x in self.source_eos_tokens])) + [tokenization.convert_to_printable(x) for x in self.source_eos_tokens])) s += "target sos tokens: %s\n" % (" ".join( - [tokenization.printable_text(x) for x in self.target_sos_tokens])) + [tokenization.convert_to_printable(x) for x in self.target_sos_tokens])) s += "target eos tokens: %s\n" % (" ".join( - [tokenization.printable_text(x) for x in self.target_eos_tokens])) + [tokenization.convert_to_printable(x) for x in self.target_eos_tokens])) s += "\n" return s @@ -185,9 +185,9 @@ def main(): if total_written <= 20: logging.info("*** Example ***") logging.info("source tokens: %s", " ".join( - [tokenization.printable_text(x) for x in instance.source_eos_tokens])) + [tokenization.convert_to_printable(x) for x in instance.source_eos_tokens])) logging.info("target tokens: %s", " ".join( - [tokenization.printable_text(x) for x in instance.target_sos_tokens])) + [tokenization.convert_to_printable(x) for x in instance.target_sos_tokens])) for feature_name in features.keys(): feature = features[feature_name] diff --git a/model_zoo/Transformer/scripts/replace-quote.perl b/model_zoo/Transformer/scripts/replace-quote.perl index 95f9abcc91..5e6e715daf 100644 --- a/model_zoo/Transformer/scripts/replace-quote.perl +++ b/model_zoo/Transformer/scripts/replace-quote.perl @@ -1,4 +1,19 @@ #!/usr/bin/env perl +#!/bin/bash +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ use warnings; use strict; diff --git a/model_zoo/Transformer/src/tokenization.py b/model_zoo/Transformer/src/tokenization.py index fd0fc97955..6c4f4fec20 100644 --- a/model_zoo/Transformer/src/tokenization.py +++ b/model_zoo/Transformer/src/tokenization.py @@ -1,193 +1,158 @@ -# coding=utf-8 -# Copyright 2018 The Google AI Language Team Authors. +# Copyright 2020 Huawei Technologies Co., Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -############################################################################### -# Modified by Huawei Technologies Co., Ltd, May, 2020, with following changes: -# - Remove some unused classes and functions -# - Modify load_vocab, convert_to_unicode, printable_text function -# - Modify BasicTokenizer class -# - Add WhiteSpaceTokenizer class -############################################################################### - +# ============================================================================ """Tokenization utilities.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - +import sys import collections import unicodedata -import six -def convert_to_unicode(text): - """Converts `text` to Unicode (if it's not already), assuming utf-8 input.""" - if six.PY3: +def convert_to_printable(text): + """ + Converts `text` to a printable coding format. + """ + if sys.version_info[0] == 3: if isinstance(text, str): return text if isinstance(text, bytes): return text.decode("utf-8", "ignore") - raise ValueError("Unsupported string type: %s" % (type(text))) - if six.PY2: + raise ValueError("Only support type `str` or `bytes`, while text type is `%s`" % (type(text))) + if sys.version_info[0] == 2: if isinstance(text, str): - return text.decode("utf-8", "ignore") - if isinstance(text, unicode): return text - raise ValueError("Unsupported string type: %s" % (type(text))) - raise ValueError("Not running on Python2 or Python 3?") - + if isinstance(text, unicode): + return text.encode("utf-8") + raise ValueError("Only support type `str` or `unicode`, while text type is `%s`" % (type(text))) + raise ValueError("Only supported when running on Python2 or Python3.") -def printable_text(text): - """Returns text encoded in a way suitable for print or `logging`.""" - # These functions want `str` for both Python2 and Python3, but in one case - # it's a Unicode string and in the other it's a byte string. - if six.PY3: +def convert_to_unicode(text): + """ + Converts `text` to Unicode format. + """ + if sys.version_info[0] == 3: if isinstance(text, str): return text if isinstance(text, bytes): return text.decode("utf-8", "ignore") - raise ValueError("Unsupported string type: %s" % (type(text))) - if six.PY2: + raise ValueError("Only support type `str` or `bytes`, while text type is `%s`" % (type(text))) + if sys.version_info[0] == 2: if isinstance(text, str): - return text + return text.decode("utf-8", "ignore") if isinstance(text, unicode): - return text.encode("utf-8") - raise ValueError("Unsupported string type: %s" % (type(text))) - raise ValueError("Not running on Python2 or Python 3?") + return text + raise ValueError("Only support type `str` or `unicode`, while text type is `%s`" % (type(text))) + raise ValueError("Only supported when running on Python2 or Python3.") -def load_vocab(vocab_file): - """Loads a vocabulary file into a dictionary.""" - vocab = collections.OrderedDict() +def load_vocab_file(vocab_file): + """ + Loads a vocabulary file and turns into a {token:id} dictionary. + """ + vocab_dict = collections.OrderedDict() index = 0 - with open(vocab_file, "r") as reader: + with open(vocab_file, "r") as vocab: while True: - token = convert_to_unicode(reader.readline()) + token = convert_to_unicode(vocab.readline()) if not token: break token = token.strip() - vocab[token] = index + vocab_dict[token] = index index += 1 - return vocab + return vocab_dict -def convert_by_vocab(vocab, items): - """Converts a sequence of [tokens|ids] using the vocab.""" +def convert_by_vocab_dict(vocab_dict, items): + """ + Converts a sequence of [tokens|ids] according to the vocab dict. + """ output = [] for item in items: - if item in vocab: - output.append(vocab[item]) + if item in vocab_dict: + output.append(vocab_dict[item]) else: - output.append(vocab[""]) + output.append(vocab_dict[""]) return output -def convert_tokens_to_ids(vocab, tokens): - return convert_by_vocab(vocab, tokens) - - -def convert_ids_to_tokens(inv_vocab, ids): - return convert_by_vocab(inv_vocab, ids) - - -def whitespace_tokenize(text): - """Runs basic whitespace cleaning and splitting on a piece of text.""" - text = text.strip() - if not text: - return [] - tokens = text.split() - return tokens - - class WhiteSpaceTokenizer(): - """Runs end-to-end tokenziation.""" + """ + Whitespace tokenizer. + """ def __init__(self, vocab_file): - self.vocab = load_vocab(vocab_file) - self.inv_vocab = {v: k for k, v in self.vocab.items()} - self.basic_tokenizer = BasicTokenizer() - - def tokenize(self, text): - return self.basic_tokenizer.tokenize(text) - - def convert_tokens_to_ids(self, tokens): - return convert_by_vocab(self.vocab, tokens) - - def convert_ids_to_tokens(self, ids): - return convert_by_vocab(self.inv_vocab, ids) - - -class BasicTokenizer(): - """Runs basic tokenization (punctuation splitting, lower casing, etc.).""" - - def __init__(self): - """Constructs a BasicTokenizer.""" + self.vocab_dict = load_vocab_file(vocab_file) + self.inv_vocab_dict = {index: token for token, index in self.vocab_dict.items()} + + def _is_whitespace_char(self, char): + """ + Checks if it is a whitespace character(regard "\t", "\n", "\r" as whitespace here). + """ + if char in (" ", "\t", "\n", "\r"): + return True + uni = unicodedata.category(char) + if uni == "Zs": + return True + return False - def tokenize(self, text): - """Tokenizes a piece of text.""" - text = convert_to_unicode(text) - text = self._clean_text(text) - return whitespace_tokenize(text) + def _is_control_char(self, char): + """ + Checks if it is a control character. + """ + if char in ("\t", "\n", "\r"): + return False + uni = unicodedata.category(char) + if uni in ("Cc", "Cf"): + return True + return False def _clean_text(self, text): - """Performs invalid character removal and whitespace cleanup on text.""" + """ + Remove invalid characters and cleanup whitespace. + """ output = [] for char in text: cp = ord(char) - if cp == 0 or cp == 0xfffd or _is_control(char): + if cp == 0 or cp == 0xfffd or self._is_control_char(char): continue - if _is_whitespace(char): + if self._is_whitespace_char(char): output.append(" ") else: output.append(char) return "".join(output) + def _whitespace_tokenize(self, text): + """ + Clean whitespace and split text into tokens. + """ + text = text.strip() + if not text: + tokens = [] + else: + tokens = text.split() + return tokens -def _is_whitespace(char): - """Checks whether `chars` is a whitespace character.""" - # \t, \n, and \r are technically contorl characters but we treat them - # as whitespace since they are generally considered as such. - if char in (" ", "\t", "\n", "\r"): - return True - cat = unicodedata.category(char) - if cat == "Zs": - return True - return False + def tokenize(self, text): + """ + Tokenizes text. + """ + text = convert_to_unicode(text) + text = self._clean_text(text) + tokens = self._whitespace_tokenize(text) + return tokens + def convert_tokens_to_ids(self, tokens): + return convert_by_vocab_dict(self.vocab_dict, tokens) -def _is_control(char): - """Checks whether `chars` is a control character.""" - # These are technically control characters but we count them as whitespace - # characters. - if char in ("\t", "\n", "\r"): - return False - cat = unicodedata.category(char) - if cat in ("Cc", "Cf"): - return True - return False - - -def _is_punctuation(char): - """Checks whether `chars` is a punctuation character.""" - cp = ord(char) - # We treat all non-letter/number ASCII as punctuation. - # Characters such as "^", "$", and "`" are not in the Unicode - # Punctuation class but we treat them as punctuation anyways, for - # consistency. - if ((33 <= cp <= 47) or (58 <= cp <= 64) or (91 <= cp <= 96) or (123 <= cp <= 126)): - return True - cat = unicodedata.category(char) - if cat.startswith("P"): - return True - return False + def convert_ids_to_tokens(self, ids): + return convert_by_vocab_dict(self.inv_vocab_dict, ids) From f9c6d12bc4a02711acddeb7b42eac16dfef87f2d Mon Sep 17 00:00:00 2001 From: Li Hongzhang Date: Tue, 30 Jun 2020 21:07:30 +0800 Subject: [PATCH 182/254] capture the time before hand over to processes pool to ensure time order --- mindspore/train/summary/_summary_adapter.py | 4 ++-- mindspore/train/summary/_writer_pool.py | 9 +++++---- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/mindspore/train/summary/_summary_adapter.py b/mindspore/train/summary/_summary_adapter.py index 1ae5bdd2d5..4cbb7b8fd5 100644 --- a/mindspore/train/summary/_summary_adapter.py +++ b/mindspore/train/summary/_summary_adapter.py @@ -91,7 +91,7 @@ def package_graph_event(data): return graph_event -def package_summary_event(data_list, step): +def package_summary_event(data_list, step, wall_time): """ Package the summary to event protobuffer. @@ -105,7 +105,7 @@ def package_summary_event(data_list, step): # create the event of summary summary_event = Event() summary = summary_event.summary - summary_event.wall_time = time.time() + summary_event.wall_time = wall_time summary_event.step = int(step) for value in data_list: diff --git a/mindspore/train/summary/_writer_pool.py b/mindspore/train/summary/_writer_pool.py index f2ebca35cd..d9cdfd3c8c 100644 --- a/mindspore/train/summary/_writer_pool.py +++ b/mindspore/train/summary/_writer_pool.py @@ -14,15 +14,16 @@ # ============================================================================ """Write events to disk in a base directory.""" import os +import time from collections import deque from multiprocessing import Pool, Process, Queue, cpu_count from ._lineage_adapter import serialize_to_lineage_event from ._summary_adapter import package_graph_event, package_summary_event -from ._summary_writer import SummaryWriter, LineageWriter +from ._summary_writer import LineageWriter, SummaryWriter -def _pack_data(datadict): +def _pack_data(datadict, wall_time): """Pack data according to which plugin.""" result = [] summaries, step, mode = [], None, None @@ -37,7 +38,7 @@ def _pack_data(datadict): step = data.get('step') mode = data.get('mode') if summaries: - result.append(['summary', mode, package_summary_event(summaries, step).SerializeToString()]) + result.append(['summary', mode, package_summary_event(summaries, step, wall_time).SerializeToString()]) return result @@ -70,7 +71,7 @@ class WriterPool(Process): if not self._queue.empty(): action, data = self._queue.get() if action == 'WRITE': - deq.append(pool.apply_async(_pack_data, (data,))) + deq.append(pool.apply_async(_pack_data, (data, time.time()))) elif action == 'FLUSH': for writer in writers: writer.flush() From 299469babb62b33fcaa9debae65e632ef4975ded Mon Sep 17 00:00:00 2001 From: Li Hongzhang Date: Tue, 30 Jun 2020 22:20:58 +0800 Subject: [PATCH 183/254] address the importance of closing the SummaryRecord and illustrate how --- mindspore/train/summary/summary_record.py | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/mindspore/train/summary/summary_record.py b/mindspore/train/summary/summary_record.py index bcb7334e7a..686d8afa9d 100644 --- a/mindspore/train/summary/summary_record.py +++ b/mindspore/train/summary/summary_record.py @@ -61,12 +61,14 @@ def _dictlist(): class SummaryRecord: """ SummaryRecord is used to record the summary data and lineage data. + The API will create a summary file and a lineage file lazily in a given directory and writes data to them. + It writes the data to files by executing the record method. In addition to record the data bubbled up from + the network by defining the summary operators, SummaryRecord also supports to record extra data which + can be added by calling add_value. Note: - The API will create a summary file and a lineage file lazily in a given directory and writes data to them. - It writes the data to files by executing the record method. In addition to record the data bubbled up from - the network by defining the summary operators, SummaryRecord also supports to record extra data which - can be added by calling add_value. Finally, make sure to close the SummaryRecord object at the end. + Make sure to close the SummaryRecord at the end, or the process will NOT exit. + Please see the Example section below on how to properly close with two ways. Args: log_dir (str): The log_dir is a directory location to save the summary. @@ -81,8 +83,15 @@ class SummaryRecord: RuntimeError: If the log_dir can not be resolved to a canonicalized absolute pathname. Examples: + >>> # use in with statement to auto close >>> with SummaryRecord(log_dir="/opt/log", file_prefix="xxx_", file_suffix="_yyy") as summary_record: >>> pass + >>> + >>> # use in try .. finally .. to ensure closing + >>> try: + >>> summary_record = SummaryRecord(log_dir="/opt/log") + >>> finally: + >>> summary_record.close() """ def __init__(self, @@ -310,8 +319,10 @@ class SummaryRecord: Flush all events and close summary records. Please use with statement to autoclose. Examples: - >>> with SummaryRecord(log_dir="/opt/log", file_prefix="xxx_", file_suffix="_yyy") as summary_record: - >>> pass # summary_record autoclosed + >>> try: + >>> summary_record = SummaryRecord(log_dir="/opt/log") + >>> finally: + >>> summary_record.close() """ if not self._closed and self._event_writer: # event writer flush and close From 3b42c360b6e7d157b2157bd6997e8fc8185b4d8d Mon Sep 17 00:00:00 2001 From: Zirui Wu Date: Tue, 30 Jun 2020 13:23:15 -0400 Subject: [PATCH 184/254] set dataset_size in generator when source has len --- mindspore/dataset/engine/datasets.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mindspore/dataset/engine/datasets.py b/mindspore/dataset/engine/datasets.py index d8fda008e9..f2c1642df5 100644 --- a/mindspore/dataset/engine/datasets.py +++ b/mindspore/dataset/engine/datasets.py @@ -3157,6 +3157,9 @@ class GeneratorDataset(MappableDataset): self.column_names.append(col["name"]) self.column_types.append(DataType(col["type"])) + if source is not None and hasattr(source, "__len__"): + self._dataset_size = len(source) + def get_args(self): args = super().get_args() args["source"] = self.source @@ -3177,6 +3180,7 @@ class GeneratorDataset(MappableDataset): return self._dataset_size if self._dataset_size is None: return None + return min(rows_from_sampler, self._dataset_size) # manually set dataset_size as a temporary solution. From 2ffe76981dee627968ec3483ec5790c2a7f12b2a Mon Sep 17 00:00:00 2001 From: Jamie Nisbet Date: Tue, 30 Jun 2020 10:57:25 -0400 Subject: [PATCH 185/254] added a pre pass for node removals cpplint --- .../dataset/engine/datasetops/batch_op.cc | 2 +- .../dataset/engine/datasetops/dataset_op.cc | 51 ++++++++++++++++++ .../dataset/engine/datasetops/dataset_op.h | 30 +++++++++-- .../engine/datasetops/device_queue_op.cc | 2 +- .../dataset/engine/datasetops/filter_op.cc | 2 +- .../ccsrc/dataset/engine/datasetops/map_op.cc | 2 +- .../dataset/engine/datasetops/project_op.cc | 2 +- .../dataset/engine/datasetops/rename_op.cc | 2 +- .../dataset/engine/datasetops/repeat_op.cc | 2 +- .../dataset/engine/datasetops/shuffle_op.cc | 2 +- .../dataset/engine/datasetops/skip_op.cc | 2 +- .../engine/datasetops/source/generator_op.cc | 2 +- .../datasetops/source/image_folder_op.cc | 2 +- .../engine/datasetops/source/mindrecord_op.cc | 2 +- .../engine/datasetops/source/tf_reader_op.cc | 2 +- .../dataset/engine/datasetops/take_op.cc | 2 +- .../ccsrc/dataset/engine/datasetops/zip_op.cc | 2 +- .../ccsrc/dataset/engine/execution_tree.cc | 4 +- .../ccsrc/dataset/engine/opt/CMakeLists.txt | 6 ++- mindspore/ccsrc/dataset/engine/opt/pass.cc | 2 +- mindspore/ccsrc/dataset/engine/opt/pass.h | 35 +++++++----- .../dataset/engine/opt/pre/removal_nodes.cc | 42 +++++++++++++++ .../dataset/engine/opt/pre/removal_nodes.h | 51 ++++++++++++++++++ .../dataset/engine/opt/pre/removal_pass.cc | 45 ++++++++++++++++ .../dataset/engine/opt/pre/removal_pass.h | 53 +++++++++++++++++++ 25 files changed, 314 insertions(+), 35 deletions(-) create mode 100644 mindspore/ccsrc/dataset/engine/opt/pre/removal_nodes.cc create mode 100644 mindspore/ccsrc/dataset/engine/opt/pre/removal_nodes.h create mode 100644 mindspore/ccsrc/dataset/engine/opt/pre/removal_pass.cc create mode 100644 mindspore/ccsrc/dataset/engine/opt/pre/removal_pass.h diff --git a/mindspore/ccsrc/dataset/engine/datasetops/batch_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/batch_op.cc index f311c90c33..8bfa8c287c 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/batch_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/batch_op.cc @@ -409,7 +409,7 @@ Status BatchOp::UnpackPadInfo(const PadInfo &pad_info, // Visitor accept method for NodePass Status BatchOp::Accept(NodePass *p, bool *modified) { // Downcast shared pointer then call visitor - return p->RunOnNode(std::static_pointer_cast(shared_from_this()), modified); + return p->RunOnNode(shared_from_base(), modified); } } // namespace dataset diff --git a/mindspore/ccsrc/dataset/engine/datasetops/dataset_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/dataset_op.cc index 91ed7fbc5f..170d9a7ce8 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/dataset_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/dataset_op.cc @@ -111,6 +111,51 @@ void DatasetOp::RemoveParent(const DatasetOp *parent) { parent_.erase(std::remove(parent_.begin(), parent_.end(), parent), parent_.end()); } +// Removes this node from the tree and connects it's parent/child together +Status DatasetOp::Remove() { + if (parent_.size() > 1) { + std::string err_msg("No support for op removal if the operator has more than one parent"); + RETURN_STATUS_UNEXPECTED(err_msg); + } + if (child_.size() > 1) { + std::string err_msg("No support for op removal if the operator has more than one child"); + RETURN_STATUS_UNEXPECTED(err_msg); + } + + // Scenario's when removing node B: + // A -> B -> C + // A -> B + // B -> C + // + // If we remove B, then first take our child A and update it's parent to be C + // It's possible the parent is null if we are the root node being removed. + if (!child_.empty()) { + // If we have a parent, then assign chlid's parent to point to our parent. + if (!parent_.empty()) { + child_[0]->parent_[0] = parent_[0]; + } else { + // We don't have a parent, so we are the root node being removed. + // clear the parent list of our child so that it becomes the new root. + child_[0]->parent_.clear(); + tree_->AssignRoot(child_[0]); + } + } + + // Next, if we had a parent, then set it's child to be our child. + if (!parent_.empty()) { + // if we have a child, then set our parent to point to it + if (!child_.empty()) { + parent_[0]->child_[0] = child_[0]; + } else { + // We don't have a child, so clear the child list of the current + // parent because it will be empty once we are removed. + parent_[0]->child_.clear(); + } + } + + return Status::OK(); +} + // Getter function to get a shared pointer to our childAdds a operator to become our child. std::shared_ptr DatasetOp::child(int32_t child_index) const { MS_ASSERT(child_index < static_cast(child_.size())); @@ -289,6 +334,12 @@ Status DatasetOp::ComputeColMap() { return Status::OK(); } +Status DatasetOp::PreAccept(NodePass *p, bool *modified) { + // DatasetOp is the base class of visitor target pre-visit. + // This method will only be called if its derived class does not implement one. + return p->PreRunOnNode(shared_from_this(), modified); +} + Status DatasetOp::Accept(NodePass *p, bool *modified) { // DatasetOp is the base class of visitor target. // This method will only be called if its derived class does not implement one. diff --git a/mindspore/ccsrc/dataset/engine/datasetops/dataset_op.h b/mindspore/ccsrc/dataset/engine/datasetops/dataset_op.h index 254cd411c5..9ee287d050 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/dataset_op.h +++ b/mindspore/ccsrc/dataset/engine/datasetops/dataset_op.h @@ -71,6 +71,10 @@ class DatasetOp : public std::enable_shared_from_this { // @param child - shared pointer to the child to remove. Status RemoveChild(std::shared_ptr child); + /// \brief Removes this node from the tree and connects it's parent/child together. + /// \return Status eerror code returned + Status Remove(); + // Getter function to get a shared pointer to our child // @param child_index - An operator can have n children. Indicates choose which child to return. std::shared_ptr child(int32_t child_index) const; @@ -264,10 +268,20 @@ class DatasetOp : public std::enable_shared_from_this { // @return Vector of Children std::vector> Children() const { return child_; } - // Base method for NodePass visit. - // Subclass needs to override this if it requires special node visit access. - // Check "dataset/engine/opt/pass.h" for more details. - // @return Statue of the node visit + /// \brief Base method for NodePass pre-visit. A tree walk consists of walking down the tree and also walking back up + /// in a depth-first order. PreAccept is the node visit on the way down, whereas the regular Accept is the main + /// visit on the way back up the tree during a post-order traversal. Subclass needs to override this if it + /// requires special node visit access. Check "dataset/engine/opt/pass.h" for more details. + /// \param[in] p The node to visit + /// \param[out] modified Indicator if the node was modified + /// \return Status of the node visit + virtual Status PreAccept(NodePass *p, bool *modified); + + /// \brief Base method for NodePass visit. Subclass needs to override this if it requires special node visit access. + /// Check "dataset/engine/opt/pass.h" for more details. + /// \param[in] p The node to visit + /// \param[out] modified Indicator if the node was modified + /// \return Status of the node visit virtual Status Accept(NodePass *p, bool *modified); // Op name getter @@ -285,6 +299,14 @@ class DatasetOp : public std::enable_shared_from_this { // Computes a CRC value for the operator static uint32_t GenerateCRC(const std::shared_ptr &op); + /// \brief A helper templated function for casting "this" pointer to shared_ptr + /// Similar to shared_from_this, except this one will give you the derived class as shared_ptr + /// \return A shared_ptr casted to the derived class + template + std::shared_ptr shared_from_base() { + return std::static_pointer_cast(shared_from_this()); + } + protected: // Adds a parent operator to this operator // @notes External callers do not have access to this function. diff --git a/mindspore/ccsrc/dataset/engine/datasetops/device_queue_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/device_queue_op.cc index 84bad9db1a..0f1fefc0f0 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/device_queue_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/device_queue_op.cc @@ -313,7 +313,7 @@ void DeviceQueueOp::Print(std::ostream &out, bool show_all) const { // Visitor accept method for NodePass Status DeviceQueueOp::Accept(NodePass *p, bool *modified) { // Downcast shared pointer then call visitor - return p->RunOnNode(std::static_pointer_cast(shared_from_this()), modified); + return p->RunOnNode(shared_from_base(), modified); } } // namespace dataset diff --git a/mindspore/ccsrc/dataset/engine/datasetops/filter_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/filter_op.cc index a1c5ed0070..81c93c6e1c 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/filter_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/filter_op.cc @@ -261,7 +261,7 @@ Status FilterOp::InvokePredicateFunc(const TensorRow &input, bool *out_predicate // Visitor accept method for NodePass Status FilterOp::Accept(NodePass *p, bool *modified) { // Downcast shared pointer then call visitor - return p->RunOnNode(std::static_pointer_cast(shared_from_this()), modified); + return p->RunOnNode(shared_from_base(), modified); } } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/map_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/map_op.cc index 020f40d268..05a1ac7925 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/map_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/map_op.cc @@ -367,7 +367,7 @@ void MapOp::CreateFinalColMap(std::unordered_map *col_name // Visitor accept method for NodePass Status MapOp::Accept(NodePass *p, bool *modified) { // Downcast shared pointer then call visitor - return p->RunOnNode(std::static_pointer_cast(shared_from_this()), modified); + return p->RunOnNode(shared_from_base(), modified); } } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/project_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/project_op.cc index 14b064bab9..5ce4056024 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/project_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/project_op.cc @@ -131,7 +131,7 @@ Status ProjectOp::EofReceived(int32_t worker_id) { return Status::OK(); } // Visitor accept method for NodePass Status ProjectOp::Accept(NodePass *p, bool *modified) { // Downcast shared pointer then call visitor - return p->RunOnNode(std::static_pointer_cast(shared_from_this()), modified); + return p->RunOnNode(shared_from_base(), modified); } // Compute the column map and save it into our own column name map diff --git a/mindspore/ccsrc/dataset/engine/datasetops/rename_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/rename_op.cc index bebca780ff..23cd29d295 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/rename_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/rename_op.cc @@ -176,7 +176,7 @@ Status RenameOp::EoeReceived(int32_t) { // Visitor accept method for NodePass Status RenameOp::Accept(NodePass *p, bool *modified) { // Downcast shared pointer then call visitor - return p->RunOnNode(std::static_pointer_cast(shared_from_this()), modified); + return p->RunOnNode(shared_from_base(), modified); } } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/repeat_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/repeat_op.cc index 86903e540a..4999dddd02 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/repeat_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/repeat_op.cc @@ -190,7 +190,7 @@ int32_t RepeatOp::num_producers() const { // Visitor accept method for NodePass Status RepeatOp::Accept(NodePass *p, bool *modified) { // Downcast shared pointer then call visitor - return p->RunOnNode(std::static_pointer_cast(shared_from_this()), modified); + return p->RunOnNode(shared_from_base(), modified); } } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/shuffle_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/shuffle_op.cc index c16f3f9625..f86fcc602b 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/shuffle_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/shuffle_op.cc @@ -298,7 +298,7 @@ Status ShuffleOp::EoeReceived(int32_t worker_id) { // Visitor accept method for NodePass Status ShuffleOp::Accept(NodePass *p, bool *modified) { // Downcast shared pointer then call visitor - return p->RunOnNode(std::static_pointer_cast(shared_from_this()), modified); + return p->RunOnNode(shared_from_base(), modified); } } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/skip_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/skip_op.cc index c00fd486b7..f6b0fe689c 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/skip_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/skip_op.cc @@ -130,7 +130,7 @@ Status SkipOp::EofReceived(int32_t worker_id) { // Visitor accept method for NodePass Status SkipOp::Accept(NodePass *p, bool *modified) { // Downcast shared pointer then call visitor - return p->RunOnNode(std::static_pointer_cast(shared_from_this()), modified); + return p->RunOnNode(shared_from_base(), modified); } } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/generator_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/generator_op.cc index eb5ba32642..36c221fc16 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/generator_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/generator_op.cc @@ -249,7 +249,7 @@ Status GeneratorOp::Reset() { // Visitor accept method for NodePass Status GeneratorOp::Accept(NodePass *p, bool *modified) { // Downcast shared pointer then call visitor - return p->RunOnNode(std::static_pointer_cast(shared_from_this()), modified); + return p->RunOnNode(shared_from_base(), modified); } Status GeneratorOp::ComputeColMap() { diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/image_folder_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/image_folder_op.cc index cb17158bff..837eae1e3c 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/image_folder_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/image_folder_op.cc @@ -411,7 +411,7 @@ Status ImageFolderOp::CountRowsAndClasses(const std::string &path, const std::se // Visitor accept method for NodePass Status ImageFolderOp::Accept(NodePass *p, bool *modified) { // Downcast shared pointer then call visitor - return p->RunOnNode(std::static_pointer_cast(shared_from_this()), modified); + return p->RunOnNode(shared_from_base(), modified); } Status ImageFolderOp::ComputeColMap() { diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/mindrecord_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/mindrecord_op.cc index 3c95b9b054..2b9d010ebb 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/mindrecord_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/mindrecord_op.cc @@ -496,7 +496,7 @@ Status MindRecordOp::CountTotalRows(const std::vector dataset_path, // Visitor accept method for NodePass Status MindRecordOp::Accept(NodePass *p, bool *modified) { // Downcast shared pointer then call visitor - return p->RunOnNode(std::static_pointer_cast(shared_from_this()), modified); + return p->RunOnNode(shared_from_base(), modified); } Status MindRecordOp::ComputeColMap() { diff --git a/mindspore/ccsrc/dataset/engine/datasetops/source/tf_reader_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/source/tf_reader_op.cc index a2b04bcc01..48f13ff766 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/source/tf_reader_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/source/tf_reader_op.cc @@ -1004,7 +1004,7 @@ int64_t TFReaderOp::CountTotalRowsSectioned(const std::vector &file // Visitor accept method for NodePass Status TFReaderOp::Accept(NodePass *p, bool *modified) { // Downcast shared pointer then call visitor - return p->RunOnNode(std::static_pointer_cast(shared_from_this()), modified); + return p->RunOnNode(shared_from_base(), modified); } Status TFReaderOp::ComputeColMap() { diff --git a/mindspore/ccsrc/dataset/engine/datasetops/take_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/take_op.cc index 259ae8e62b..8bc449cdc9 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/take_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/take_op.cc @@ -136,7 +136,7 @@ Status TakeOp::PrepareNodePostAction() { // Visitor accept method for NodePass Status TakeOp::Accept(NodePass *p, bool *modified) { // Downcast shared pointer then call visitor - return p->RunOnNode(std::static_pointer_cast(shared_from_this()), modified); + return p->RunOnNode(shared_from_base(), modified); } } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/datasetops/zip_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/zip_op.cc index 55734324fc..70bce16a89 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/zip_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/zip_op.cc @@ -237,7 +237,7 @@ Status ZipOp::EoeReceived(int32_t) { // Visitor accept method for NodePass Status ZipOp::Accept(NodePass *p, bool *modified) { // Downcast shared pointer then call visitor - return p->RunOnNode(std::static_pointer_cast(shared_from_this()), modified); + return p->RunOnNode(shared_from_base(), modified); } Status ZipOp::ComputeColMap() { diff --git a/mindspore/ccsrc/dataset/engine/execution_tree.cc b/mindspore/ccsrc/dataset/engine/execution_tree.cc index 2f88ee1795..385722e257 100644 --- a/mindspore/ccsrc/dataset/engine/execution_tree.cc +++ b/mindspore/ccsrc/dataset/engine/execution_tree.cc @@ -20,6 +20,7 @@ #include "dataset/engine/datasetops/shuffle_op.h" #include "dataset/util/task_manager.h" #include "dataset/engine/opt/pass.h" +#include "dataset/engine/opt/pre/removal_pass.h" #include "dataset/engine/perf/profiling.h" #include "dataset/engine/perf/monitor.h" @@ -214,7 +215,8 @@ Status ExecutionTree::PrepareTreePreAction() { bool modified = false; std::vector> pre_actions; // Construct pre actions - // example: pre_actions.push_back(new SomePass()); + MS_LOG(INFO) << "Running pre pass"; + pre_actions.push_back(std::make_unique(RemovalPass())); // Apply pre action passes for (auto &pass : pre_actions) { RETURN_IF_NOT_OK(pass->Run(this, &modified)); diff --git a/mindspore/ccsrc/dataset/engine/opt/CMakeLists.txt b/mindspore/ccsrc/dataset/engine/opt/CMakeLists.txt index af0a8918db..080d968cfc 100644 --- a/mindspore/ccsrc/dataset/engine/opt/CMakeLists.txt +++ b/mindspore/ccsrc/dataset/engine/opt/CMakeLists.txt @@ -1,6 +1,8 @@ file(GLOB_RECURSE _CURRENT_SRC_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cc") set_property(SOURCE ${_CURRENT_SRC_FILES} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_MD) add_library(engine-opt OBJECT - pass.cc - util/printer_pass.cc + pass.cc + pre/removal_nodes.cc + pre/removal_pass.cc + util/printer_pass.cc ) diff --git a/mindspore/ccsrc/dataset/engine/opt/pass.cc b/mindspore/ccsrc/dataset/engine/opt/pass.cc index a032d46cba..27769f056b 100644 --- a/mindspore/ccsrc/dataset/engine/opt/pass.cc +++ b/mindspore/ccsrc/dataset/engine/opt/pass.cc @@ -61,6 +61,7 @@ Status NodePass::Run(ExecutionTree *tree, bool *modified) { // Helper function to perform DFS visit Status NodePass::DFSNodeVisit(std::shared_ptr node, bool *modified) { + RETURN_IF_NOT_OK(node->PreAccept(this, modified)); for (const auto &c : node->Children()) { RETURN_IF_NOT_OK(this->DFSNodeVisit(c, modified)); } @@ -159,6 +160,5 @@ Status NodePass::RunOnNode(std::shared_ptr node, bool *modified) // Fallback to base class visitor by default return RunOnNode(std::static_pointer_cast(node), modified); } - } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/opt/pass.h b/mindspore/ccsrc/dataset/engine/opt/pass.h index 39682b22f7..129c2fab37 100644 --- a/mindspore/ccsrc/dataset/engine/opt/pass.h +++ b/mindspore/ccsrc/dataset/engine/opt/pass.h @@ -66,14 +66,16 @@ class Pass : public std::enable_shared_from_this { // TreePass is a basic Pass class which performs transformation on ExecutionTree directly. class TreePass : public Pass { public: - // Run the transformation pass against the execution tree. - // @param tree - Pointer to the execution tree to be transformed. - // @param modified - Pointer to the modified flag, + /// \brief Run the transformation pass against the execution tree. + /// \param[inout] tree Pointer to the execution tree to be transformed. + /// \param[inout] modified Indicate if the tree was modified Status Run(ExecutionTree *tree, bool *modified) final; - // Derived classes may implement the runOnTree function to implement tree transformation. - // "modified" flag needs to be set to true if tree is modified during the pass execution. - // @return Status - The error code return + /// \brief Derived classes may implement the runOnTree function to implement tree transformation. + /// "modified" flag needs to be set to true if tree is modified during the pass execution. + /// \param[inout] tree The tree to operate on. + /// \param[inout] Indicate of the tree was modified. + /// \return Status The error code return virtual Status RunOnTree(ExecutionTree *tree, bool *modified) { return Status::OK(); } }; @@ -90,14 +92,23 @@ class NodePass : public Pass { ~NodePass() = default; - // Run the transformation pass against the execution tree. - // @param tree - Pointer to the execution tree to be transformed. - // @param modified - Pointer to the modified flag, + /// \brief Run the transformation pass against the execution tree + /// \param[inout] tree Pointer to the execution tree to be transformed + /// \param[inout] modified Indicator if the tree was changed Status Run(ExecutionTree *tree, bool *modified) final; - // Derived classes may implement the runOnNode function to implement node level tree transformation. - // "modified" flag needs to be set to true if tree is modified during the pass execution. - // @return Status - The error code return + /// \brief Derived classes may implement the PreRunOnNode function to implement any initial visit work on the way down + /// a tree traversal. "modified" flag needs to be set to true if tree is modified during the pass execution + /// \param[in] node The node being visited + /// \param[out] modified Indicator if the node was changed at all + /// \return Status The error code return + virtual Status PreRunOnNode(std::shared_ptr node, bool *modified) { return Status::OK(); } + + /// \brief Derived classes may implement the RunOnNode function to implement node level tree transformation + /// "modified" flag needs to be set to true if tree is modified during the pass execution + /// \param[in] node The node being visited + /// \param[out] modified Indicator if the node was changed at all. + /// \return Status The error code return virtual Status RunOnNode(std::shared_ptr node, bool *modified) { return Status::OK(); } // Visit methods to be overridden. diff --git a/mindspore/ccsrc/dataset/engine/opt/pre/removal_nodes.cc b/mindspore/ccsrc/dataset/engine/opt/pre/removal_nodes.cc new file mode 100644 index 0000000000..831a2a76ba --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/opt/pre/removal_nodes.cc @@ -0,0 +1,42 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include "dataset/engine/opt/pre/removal_nodes.h" +#include "dataset/engine/opt/pre/removal_pass.h" +#include "dataset/engine/datasetops/shuffle_op.h" + +namespace mindspore { +namespace dataset { + +RemovalNodes::RemovalNodes(RemovalPass *removal_pass) : removal_pass_(removal_pass), is_caching_(false) {} + +// Perform ShuffleOp removal check. +Status RemovalNodes::RunOnNode(std::shared_ptr node, bool *modified) { + *modified = false; + // If we are in a cache descendant tree, then this shuffle op needs to be removed + if (is_caching_) { + MS_LOG(DEBUG) << "ShuffleOp identified for removal (CacheOp is in ascendant tree)"; + if (removal_pass_) { + removal_pass_->AddToRemovalList(std::static_pointer_cast(node)); + } else { + return Status(StatusCode::kUnexpectedError, __LINE__, __FILE__, "Back reference to removal pass is missing!"); + } + } + return Status::OK(); +} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/opt/pre/removal_nodes.h b/mindspore/ccsrc/dataset/engine/opt/pre/removal_nodes.h new file mode 100644 index 0000000000..11ef37d80c --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/opt/pre/removal_nodes.h @@ -0,0 +1,51 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef DATASET_ENGINE_OPT_PASS_PRE_REMOVAL_NODES_H_ +#define DATASET_ENGINE_OPT_PASS_PRE_REMOVAL_NODES_H_ + +#include +#include "dataset/engine/opt/pass.h" + +namespace mindspore { +namespace dataset { + +class RemovalPass; + +/// \class RemovalNodes removal_nodes.h +/// \brief This is a NodePass who's job is to identify which nodes should be removed. +/// It works in conjunction with the removal_pass. +class RemovalNodes : public NodePass { + public: + /// \brief Constructor + /// \param[in] removal_pass Raw pointer back to controlling tree pass + explicit RemovalNodes(RemovalPass *removal_pass); + + /// \brief Perform ShuffleOp removal check + /// \param[in] node The node being visited + /// \param[inout] modified Indicator if the node was changed at all + /// \return Status The error code return + Status RunOnNode(std::shared_ptr node, bool *modified) override; + + private: + bool is_caching_; + RemovalPass *removal_pass_; // Back pointer to the owning removal pass +}; + +} // namespace dataset +} // namespace mindspore + +#endif // DATASET_ENGINE_OPT_PASS_PRE_REMOVAL_NODES_ diff --git a/mindspore/ccsrc/dataset/engine/opt/pre/removal_pass.cc b/mindspore/ccsrc/dataset/engine/opt/pre/removal_pass.cc new file mode 100644 index 0000000000..31ec31234f --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/opt/pre/removal_pass.cc @@ -0,0 +1,45 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include "dataset/engine/opt/pre/removal_nodes.h" +#include "dataset/engine/opt/pre/removal_pass.h" +#include "dataset/engine/execution_tree.h" + +namespace mindspore { +namespace dataset { + +// constructor +RemovalPass::RemovalPass() {} + +// Runs a removal_nodes pass first to find out which nodes to remove, then removes them. +Status RemovalPass::RunOnTree(ExecutionTree *tree, bool *modified) { + // Create the removal node pass which can identify which nodes need to be removed. + std::unique_ptr removal_nodes = std::make_unique(this); + RETURN_IF_NOT_OK(removal_nodes->Run(tree, modified)); + + // Then, execute the removal of any nodes that were set up for removal + for (auto node : removal_nodes_) { + node->Remove(); + } + return Status::OK(); +} + +// Adds an operator to the list of operators to be removed +void RemovalPass::AddToRemovalList(std::shared_ptr dataset_op) { removal_nodes_.push_back(dataset_op); } +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/dataset/engine/opt/pre/removal_pass.h b/mindspore/ccsrc/dataset/engine/opt/pre/removal_pass.h new file mode 100644 index 0000000000..6523ca69b2 --- /dev/null +++ b/mindspore/ccsrc/dataset/engine/opt/pre/removal_pass.h @@ -0,0 +1,53 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef DATASET_ENGINE_OPT_PASS_PRE_REMOVAL_PASS_H_ +#define DATASET_ENGINE_OPT_PASS_PRE_REMOVAL_PASS_H_ + +#include +#include +#include "dataset/engine/opt/pass.h" + +namespace mindspore { +namespace dataset { + +class DatasetOp; + +/// \class RemovalPass removal_pass.h +/// \brief This is a tree pass that will remove nodes. It uses removal_nodes to first identify which +/// nodes should be removed, and then removes them. +class RemovalPass : public TreePass { + public: + /// \brief Constructor + RemovalPass(); + + /// \brief Runs a removal_nodes pass first to find out which nodes to remove, then removes them. + /// \param[inout] tree The tree to operate on. + /// \param[inout] Indicate of the tree was modified. + /// \return Status The error code return + Status RunOnTree(ExecutionTree *tree, bool *modified) override; + + /// \brief Adds an operator to the list of operators to be removed + /// \param[in] dataset_op The operator to add to the removal list + void AddToRemovalList(std::shared_ptr dataset_op); + + private: + std::vector> removal_nodes_; +}; +} // namespace dataset +} // namespace mindspore + +#endif // DATASET_ENGINE_OPT_PASS_PRE_REMOVAL_PASS_H_ From 593ee1eb344b02f985221c94b3004b069da43303 Mon Sep 17 00:00:00 2001 From: islam_amin Date: Mon, 29 Jun 2020 13:01:19 -0400 Subject: [PATCH 186/254] C++ UT common files for object detection tests --- tests/ut/cpp/dataset/CMakeLists.txt | 1 + tests/ut/cpp/dataset/common/bboxop_common.cc | 230 +++++++++++++++++++ tests/ut/cpp/dataset/common/bboxop_common.h | 74 ++++++ 3 files changed, 305 insertions(+) create mode 100644 tests/ut/cpp/dataset/common/bboxop_common.cc create mode 100644 tests/ut/cpp/dataset/common/bboxop_common.h diff --git a/tests/ut/cpp/dataset/CMakeLists.txt b/tests/ut/cpp/dataset/CMakeLists.txt index bfdc2b4cb3..eda1ca88ac 100644 --- a/tests/ut/cpp/dataset/CMakeLists.txt +++ b/tests/ut/cpp/dataset/CMakeLists.txt @@ -3,6 +3,7 @@ include(GoogleTest) SET(DE_UT_SRCS common/common.cc common/cvop_common.cc + common/bboxop_common.cc batch_op_test.cc bit_functions_test.cc storage_container_test.cc diff --git a/tests/ut/cpp/dataset/common/bboxop_common.cc b/tests/ut/cpp/dataset/common/bboxop_common.cc new file mode 100644 index 0000000000..70e6b5a339 --- /dev/null +++ b/tests/ut/cpp/dataset/common/bboxop_common.cc @@ -0,0 +1,230 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "bboxop_common.h" + +#include +#include +#include +#include + +#include + +#include "./tinyxml2.h" +#include "opencv2/opencv.hpp" +#include "common/utils.h" +#include "dataset/core/cv_tensor.h" +#include "dataset/util/path.h" +#include "dataset/core/constants.h" +#include "utils/log_adapter.h" + +using namespace mindspore::dataset; +using namespace UT::CVOP::BBOXOP; +using tinyxml2::XMLDocument; +using tinyxml2::XMLElement; +using tinyxml2::XMLError; + +const char kAnnotationsFolder[] = "/Annotations/"; +const char kImagesFolder[] = "/JPEGImages/"; +const char kExpectedName[] = "Expected"; +const char kActualName[] = "Actual"; +const char kAnnotExt[] = ".xml"; +const char kImageExt[] = ".jpg"; + +BBoxOpCommon::BBoxOpCommon() {} + +BBoxOpCommon::~BBoxOpCommon() {} + +void BBoxOpCommon::SetUp() { + MS_LOG(INFO) << "starting test."; + image_folder_build_ = "data/dataset/imagefolder/"; + image_folder_src_ = "../../../../../tests/ut/data/dataset/imagefolder/"; + std::string dir_path = "data/dataset/testVOC2012_2"; + GetInputImagesAndAnnotations(dir_path); +} + +void BBoxOpCommon::GetInputImagesAndAnnotations(const std::string &dir, std::size_t num_of_samples) { + std::string images_path = dir + std::string(kImagesFolder); + std::string annots_path = dir + std::string(kAnnotationsFolder); + Path dir_path(images_path); + std::shared_ptr image_dir_itr = Path::DirIterator::OpenDirectory(&dir_path); + std::vector paths_to_fetch; + if (!dir_path.Exists()) { + MS_LOG(ERROR) << "Images folder was not found : " + images_path; + EXPECT_TRUE(dir_path.Exists()); + } + std::size_t files_fetched = 0; + // get image file paths + while (image_dir_itr->hasNext() && files_fetched < num_of_samples) { + Path image_path = image_dir_itr->next(); + if (image_path.Extension() == std::string(kImageExt)) { + paths_to_fetch.push_back(image_path.toString()); + files_fetched++; + } + } + // sort fetched files + std::sort(paths_to_fetch.begin(), paths_to_fetch.end()); + for (const auto &image_file : paths_to_fetch) { + std::string image_ext = std::string(kImageExt); + std::string annot_file = image_file; + std::size_t pos = 0; + // first replace the Image dir with the Annotation dir. + if ((pos = image_file.find(std::string(kImagesFolder), 0)) != std::string::npos) { + annot_file.replace(pos, std::string(kImagesFolder).length(), std::string(kAnnotationsFolder)); + } + // then replace the extensions. the image extension to annotation extension + if ((pos = annot_file.find(image_ext, 0)) != std::string::npos) { + annot_file.replace(pos, std::string(kAnnotExt).length(), std::string(kAnnotExt)); + } + std::shared_ptr annotation_tensor; + // load annotations and log failure + if (!LoadAnnotationFile(annot_file, &annotation_tensor)) { + MS_LOG(ERROR) << "Loading Annotations failed in GetInputImagesAndAnnotations"; + EXPECT_EQ(0, 1); + } + // load image + GetInputImage(image_file); + // add image and annotation to the tensor table + TensorRow row_data({std::move(input_tensor_), std::move(annotation_tensor)}); + images_and_annotations_.push_back(row_data); + } +} + +void BBoxOpCommon::SaveImagesWithAnnotations(BBoxOpCommon::FileType type, const std::string &op_name, + const TensorTable &table) { + int i = 0; + for (auto &row : table) { + std::shared_ptr row_to_save; + Status swap_status = SwapRedAndBlue(row[0], &row_to_save); + if (!swap_status.IsOk()) { + MS_LOG(ERROR) << "Swaping red and blue channels failed in SaveImagesWithAnnotations."; + EXPECT_TRUE(swap_status.IsOk()); + } + cv::Mat image = std::static_pointer_cast(row_to_save)->mat(); + uint32_t num_of_boxes = row[1]->shape()[0]; + bool passing_data_fetch = true; + // For each bounding box draw on the image. + for (uint32_t i = 0; i < num_of_boxes; i++) { + uint32_t x = 0; + uint32_t y = 0; + uint32_t w = 0; + uint32_t h = 0; + passing_data_fetch &= row[1]->GetUnsignedIntAt(&x, {i, 0}).IsOk(); + passing_data_fetch &= row[1]->GetUnsignedIntAt(&y, {i, 1}).IsOk(); + passing_data_fetch &= row[1]->GetUnsignedIntAt(&w, {i, 2}).IsOk(); + passing_data_fetch &= row[1]->GetUnsignedIntAt(&h, {i, 3}).IsOk(); + if (!passing_data_fetch) { + MS_LOG(ERROR) << "Fetching bbox coordinates failed in SaveImagesWithAnnotations."; + EXPECT_TRUE(passing_data_fetch); + } + cv::Rect bbox(x, y, w, h); + cv::rectangle(image, bbox, cv::Scalar(255, 0, 0), 10, 8, 0); + } + bool im_write_success = false; + // if user wants to save an expected image, use the path to the source folder. + if (type == FileType::kExpected) { + im_write_success = cv::imwrite( + image_folder_src_ + std::string(kExpectedName) + op_name + std::to_string(i) + std::string(kImageExt), image); + } else { + // otherwise if we are saving actual images only for comparison, save in current working dir in build folders. + im_write_success = + cv::imwrite(std::string(kActualName) + op_name + std::to_string(i) + std::string(kImageExt), image); + } + if (!im_write_success) { + MS_LOG(ERROR) << "Image write failed in SaveImagesWithAnnotations."; + EXPECT_TRUE(im_write_success); + } + i += 1; + } +} + +void BBoxOpCommon::CompareActualAndExpected(const std::string &op_name) { + size_t num_of_images = images_and_annotations_.size(); + for (size_t i = 0; i < num_of_images; i++) { + // load actual and expected images. + std::string actual_path = std::string(kActualName) + op_name + std::to_string(i) + std::string(kImageExt); + std::string expected_path = + image_folder_build_ + std::string(kExpectedName) + op_name + std::to_string(i) + std::string(kImageExt); + cv::Mat expect_img = cv::imread(expected_path, cv::IMREAD_COLOR); + cv::Mat actual_img = cv::imread(actual_path, cv::IMREAD_COLOR); + // after comparison is done remove temporary file + EXPECT_TRUE(remove(actual_path.c_str()) == 0); + // compare using ==operator by Tensor + if (actual_img.data) { + EXPECT_EQ(CVTensor(expect_img) == CVTensor(actual_img), true); + } else { + MS_LOG(ERROR) << "Not pass verification! Image data is null."; + EXPECT_EQ(0, 1); + } + } +} + +bool BBoxOpCommon::LoadAnnotationFile(const std::string &path, std::shared_ptr *target_BBox) { + if (!Path(path).Exists()) { + MS_LOG(ERROR) << "File is not found : " + path; + return false; + } + XMLDocument doc; + XMLError e = doc.LoadFile(mindspore::common::SafeCStr(path)); + if (e != XMLError::XML_SUCCESS) { + MS_LOG(ERROR) << "Xml load failed"; + return false; + } + XMLElement *root = doc.RootElement(); + if (root == nullptr) { + MS_LOG(ERROR) << "Xml load root element error"; + return false; + } + XMLElement *object = root->FirstChildElement("object"); + if (object == nullptr) { + MS_LOG(ERROR) << "No object find in " + path; + return false; + } + std::vector return_value_list; + dsize_t bbox_count = 0; // keep track of number of bboxes in file + dsize_t bbox_val_count = 4; // creating bboxes of size 4 to test function + // FILE OK TO READ + while (object != nullptr) { + bbox_count += 1; + std::string label_name; + uint32_t xmin = 0, ymin = 0, xmax = 0, ymax = 0; + XMLElement *bbox_node = object->FirstChildElement("bndbox"); + if (bbox_node != nullptr) { + XMLElement *xmin_node = bbox_node->FirstChildElement("xmin"); + if (xmin_node != nullptr) xmin = xmin_node->UnsignedText(); + XMLElement *ymin_node = bbox_node->FirstChildElement("ymin"); + if (ymin_node != nullptr) ymin = ymin_node->UnsignedText(); + XMLElement *xmax_node = bbox_node->FirstChildElement("xmax"); + if (xmax_node != nullptr) xmax = xmax_node->UnsignedText(); + XMLElement *ymax_node = bbox_node->FirstChildElement("ymax"); + if (ymax_node != nullptr) ymax = ymax_node->UnsignedText(); + } else { + MS_LOG(ERROR) << "bndbox dismatch in " + path; + return false; + } + if (xmin > 0 && ymin > 0 && xmax > xmin && ymax > ymin) { + for (auto item : {xmin, ymin, xmax - xmin, ymax - ymin}) { + return_value_list.push_back(item); + } + } + object = object->NextSiblingElement("object"); // Read next BBox if exists + } + std::shared_ptr ret_value; + Status s = Tensor::CreateTensor(&ret_value, return_value_list, TensorShape({bbox_count, bbox_val_count})); + EXPECT_TRUE(s.IsOk()); + (*target_BBox) = ret_value; // load bbox from file into return + return true; +} diff --git a/tests/ut/cpp/dataset/common/bboxop_common.h b/tests/ut/cpp/dataset/common/bboxop_common.h new file mode 100644 index 0000000000..ba3ceb62d9 --- /dev/null +++ b/tests/ut/cpp/dataset/common/bboxop_common.h @@ -0,0 +1,74 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef TESTS_DATASET_UT_CORE_COMMON_DE_UT_BBOXOP_COMMON_H_ +#define TESTS_DATASET_UT_CORE_COMMON_DE_UT_BBOXOP_COMMON_H_ + +#include "cvop_common.h" +#include "dataset/util/path.h" + +namespace UT { +namespace CVOP { +namespace BBOXOP { + +class BBoxOpCommon : public CVOpCommon { + public: + enum FileType { + kExpected, + kActual + }; + + BBoxOpCommon(); + + ~BBoxOpCommon(); + + /// \brief Sets up the class's variable, images_and_annotations + void SetUp() override; + + /// \brief Get all images and annotations in images_and_annotations TensorTable from dir + /// \param[in] dir directory containing images and annotation folders + /// \param[in] num_of_samples number of rows to fetch (default = 1) + void GetInputImagesAndAnnotations(const std::string &dir, std::size_t num_of_samples = 1); + + /// \brief Save the given tensor table + /// \param[in] type type of images to be stored (e.g. Expected or Actual) + /// \param[in] op_name name of op being tested + /// \param[in] table rows of images and corresponding annotations + void SaveImagesWithAnnotations(FileType type, const std::string &op_name, const TensorTable &table); + + /// \brief Compare actual and expected results. The images will have the bounding boxes on them + /// Log if images don't match + /// \param[in] op_name name of op being tested + void CompareActualAndExpected(const std::string &op_name); + + /// \brief Load BBox data from an XML file into a Tensor + /// \param[in] path path to XML bbox data file + /// \param[inout] target_BBox pointer to a Tensor to load + /// \return True if file loaded successfully, false if error -> logged to STD out + bool LoadAnnotationFile(const std::string &path, std::shared_ptr *target_BBox); + + TensorTable images_and_annotations_; + + private: + // directory of image_folder when the dataset/data gets transferred to build + std::string image_folder_build_; + // directory of image_folder in the source project (used to store expected results) + std::string image_folder_src_; +}; +} // namespace BBOXOP +} // namespace CVOP +} // namespace UT + +#endif // TESTS_DATASET_UT_CORE_COMMON_DE_UT_BBOXOP_COMMON_H_ From fd2fa2bcba9a74a2d9d274d9814058e8cf8495e9 Mon Sep 17 00:00:00 2001 From: wilfChen Date: Wed, 1 Jul 2020 09:33:42 +0800 Subject: [PATCH 187/254] GPU Lstm network --- model_zoo/lstm/src/lstm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/model_zoo/lstm/src/lstm.py b/model_zoo/lstm/src/lstm.py index f014eef8df..c3ca0bbf7c 100644 --- a/model_zoo/lstm/src/lstm.py +++ b/model_zoo/lstm/src/lstm.py @@ -88,6 +88,6 @@ class SentimentNet(nn.Cell): embeddings = self.trans(embeddings, self.perm) output, _ = self.encoder(embeddings, (self.h, self.c)) # states[i] size(64,200) -> encoding.size(64,400) - encoding = self.concat((output[0], output[-1])) + encoding = self.concat((output[0], output[499])) outputs = self.decoder(encoding) return outputs From ffd035216223b2b6119d4d79db571abdabfdc076 Mon Sep 17 00:00:00 2001 From: jiangjinsheng Date: Sun, 28 Jun 2020 12:03:55 +0800 Subject: [PATCH 188/254] vm for mod --- mindspore/ops/_grad/grad_math_ops.py | 12 +++++++ mindspore/ops/_op_impl/tbe/__init__.py | 1 + mindspore/ops/_op_impl/tbe/mod.py | 45 ++++++++++++++++++++++++++ mindspore/ops/operations/__init__.py | 5 +-- mindspore/ops/operations/math_ops.py | 43 ++++++++++++++++++++++-- mindspore/ops/operations/nn_ops.py | 2 +- tests/ut/python/ops/test_ops.py | 4 +++ 7 files changed, 106 insertions(+), 6 deletions(-) create mode 100644 mindspore/ops/_op_impl/tbe/mod.py diff --git a/mindspore/ops/_grad/grad_math_ops.py b/mindspore/ops/_grad/grad_math_ops.py index 1e4f932442..975e918817 100755 --- a/mindspore/ops/_grad/grad_math_ops.py +++ b/mindspore/ops/_grad/grad_math_ops.py @@ -306,6 +306,18 @@ def get_bprop_floormod(self): return bprop +@bprop_getters.register(P.Mod) +def get_bprop_mod(self): + """Grad definition for `Mod` operation.""" + + def bprop(x, y, out, dout): + bc_x = dout + bc_y = -dout * (x // y) + return binop_grad_common(x, y, bc_x, bc_y) + + return bprop + + @bprop_getters.register(P.Square) def get_bprop_square(self): """Grad definition for `Square` operation.""" diff --git a/mindspore/ops/_op_impl/tbe/__init__.py b/mindspore/ops/_op_impl/tbe/__init__.py index 0c2618aded..6067295010 100644 --- a/mindspore/ops/_op_impl/tbe/__init__.py +++ b/mindspore/ops/_op_impl/tbe/__init__.py @@ -276,3 +276,4 @@ from .lrn_grad import _lrn_grad_tbe from .scatter_max import _scatter_max_tbe from .scatter_min import _scatter_min_tbe from .scatter_sub import _scatter_sub_tbe +from .mod import _mod_tbe diff --git a/mindspore/ops/_op_impl/tbe/mod.py b/mindspore/ops/_op_impl/tbe/mod.py new file mode 100644 index 0000000000..c8fecd697a --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/mod.py @@ -0,0 +1,45 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Mod op""" +from mindspore.ops.op_info_register import op_info_register, TBERegOp, DataType + +mod_op_info = TBERegOp("Mod") \ + .fusion_type("ELEMWISE") \ + .async_flag(False) \ + .binfile_name("mod.so") \ + .compute_cost(10) \ + .kernel_name("mod") \ + .partial_flag(True) \ + .input(0, "x1", False, "required", "all") \ + .input(1, "x2", False, "required", "all") \ + .output(0, "y", False, "required", "all") \ + .dtype_format(DataType.I8_Default, DataType.I8_Default, DataType.I8_Default) \ + .dtype_format(DataType.I8_5HD, DataType.I8_5HD, DataType.I8_5HD) \ + .dtype_format(DataType.U8_Default, DataType.U8_Default, DataType.U8_Default) \ + .dtype_format(DataType.U8_5HD, DataType.U8_5HD, DataType.U8_5HD) \ + .dtype_format(DataType.I32_Default, DataType.I32_Default, DataType.I32_Default) \ + .dtype_format(DataType.I32_5HD, DataType.I32_5HD, DataType.I32_5HD) \ + .dtype_format(DataType.F16_Default, DataType.F16_Default, DataType.F16_Default) \ + .dtype_format(DataType.F16_5HD, DataType.F16_5HD, DataType.F16_5HD) \ + .dtype_format(DataType.F32_Default, DataType.F32_Default, DataType.F32_Default) \ + .dtype_format(DataType.F32_5HD, DataType.F32_5HD, DataType.F32_5HD) \ + .get_op_info() + + +@op_info_register(mod_op_info) +def _mod_tbe(): + """Mod TBE register""" + return diff --git a/mindspore/ops/operations/__init__.py b/mindspore/ops/operations/__init__.py index 552a980a0d..4e76f55cd4 100644 --- a/mindspore/ops/operations/__init__.py +++ b/mindspore/ops/operations/__init__.py @@ -45,7 +45,7 @@ from .math_ops import (Abs, ACos, Asin, Asinh, AddN, AccumulateNV2, AssignAdd, A BitwiseXor, Inv, Invert, ApproximateEqual, InplaceAdd, InplaceSub, ReduceMax, ReduceMin, ReduceMean, ReduceSum, ReduceAll, ReduceProd, CumProd, Cos, Div, DivNoNan, Equal, EqualCount, Exp, Expm1, Erf, Erfc, Floor, FloorDiv, FloorMod, Ceil, - Acosh, Greater, GreaterEqual, Less, LessEqual, Log, Log1p, LogicalAnd, + Acosh, Greater, GreaterEqual, Less, LessEqual, Log, Log1p, LogicalAnd, Mod, LogicalNot, LogicalOr, MatMul, Maximum, Minimum, Mul, Neg, NMSWithMask, NotEqual, NPUAllocFloatStatus, NPUClearFloatStatus, @@ -322,7 +322,8 @@ __all__ = [ "ApproximateEqual", "InplaceUpdate", "InTopK", - "LRN" + "LRN", + "Mod" ] __all__.sort() diff --git a/mindspore/ops/operations/math_ops.py b/mindspore/ops/operations/math_ops.py index c222f5326a..5198e912fa 100644 --- a/mindspore/ops/operations/math_ops.py +++ b/mindspore/ops/operations/math_ops.py @@ -1361,7 +1361,7 @@ class HistogramFixedWidth(PrimitiveWithInfer): Inputs: - **x** (Tensor) - Numeric Tensor. Must be one of the following types: int32, float32, float16. - **range** (Tensor) - Must have the same type as x. Shape [2] Tensor of same dtype as x. - x <= range[0] will be mapped to hist[0], x >= range[1] will be mapped to hist[-1]. + x <= range[0] will be mapped to hist[0], x >= range[1] will be mapped to hist[-1]. Outputs: Tensor, the type is int32. @@ -1645,8 +1645,9 @@ class Div(_MathBinaryOp): Inputs: - **input_x** (Union[Tensor, Number, bool]) - The first input is a number or a bool or a tensor whose data type is number or bool. - - **input_y** (Union[Tensor, Number, bool]) - The second input is a number or - a bool when the first input is a tensor or a tensor whose data type is number or bool. + - **input_y** (Union[Tensor, Number, bool]) - When the first input is a tensor, The second input + could be a number or a bool, or a tensor whose data type is number or bool. When the first input + is a number or a bool, the second input should be a tensor whose data type is number or bool. Outputs: Tensor, the shape is same as the shape after broadcasting, @@ -1742,6 +1743,42 @@ class FloorDiv(_MathBinaryOp): """ +class Mod(_MathBinaryOp): + """ + Computes the remainder of dividing the first input tensor by the second input tensor element-wise. + + The inputs must be two tensors or one tensor and one scalar. When the inputs are two tensors, + both dtypes cannot be bool, and the shapes of them could be broadcast. When the inputs are one tensor + and one scalar, the scalar only could be a constant. + + Inputs: + - **input_x** (Union[Tensor, Number]) - The first input is a number or a tensor whose data type is number. + - **input_y** (Union[Tensor, Number]) - When the first input is a tensor, The second input + could be a number or a tensor whose data type is number. When the first input is a number, + the second input should be a tensor whose data type is number. + + Outputs: + Tensor, the shape is same as the shape after broadcasting, + and the data type is the one with high precision or high digits among the two inputs. + + Raises: + ValueError: When `input_x` and `input_y` are not the same dtype. + + Examples: + >>> input_x = Tensor(np.array([-4.0, 5.0, 6.0]), mindspore.float32) + >>> input_y = Tensor(np.array([3.0, 2.0, 3.0]), mindspore.float32) + >>> mod = P.Mod() + >>> mod(input_x, input_y) + """ + + def infer_value(self, x, y): + if x is not None and y is not None: + x = x.asnumpy() + y = y.asnumpy() + return Tensor(np.fmod(x, y)) + return None + + class Floor(PrimitiveWithInfer): """ Round a tensor down to the closest integer element-wise. diff --git a/mindspore/ops/operations/nn_ops.py b/mindspore/ops/operations/nn_ops.py index 848154cc0e..c07f072f38 100644 --- a/mindspore/ops/operations/nn_ops.py +++ b/mindspore/ops/operations/nn_ops.py @@ -1669,7 +1669,7 @@ class DataFormatDimMap(PrimitiveWithInfer): Inputs: - **input_x** (Tensor) - A Tensor with each element as a dimension index in source data format. - Must be in the range [-4, 4). It's type is int32. + Must be in the range [-4, 4). It's type is int32. Outputs: Tensor, has the same type as the `input_x`. diff --git a/tests/ut/python/ops/test_ops.py b/tests/ut/python/ops/test_ops.py index cd829e362e..9d0dfd32a3 100755 --- a/tests/ut/python/ops/test_ops.py +++ b/tests/ut/python/ops/test_ops.py @@ -996,6 +996,10 @@ test_case_math_ops = [ 'block': NormalNet((3, 2, 4), 0.0, 1.0, 0), 'desc_inputs': [], 'skip': ['backward']}), + ('Mod', { + 'block': P.Mod(), + 'desc_inputs': [[3, 4, 5], [2, 3, 4, 5]], + 'desc_bprop': [[2, 3, 4, 5]]}), ] test_case_nn_ops = [ From 274bd25386a538f465487d2099b7adae52b6b477 Mon Sep 17 00:00:00 2001 From: wuyongkang Date: Mon, 29 Jun 2020 20:39:52 +0800 Subject: [PATCH 189/254] Optimize parser --- mindspore/_extends/parse/parser.py | 29 ++++++++++++++++++----- mindspore/ccsrc/pipeline/parse/resolve.cc | 2 +- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/mindspore/_extends/parse/parser.py b/mindspore/_extends/parse/parser.py index 2a1c9e0943..30731316e2 100644 --- a/mindspore/_extends/parse/parser.py +++ b/mindspore/_extends/parse/parser.py @@ -19,6 +19,7 @@ import ast import types import inspect +import hashlib from textwrap import dedent from dataclasses import is_dataclass import asttokens @@ -319,7 +320,6 @@ def get_dataclass_methods(cls): if isinstance(getattr(cls, name), (types.FunctionType,))} return methods - class Parser: """ Parser python code to ast tree. @@ -327,7 +327,10 @@ class Parser: Args: fn(FunctionType/MethodType): Need parse object instance. parse_method(ExtendInfoOfParseObj): Extend information for parse the function. + ast_cache: Dictionary for caching ast tree. """ + ast_cache = {} + def __init__(self, fn: (types.FunctionType, types.MethodType), parse_method=None) -> None: self.fn = fn self.parse_method = parse_method @@ -342,17 +345,31 @@ class Parser: self.function_name = fn.__name__ self.col_offset = 0 + @classmethod + def get_cache(cls, key): + """Get the value of the ast_cache dictionary""" + return cls.ast_cache.get(key) + + @classmethod + def insert_cache(cls, key, value): + """Insert elements to the ast_cache dictionary""" + cls.ast_cache[key] = value + def parse(self): """Parse the function or method.""" logger.debug("fn = %r", self.fn) tree = None if isinstance(self.fn, (types.FunctionType, types.MethodType)): original_src = inspect.getsource(self.fn) - src = dedent(original_src) - self.col_offset = \ - len(original_src.split('\n')[0]) - len(src.split('\n')[0]) - logger.debug("get source = %s", src) - tree = asttokens.ASTTokens(src, parse=True).tree + hexstr = hashlib.sha256(original_src.encode()).hexdigest() + tree = Parser.get_cache(hexstr) + if not tree: + src = dedent(original_src) + self.col_offset = \ + len(original_src.split('\n')[0]) - len(src.split('\n')[0]) + logger.debug("get source = %s", src) + tree = asttokens.ASTTokens(src, parse=True).tree + Parser.insert_cache(hexstr, tree) else: logger.error("Fn type is invalid") return tree diff --git a/mindspore/ccsrc/pipeline/parse/resolve.cc b/mindspore/ccsrc/pipeline/parse/resolve.cc index d5e1f828cc..87c2f78b42 100644 --- a/mindspore/ccsrc/pipeline/parse/resolve.cc +++ b/mindspore/ccsrc/pipeline/parse/resolve.cc @@ -94,7 +94,7 @@ AnfNodePtr ResolveParameterObj(const FuncGraphPtr &func_graph, const py::object auto top_graph = Parser::GetTopFuncGraph(); // if the parameter node has been created , return it AnfNodePtr para_node = nullptr; - for (auto param : top_graph->parameters()) { + for (auto const ¶m : top_graph->parameters()) { auto param_node = dyn_cast(param); if (param_node != nullptr && param_node->name() == param_name) { para_node = param; From be771aa9e482e908db2b6d1fa7da186fa184ffb6 Mon Sep 17 00:00:00 2001 From: buxue Date: Sat, 20 Jun 2020 18:20:33 +0800 Subject: [PATCH 190/254] refactor StridedSlice op --- mindspore/ops/operations/array_ops.py | 220 +++++++++++++++++------ tests/ut/python/ops/test_ops.py | 95 +++++++--- tests/ut/python/ops/test_tensor_slice.py | 8 +- 3 files changed, 238 insertions(+), 85 deletions(-) diff --git a/mindspore/ops/operations/array_ops.py b/mindspore/ops/operations/array_ops.py index dc387353af..b02d1b3c87 100644 --- a/mindspore/ops/operations/array_ops.py +++ b/mindspore/ops/operations/array_ops.py @@ -37,6 +37,7 @@ from ..._c_expression import signature_kind as sig_kind from ..._c_expression import signature_dtype as sig_dtype from ..._c_expression import typing + def _check_infer_attr_reduce(axis, keep_dims, prim_name): validator.check_value_type('keep_dims', keep_dims, [bool], prim_name) validator.check_value_type('axis', axis, [int, tuple], prim_name) @@ -193,7 +194,7 @@ class Cast(PrimitiveWithInfer): self.init_prim_io_names(inputs=['x', 'dst_type'], outputs=['output']) def check_elim(self, x, dtype): - if isinstance(x, (Tensor, numbers.Number, Parameter)): + if isinstance(x, (Tensor, numbers.Number, Parameter)): if isinstance(x, Tensor) and x.dtype == dtype: return (True, x) if isinstance(x, numbers.Number): @@ -987,10 +988,10 @@ class InvertPermutation(PrimitiveWithInfer): z.sort() for i in range(1, len(z)): - if z[i-1] == z[i]: + if z[i - 1] == z[i]: raise ValueError(f"For {self.name}, {z[i]} is duplicated in the input.") validator.check(f'value min', min(x_value), '', 0, Rel.EQ, self.name) - validator.check(f'value max', max(x_value), '', len(x_value)-1, Rel.EQ, self.name) + validator.check(f'value max', max(x_value), '', len(x_value) - 1, Rel.EQ, self.name) y = [None] * len(x_value) for i, value in enumerate(x_value): @@ -1693,6 +1694,57 @@ class Select(PrimitiveWithInfer): return None +def _compute_slicing_length(begin, end, stride, x_shape, i): + """Compute the length of the slicing.""" + if i >= len(x_shape): + raise ValueError(f"For 'StridedSlice', When their is no new axis, the index length must be less or " + f"equal than the dim of x.") + x_dim = x_shape[i] + if stride > 0: + # When slicing forward, convert begin and end to positive numbers. + if begin >= x_dim or end < -x_dim: + # When slicing forward, if begin >= x_dim or end < -x_dim, the length of the slicing is 0. + slicing_length = 0 + else: + if -x_dim <= begin < 0: + begin += x_dim + if begin < -x_dim: + # When slicing forward, if begin < -x_dim, set begin = 0, which means start from the 0th element. + begin = 0 + if -x_dim <= end < 0: + end += x_dim + if end > x_dim: + # When slicing forward, if end > x_dim, set end = x_dims, which means slice to the last element. + end = x_dim + if begin >= end: + # When slicing forward, if begin >= end, the length of the slicing is 0. + slicing_length = 0 + else: + slicing_length = 1 + (end - 1 - begin) // stride + else: + # When slicing backward, convert begin and end to negative numbers. + if begin < -x_dim or end >= x_dim: + # When slicing backward, if begin < -x_dim or end >= x_dim, the length of the slicing is 0. + slicing_length = 0 + else: + if 0 <= begin < x_dim: + begin += -x_dim + if begin >= x_dim: + # When slicing backward, if begin >= x_dim, set begin = -1, which means start from the last element. + begin = -1 + if 0 < end < x_dim: + end += -x_dim + if end < -x_dim - 1: + # When slicing backward, if end < -x_dim - 1, set end = -x_dim - 1, which means + # slicing to the 0th element. + end = -x_dim - 1 + if begin <= end: + # When slicing backward, if begin <= end, the length of the slicing is 0. + slicing_length = 0 + else: + slicing_length = 1 + (end + 1 - begin) // stride + return slicing_length + class StridedSlice(PrimitiveWithInfer): r""" @@ -1756,13 +1808,15 @@ class StridedSlice(PrimitiveWithInfer): ellipsis_mask=0, new_axis_mask=0, shrink_axis_mask=0): - """init StrideSlice""" + """Init StrideSlice""" self.init_prim_io_names(inputs=['x', 'begin', 'end', 'strides'], outputs=['output']) - validator.check_value_type('begin_mask', begin_mask, [int], self.name) - validator.check_value_type('end_mask', end_mask, [int], self.name) - validator.check_value_type('ellipsis_mask', ellipsis_mask, [int], self.name) - validator.check_value_type('new_axis_mask', new_axis_mask, [int], self.name) - validator.check_value_type('shrink_axis_mask', shrink_axis_mask, [int], self.name) + validator.check_integer('begin_mask', begin_mask, 0, Rel.GE, self.name) + validator.check_integer('end_mask', end_mask, 0, Rel.GE, self.name) + validator.check_integer('ellipsis_mask', ellipsis_mask, 0, Rel.GE, self.name) + if len(tuple(filter(lambda x: x == '1', bin(ellipsis_mask)[-1:1:-1]))) > 1: + raise ValueError(f"For '{self.name}', only support one ellipsis in the index, but got {end_mask}.") + validator.check_integer('new_axis_mask', new_axis_mask, 0, Rel.GE, self.name) + validator.check_integer('shrink_axis_mask', shrink_axis_mask, 0, Rel.GE, self.name) def __infer__(self, x, begin, end, strides): begin_v, end_v, strides_v = begin['value'], end['value'], strides['value'] @@ -1770,58 +1824,103 @@ class StridedSlice(PrimitiveWithInfer): validator.check_value_type("end", end_v, [tuple], self.name) validator.check_value_type("strides", strides_v, [tuple], self.name) - x_shape = x['shape'] - x_shp_len = len(x_shape) - if len(begin_v) != x_shp_len or len(end_v) != x_shp_len or len(strides_v) != x_shp_len: - raise ValueError(f"For \'{self.name}\' the length of begin index{begin_v}, end index{end_v} and " - f"strides{strides_v} must be equal to the dims({x_shp_len}) of input.") + if tuple(filter(lambda x: not isinstance(x, int), begin_v + end_v + strides_v)): + raise ValueError(f"For {self.name}, both the begins, ends, and strides must be a tuple of int, " + f"but got begins: {begin_v}, ends: {end_v}, strides: {strides_v}.") - ret_shape = [] - append_dimensions = [] - shrink_pos = bin(self.shrink_axis_mask)[::-1] - new_pos = bin(self.new_axis_mask)[::-1] - for i in range(x_shp_len): - # After the integer is converted to binary, it is a str and the first two chars are the flag char '0b' - if i < (len(new_pos) - 2) and new_pos[i] == '1': - ret_shape.append(1) - append_dimensions.append(x_shape[x_shp_len - 1 - len(append_dimensions)]) - continue - if i < (len(shrink_pos) - 2) and shrink_pos[i] == '1': - validator.check_integer(f'begin[{i}]', begin_v[i], -x_shape[i], Rel.GE, self.name) - validator.check_integer(f'begin[{i}]', begin_v[i], x_shape[i], Rel.LT, self.name) - continue - - begin_idx = begin_v[i] - end_idx = end_v[i] - strides_idx = strides_v[i] - if self.begin_mask: - begin_idx = 0 - if self.end_mask: - end_idx = x_shape[i] - validator.check_integer(f'begin[{i}]', begin_idx, x_shape[i], Rel.LE, self.name) - validator.check_integer(f'end[{i}]', end_idx, x_shape[i], Rel.LE, self.name) - validator.check_integer(f'strides[{i}]', strides_idx, 0, Rel.NE, self.name) - if strides_idx > 0: - # If sliced forward , end_idx >= begin_idx - validator.check(f'begin[{i}]', begin_idx, f'end[{i}]', end_idx, Rel.LE) - if begin_idx < 0 < end_idx: - # Turn negative begin_idx into positive values - begin_idx = x_shape[i] + begin_idx - num_elems = (end_idx - begin_idx + strides_idx - 1) // strides_idx - else: - # If sliced backwards, end_idx <= begin_idx - validator.check(f'begin[{i}]', begin_idx, f'end[{i}]', end_idx, Rel.GE) - if end_idx < 0 < begin_idx: - # Turn negative end_idx into positive values - end_idx = x_shape[i] + end_idx - num_elems = (end_idx - begin_idx + strides_idx + 1) // strides_idx - - ret_shape.append(num_elems) - if append_dimensions: - ret_shape += append_dimensions[::-1] + if tuple(filter(lambda x: x == 0, strides_v)): + raise ValueError(f"For '{self.name}', the strides cannot contain 0, but got strides: {strides_v}.") + + if len(end_v) != len(begin_v) or len(strides_v) != len(begin_v): + raise ValueError(f"For '{self.name}' the length of begin index: {begin_v}, end index: {end_v} and " + f"strides: {strides_v} must be equal.") + + ret_shape = self._compute_slicing_shape(x['shape'], begin_v, end_v, strides_v) + + value = None if all(ret_shape) else Tensor(np.array([]).reshape(ret_shape), x['dtype'].element_type()) return {'shape': ret_shape, 'dtype': x['dtype'], - 'value': None} + 'value': value} + + def _compute_slicing_shape(self, x_shape, begin_v, end_v, strides_v): + """Compute the shape of the slicing.""" + x_rank = len(x_shape) + slice_len = len(begin_v) + + # After the integer is converted to binary, it is a str and the first two chars are the flag char '0b'. + begin_pos = bin(self.begin_mask)[-1:1:-1] + end_pos = bin(self.end_mask)[-1:1:-1] + ellipsis_pos = bin(self.ellipsis_mask)[-1:1:-1] + new_axis_pos = bin(self.new_axis_mask)[-1:1:-1] + shrink_axis_pos = bin(self.shrink_axis_mask)[-1:1:-1] + + ret_shape = [] + i, j = 0, 0 + has_ellipsis = False + while i < x_rank or j < slice_len: + if j < slice_len: + begin, end, stride = begin_v[j], end_v[j], strides_v[j] + + if j < len(ellipsis_pos) and ellipsis_pos[j] == '1': + # When there is ellipsis, the latter part of the ellipsis will be processed separately. + has_ellipsis = True + break + if j < len(begin_pos) and begin_pos[j] == '1': + begin = -1 if strides_v[j] < 0 else 0 + if j < len(end_pos) and end_pos[j] == '1': + end = -(x_shape[i] + 1) if strides_v[j] < 0 else x_shape[i] + if j < len(new_axis_pos) and new_axis_pos[j] == '1': + ret_shape.append(1) + j += 1 + continue + if j < len(shrink_axis_pos) and shrink_axis_pos[j] == '1': + if (not -x_shape[i] <= begin < x_shape[i]) or stride < 0: + raise ValueError(f"For {self.name}, when shrink axis, the stride cannot be negative number, " + f"and begin should be in [-{x_shape[i]}, {x_shape[i]}), " + f"but got stride: {stride}, begin: {begin}.") + j += 1 + i += 1 + continue + else: + begin, end, stride = 0, x_shape[i], 1 + + slicing_length = _compute_slicing_length(begin, end, stride, x_shape, i) + ret_shape.append(slicing_length) + i += 1 + j += 1 + if has_ellipsis: + # When there is ellipsis, handle the second half of the ellipsis split. + ellipsis_occupied_dims = x_rank - i - (slice_len - (j + 1)) + \ + len(tuple(filter(lambda x: x == '1', new_axis_pos[j + 1:slice_len]))) + ret_shape.extend(x_shape[i:i + ellipsis_occupied_dims]) + j += 1 + i += ellipsis_occupied_dims + + while i < x_rank or j < slice_len: + begin, end, stride = begin_v[j], end_v[j], strides_v[j] + + if j < len(begin_pos) and begin_pos[j] == '1': + begin = -1 if strides_v[j] < 0 else 0 + if j < len(end_pos) and end_pos[j] == '1': + end = -(x_shape[i] + 1) if strides_v[j] < 0 else x_shape[i] + if j < len(new_axis_pos) and new_axis_pos[j] == '1': + ret_shape.append(1) + j += 1 + continue + if j < len(shrink_axis_pos) and shrink_axis_pos[j] == '1': + if (not -x_shape[i] <= begin < x_shape[i]) or stride < 0: + raise ValueError(f"For {self.name}, when shrink axis, the stride cannot be negative number, " + f"and begin should be in [-{x_shape[i]}, {x_shape[i]}), " + f"but got stride: {stride}, begin: {begin}.") + j += 1 + i += 1 + continue + + slicing_length = _compute_slicing_length(begin, end, stride, x_shape, i) + ret_shape.append(slicing_length) + i += 1 + j += 1 + return ret_shape class Diag(PrimitiveWithInfer): @@ -2102,6 +2201,7 @@ class TensorScatterUpdate(PrimitiveWithInfer): >>> op = P.TensorScatterUpdate() >>> output = op(input_x, indices, update) """ + @prim_attr_register def __init__(self): """Init TensorScatterUpdate""" @@ -2153,6 +2253,7 @@ class ScatterUpdate(PrimitiveWithInfer): ('indices', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T1), ('value', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T) ) + @prim_attr_register def __init__(self, use_locking=True): """Init ScatterUpdate""" @@ -2201,6 +2302,7 @@ class ScatterNdUpdate(PrimitiveWithInfer): ('indices', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T1), ('value', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T) ) + @prim_attr_register def __init__(self, use_locking=True): """Init ScatterNdUpdate""" @@ -2220,6 +2322,7 @@ class ScatterNdUpdate(PrimitiveWithInfer): validator.check_tensor_type_same(args, (mstype.bool_,) + mstype.number_type, self.name) return x_dtype + def _check_scatter_shape(x_shape, indices_shape, updates_shape, prim_name): if updates_shape and updates_shape != indices_shape + x_shape[1:]: raise ValueError(f"For '{prim_name}', the shape of updates should be [] or " @@ -2912,6 +3015,7 @@ class InplaceUpdate(PrimitiveWithInfer): [ 4. 5.] [ 6. 7.]]] """ + @prim_attr_register def __init__(self, indices): """Init InplaceUpdate""" diff --git a/tests/ut/python/ops/test_ops.py b/tests/ut/python/ops/test_ops.py index cd829e362e..907235d6e0 100755 --- a/tests/ut/python/ops/test_ops.py +++ b/tests/ut/python/ops/test_ops.py @@ -35,25 +35,6 @@ from ....mindspore_test_framework.pipeline.gradient.compile_gradient \ import pipeline_for_compile_grad_ge_graph_for_case_by_case_config -def test_tensor_scatter_update(): - class TensorScatterUpdateNet(nn.Cell): - """TensorScatterUpdate net definition""" - - def __init__(self): - super(TensorScatterUpdateNet, self).__init__() - self.tensor_scatter_update = P.TensorScatterUpdate() - - def construct(self, x, i, u): - out = self.tensor_scatter_update(x, i, u) - return out - net = TensorScatterUpdateNet() - context.set_context(mode=context.GRAPH_MODE, save_graphs=True) - x = Tensor(np.arange(3 * 4 * 5).reshape((3, 4, 5)), mstype.float32) - indices = Tensor(np.array([[0, 0], [1, 1]], np.int32)) - updates = Tensor(np.ones([2, 5], np.float32)) - net(x, indices, updates) - - class InputBackward(nn.Cell): def __init__(self, network): super(InputBackward, self).__init__() @@ -446,6 +427,7 @@ class SparseApplyAdagradNet(nn.Cell): out = self.sparse_apply_adagrad(self.var, self.accum, grad, indices) return out + class ApplyRMSNet(nn.Cell): def __init__(self): super(ApplyRMSNet, self).__init__() @@ -496,6 +478,60 @@ class NormalNet(nn.Cell): return out +class StridedSliceNet(nn.Cell): + def __init__(self): + super(StridedSliceNet, self).__init__() + self.begins = (1, 2, 3, 2, 1) + self.ends = (5, 6, 7, 8, 9) + self.strides = (1, 2, 3, 2, 1) + self.strided_slice_0 = P.StridedSlice(begin_mask=3, end_mask=5, ellipsis_mask=4, + shrink_axis_mask=2, new_axis_mask=8) + self.strided_slice_1 = P.StridedSlice(begin_mask=5, end_mask=2, ellipsis_mask=2, + shrink_axis_mask=6, new_axis_mask=10) + self.strided_slice_2 = P.StridedSlice(begin_mask=3, end_mask=3, ellipsis_mask=4, + shrink_axis_mask=5, new_axis_mask=13) + self.strided_slice_3 = P.StridedSlice(begin_mask=0, end_mask=0, ellipsis_mask=4, + shrink_axis_mask=12, new_axis_mask=15) + self.const_0 = Tensor(np.ones([6, 8, 9, 1, 8], np.float32)) + self.const_1 = Tensor(np.ones([5, 7, 8, 1, 8], np.float32)) + self.const_2 = Tensor(np.ones([1, 3, 7, 8, 9, 1, 8], np.float32)) + self.const_3 = Tensor(np.ones([1, 1, 6, 7, 8, 9, 1, 8], np.float32)) + + def construct(self, x): + out_0 = self.strided_slice_0(x, self.begins, self.ends, self.strides) + self.const_0 + out_1 = self.strided_slice_1(x, self.begins, self.ends, self.strides) + self.const_1 + out_2 = self.strided_slice_2(x, self.begins, self.ends, self.strides) + self.const_2 + out_3 = self.strided_slice_3(x, self.begins, self.ends, self.strides) + self.const_3 + return out_0, out_1, out_2, out_3 + + +def test_strided_slice_const(): + class StridedSLiceConstNet(nn.Cell): + """StridedSLiceConstNet net definition""" + + def __init__(self): + super(StridedSLiceConstNet, self).__init__() + self.begins = (0, 2, -5, 2, 1) + self.ends = (0, 6, 9, 8, 9) + self.strides = (1, 2, 1, 2, 1) + self.strided_slice = P.StridedSlice(begin_mask=2, + end_mask=6, + ellipsis_mask=4, + shrink_axis_mask=6, + new_axis_mask=18) + + def construct(self, x): + out = self.strided_slice(x, self.begins, self.ends, self.strides) + return out + + net = StridedSLiceConstNet() + context.set_context(mode=context.GRAPH_MODE, save_graphs=True) + x = Tensor(np.ones([6, 7, 8, 9, 10]), mstype.float32) + ret = net(x) + assert ret.shape == (0, 1, 7, 8, 9, 3, 1) + assert (ret.asnumpy() == np.array([], np.float32).reshape([0, 1, 7, 8, 9, 3, 1])).all() + + test_case_math_ops = [ ('BitwiseAnd', { 'block': P.BitwiseAnd(), @@ -1366,6 +1402,10 @@ test_case_nn_ops = [ 'desc_inputs': [Tensor(np.array([[128, 32, 32, 64], [128, 32, 32, 64]]).astype(np.float16))], 'desc_bprop': [Tensor(np.array([[128, 32, 32, 64], [128, 32, 32, 64]]).astype(np.float16))], 'skip': ['backward']}), + ('StridedSliceNet', { + 'block': StridedSliceNet(), + 'desc_inputs': [[6, 7, 8, 9, 10]], + 'skip': ['backward']}), ('OneHot', { 'block': P.OneHot(), 'desc_const': [3, Tensor(1.0, mstype.float32), Tensor(0.0, mstype.float32)], @@ -1763,7 +1803,7 @@ test_case_other_ops = [ 'desc_bprop': [([3, 3], {'dtype': np.int32})]}), ('TensorScatterUpdate', { 'block': P.TensorScatterUpdate(), - 'desc_inputs': (Tensor(np.arange(3 * 4 * 5).reshape((3, 4, 5)), mstype.float32), + 'desc_inputs': (Tensor(np.arange(3 * 4 * 5).reshape((3, 4, 5)), mstype.float32), Tensor(np.array([[0, 1], [1, 2]], np.int32)), Tensor(np.ones([2, 5], np.float32) * 99)), 'desc_bprop': [([3, 4, 5], {'dtype': np.float32})]}), @@ -1930,11 +1970,10 @@ test_case_other_ops = [ ] - test_case_quant_ops = [ ('AscendQuant_1', { 'block': inner.AscendQuant(0.5, 0.0, False, "Round"), - 'desc_inputs': [Tensor(np.random.rand(1,2,4,4), mstype.float32)], + 'desc_inputs': [Tensor(np.random.rand(1, 2, 4, 4), mstype.float32)], 'skip': ['backward']}), ('AscendQuant_2', { 'block': inner.AscendQuant(80.0, 10.0, True, "Round"), @@ -2027,6 +2066,18 @@ raise_set = [ 'block': (nn.SSIM(), {'exception': ValueError}), 'desc_inputs': [Tensor(np.ones((1, 3, 8, 8)), mstype.float32), Tensor(np.ones((1, 3, 8, 8)), mstype.float32)]}), + ('StridedSlice_0', { + 'block': (P.StridedSlice(), {'exception': ValueError}), + 'desc_const': [(1, 2.2, 3), (3, 4, 5), (1, 1, 1)], + 'desc_inputs': [[4, 5, 6, 7]]}), + ('StridedSlice_1', { + 'block': (P.StridedSlice(), {'exception': ValueError}), + 'desc_const': [(1, 2, 3), (3, 4, 5), (1, 1)], + 'desc_inputs': [[4, 5, 6, 7]]}), + ('StridedSlice_2', { + 'block': (P.StridedSlice(), {'exception': ValueError}), + 'desc_const': [(1, 2, 3), (3, 4, 5), (1, 1, 0)], + 'desc_inputs': [[4, 5, 6, 7]]}), ] diff --git a/tests/ut/python/ops/test_tensor_slice.py b/tests/ut/python/ops/test_tensor_slice.py index 73f705e009..66590945da 100644 --- a/tests/ut/python/ops/test_tensor_slice.py +++ b/tests/ut/python/ops/test_tensor_slice.py @@ -25,6 +25,7 @@ from ....mindspore_test_framework.pipeline.forward.compile_forward \ import pipeline_for_compile_forward_ge_graph_for_case_by_case_config, \ pipeline_for_compile_forward_ge_graph_for_case_by_case_config_exception + class NetWorkSlicePositive(Cell): def __init__(self): super(NetWorkSlicePositive, self).__init__() @@ -1159,10 +1160,8 @@ def test_tensor_slice_reduce_out_of_bounds_neg(): input_tensor = Tensor(np.ones([6, 8, 10], np.int32)) net = NetWork() - with pytest.raises(ValueError) as ex: + with pytest.raises(ValueError): net(input_tensor) - assert "For 'StridedSlice' the `begin[0]` should be an int and must greater or equal to -6, but got `-7`" in str( - ex.value) def test_tensor_slice_reduce_out_of_bounds_positive(): @@ -1177,6 +1176,5 @@ def test_tensor_slice_reduce_out_of_bounds_positive(): input_tensor = Tensor(np.ones([6, 8, 10], np.int32)) net = NetWork() - with pytest.raises(ValueError) as ex: + with pytest.raises(ValueError): net(input_tensor) - assert "For 'StridedSlice' the `begin[0]` should be an int and must less than 6, but got `6`" in str(ex.value) From 5d301092f6d7e299df8c2b375770fc7f9ca8003d Mon Sep 17 00:00:00 2001 From: kingfo Date: Tue, 30 Jun 2020 20:58:55 +0800 Subject: [PATCH 191/254] fix hook operator compare issue --- mindspore/ccsrc/optimizer/ad/kprim.cc | 29 +++++++++++---------------- mindspore/common/tensor.py | 3 +++ 2 files changed, 15 insertions(+), 17 deletions(-) diff --git a/mindspore/ccsrc/optimizer/ad/kprim.cc b/mindspore/ccsrc/optimizer/ad/kprim.cc index 791279b1a1..4141fb5413 100644 --- a/mindspore/ccsrc/optimizer/ad/kprim.cc +++ b/mindspore/ccsrc/optimizer/ad/kprim.cc @@ -106,28 +106,23 @@ FuncGraphPtr KPrim::KPrimitive(const ValueNodePtr &value_node, const pipeline::R } FuncGraphPtr bprop_fg = nullptr; - auto iter = bprop_registry_.find(prim); - if (iter != bprop_registry_.end()) { - bprop_fg = iter->second; - } + if (prim->Hash() == prim::kPrimHookBackward->Hash() && prim->name() == prim::kPrimHookBackward->name()) { + bprop_fg = BpropCut(value_node, resources); + } else { + auto iter = bprop_registry_.find(prim); + if (iter != bprop_registry_.end()) { + bprop_fg = iter->second; + } - if (bprop_fg == nullptr) { - bool is_faked_bprop = false; - if (prim->Hash() == prim::kPrimHookBackward->Hash() && prim->name() == prim::kPrimHookBackward->name()) { - bprop_fg = BpropCut(value_node, resources); - } else { + if (bprop_fg == nullptr) { bprop_fg = GetBprop(prim); - if (bprop_fg == nullptr) { + if (bprop_fg != nullptr) { + // Set bprop_g graph cache + bprop_registry_[prim] = bprop_fg; + } else { bprop_fg = FakeBprop(value_node, resources); - is_faked_bprop = true; } } - - // To support primitives with variable params, do not cache faked bprop - if (!is_faked_bprop) { - // Set bprop_g graph cache - bprop_registry_[prim] = bprop_fg; - } } auto expanded_fg = BpropToK(prim, bprop_fg); diff --git a/mindspore/common/tensor.py b/mindspore/common/tensor.py index 4bb845af55..043ab4f6cf 100644 --- a/mindspore/common/tensor.py +++ b/mindspore/common/tensor.py @@ -109,6 +109,9 @@ class Tensor(Tensor_): out = tensor_operator_registry.get('__neg__')(self) return out + def __pos__(self): + return self + def __iadd__(self, other): return self.__add__(other) From b7667628fa16af7b397aead18aaec50aee7225ce Mon Sep 17 00:00:00 2001 From: dinghao Date: Wed, 1 Jul 2020 14:52:34 +0800 Subject: [PATCH 192/254] fix serving exit signal handle --- serving/core/server.cc | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/serving/core/server.cc b/serving/core/server.cc index c07558a5c2..273f0920c3 100644 --- a/serving/core/server.cc +++ b/serving/core/server.cc @@ -227,6 +227,7 @@ class MSServiceImpl final : public MSService::Service { Status Server::BuildAndStart() { // handle exit signal signal(SIGINT, HandleSignal); + signal(SIGTERM, HandleSignal); Status res; auto option_args = Options::Instance().GetArgs(); std::string server_address = "0.0.0.0:" + std::to_string(option_args->grpc_port); @@ -258,21 +259,17 @@ Status Server::BuildAndStart() { } g_ctx = ctx; #endif - MSServiceImpl service; + MSServiceImpl msService; grpc::EnableDefaultHealthCheckService(true); grpc::reflection::InitProtoReflectionServerBuilderPlugin(); // Set the port is not reuseable auto option = grpc::MakeChannelArgumentOption(GRPC_ARG_ALLOW_REUSEPORT, 0); - grpc::ServerBuilder builder; - builder.SetOption(std::move(option)); - builder.SetMaxMessageSize(uint32max); - // Listen on the given address without any authentication mechanism. - builder.AddListeningPort(server_address, grpc::InsecureServerCredentials()); - // Register "service" as the instance through which we'll communicate with - // clients. In this case it corresponds to an *synchronous* service. - builder.RegisterService(&service); - // Finally assemble the server. - std::unique_ptr server(builder.BuildAndStart()); + grpc::ServerBuilder serverBuilder; + serverBuilder.SetOption(std::move(option)); + serverBuilder.SetMaxMessageSize(uint32max); + serverBuilder.AddListeningPort(server_address, grpc::InsecureServerCredentials()); + serverBuilder.RegisterService(&msService); + std::unique_ptr server(serverBuilder.BuildAndStart()); if (server == nullptr) { MS_LOG(ERROR) << "The serving server create failed"; ClearEnv(); @@ -280,7 +277,7 @@ Status Server::BuildAndStart() { } auto grpc_server_run = [&server]() { server->Wait(); }; std::thread serving_thread(grpc_server_run); - MS_LOG(INFO) << "Server listening on " << server_address << std::endl; + MS_LOG(INFO) << "MS Serving listening on " << server_address; auto exit_future = exit_requested.get_future(); exit_future.wait(); ClearEnv(); From f797d17a6ec7cd20ec7e88ce288c00ad17b949f8 Mon Sep 17 00:00:00 2001 From: wuyongkang Date: Wed, 1 Jul 2020 15:14:52 +0800 Subject: [PATCH 193/254] Optimization for ast_cache --- mindspore/_extends/parse/parser.py | 14 ++------------ mindspore/ccsrc/optimizer/optimizer.h | 2 +- 2 files changed, 3 insertions(+), 13 deletions(-) diff --git a/mindspore/_extends/parse/parser.py b/mindspore/_extends/parse/parser.py index 30731316e2..a6043eb787 100644 --- a/mindspore/_extends/parse/parser.py +++ b/mindspore/_extends/parse/parser.py @@ -345,16 +345,6 @@ class Parser: self.function_name = fn.__name__ self.col_offset = 0 - @classmethod - def get_cache(cls, key): - """Get the value of the ast_cache dictionary""" - return cls.ast_cache.get(key) - - @classmethod - def insert_cache(cls, key, value): - """Insert elements to the ast_cache dictionary""" - cls.ast_cache[key] = value - def parse(self): """Parse the function or method.""" logger.debug("fn = %r", self.fn) @@ -362,14 +352,14 @@ class Parser: if isinstance(self.fn, (types.FunctionType, types.MethodType)): original_src = inspect.getsource(self.fn) hexstr = hashlib.sha256(original_src.encode()).hexdigest() - tree = Parser.get_cache(hexstr) + tree = Parser.ast_cache.get(hexstr) if not tree: src = dedent(original_src) self.col_offset = \ len(original_src.split('\n')[0]) - len(src.split('\n')[0]) logger.debug("get source = %s", src) tree = asttokens.ASTTokens(src, parse=True).tree - Parser.insert_cache(hexstr, tree) + Parser.ast_cache[hexstr] = tree else: logger.error("Fn type is invalid") return tree diff --git a/mindspore/ccsrc/optimizer/optimizer.h b/mindspore/ccsrc/optimizer/optimizer.h index 3e77edc1e9..5f21de43cd 100644 --- a/mindspore/ccsrc/optimizer/optimizer.h +++ b/mindspore/ccsrc/optimizer/optimizer.h @@ -136,7 +136,7 @@ class Optimizer : public std::enable_shared_from_this { return func_graph; } // Optimizer step counter; - int counter = -1; + int counter = 1; bool changes = true; while (changes) { From 88afef5e90814c191e89cd2e32c5b3cdb52b3759 Mon Sep 17 00:00:00 2001 From: yanghaitao Date: Wed, 1 Jul 2020 15:43:01 +0800 Subject: [PATCH 194/254] fix fastrcnn accuracy --- model_zoo/faster_rcnn/src/dataset.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/model_zoo/faster_rcnn/src/dataset.py b/model_zoo/faster_rcnn/src/dataset.py index 133824dd24..346ed5a6cd 100644 --- a/model_zoo/faster_rcnn/src/dataset.py +++ b/model_zoo/faster_rcnn/src/dataset.py @@ -441,6 +441,7 @@ def create_fasterrcnn_dataset(mindrecord_file, batch_size=2, repeat_num=12, devi hwc_to_chw = C.HWC2CHW() normalize_op = C.Normalize((123.675, 116.28, 103.53), (58.395, 57.12, 57.375)) horizontally_op = C.RandomHorizontalFlip(1) + type_cast0 = CC.TypeCast(mstype.float32) type_cast1 = CC.TypeCast(mstype.float16) type_cast2 = CC.TypeCast(mstype.int32) type_cast3 = CC.TypeCast(mstype.bool_) @@ -453,13 +454,15 @@ def create_fasterrcnn_dataset(mindrecord_file, batch_size=2, repeat_num=12, devi flip = (np.random.rand() < config.flip_ratio) if flip: - ds = ds.map(input_columns=["image"], operations=[normalize_op, horizontally_op, hwc_to_chw, type_cast1], - num_parallel_workers=24) + ds = ds.map(input_columns=["image"], operations=[normalize_op, type_cast0, horizontally_op], + num_parallel_workers=12) ds = ds.map(input_columns=["image", "image_shape", "box", "label", "valid_num"], operations=flipped_generation, num_parallel_workers=num_parallel_workers) else: - ds = ds.map(input_columns=["image"], operations=[normalize_op, hwc_to_chw, type_cast1], - num_parallel_workers=24) + ds = ds.map(input_columns=["image"], operations=[normalize_op, type_cast0], + num_parallel_workers=12) + ds = ds.map(input_columns=["image"], operations=[hwc_to_chw, type_cast1], + num_parallel_workers=12) else: ds = ds.map(input_columns=["image", "annotation"], From 8d9939eaa41dbd7b5bbd78a59896d68701acdd5b Mon Sep 17 00:00:00 2001 From: zhaozhenlong Date: Tue, 30 Jun 2020 19:39:22 +0800 Subject: [PATCH 195/254] ScatterMull and ScatterDiv vm --- mindspore/nn/loss/loss.py | 2 +- mindspore/ops/_op_impl/tbe/__init__.py | 2 + mindspore/ops/_op_impl/tbe/scatter_div.py | 42 +++++ mindspore/ops/_op_impl/tbe/scatter_mul.py | 42 +++++ mindspore/ops/operations/__init__.py | 6 +- mindspore/ops/operations/array_ops.py | 180 ++++++++++++---------- tests/ut/python/ops/test_ops.py | 111 +++++++++++++ 7 files changed, 301 insertions(+), 84 deletions(-) create mode 100644 mindspore/ops/_op_impl/tbe/scatter_div.py create mode 100644 mindspore/ops/_op_impl/tbe/scatter_mul.py diff --git a/mindspore/nn/loss/loss.py b/mindspore/nn/loss/loss.py index 4639229c41..3f97fbf83c 100644 --- a/mindspore/nn/loss/loss.py +++ b/mindspore/nn/loss/loss.py @@ -369,7 +369,7 @@ class CosineEmbeddingLoss(_Loss): >>> x2 = Tensor(np.array([[0.4, 1.2], [-0.4, -0.9]]), mindspore.float32) >>> y = Tensor(np.array([1,-1]), mindspore.int32) >>> cosine_embedding_loss = P.CosineEmbeddingLoss() - >>> cosine_embedding_loss(x1, x2, target) + >>> cosine_embedding_loss(x1, x2, y) [0.0003426671] """ def __init__(self, margin=0.0, reduction="mean"): diff --git a/mindspore/ops/_op_impl/tbe/__init__.py b/mindspore/ops/_op_impl/tbe/__init__.py index 6067295010..be3d313554 100644 --- a/mindspore/ops/_op_impl/tbe/__init__.py +++ b/mindspore/ops/_op_impl/tbe/__init__.py @@ -276,4 +276,6 @@ from .lrn_grad import _lrn_grad_tbe from .scatter_max import _scatter_max_tbe from .scatter_min import _scatter_min_tbe from .scatter_sub import _scatter_sub_tbe +from .scatter_mul import _scatter_mul_tbe +from .scatter_div import _scatter_div_tbe from .mod import _mod_tbe diff --git a/mindspore/ops/_op_impl/tbe/scatter_div.py b/mindspore/ops/_op_impl/tbe/scatter_div.py new file mode 100644 index 0000000000..5c6572dfd7 --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/scatter_div.py @@ -0,0 +1,42 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""ScatterDiv op""" +from mindspore.ops.op_info_register import op_info_register, TBERegOp, DataType + +scatter_div_op_info = TBERegOp("ScatterDiv") \ + .fusion_type("ELEMWISE") \ + .async_flag(False) \ + .binfile_name("scatter_div.so") \ + .compute_cost(10) \ + .kernel_name("scatter_div") \ + .partial_flag(True) \ + .attr("use_locking", "optional", "bool", "all") \ + .input(0, "var", False, "required", "all") \ + .input(1, "indices", False, "required", "all") \ + .input(2, "updates", False, "required", "all") \ + .output(0, "var", False, "required", "all") \ + .dtype_format(DataType.F16_Default, DataType.I32_Default, DataType.F16_Default, DataType.F16_Default) \ + .dtype_format(DataType.F32_Default, DataType.I32_Default, DataType.F32_Default, DataType.F32_Default) \ + .dtype_format(DataType.I32_Default, DataType.I32_Default, DataType.I32_Default, DataType.I32_Default) \ + .dtype_format(DataType.I8_Default, DataType.I32_Default, DataType.I8_Default, DataType.I8_Default) \ + .dtype_format(DataType.U8_Default, DataType.I32_Default, DataType.U8_Default, DataType.U8_Default) \ + .get_op_info() + + +@op_info_register(scatter_div_op_info) +def _scatter_div_tbe(): + """ScatterDiv TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/scatter_mul.py b/mindspore/ops/_op_impl/tbe/scatter_mul.py new file mode 100644 index 0000000000..dd77e33f6d --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/scatter_mul.py @@ -0,0 +1,42 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""ScatterMul op""" +from mindspore.ops.op_info_register import op_info_register, TBERegOp, DataType + +scatter_mul_op_info = TBERegOp("ScatterMul") \ + .fusion_type("ELEMWISE") \ + .async_flag(False) \ + .binfile_name("scatter_mul.so") \ + .compute_cost(10) \ + .kernel_name("scatter_mul") \ + .partial_flag(True) \ + .attr("use_locking", "optional", "bool", "all") \ + .input(0, "var", False, "required", "all") \ + .input(1, "indices", False, "required", "all") \ + .input(2, "updates", False, "required", "all") \ + .output(0, "var", False, "required", "all") \ + .dtype_format(DataType.F16_Default, DataType.I32_Default, DataType.F16_Default, DataType.F16_Default) \ + .dtype_format(DataType.F32_Default, DataType.I32_Default, DataType.F32_Default, DataType.F32_Default) \ + .dtype_format(DataType.I32_Default, DataType.I32_Default, DataType.I32_Default, DataType.I32_Default) \ + .dtype_format(DataType.I8_Default, DataType.I32_Default, DataType.I8_Default, DataType.I8_Default) \ + .dtype_format(DataType.U8_Default, DataType.I32_Default, DataType.U8_Default, DataType.U8_Default) \ + .get_op_info() + + +@op_info_register(scatter_mul_op_info) +def _scatter_mul_tbe(): + """ScatterMul TBE register""" + return diff --git a/mindspore/ops/operations/__init__.py b/mindspore/ops/operations/__init__.py index 4e76f55cd4..bc4edce193 100644 --- a/mindspore/ops/operations/__init__.py +++ b/mindspore/ops/operations/__init__.py @@ -25,8 +25,8 @@ from .array_ops import (Argmax, Argmin, Cast, Concat, Pack, Unpack, Fill, GatherNd, GatherV2, SparseGatherV2, InvertPermutation, IsInstance, IsSubClass, ArgMaxWithValue, OnesLike, ZerosLike, Rank, Reshape, ResizeNearestNeighbor, ArgMinWithValue, - SameTypeShape, ScatterAdd, ScatterSub, ScatterMax, ScatterMin, ScatterUpdate, - ScalarToArray, ScalarToTensor, ScatterNd, ScatterNdUpdate, Select, + SameTypeShape, ScatterAdd, ScatterSub, ScatterMul, ScatterDiv, ScatterMax, ScatterMin, + ScatterUpdate, ScalarToArray, ScalarToTensor, ScatterNd, ScatterNdUpdate, Select, Shape, Size, Slice, Split, Squeeze, StridedSlice, Tile, TensorScatterUpdate, Transpose, TruncatedNormal, TupleToArray, UnsortedSegmentMin, @@ -215,6 +215,8 @@ __all__ = [ 'L2Normalize', 'ScatterAdd', 'ScatterSub', + 'ScatterMul', + 'ScatterDiv', 'ScatterNd', 'ScatterMax', 'ScatterMin', diff --git a/mindspore/ops/operations/array_ops.py b/mindspore/ops/operations/array_ops.py index b02d1b3c87..daefea64c8 100644 --- a/mindspore/ops/operations/array_ops.py +++ b/mindspore/ops/operations/array_ops.py @@ -38,6 +38,39 @@ from ..._c_expression import signature_dtype as sig_dtype from ..._c_expression import typing +class _ScatterOp(PrimitiveWithInfer): + """ + Define Scatter operators + """ + __mindspore_signature__ = ( + ('x', sig_rw.RW_WRITE, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), + ('indices', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T1), + ('updates', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T) + ) + @staticmethod + def _check_scatter_shape(x_shape, indices_shape, updates_shape, prim_name): + if updates_shape and updates_shape != indices_shape + x_shape[1:]: + raise ValueError(f"For '{prim_name}', the shape of updates should be [] or " + f"updates_shape = indices_shape + x_shape[1:], but got x_shape: {x_shape}, " + f"indices_shape: {indices_shape}, updates_shape: {updates_shape}.") + + @prim_attr_register + def __init__(self, use_locking=False): + """Init _ScatterOp""" + validator.check_value_type('use_locking', use_locking, [bool], self.name) + self.init_prim_io_names(inputs=['x', 'indices', 'updates'], outputs=['y']) + + def infer_shape(self, x_shape, indices_shape, updates_shape): + _ScatterOp._check_scatter_shape(x_shape, indices_shape, updates_shape, self.name) + return x_shape + + def infer_dtype(self, x_dtype, indices_dtype, updates_dtype): + validator.check_tensor_type_same({'indices': indices_dtype}, [mstype.int32], self.name) + args = {"x": x_dtype, "updates": updates_dtype} + validator.check_tensor_type_same(args, mstype.number_type, self.name) + return x_dtype + + def _check_infer_attr_reduce(axis, keep_dims, prim_name): validator.check_value_type('keep_dims', keep_dims, [bool], prim_name) validator.check_value_type('axis', axis, [int, tuple], prim_name) @@ -2221,7 +2254,7 @@ class TensorScatterUpdate(PrimitiveWithInfer): return x_dtype -class ScatterUpdate(PrimitiveWithInfer): +class ScatterUpdate(_ScatterOp): """ Update tensor value by using input indices and value. @@ -2233,8 +2266,8 @@ class ScatterUpdate(PrimitiveWithInfer): Inputs: - **input_x** (Parameter) - The target tensor, with data type of Parameter. - **indices** (Tensor) - The index of input tensor. With int32 data type. - - **update** (Tensor) - The tensor to update the input tensor, has the same type as input, - and update.shape = indices.shape + input_x.shape[1:]. + - **updates** (Tensor) - The tensor to update the input tensor, has the same type as input, + and updates.shape = indices.shape + input_x.shape[1:]. Outputs: Tensor, has the same shape and type as `input_x`. @@ -2243,27 +2276,17 @@ class ScatterUpdate(PrimitiveWithInfer): >>> np_x = np.array([[-0.1, 0.3, 3.6], [0.4, 0.5, -3.2]]) >>> input_x = mindspore.Parameter(Tensor(np_x, mindspore.float32), name="x") >>> indices = Tensor(np.array([[0, 0], [1, 1]]), mindspore.int32) - >>> np_update = np.array([[[1.0, 2.2, 1.0], [2.0, 1.2, 1.0]], [[2.0, 2.2, 1.0], [3.0, 1.2, 1.0]]]) - >>> update = Tensor(np_update, mindspore.float32) + >>> np_updates = np.array([[[1.0, 2.2, 1.0], [2.0, 1.2, 1.0]], [[2.0, 2.2, 1.0], [3.0, 1.2, 1.0]]]) + >>> updates = Tensor(np_updates, mindspore.float32) >>> op = P.ScatterUpdate() - >>> output = op(input_x, indices, update) + >>> output = op(input_x, indices, updates) """ - __mindspore_signature__ = ( - ('x', sig_rw.RW_WRITE, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), - ('indices', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T1), - ('value', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T) - ) @prim_attr_register def __init__(self, use_locking=True): """Init ScatterUpdate""" validator.check_value_type('use_locking', use_locking, [bool], self.name) - self.init_prim_io_names(inputs=['x', 'indices', 'value'], outputs=['y']) - - def infer_shape(self, x_shape, indices_shape, value_shape): - if indices_shape + x_shape[1:] != value_shape: - raise ValueError("For 'ScatterUpdate', input value are not match with input indices.") - return x_shape + self.init_prim_io_names(inputs=['x', 'indices', 'updates'], outputs=['y']) def infer_dtype(self, x_dtype, indices_dtype, value_dtype): validator.check_tensor_type_same({'indices': indices_dtype}, [mstype.int32], self.name) @@ -2323,14 +2346,7 @@ class ScatterNdUpdate(PrimitiveWithInfer): return x_dtype -def _check_scatter_shape(x_shape, indices_shape, updates_shape, prim_name): - if updates_shape and updates_shape != indices_shape + x_shape[1:]: - raise ValueError(f"For '{prim_name}', the shape of updates should be [] or " - f"updates_shape = indices_shape + x_shape[1:], but got x_shape: {x_shape}, " - f"indices_shape: {indices_shape}, updates_shape: {updates_shape}.") - - -class ScatterMax(PrimitiveWithInfer): +class ScatterMax(_ScatterOp): """ Update the value of the input tensor through the max operation. @@ -2364,18 +2380,8 @@ class ScatterMax(PrimitiveWithInfer): self.init_prim_io_names(inputs=['x', 'indices', 'updates'], outputs=['y']) validator.check_value_type('use_locking', use_locking, (bool,), self.name) - def infer_shape(self, x_shape, indices_shape, updates_shape): - _check_scatter_shape(x_shape, indices_shape, updates_shape, self.name) - return x_shape - def infer_dtype(self, x_dtype, indices_dtype, updates_dtype): - validator.check_tensor_type_same({'indices': indices_dtype}, (mstype.int32,), self.name) - args = {"x": x_dtype, "updates": updates_dtype} - validator.check_tensor_type_same(args, mstype.number_type, self.name) - return x_dtype - - -class ScatterMin(PrimitiveWithInfer): +class ScatterMin(_ScatterOp): """ Update the value of the input tensor through the min operation. @@ -2403,24 +2409,8 @@ class ScatterMin(PrimitiveWithInfer): [[0.0, 1.0, 1.0], [0.0, 0.0, 0.0]] """ - @prim_attr_register - def __init__(self, use_locking=False): - """Init ScatterMin""" - self.init_prim_io_names(inputs=['x', 'indices', 'updates'], outputs=['y']) - validator.check_value_type('use_locking', use_locking, (bool,), self.name) - - def infer_shape(self, x_shape, indices_shape, updates_shape): - _check_scatter_shape(x_shape, indices_shape, updates_shape, self.name) - return x_shape - - def infer_dtype(self, x_dtype, indices_dtype, updates_dtype): - validator.check_tensor_type_same({'indices': indices_dtype}, (mstype.int32,), self.name) - args = {"x": x_dtype, "updates": updates_dtype} - validator.check_tensor_type_same(args, mstype.number_type, self.name) - return x_dtype - -class ScatterAdd(PrimitiveWithInfer): +class ScatterAdd(_ScatterOp): """ Update the value of the input tensor through the add operation. @@ -2448,23 +2438,8 @@ class ScatterAdd(PrimitiveWithInfer): [[1.0, 1.0, 1.0], [3.0, 3.0, 3.0]] """ - @prim_attr_register - def __init__(self, use_locking=False): - """Init ScatterAdd""" - validator.check_value_type('use_locking', use_locking, (bool,), self.name) - - def infer_shape(self, x_shape, indices_shape, updates_shape): - _check_scatter_shape(x_shape, indices_shape, updates_shape, self.name) - return x_shape - - def infer_dtype(self, x_dtype, indices_dtype, updates_dtype): - validator.check_tensor_type_same({'indices': indices_dtype}, (mstype.int32,), self.name) - args = {'x': x_dtype, 'updates': updates_dtype} - validator.check_tensor_type_same(args, mstype.number_type, self.name) - return x_dtype - -class ScatterSub(PrimitiveWithInfer): +class ScatterSub(_ScatterOp): """ Update the value of the input tensor through the sub operation. @@ -2492,20 +2467,63 @@ class ScatterSub(PrimitiveWithInfer): [[-1.0, -1.0, -1.0], [-1.0, -1.0, -1.0]] """ - @prim_attr_register - def __init__(self, use_locking=False): - """Init ScatterSub""" - validator.check_value_type('use_locking', use_locking, (bool,), self.name) - def infer_shape(self, x_shape, indices_shape, updates_shape): - _check_scatter_shape(x_shape, indices_shape, updates_shape, self.name) - return x_shape +class ScatterMul(_ScatterOp): + """ + Update the value of the input tensor through the mul operation. - def infer_dtype(self, x_dtype, indices_dtype, updates_dtype): - validator.check_tensor_type_same({'indices': indices_dtype}, (mstype.int32,), self.name) - args = {'x': x_dtype, 'updates': updates_dtype} - validator.check_tensor_type_same(args, mstype.number_type, self.name) - return x_dtype + Using given values to update tensor value through the mul operation, along with the input indices. + This operation outputs the `input_x` after the update is done, which makes it convenient to use the updated value. + + Args: + use_locking (bool): Whether protect the assignment by a lock. Default: False. + + Inputs: + - **input_x** (Parameter) - The target parameter. + - **indices** (Tensor) - The index to do mul operation whose data type should be mindspore.int32. + - **updates** (Tensor) - The tensor doing the mul operation with `input_x`, + the data type is same as `input_x`, the shape is `indices_shape + x_shape[1:]`. + + Outputs: + Parameter, the updated `input_x`. + + Examples: + >>> input_x = Parameter(Tensor(np.array([[1.0, 1.0, 1.0], [2.0, 2.0, 2.0]]), mindspore.float32), name="x") + >>> indices = Tensor(np.array([0, 1]), mindspore.int32) + >>> updates = Tensor(np.ones([[2.0, 2.0, 2.0], [2.0, 2.0, 2.0]]), mindspore.float32) + >>> scatter_mul = P.ScatterMul() + >>> output = scatter_mul(input_x, indices, updates) + [[2.0, 2.0, 2.0], [4.0, 4.0, 4.0]] + """ + + +class ScatterDiv(_ScatterOp): + """ + Update the value of the input tensor through the div operation. + + Using given values to update tensor value through the div operation, along with the input indices. + This operation outputs the `input_x` after the update is done, which makes it convenient to use the updated value. + + Args: + use_locking (bool): Whether protect the assignment by a lock. Default: False. + + Inputs: + - **input_x** (Parameter) - The target parameter. + - **indices** (Tensor) - The index to do div operation whose data type should be mindspore.int32. + - **updates** (Tensor) - The tensor doing the div operation with `input_x`, + the data type is same as `input_x`, the shape is `indices_shape + x_shape[1:]`. + + Outputs: + Parameter, the updated `input_x`. + + Examples: + >>> input_x = Parameter(Tensor(np.array([[6.0, 6.0, 6.0], [2.0, 2.0, 2.0]]), mindspore.float32), name="x") + >>> indices = Tensor(np.array([0, 1]), mindspore.int32) + >>> updates = Tensor(np.ones([[2.0, 2.0, 2.0], [2.0, 2.0, 2.0]]), mindspore.float32) + >>> scatter_div = P.ScatterDiv() + >>> output = scatter_div(input_x, indices, updates) + [[3.0, 3.0, 3.0], [1.0, 1.0, 1.0]] + """ class SpaceToDepth(PrimitiveWithInfer): diff --git a/tests/ut/python/ops/test_ops.py b/tests/ut/python/ops/test_ops.py index b3f4e50c38..98a7b766e7 100755 --- a/tests/ut/python/ops/test_ops.py +++ b/tests/ut/python/ops/test_ops.py @@ -185,6 +185,19 @@ class HistogramSummaryNet(nn.Cell): return out +class ScatterUpdate(nn.Cell): + """ScatterUpdate net definition""" + + def __init__(self, ref_shape, dtype=np.float32, use_locking=False): + super(ScatterUpdate, self).__init__() + self.scatter_update = P.ScatterUpdate(use_locking) + self.ref = Parameter(Tensor(np.ones(ref_shape, dtype)), name="ref") + + def construct(self, indices, updates): + out = self.scatter_update(self.ref, indices, updates) + return out + + class ScatterMax(nn.Cell): """ScatterMax net definition""" @@ -237,6 +250,32 @@ class ScatterSub(nn.Cell): return out +class ScatterMul(nn.Cell): + """ScatterMul net definition""" + + def __init__(self, ref_shape, dtype=np.float32, use_locking=False): + super(ScatterMul, self).__init__() + self.scatter_mul = P.ScatterMul(use_locking) + self.ref = Parameter(Tensor(np.ones(ref_shape, dtype)), name="ref") + + def construct(self, indices, updates): + out = self.scatter_mul(self.ref, indices, updates) + return out + + +class ScatterDiv(nn.Cell): + """ScatterDiv net definition""" + + def __init__(self, ref_shape, dtype=np.float32, use_locking=False): + super(ScatterDiv, self).__init__() + self.scatter_div = P.ScatterDiv(use_locking) + self.ref = Parameter(Tensor(np.ones(ref_shape, dtype)*10), name="ref") + + def construct(self, indices, updates): + out = self.scatter_div(self.ref, indices, updates) + return out + + class ApplyFtrlNet(nn.Cell): def __init__(self): super(ApplyFtrlNet, self).__init__() @@ -1861,6 +1900,11 @@ test_case_other_ops = [ 'desc_inputs': (Tensor(np.array([[0, 0], [1, 1]], np.int32)), Tensor(np.ones([2, 2, 3], np.int32))), 'skip': ['backward']}), + ('ScatterUpdate', { + 'block': ScatterUpdate((6,)), + 'desc_inputs': (Tensor(np.array([2, 0, 5], np.int32)), + Tensor(np.array([2.0, 3.0, 4.0], np.float32))), + 'skip': ['backward']}), ('ScatterAddUseLocking', { 'block': ScatterAdd((6,), use_locking=True), 'desc_inputs': (Tensor(np.array([2, 0, 5], np.int32)), @@ -1902,6 +1946,73 @@ test_case_other_ops = [ 'desc_inputs': (Tensor(np.array([2, 0, 5], np.int32)), Tensor(np.array([2, 3, 4], np.uint8))), 'skip': ['backward']}), + ('ScatterMulUseLocking', { + 'block': ScatterMul((6,), use_locking=True), + 'desc_inputs': (Tensor(np.array([2], np.int32)), + Tensor(np.array([2.0], np.float32))), + 'skip': ['backward']}), + ('ScatterMulScalar', { + 'block': ScatterMul((6,)), + 'desc_inputs': (Tensor(np.array([2], np.int32)), + Tensor(np.array([2.0], np.float32))), + 'skip': ['backward']}), + ('ScatterMul2d', { + 'block': ScatterMul((3, 4)), + 'desc_inputs': (Tensor(np.array([[0, 1], [1, 2]], np.int32)), + Tensor(np.array([[[1, 1, 1, 1], [2, 2, 2, 2]], + [[3, 3, 3, 3], [4, 4, 4, 4]]], np.float32))), + 'skip': ['backward']}), + ('ScatterMulF16', { + 'block': ScatterMul((6,), np.float16), + 'desc_inputs': (Tensor(np.array([2, 0, 5], np.int32)), + Tensor(np.array([2.0, 3.0, 4.0], np.float16))), + 'skip': ['backward']}), + ('ScatterMulI8', { + 'block': ScatterMul((6,), np.int8), + 'desc_inputs': (Tensor(np.array([2, 0, 5], np.int32)), + Tensor(np.array([2, 3, 4], np.int8))), + 'skip': ['backward']}), + ('ScatterMulI32', { + 'block': ScatterMul((6,), np.int32), + 'desc_inputs': (Tensor(np.array([2, 0, 5], np.int32)), + Tensor(np.array([2, 3, 4], np.int32))), + 'skip': ['backward']}), + ('ScatterMulU8', { + 'block': ScatterMul((6,), np.uint8), + 'desc_inputs': (Tensor(np.array([2, 0, 5], np.int32)), + Tensor(np.array([2, 3, 4], np.uint8))), + 'skip': ['backward']}), + ('ScatterDivUseLocking', { + 'block': ScatterDiv((6,), use_locking=True), + 'desc_inputs': (Tensor(np.array([2], np.int32)), + Tensor(np.array([2.0], np.float32))), + 'skip': ['backward']}), + ('ScatterDivScalar', { + 'block': ScatterDiv((6,)), + 'desc_inputs': (Tensor(np.array([2], np.int32)), + Tensor(np.array([2.0], np.float32))), + 'skip': ['backward']}), + ('ScatterDiv2d', { + 'block': ScatterDiv((3, 4)), + 'desc_inputs': (Tensor(np.array([[0, 1], [1, 2]], np.int32)), + Tensor(np.array([[[1, 1, 1, 1], [2, 2, 2, 2]], + [[3, 3, 3, 3], [4, 4, 4, 4]]], np.float32))), + 'skip': ['backward']}), + ('ScatterDivF16', { + 'block': ScatterDiv((6,), np.float16), + 'desc_inputs': (Tensor(np.array([2, 0, 5], np.int32)), + Tensor(np.array([2.0, 3.0, 4.0], np.float16))), + 'skip': ['backward']}), + ('ScatterDivI8', { + 'block': ScatterDiv((6,), np.int8), + 'desc_inputs': (Tensor(np.array([2, 0, 5], np.int32)), + Tensor(np.array([2, 3, 4], np.int8))), + 'skip': ['backward']}), + ('ScatterDivU8', { + 'block': ScatterDiv((6,), np.uint8), + 'desc_inputs': (Tensor(np.array([2, 0, 5], np.int32)), + Tensor(np.array([2, 3, 4], np.uint8))), + 'skip': ['backward']}), ('ScatterSubUseLocking', { 'block': ScatterSub((6,), use_locking=True), 'desc_inputs': (Tensor(np.array([2], np.int32)), From 18bc81e88c7a6c9e65dfda9c3191f39d01efe5d8 Mon Sep 17 00:00:00 2001 From: zhoufeng Date: Wed, 1 Jul 2020 15:53:03 +0800 Subject: [PATCH 196/254] Ascend control sink testcase Signed-off-by: zhoufeng --- tests/st/control/test_ascend_control_sink.py | 189 +++++++++++++++++++ 1 file changed, 189 insertions(+) create mode 100644 tests/st/control/test_ascend_control_sink.py diff --git a/tests/st/control/test_ascend_control_sink.py b/tests/st/control/test_ascend_control_sink.py new file mode 100644 index 0000000000..2c206c9768 --- /dev/null +++ b/tests/st/control/test_ascend_control_sink.py @@ -0,0 +1,189 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" test_ascend_control_sink """ +import pytest +import numpy as np +import mindspore.context as context +import mindspore.nn as nn +from mindspore.ops import operations as op +from mindspore.common import dtype as mstype +from mindspore.common.tensor import Tensor +from mindspore.common.parameter import Parameter +from mindspore.common.initializer import initializer + + +class ControlSimpleIf(nn.Cell): + def __init__(self): + super().__init__() + self.addn = op.AddN() + + def construct(self, x, y, z, input1, input2): + addn1 = self.addn([input1, input1, input1]) + addn2 = self.addn([input2, input2, input2]) + addn11 = self.addn([addn1, addn1, addn1]) + addn22 = self.addn([addn2, addn2, addn2]) + cond1 = x > y + cond2 = y > z + # dodge pylint + if cond1 and cond2: + out = self.addn([addn11, addn11]) + else: + out = self.addn([addn22, addn22]) + out_me = self.addn([out, input1]) + return out_me + + +class ControlSimpleIfWithAssign(nn.Cell): + def __init__(self, input_shape): + super().__init__() + self.addn = op.AddN() + self.assign = op.Assign() + self.input_data = Parameter(initializer(1, input_shape, mstype.float32), name="var") + + def construct(self, x, y, input_data): + if x > y: + out = self.addn([input_data, input_data, input_data]) + else: + out = self.assign(self.input_data, input_data) + return out + + +class ControlIfinIf(nn.Cell): + def __init__(self): + super().__init__() + + def construct(self, x, y): + if x > y: + x = x + 1 + if y < 0: + y = y + 1 + else: + y = y + 2 + else: + x = x + 2 + x = x + y + return x + + +class ControlIfbyIfbyIf(nn.Cell): + def __init__(self): + super().__init__() + self.addn = op.AddN() + + def construct(self, x, y, cond1, cond2, input_data): + tri_in = self.addn([input_data, input_data, input_data]) + if x > y: + addn_1 = self.addn([tri_in, tri_in]) + else: + addn_1 = self.addn([tri_in, tri_in, tri_in]) + if cond1: + addn_2 = self.addn([addn_1, addn_1]) + else: + addn_2 = self.addn([addn_1, addn_1, addn_1]) + if cond2: + out = self.addn([addn_2, addn_2, addn_2]) + else: + out = self.addn([addn_2, addn_2]) + return out + + +class ControlMixedWhileIf(nn.Cell): + def __init__(self): + super().__init__() + + def construct(self, x, y): + y = y + 4 + while x < y: + if 2 * x < y: + x = x + 1 + else: + x = x + 2 + x = x + 3 + return x + +@pytest.mark.level0 +@pytest.mark.platform_arm_ascend_training +@pytest.mark.platform_x86_ascend_training +@pytest.mark.env_onecard +def test_simple_if(): + context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") + x = np.array(3).astype(np.float32) + y = np.array(2).astype(np.float32) + z = np.array(3).astype(np.float32) + input_shape = (127, 7, 53, 31) + input1 = np.random.randn(*input_shape).astype(np.float32) + input2 = np.random.randn(*input_shape).astype(np.float32) + net = ControlSimpleIf() + output = net(Tensor(x), Tensor(y), Tensor(z), Tensor(input1), Tensor(input2)) + expect = input2 * 3 * 3 * 2 + input1 + assert np.allclose(expect, output.asnumpy(), 0.0001, 0.0001) + +@pytest.mark.level0 +@pytest.mark.platform_arm_ascend_training +@pytest.mark.platform_x86_ascend_training +@pytest.mark.env_onecard +def test_simple_if_with_assign(): + context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") + x = np.array(0).astype(np.float32) + y = np.array(1).astype(np.float32) + input_shape = (127, 7, 53, 31) + input_data = np.random.randn(*input_shape).astype(np.float32) + net = ControlSimpleIfWithAssign(input_shape) + output = net(Tensor(x), Tensor(y), Tensor(input_data)) + expect = input_data + assert np.allclose(expect, output.asnumpy(), 0.0001, 0.0001) + +@pytest.mark.level0 +@pytest.mark.platform_arm_ascend_training +@pytest.mark.platform_x86_ascend_training +@pytest.mark.env_onecard +def test_if_in_if(): + context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") + x = np.array(2.345678).astype(np.float32) + y = np.array(1.234567).astype(np.float32) + net = ControlIfinIf() + output = net(Tensor(x), Tensor(y)) + expect = x + y + 3 + assert np.allclose(expect, output.asnumpy(), 0.0001, 0.0001) + +@pytest.mark.level0 +@pytest.mark.platform_arm_ascend_training +@pytest.mark.platform_x86_ascend_training +@pytest.mark.env_onecard +def test_if_by_if_by_if(): + context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") + x = np.array(2.345678).astype(np.float32) + y = np.array(1.234567).astype(np.float32) + cond1 = np.array(True).astype(np.bool) + cond2 = np.array(False).astype(np.bool) + input_shape = (127, 7, 53, 31) + input_data = np.random.randn(*input_shape).astype(np.float32) + net = ControlIfbyIfbyIf() + output = net(Tensor(x), Tensor(y), Tensor(cond1), Tensor(cond2), Tensor(input_data)) + expect = input_data * 3 * 2 * 2 * 2 + assert np.allclose(expect, output.asnumpy(), 0.0001, 0.0001) + +@pytest.mark.level0 +@pytest.mark.platform_arm_ascend_training +@pytest.mark.platform_x86_ascend_training +@pytest.mark.env_onecard +def test_mixed_while_if(): + context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") + x = np.array(2).astype(np.int32) + y = np.array(14).astype(np.int32) + net = ControlMixedWhileIf() + output = net(Tensor(x), Tensor(y)) + expect = np.array(22).astype(np.int32) + assert np.allclose(expect, output.asnumpy(), 0.0001, 0.0001) From 2351845edee37537018f0a23c3b428ed84abab01 Mon Sep 17 00:00:00 2001 From: yangzhenzhang <285824651@qq.com> Date: Wed, 1 Jul 2020 15:24:17 +0800 Subject: [PATCH 197/254] delete shape validation in initializer --- mindspore/common/initializer.py | 4 ---- tests/ut/python/utils/test_initializer_fuzz.py | 9 +-------- 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/mindspore/common/initializer.py b/mindspore/common/initializer.py index 83586272ee..48a142a23f 100644 --- a/mindspore/common/initializer.py +++ b/mindspore/common/initializer.py @@ -343,10 +343,6 @@ def initializer(init, shape=None, dtype=mstype.float32): shape = tuple(shape) elif isinstance(shape, numbers.Number): shape = (shape,) - try: - np.ndarray(shape) - except ValueError: - raise ValueError("Error shape={}".format(shape)) if isinstance(init, Initializer): init.shape = init.shape if init.shape is not None else shape diff --git a/tests/ut/python/utils/test_initializer_fuzz.py b/tests/ut/python/utils/test_initializer_fuzz.py index 3a6fb4833f..5cd76a1cd8 100644 --- a/tests/ut/python/utils/test_initializer_fuzz.py +++ b/tests/ut/python/utils/test_initializer_fuzz.py @@ -49,13 +49,6 @@ class Net(nn.Cell): return out -def test_shape_error(): - """ for fuzz test""" - in_str = "3 22222222222222222222222222264 3 64 64 222 222 3" - with pytest.raises(ValueError): - Net(in_str) - - class LeNet5(nn.Cell): """ LeNet5 definition """ @@ -98,7 +91,7 @@ class LeNet5(nn.Cell): return x -def test_shape_error_2(): +def test_shape_error(): """ for fuzz test""" in_str = "3 6 5 6 -6 5 16 5 5 120 120 84 84 3 2" with pytest.raises(ValueError): From 83c55057272fc949246af718375b1dc95be51bd5 Mon Sep 17 00:00:00 2001 From: hanjun996 Date: Wed, 1 Jul 2020 11:44:10 +0800 Subject: [PATCH 198/254] clean codedex warning --- model_zoo/wide_and_deep/src/preprocess_data.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/model_zoo/wide_and_deep/src/preprocess_data.py b/model_zoo/wide_and_deep/src/preprocess_data.py index 35d13b841d..75562aa71b 100644 --- a/model_zoo/wide_and_deep/src/preprocess_data.py +++ b/model_zoo/wide_and_deep/src/preprocess_data.py @@ -17,6 +17,8 @@ import os import pickle import collections import argparse +import urllib.request +import tarfile import numpy as np from mindspore.mindrecord import FileWriter @@ -257,10 +259,15 @@ if __name__ == '__main__': download_data_path = data_path + "origin_data/" mkdir_path(download_data_path) - os.system( - "wget -P {} -c https://s3-eu-west-1.amazonaws.com/kaggle-display-advertising-challenge-dataset/dac.tar.gz --no-check-certificate".format( - download_data_path)) - os.system("tar -zxvf {}dac.tar.gz".format(download_data_path)) + url = "https://s3-eu-west-1.amazonaws.com/kaggle-display-advertising-challenge-dataset/dac.tar.gz" + file_name = download_data_path + '/' + url.split('/')[-1] + urllib.request.urlretrieve(url, filename=file_name) + + tar = tarfile.open(file_name) + names = tar.getnames() + for name in names: + tar.extract(name, path=download_data_path) + tar.close() criteo_stats = CriteoStatsDict() data_file_path = data_path + "origin_data/train.txt" From ad8731944cf8925967dbc96f76d7fce741baa576 Mon Sep 17 00:00:00 2001 From: Margaret_wangrui Date: Wed, 1 Jul 2020 10:31:55 +0800 Subject: [PATCH 199/254] handle partial node in CreateNewCNode --- mindspore/ccsrc/session/session_basic.cc | 44 +++++++++++++++++------- 1 file changed, 32 insertions(+), 12 deletions(-) diff --git a/mindspore/ccsrc/session/session_basic.cc b/mindspore/ccsrc/session/session_basic.cc index 8935d3df2f..8382d6af9b 100644 --- a/mindspore/ccsrc/session/session_basic.cc +++ b/mindspore/ccsrc/session/session_basic.cc @@ -447,6 +447,37 @@ CNodePtr SessionBasic::CreateNewCNode(const CNodePtr &cnode, bool valid_input, K return new_cnode; } +static std::vector CreateSwitchOrPartialNode(const CNodePtr &cnode, KernelGraph *graph) { + MS_EXCEPTION_IF_NULL(cnode); + MS_EXCEPTION_IF_NULL(graph); + // create primitive of cnode:call(partial or switch) + std::vector cnode_inputs = { + graph->NewValueNode(NewValueNode(std::make_shared(prim::kPrimCall->name())))}; + auto attr_input = cnode->input(kAnfPrimitiveIndex); + MS_EXCEPTION_IF_NULL(attr_input); + auto cnode_input = graph->GetBackendAnfByFrontAnf(attr_input); + if (cnode_input == nullptr) { + MS_LOG(EXCEPTION) << "CNode input[0] is CNode:" << attr_input->DebugString() + << ", but input[0] has not been created."; + } + // if the node is partial, insert the inputs of partial to the call + if (AnfAlgo::CheckPrimitiveType(cnode_input, prim::kPrimPartial)) { + auto partial_node = attr_input->cast(); + MS_EXCEPTION_IF_NULL(partial_node); + auto partial_inputs = partial_node->inputs(); + std::transform(partial_inputs.begin() + kFirstDataInputIndex, partial_inputs.end(), + std::back_inserter(cnode_inputs), [&graph](const AnfNodePtr &node) { + MS_EXCEPTION_IF_NULL(graph->GetBackendAnfByFrontAnf(node)); + return graph->GetBackendAnfByFrontAnf(node); + }); + return cnode_inputs; + } else if (AnfAlgo::CheckPrimitiveType(cnode_input, prim::kPrimSwitch)) { + cnode_inputs.emplace_back(cnode_input); + return cnode_inputs; + } + MS_LOG(EXCEPTION) << "CNode input[0] must be partial or switch."; +} + CNodePtr SessionBasic::CreateNewCNode(const CNodePtr &cnode, KernelGraph *graph) { MS_EXCEPTION_IF_NULL(cnode); MS_EXCEPTION_IF_NULL(graph); @@ -471,18 +502,7 @@ CNodePtr SessionBasic::CreateNewCNode(const CNodePtr &cnode, KernelGraph *graph) } } } else if (attr_input->isa()) { - // create primitive of cnode:call(switch) - cnode_inputs = {graph->NewValueNode(NewValueNode(std::make_shared(prim::kPrimCall->name())))}; - if (graph->GetBackendAnfByFrontAnf(attr_input) != nullptr) { - auto cnode_input = graph->GetBackendAnfByFrontAnf(attr_input); - if (!AnfAlgo::CheckPrimitiveType(cnode_input, prim::kPrimSwitch)) { - MS_LOG(EXCEPTION) << "CNode input[0] must be switch."; - } - cnode_inputs.emplace_back(cnode_input); - } else { - MS_LOG(EXCEPTION) << "CNode input[0] is CNode:" << attr_input->DebugString() - << ", but input[0] has not been created."; - } + cnode_inputs = CreateSwitchOrPartialNode(cnode, graph); } else { // get primitive of old node auto prim = AnfAlgo::GetCNodePrimitive(cnode); From a057639abcc12bb8ab08ae699572f004ff1f63b5 Mon Sep 17 00:00:00 2001 From: ZPaC Date: Mon, 8 Jun 2020 15:31:37 +0800 Subject: [PATCH 200/254] Add pslite dependency --- CMakeLists.txt | 8 +- build.sh | 13 +- cmake/external_libs/pslite.cmake | 14 ++ cmake/external_libs/zeromq.cmake | 5 + cmake/mind_expression.cmake | 4 + cmake/options.cmake | 1 + cmake/utils.cmake | 14 +- mindspore/ccsrc/CMakeLists.txt | 4 + third_party/patch/pslite/ps_lite.patch001 | 264 ++++++++++++++++++++++ 9 files changed, 319 insertions(+), 8 deletions(-) create mode 100644 cmake/external_libs/pslite.cmake create mode 100644 cmake/external_libs/zeromq.cmake create mode 100644 third_party/patch/pslite/ps_lite.patch001 diff --git a/CMakeLists.txt b/CMakeLists.txt index 324eca867b..987e4ae709 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -38,6 +38,10 @@ if (NOT Patch_FOUND) endif () message(PATCH_EXECUTABLE = ${Patch_EXECUTABLE}) +if (ENABLE_AKG AND ENABLE_D) + add_subdirectory("${CMAKE_SOURCE_DIR}/akg") +endif() + include(${CMAKE_SOURCE_DIR}/cmake/mind_expression.cmake) include_directories(${CMAKE_CURRENT_SOURCE_DIR}) include_directories(${CMAKE_CURRENT_SOURCE_DIR}/third_party/flatbuffers/include) @@ -86,10 +90,6 @@ if (ENABLE_GE OR ENABLE_D OR ENABLE_TESTCASES) include_directories(${CMAKE_CURRENT_SOURCE_DIR}/graphengine/third_party/fwkacllib/inc/toolchain) endif() -if (ENABLE_AKG AND ENABLE_D) - add_subdirectory("${CMAKE_SOURCE_DIR}/akg") -endif() - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fvisibility=hidden") add_subdirectory(mindspore/ccsrc) if (ENABLE_TESTCASES) diff --git a/build.sh b/build.sh index 56d2294e7e..059478b9af 100755 --- a/build.sh +++ b/build.sh @@ -25,7 +25,7 @@ usage() echo "Usage:" echo "bash build.sh [-d] [-r] [-v] [-c on|off] [-t on|off] [-g on|off] [-h] [-b ge] [-m infer|train] \\" echo " [-a on|off] [-Q on|off] [-p on|off] [-i] [-L] [-R] [-D on|off] [-j[n]] [-e gpu|d|cpu] \\" - echo " [-P on|off] [-z [on|off]] [-M on|off] [-V 9.2|10.1] [-I] [-K] [-B on|off]" + echo " [-P on|off] [-z [on|off]] [-M on|off] [-V 9.2|10.1] [-I] [-K] [-B on|off] [-E]" echo "" echo "Options:" echo " -d Debug mode" @@ -55,6 +55,7 @@ usage() echo " -K Compile with AKG, default on" echo " -s Enable serving module, default off" echo " -B Enable debugger, default off" + echo " -E Enable IBVERBS for parameter server, default off" } # check value of input is 'on' or 'off' @@ -96,9 +97,10 @@ checkopts() ENABLE_AKG="on" ENABLE_SERVING="off" ENABLE_DEBUGGER="off" + ENABLE_IBVERBS="off" # Process the options - while getopts 'drvj:c:t:hsb:a:g:p:ie:m:I:LRP:Q:D:zM:V:K:sB:' opt + while getopts 'drvj:c:t:hsb:a:g:p:ie:m:I:LRP:Q:D:zM:V:K:sB:E' opt do OPTARG=$(echo ${OPTARG} | tr '[A-Z]' '[a-z]') case "${opt}" in @@ -252,6 +254,10 @@ checkopts() ENABLE_DEBUGGER="on" echo "enable debugger" ;; + E) + ENABLE_IBVERBS="on" + echo "enable IBVERBS for parameter server" + ;; *) echo "Unknown option ${opt}!" usage @@ -338,6 +344,9 @@ build_mindspore() CMAKE_ARGS="${CMAKE_ARGS} -DENABLE_DEBUGGER=ON" fi + if [[ "X$ENABLE_IBVERBS" = "Xon" ]]; then + CMAKE_ARGS="${CMAKE_ARGS} -DENABLE_IBVERBS=ON" + fi echo "${CMAKE_ARGS}" if [[ "X$INC_BUILD" = "Xoff" ]]; then cmake ${CMAKE_ARGS} ../.. diff --git a/cmake/external_libs/pslite.cmake b/cmake/external_libs/pslite.cmake new file mode 100644 index 0000000000..28c851b094 --- /dev/null +++ b/cmake/external_libs/pslite.cmake @@ -0,0 +1,14 @@ +set(pslite_USE_STATIC_LIBS ON) +if (${ENABLE_IBVERBS} STREQUAL "ON") + set(pslite_CXXFLAGS "USE_IBVERBS=1") +endif() +mindspore_add_pkg(pslite + LIBS ps + URL https://github.com/dmlc/ps-lite/archive/34fd45cae457d59850fdcb2066467778d0673f21.zip + MD5 393c0e27b68bfaf96718caa3aa96f5a3 + PATCHES ${CMAKE_SOURCE_DIR}/third_party/patch/pslite/ps_lite.patch001 + ONLY_MAKE True + ONLY_MAKE_INCS include/* + ONLY_MAKE_LIBS build/*) +include_directories(${pslite_INC}) +add_library(mindspore::pslite ALIAS pslite::ps) diff --git a/cmake/external_libs/zeromq.cmake b/cmake/external_libs/zeromq.cmake new file mode 100644 index 0000000000..122f1ee90c --- /dev/null +++ b/cmake/external_libs/zeromq.cmake @@ -0,0 +1,5 @@ +mindspore_add_pkg(zeromq + VER 4.1.4 + HEAD_ONLY ./ + URL https://raw.githubusercontent.com/mli/deps/master/build/zeromq-4.1.4.tar.gz + MD5 a611ecc93fffeb6d058c0e6edf4ad4fb) diff --git a/cmake/mind_expression.cmake b/cmake/mind_expression.cmake index 403316ac47..63a65cd533 100644 --- a/cmake/mind_expression.cmake +++ b/cmake/mind_expression.cmake @@ -30,6 +30,10 @@ include(${CMAKE_SOURCE_DIR}/cmake/external_libs/flatbuffers.cmake) if(USE_GLOG) include(${CMAKE_SOURCE_DIR}/cmake/external_libs/glog.cmake) endif() +if (NOT ${CMAKE_SYSTEM_NAME} MATCHES "Windows") + include(${CMAKE_SOURCE_DIR}/cmake/external_libs/zeromq.cmake) + include(${CMAKE_SOURCE_DIR}/cmake/external_libs/pslite.cmake) +endif() find_package(Python3) include_directories(${Python3_INCLUDE_DIRS}) diff --git a/cmake/options.cmake b/cmake/options.cmake index 33e4b47ef3..18db942d68 100644 --- a/cmake/options.cmake +++ b/cmake/options.cmake @@ -18,6 +18,7 @@ option(ENABLE_DUMP_IR "Enable dump funciton graph ir, default on" ON) option(ENABLE_MPI "enable mpi" OFF) option(ENABLE_AKG "enable akg" OFF) option(ENABLE_DEBUGGER "enable debugger" OFF) +option(ENABLE_IBVERBS "enable IBVERBS for parameter server" OFF) if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") if (WIN32) diff --git a/cmake/utils.cmake b/cmake/utils.cmake index f0a5dc594c..cf8f6ebb46 100644 --- a/cmake/utils.cmake +++ b/cmake/utils.cmake @@ -206,7 +206,7 @@ function(mindspore_add_pkg pkg_name ) set(options ) set(oneValueArgs URL MD5 GIT_REPOSITORY GIT_TAG VER EXE DIR HEAD_ONLY CMAKE_PATH RELEASE LIB_PATH CUSTOM_CMAKE) - set(multiValueArgs CMAKE_OPTION LIBS PRE_CONFIGURE_COMMAND CONFIGURE_COMMAND BUILD_OPTION INSTALL_INCS INSTALL_LIBS PATCHES SUBMODULES SOURCEMODULES) + set(multiValueArgs CMAKE_OPTION LIBS PRE_CONFIGURE_COMMAND CONFIGURE_COMMAND BUILD_OPTION INSTALL_INCS INSTALL_LIBS PATCHES SUBMODULES SOURCEMODULES ONLY_MAKE ONLY_MAKE_INCS ONLY_MAKE_LIBS) cmake_parse_arguments(PKG "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN} ) if (NOT PKG_LIB_PATH) @@ -290,7 +290,7 @@ function(mindspore_add_pkg pkg_name ) foreach(_PATCH_FILE ${PKG_PATCHES}) get_filename_component(_PATCH_FILE_NAME ${_PATCH_FILE} NAME) set(_LF_PATCH_FILE ${CMAKE_BINARY_DIR}/_ms_patch/${_PATCH_FILE_NAME}) - configure_file(${_PATCH_FILE} ${_LF_PATCH_FILE} NEWLINE_STYLE LF) + configure_file(${_PATCH_FILE} ${_LF_PATCH_FILE} NEWLINE_STYLE LF @ONLY) message("patching ${${pkg_name}_SOURCE_DIR} -p1 < ${_LF_PATCH_FILE}") execute_process(COMMAND ${Patch_EXECUTABLE} -p1 INPUT_FILE ${_LF_PATCH_FILE} @@ -324,6 +324,16 @@ function(mindspore_add_pkg pkg_name ) target_include_directories(${pkg_name} INTERFACE ${${pkg_name}_INC}) endif () + elseif (PKG_ONLY_MAKE) + __exec_cmd(COMMAND ${CMAKE_MAKE_PROGRAM} ${${pkg_name}_CXXFLAGS} -j${THNUM} + WORKING_DIRECTORY ${${pkg_name}_SOURCE_DIR}) + set(PKG_INSTALL_INCS ${PKG_ONLY_MAKE_INCS}) + set(PKG_INSTALL_LIBS ${PKG_ONLY_MAKE_LIBS}) + file(GLOB ${pkg_name}_INSTALL_INCS ${${pkg_name}_SOURCE_DIR}/${PKG_INSTALL_INCS}) + file(GLOB ${pkg_name}_INSTALL_LIBS ${${pkg_name}_SOURCE_DIR}/${PKG_INSTALL_LIBS}) + file(COPY ${${pkg_name}_INSTALL_INCS} DESTINATION ${${pkg_name}_BASE_DIR}/include) + file(COPY ${${pkg_name}_INSTALL_LIBS} DESTINATION ${${pkg_name}_BASE_DIR}/lib) + elseif (PKG_CMAKE_OPTION) # in cmake file(MAKE_DIRECTORY ${${pkg_name}_SOURCE_DIR}/_build) diff --git a/mindspore/ccsrc/CMakeLists.txt b/mindspore/ccsrc/CMakeLists.txt index cc5845cbf1..e27a2049f6 100644 --- a/mindspore/ccsrc/CMakeLists.txt +++ b/mindspore/ccsrc/CMakeLists.txt @@ -230,6 +230,10 @@ else () target_link_libraries(_c_expression PRIVATE -Wl,--whole-archive mindspore -Wl,--no-whole-archive) target_link_libraries(_c_expression PRIVATE mindspore::pybind11_module) target_link_libraries(_c_expression PRIVATE mindspore_gvar) + target_link_libraries(_c_expression PRIVATE mindspore::pslite mindspore::protobuf ${zeromq_DIRPATH}/zmq_install/lib/libzmq.a) + if (${ENABLE_IBVERBS} STREQUAL "ON") + target_link_libraries(_c_expression PRIVATE ibverbs rdmacm) + endif() endif () if (USE_GLOG) diff --git a/third_party/patch/pslite/ps_lite.patch001 b/third_party/patch/pslite/ps_lite.patch001 new file mode 100644 index 0000000000..bdc7b11a4b --- /dev/null +++ b/third_party/patch/pslite/ps_lite.patch001 @@ -0,0 +1,264 @@ +diff -Npur ps-lite-master/include/dmlc/base.h ps-lite-master-new/include/dmlc/base.h +--- ps-lite-master/include/dmlc/base.h 2020-02-29 13:59:55.000000000 +0800 ++++ ps-lite-master-new/include/dmlc/base.h 2020-07-01 11:56:50.444833389 +0800 +@@ -8,7 +8,7 @@ + + /*! \brief whether use glog for logging */ + #ifndef DMLC_USE_GLOG +-#define DMLC_USE_GLOG 0 ++#define DMLC_USE_GLOG 1 + #endif + + /*! +diff -Npur ps-lite-master/include/dmlc/logging.h ps-lite-master-new/include/dmlc/logging.h +--- ps-lite-master/include/dmlc/logging.h 2020-02-29 13:59:55.000000000 +0800 ++++ ps-lite-master-new/include/dmlc/logging.h 2020-07-01 11:58:00.015919207 +0800 +@@ -13,7 +13,7 @@ + #include + #include + #include +-#include "./base.h" ++//#include "./base.h" + + #if DMLC_LOG_STACK_TRACE + #include +@@ -52,7 +52,7 @@ struct Error : public std::runtime_error + + namespace dmlc { + inline void InitLogging(const char* argv0) { +- google::InitGoogleLogging(argv0); ++ //google::InitGoogleLogging(argv0); + } + } // namespace dmlc + +diff -Npur ps-lite-master/make/deps.mk ps-lite-master-new/make/deps.mk +--- ps-lite-master/make/deps.mk 2020-02-29 13:59:55.000000000 +0800 ++++ ps-lite-master-new/make/deps.mk 2020-06-17 10:35:46.253837426 +0800 +@@ -1,69 +1,7 @@ + # Install dependencies +- +-URL1=https://raw.githubusercontent.com/mli/deps/master/build +-URL2=https://github.com/google/protobuf/releases/download/v3.5.1 +-ifndef WGET +-WGET = wget +-endif +- +-# protobuf +-PROTOBUF = ${DEPS_PATH}/include/google/protobuf/message.h +-${PROTOBUF}: +- $(eval FILE=protobuf-cpp-3.5.1.tar.gz) +- $(eval DIR=protobuf-3.5.1) +- rm -rf $(FILE) $(DIR) +- $(WGET) $(URL2)/$(FILE) && tar --no-same-owner -zxf $(FILE) +- cd $(DIR) && export CFLAGS=-fPIC && export CXXFLAGS=-fPIC && ./configure -prefix=$(DEPS_PATH) && $(MAKE) && $(MAKE) install +- rm -rf $(FILE) $(DIR) +- + # zmq +-ZMQ = ${DEPS_PATH}/include/zmq.h ++ZMQ = $(MS_ZMQ_INSTALL_PATH)/lib/libzmq.a + + ${ZMQ}: +- $(eval FILE=zeromq-4.1.4.tar.gz) +- $(eval DIR=zeromq-4.1.4) +- rm -rf $(FILE) $(DIR) +- $(WGET) $(URL1)/$(FILE) && tar --no-same-owner -zxf $(FILE) +- cd $(DIR) && export CFLAGS=-fPIC && export CXXFLAGS=-fPIC && ./configure -prefix=$(DEPS_PATH) --with-libsodium=no --with-libgssapi_krb5=no && $(MAKE) && $(MAKE) install +- rm -rf $(FILE) $(DIR) +- +-# lz4 +-LZ4 = ${DEPS_PATH}/include/lz4.h +-${LZ4}: +- $(eval FILE=lz4-r129.tar.gz) +- $(eval DIR=lz4-r129) +- rm -rf $(FILE) $(DIR) +- wget $(URL1)/$(FILE) && tar --no-same-owner -zxf $(FILE) +- cd $(DIR) && $(MAKE) && PREFIX=$(DEPS_PATH) $(MAKE) install +- rm -rf $(FILE) $(DIR) +- +-# cityhash +-CITYHASH = ${DEPS_PATH}/include/city.h +-${CITYHASH}: +- $(eval FILE=cityhash-1.1.1.tar.gz) +- $(eval DIR=cityhash-1.1.1) +- rm -rf $(FILE) $(DIR) +- wget $(URL1)/$(FILE) && tar --no-same-owner -zxf $(FILE) +- cd $(DIR) && ./configure -prefix=$(DEPS_PATH) --enable-sse4.2 && $(MAKE) CXXFLAGS="-g -O3 -msse4.2" && $(MAKE) install +- rm -rf $(FILE) $(DIR) +- +- +-# # gflags +-# ${DEPS_PATH}/include/google/gflags.h: +-# $(eval FILE=gflags-2.0-no-svn-files.tar.gz) +-# $(eval DIR=gflags-2.0) +-# rm -rf $(FILE) $(DIR) +-# wget $(URL)/$(FILE) && tar -zxf $(FILE) +-# cd $(DIR) && ./configure -prefix=$(DEPS_PATH) && $(MAKE) && $(MAKE) install +-# rm -rf $(FILE) $(DIR) +-# gflags: | ${DEPS_PATH}/include/google/gflags.h ++ cd $(MS_ZMQ_DIR) && export CFLAGS="-fPIC -D_GLIBCXX_USE_CXX11_ABI=0" && export CXXFLAGS=-fPIC && ./configure -prefix=$(MS_ZMQ_INSTALL_PATH) --with-libsodium=no --with-libgssapi_krb5=no && $(MAKE) && $(MAKE) install + +-# # glog +-# ${DEPS_PATH}/include/glog/logging.h: | ${DEPS_PATH}/include/google/gflags.h +-# $(eval FILE=v0.3.4.tar.gz) +-# $(eval DIR=glog-0.3.4) +-# rm -rf $(FILE) $(DIR) +-# wget https://github.com/google/glog/archive/$(FILE) && tar -zxf $(FILE) +-# cd $(DIR) && ./configure -prefix=$(DEPS_PATH) --with-gflags=$(DEPS_PATH) && $(MAKE) && $(MAKE) install +-# rm -rf $(FILE) $(DIR) +-# glog: | ${DEPS_PATH}/include/glog/logging.h +diff -Npur ps-lite-master/make/ps.mk ps-lite-master-new/make/ps.mk +--- ps-lite-master/make/ps.mk 2020-02-29 13:59:55.000000000 +0800 ++++ ps-lite-master-new/make/ps.mk 2020-06-05 09:28:35.337740291 +0800 +@@ -9,5 +9,5 @@ ifeq ($(USE_KEY32), 1) + ADD_CFLAGS += -DUSE_KEY32=1 + endif + +-PS_LDFLAGS_SO = -L$(DEPS_PATH)/lib -lprotobuf-lite -lzmq +-PS_LDFLAGS_A = $(addprefix $(DEPS_PATH)/lib/, libprotobuf-lite.a libzmq.a) ++PS_LDFLAGS_SO = -L$(MS_ZMQ_INSTALL_PATH)/lib -lzmq -L$(MS_PROTO_LIB_DIR) -lprotobuf-lite ++PS_LDFLAGS_A = $(addprefix $(MS_ZMQ_INSTALL_PATH)/lib -L$(MS_PROTO_LIB_DIR), libprotobuf-lite.a libzmq.a) +diff -Npur ps-lite-master/Makefile ps-lite-master-new/Makefile +--- ps-lite-master/Makefile 2020-02-29 13:59:55.000000000 +0800 ++++ ps-lite-master-new/Makefile 2020-06-17 11:09:20.240322660 +0800 +@@ -12,13 +12,24 @@ ifndef DEPS_PATH + DEPS_PATH = $(shell pwd)/deps + endif + ++MS_PROTO_DIR = @protobuf_DIRPATH@ ++MS_GLOG_DIR = @glog_DIRPATH@ ++MS_ZMQ_DIR = @zeromq_DIRPATH@ ++ ++MS_PROTO_LIB_DIR = @protobuf_LIBPATH@ ++MS_GLOG_LIB_DIR = @glog_LIBPATH@ ++MS_ZMQ_INSTALL_PATH = $(MS_ZMQ_DIR)/zmq_install + + ifndef PROTOC +-PROTOC = ${DEPS_PATH}/bin/protoc ++PROTOC = $(MS_PROTO_DIR)/bin/protoc + endif + +-INCPATH = -I./src -I./include -I$(DEPS_PATH)/include +-CFLAGS = -std=c++11 -msse2 -fPIC -O3 -ggdb -Wall -finline-functions $(INCPATH) $(ADD_CFLAGS) ++INCPATH = -I./src -I./include -I$(MS_ZMQ_INSTALL_PATH)/include ++INCPATH += -I$(MS_PROTO_DIR)/include ++INCPATH += -I$(MS_GLOG_DIR)/include ++ ++CXXFLAGS = -D_GLIBCXX_USE_CXX11_ABI=0 ++CFLAGS = -std=c++11 -fPIC -O3 -ggdb -Wall -finline-functions $(INCPATH) $(ADD_CFLAGS) -D_GLIBCXX_USE_CXX11_ABI=0 + LIBS = -pthread + + ifdef USE_IBVERBS +@@ -30,6 +41,7 @@ ifdef ASAN + CFLAGS += -fsanitize=address -fno-omit-frame-pointer -fno-optimize-sibling-calls + endif + ++LIBS += -L$(MS_GLOG_LIB_DIR) -lglog + + all: ps test + +@@ -51,9 +63,9 @@ build/libps.a: $(OBJS) + build/%.o: src/%.cc ${ZMQ} src/meta.pb.h + @mkdir -p $(@D) + $(CXX) $(INCPATH) -std=c++11 -MM -MT build/$*.o $< >build/$*.d +- $(CXX) $(CFLAGS) $(LIBS) -c $< -o $@ ++ $(CXX) $(CFLAGS) $(CXXFLAGS) $(LIBS) -c $< -o $@ + +-src/%.pb.cc src/%.pb.h : src/%.proto ${PROTOBUF} ++src/%.pb.cc src/%.pb.h : src/%.proto + $(PROTOC) --cpp_out=./src --proto_path=./src $< + + -include build/*.d +diff -Npur ps-lite-master/src/ibverbs_van.h ps-lite-master-new/src/ibverbs_van.h +--- ps-lite-master/src/ibverbs_van.h 2020-02-29 13:59:55.000000000 +0800 ++++ ps-lite-master-new/src/ibverbs_van.h 2020-06-02 20:52:11.076230014 +0800 +@@ -145,15 +145,15 @@ class SimpleMempool { + total_allocated_size += new_mem_size; + } + +- CHECK_NE(free_list.end(), it) << "Not enough memory"; ++ //CHECK_NE(free_list.end(), it) << "Not enough memory"; + CHECK_GE(it->first, proper_size); + + char *addr = it->second; + size_t space_left = it->first - proper_size; + + free_list.erase(it); +- CHECK_EQ(used_list.find(addr), used_list.end()) +- << "Address is already allocated"; ++ //CHECK_EQ(used_list.find(addr), used_list.end()) ++ //<< "Address is already allocated"; + + used_list.emplace(addr, proper_size); + +@@ -173,8 +173,8 @@ class SimpleMempool { + std::lock_guard lk(mu_); + + auto it = used_list.find(addr); +- CHECK_NE(used_list.end(), it) +- << "Cannot find info about address: " << (uintptr_t)addr; ++ //CHECK_NE(used_list.end(), it) ++ //<< "Cannot find info about address: " << (uintptr_t)addr; + + size_t size = it->second; + used_list.erase(it); +@@ -208,7 +208,7 @@ class SimpleMempool { + // Convert the memory address to its associated RDMA memory region + inline struct ibv_mr *Addr2MR(char *addr) { + auto it = mr_list.lower_bound(addr); +- CHECK_NE(it, mr_list.end()) << "cannot find the associated memory region"; ++ //CHECK_NE(it, mr_list.end()) << "cannot find the associated memory region"; + return it->second; + } + }; +@@ -330,7 +330,7 @@ class AddressPool { + CHECK(ptr); + uint32_t idx = indices_.front(); + indices_.pop(); +- CHECK_EQ(table_[idx], nullptr); ++ //CHECK_EQ(table_[idx], nullptr); + table_[idx] = ptr; + return idx; + } +@@ -636,7 +636,7 @@ class IBVerbsVan : public Van { + PBMeta meta; + PackMetaPB(msg.meta, &meta); + +- CHECK_NE(endpoints_.find(remote_id), endpoints_.end()); ++ //CHECK_NE(endpoints_.find(remote_id), endpoints_.end()); + Endpoint *endpoint = endpoints_[remote_id].get(); + MessageBuffer *msg_buf = new MessageBuffer(); + +diff -Npur ps-lite-master/src/van.cc ps-lite-master-new/src/van.cc +--- ps-lite-master/src/van.cc 2020-02-29 13:59:55.000000000 +0800 ++++ ps-lite-master-new/src/van.cc 2020-06-02 20:52:43.330405828 +0800 +@@ -448,6 +448,7 @@ void Van::PackMetaPB(const Meta& meta, P + if (meta.timestamp != Meta::kEmpty) pb->set_timestamp(meta.timestamp); + if (meta.body.size()) pb->set_body(meta.body); + pb->set_push(meta.push); ++ pb->set_pull(meta.pull); + pb->set_request(meta.request); + pb->set_simple_app(meta.simple_app); + pb->set_priority(meta.priority); +diff -Npur ps-lite-master/tests/test.mk ps-lite-master-new/tests/test.mk +--- ps-lite-master/tests/test.mk 2020-02-29 13:59:55.000000000 +0800 ++++ ps-lite-master-new/tests/test.mk 2020-06-16 19:15:06.025087897 +0800 +@@ -1,10 +1,10 @@ +-TEST_SRC = $(wildcard tests/test_*.cc) +-TEST = $(patsubst tests/test_%.cc, tests/test_%, $(TEST_SRC)) ++#TEST_SRC = $(wildcard tests/test_*.cc) ++#TEST = $(patsubst tests/test_%.cc, tests/test_%, $(TEST_SRC)) + +-# -ltcmalloc_and_profiler +-LDFLAGS = -Wl,-rpath,$(DEPS_PATH)/lib $(PS_LDFLAGS_SO) -pthread +-tests/% : tests/%.cc build/libps.a +- $(CXX) $(CFLAGS) $(LIBS) -MM -MT tests/$* $< >tests/$*.d +- $(CXX) $(CFLAGS) $(LIBS) -o $@ $(filter %.cc %.a, $^) $(LDFLAGS) +- +--include tests/*.d ++## -ltcmalloc_and_profiler ++#LDFLAGS = -Wl,-rpath,$(DEPS_PATH)/lib $(PS_LDFLAGS_SO) -pthread ++#tests/% : tests/%.cc build/libps.a ++# $(CXX) $(CFLAGS) $(LIBS) -MM -MT tests/$* $< >tests/$*.d ++# $(CXX) $(CFLAGS) $(LIBS) -o $@ $(filter %.cc %.a, $^) $(LDFLAGS) ++# ++#-include tests/*.d From c880774942a36f4585e472da9ab9adf47105a869 Mon Sep 17 00:00:00 2001 From: chujinjin Date: Sun, 28 Jun 2020 10:07:19 +0800 Subject: [PATCH 201/254] change pynative batchnorm same as graph mode --- .../device/ascend/ascend_device_address.cc | 8 +++++++- .../device/ascend/ascend_device_address.h | 1 + mindspore/ccsrc/device/device_address.h | 2 ++ mindspore/ccsrc/device/kernel_runtime.cc | 5 ++++- .../ascend/ascend_backend_optimization.cc | 20 +++++++++---------- mindspore/nn/layer/normalization.py | 5 ++--- 6 files changed, 25 insertions(+), 16 deletions(-) diff --git a/mindspore/ccsrc/device/ascend/ascend_device_address.cc b/mindspore/ccsrc/device/ascend/ascend_device_address.cc index ccf9dfe405..e5f94ff849 100644 --- a/mindspore/ccsrc/device/ascend/ascend_device_address.cc +++ b/mindspore/ccsrc/device/ascend/ascend_device_address.cc @@ -174,7 +174,13 @@ bool AscendDeviceAddress::SyncDeviceToHostAndConvertFormat(const std::vector &host_shape, TypeId host_type, size_t slot, Debugger *debugger) const; #endif + private: bool SyncDeviceToHostAndConvertFormat(const std::vector &shape, size_t size, TypeId type, void *host_ptr) const; bool ConvertFormatAndSyncHostToDevice(const std::vector &shape, size_t size, TypeId type, diff --git a/mindspore/ccsrc/device/device_address.h b/mindspore/ccsrc/device/device_address.h index e02d231dd5..0447cc2539 100644 --- a/mindspore/ccsrc/device/device_address.h +++ b/mindspore/ccsrc/device/device_address.h @@ -63,6 +63,7 @@ class DeviceAddress { size_t GetSize() const { return size_; } std::string format() const { return format_; } TypeId type_id() const { return type_id_; } + void set_host_shape(const std::vector &shape) { host_shape_ = shape; } virtual void set_status(DeviceAddressStatus status) {} virtual DeviceAddressStatus status() const { return DeviceAddressStatus::kInDevice; } virtual DeviceAddressType DeviceType() const { return DeviceAddressType::kUnknown; } @@ -77,6 +78,7 @@ class DeviceAddress { string format_{"DefaultFormat"}; TypeId type_id_{kNumberTypeFloat16}; bool from_mem_pool_{false}; + std::vector host_shape_{}; friend class KernelRuntime; friend class MemoryManager; friend class mindspore::device::ascend::tasksink::TaskGenerator; diff --git a/mindspore/ccsrc/device/kernel_runtime.cc b/mindspore/ccsrc/device/kernel_runtime.cc index 4581141790..529c0d63b6 100644 --- a/mindspore/ccsrc/device/kernel_runtime.cc +++ b/mindspore/ccsrc/device/kernel_runtime.cc @@ -258,6 +258,7 @@ void KernelRuntime::RunOpAssignOutputMemory(const AnfNodePtr &kernel) { std::string output_format = AnfAlgo::GetOutputFormat(kernel, i); auto output_type = AnfAlgo::GetOutputDeviceDataType(kernel, i); auto device_address = CreateDeviceAddress(nullptr, output_sizes[i], output_format, output_type); + device_address->set_host_shape(trans::GetRuntimePaddingShape(kernel, i)); MS_EXCEPTION_IF_NULL(device_address); auto ret = mem_manager_->MallocMemFromMemPool(device_address, output_sizes[i]); if (!ret) { @@ -506,7 +507,9 @@ void KernelRuntime::AssignNodeOutputMem(int flag, const AnfNodePtr &node, int in } std::string output_format = AnfAlgo::GetOutputFormat(node, i); auto output_type = AnfAlgo::GetOutputDeviceDataType(node, i); - AnfAlgo::SetOutputAddr(CreateDeviceAddress(ptr, output_sizes[i], output_format, output_type), i, node.get()); + auto device_address = CreateDeviceAddress(ptr, output_sizes[i], output_format, output_type); + device_address->set_host_shape(trans::GetRuntimePaddingShape(node, i)); + AnfAlgo::SetOutputAddr(device_address, i, node.get()); } } diff --git a/mindspore/ccsrc/pre_activate/ascend/ascend_backend_optimization.cc b/mindspore/ccsrc/pre_activate/ascend/ascend_backend_optimization.cc index 981e2255f3..ff864401b1 100644 --- a/mindspore/ccsrc/pre_activate/ascend/ascend_backend_optimization.cc +++ b/mindspore/ccsrc/pre_activate/ascend/ascend_backend_optimization.cc @@ -238,16 +238,11 @@ void AscendBackendIRFusionOptimization(const std::shared_ptr(); auto ir_fusion_pm = std::make_shared("ir_fusion_pm"); - if (context_ptr->execution_mode() == kPynativeMode) { - ir_fusion_pm->AddPass(std::make_shared()); - ir_fusion_pm->AddPass(std::make_shared()); - } else { - ir_fusion_pm->AddPass(std::make_shared()); - ir_fusion_pm->AddPass(std::make_shared()); - ir_fusion_pm->AddPass(std::make_shared()); - ir_fusion_pm->AddPass(std::make_shared()); - ir_fusion_pm->AddPass(std::make_shared()); - } + ir_fusion_pm->AddPass(std::make_shared()); + ir_fusion_pm->AddPass(std::make_shared()); + ir_fusion_pm->AddPass(std::make_shared()); + ir_fusion_pm->AddPass(std::make_shared()); + ir_fusion_pm->AddPass(std::make_shared()); ir_fusion_pm->AddPass(std::make_shared()); if (context_ptr->ir_fusion_flag()) { AddAscendBackendOptionalIRFusion(ir_fusion_pm.get()); @@ -287,8 +282,11 @@ void RunOpAscendBackendIRFusionOptimization(const std::shared_ptr(); auto ir_fusion_pm = std::make_shared("ir_fusion_pm"); - ir_fusion_pm->AddPass(std::make_shared()); + ir_fusion_pm->AddPass(std::make_shared()); ir_fusion_pm->AddPass(std::make_shared()); + ir_fusion_pm->AddPass(std::make_shared()); + ir_fusion_pm->AddPass(std::make_shared()); + ir_fusion_pm->AddPass(std::make_shared()); ir_fusion_pm->AddPass(std::make_shared()); ir_fusion_pm->AddPass(std::make_shared()); ir_fusion_pm->AddPass(std::make_shared()); diff --git a/mindspore/nn/layer/normalization.py b/mindspore/nn/layer/normalization.py index 4c7ea9d4d6..d6c920b620 100644 --- a/mindspore/nn/layer/normalization.py +++ b/mindspore/nn/layer/normalization.py @@ -84,14 +84,13 @@ class _BatchNorm(Cell): self.dtype = P.DType() self.reshape = P.Reshape() self.is_ascend = context.get_context("device_target") == "Ascend" - self.is_graph_mode = context.get_context("mode") == context.GRAPH_MODE self.momentum = 1.0 - momentum if context.get_context("enable_ge"): self.is_ge_backend = True else: self.is_ge_backend = False - if self.is_graph_mode and (self.is_ge_backend or self.is_ascend): + if self.is_ge_backend or self.is_ascend: self.bn_train = P.BatchNorm(is_training=True, epsilon=self.eps) else: @@ -153,7 +152,7 @@ class _BatchNorm(Cell): if self.is_ge_backend and self.is_global: axes, re_shape = _shape_infer(F.shape(x), self.num_features) y = self._global_sync(x, axes, re_shape) - elif self.is_graph_mode and (self.is_ge_backend or self.is_ascend): + elif self.is_ge_backend or self.is_ascend: if self.is_global: axes, re_shape = _shape_infer(F.shape(x), self.num_features) y = self._global_sync(x, axes, re_shape) From 22dea2fc18ff40e1f43771d68e2196c4fa653b47 Mon Sep 17 00:00:00 2001 From: Li Hongzhang Date: Wed, 1 Jul 2020 15:14:18 +0800 Subject: [PATCH 202/254] SummaryRecord register close atexit --- mindspore/train/summary/summary_record.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/mindspore/train/summary/summary_record.py b/mindspore/train/summary/summary_record.py index 686d8afa9d..065d281355 100644 --- a/mindspore/train/summary/summary_record.py +++ b/mindspore/train/summary/summary_record.py @@ -13,6 +13,7 @@ # limitations under the License. # ============================================================================ """Record the summary event.""" +import atexit import os import re import threading @@ -21,7 +22,7 @@ from mindspore import log as logger from ..._c_expression import Tensor from ..._checkparam import _check_str_by_regular -from .._utils import _make_directory, _check_to_numpy, _check_lineage_value +from .._utils import _check_lineage_value, _check_to_numpy, _make_directory from ._summary_adapter import get_event_file_name, package_graph_event from ._writer_pool import WriterPool @@ -102,8 +103,8 @@ class SummaryRecord: file_suffix="_MS", network=None): - self._closed, self._mode = False, 'train' - self._data_pool = _dictlist() + self._closed, self._event_writer = False, None + self._mode, self._data_pool = 'train', _dictlist() _check_str_by_regular(file_prefix) _check_str_by_regular(file_suffix) @@ -141,6 +142,7 @@ class SummaryRecord: self._event_writer = WriterPool(log_dir, summary=self.full_file_name, lineage=get_event_file_name('events', '_lineage')) + atexit.register(self.close) def __enter__(self): """Enter the context manager.""" @@ -327,12 +329,10 @@ class SummaryRecord: if not self._closed and self._event_writer: # event writer flush and close logger.info('Please wait it may take quite some time to finish writing and closing.') + atexit.unregister(self.close) self._event_writer.close() self._closed = True - def __del__(self) -> None: - self.close() - @staticmethod def _parse_from(name: str = None): """Parse the tag and type from name.""" From 0a07f6c909c91008db48f228c05e60c2fe826b24 Mon Sep 17 00:00:00 2001 From: heleiwang Date: Tue, 30 Jun 2020 17:44:29 +0800 Subject: [PATCH 203/254] support pubmed dataset --- .../graph_to_mindrecord/pubmed/__init__.py | 0 .../graph_to_mindrecord/pubmed/mr_api.py | 105 ++++++++++++++++++ .../utils/graph_to_mindrecord/write_pubmed.sh | 12 ++ 3 files changed, 117 insertions(+) create mode 100644 model_zoo/utils/graph_to_mindrecord/pubmed/__init__.py create mode 100644 model_zoo/utils/graph_to_mindrecord/pubmed/mr_api.py create mode 100644 model_zoo/utils/graph_to_mindrecord/write_pubmed.sh diff --git a/model_zoo/utils/graph_to_mindrecord/pubmed/__init__.py b/model_zoo/utils/graph_to_mindrecord/pubmed/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/model_zoo/utils/graph_to_mindrecord/pubmed/mr_api.py b/model_zoo/utils/graph_to_mindrecord/pubmed/mr_api.py new file mode 100644 index 0000000000..4e04e90cdd --- /dev/null +++ b/model_zoo/utils/graph_to_mindrecord/pubmed/mr_api.py @@ -0,0 +1,105 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""" +User-defined API for MindRecord GNN writer. +""" +import os + +import pickle as pkl +import numpy as np +import scipy.sparse as sp + +# parse args from command line parameter 'graph_api_args' +# args delimiter is ':' +args = os.environ['graph_api_args'].split(':') +PUBMED_PATH = args[0] +dataset_str = 'pubmed' + +# profile: (num_features, feature_data_types, feature_shapes) +node_profile = (2, ["float32", "int32"], [[-1], [-1]]) +edge_profile = (0, [], []) + + +def _normalize_cora_features(features): + row_sum = np.array(features.sum(1)) + r_inv = np.power(row_sum * 1.0, -1).flatten() + r_inv[np.isinf(r_inv)] = 0. + r_mat_inv = sp.diags(r_inv) + features = r_mat_inv.dot(features) + return features + + +def _parse_index_file(filename): + """Parse index file.""" + index = [] + for line in open(filename): + index.append(int(line.strip())) + return index + + +def yield_nodes(task_id=0): + """ + Generate node data + + Yields: + data (dict): data row which is dict. + """ + print("Node task is {}".format(task_id)) + + names = ['tx', 'ty', 'allx', 'ally'] + objects = [] + for name in names: + with open("{}/ind.{}.{}".format(PUBMED_PATH, dataset_str, name), 'rb') as f: + objects.append(pkl.load(f, encoding='latin1')) + tx, ty, allx, ally = tuple(objects) + test_idx_reorder = _parse_index_file( + "{}/ind.{}.test.index".format(PUBMED_PATH, dataset_str)) + test_idx_range = np.sort(test_idx_reorder) + + features = sp.vstack((allx, tx)).tolil() + features[test_idx_reorder, :] = features[test_idx_range, :] + features = _normalize_cora_features(features) + features = features.A + + labels = np.vstack((ally, ty)) + labels[test_idx_reorder, :] = labels[test_idx_range, :] + + line_count = 0 + for i, label in enumerate(labels): + node = {'id': i, 'type': 0, 'feature_1': features[i].tolist(), + 'feature_2': label.tolist().index(1)} + line_count += 1 + yield node + print('Processed {} lines for nodes.'.format(line_count)) + + +def yield_edges(task_id=0): + """ + Generate edge data + + Yields: + data (dict): data row which is dict. + """ + print("Edge task is {}".format(task_id)) + with open("{}/ind.{}.graph".format(PUBMED_PATH, dataset_str), 'rb') as f: + graph = pkl.load(f, encoding='latin1') + line_count = 0 + for i in graph: + for dst_id in graph[i]: + edge = {'id': line_count, + 'src_id': i, 'dst_id': dst_id, 'type': 0} + line_count += 1 + yield edge + print('Processed {} lines for edges.'.format(line_count)) diff --git a/model_zoo/utils/graph_to_mindrecord/write_pubmed.sh b/model_zoo/utils/graph_to_mindrecord/write_pubmed.sh new file mode 100644 index 0000000000..309505e27d --- /dev/null +++ b/model_zoo/utils/graph_to_mindrecord/write_pubmed.sh @@ -0,0 +1,12 @@ +#!/bin/bash +SRC_PATH=/tmp/pubmed/dataset +MINDRECORD_PATH=/tmp/pubmed/mindrecord + +rm -f $MINDRECORD_PATH/* + +python writer.py --mindrecord_script pubmed \ +--mindrecord_file "$MINDRECORD_PATH/pubmed_mr" \ +--mindrecord_partitions 1 \ +--mindrecord_header_size_by_bit 18 \ +--mindrecord_page_size_by_bit 20 \ +--graph_api_args "$SRC_PATH" From ce4e15b0b3491ce4abcf36a20558ce3db365ab11 Mon Sep 17 00:00:00 2001 From: jiangjinsheng Date: Sun, 28 Jun 2020 16:55:58 +0800 Subject: [PATCH 204/254] vm for MaxPoolExt2,MaxPoolGradGrad,MaxPoolGradGradWithArgmax --- mindspore/ops/_grad/grad_nn_ops.py | 34 +++++++ mindspore/ops/_op_impl/tbe/__init__.py | 2 + .../ops/_op_impl/tbe/max_pool_grad_grad.py | 41 ++++++++ .../tbe/max_pool_grad_grad_with_argmax.py | 41 ++++++++ mindspore/ops/operations/_grad_ops.py | 93 +++++++++++++++++++ tests/ut/python/ops/test_ops.py | 14 +++ 6 files changed, 225 insertions(+) create mode 100644 mindspore/ops/_op_impl/tbe/max_pool_grad_grad.py create mode 100644 mindspore/ops/_op_impl/tbe/max_pool_grad_grad_with_argmax.py diff --git a/mindspore/ops/_grad/grad_nn_ops.py b/mindspore/ops/_grad/grad_nn_ops.py index 107de1768c..c3a9e613ef 100755 --- a/mindspore/ops/_grad/grad_nn_ops.py +++ b/mindspore/ops/_grad/grad_nn_ops.py @@ -146,6 +146,40 @@ def get_bprop_max_pool_with_argmax(self): return bprop +@bprop_getters.register(G.MaxPoolGrad) +def get_bprop_max_pool_grad_grad(self): + """Grad definition for `MaxPoolGrad` operation.""" + maxpool_grad_grad = G.MaxPoolGradGrad( + ksize=self.ksize, + strides=self.strides, + padding=self.padding) + + def bprop(x1, x2, grad, out, dout): + dx1 = zeros_like(x1) + dx2 = zeros_like(x2) + dgrad = maxpool_grad_grad(x1, x2, dout) + return (dx1, dx2, dgrad) + + return bprop + + +@bprop_getters.register(G.MaxPoolGradGrad) +def get_bprop_max_pool_grad_grad_grad(self): + """Grad definition for `MaxPoolGradGrad` operation.""" + maxpool_grad = G.MaxPoolGrad( + ksize=self.ksize, + strides=self.strides, + padding=self.padding) + + def bprop(x1, x2, grad, out, dout): + dx1 = zeros_like(x1) + dx2 = zeros_like(x2) + dgrad = maxpool_grad(x1, x2, dout) + return (dx1, dx2, dgrad) + + return bprop + + @bprop_getters.register(P.MaxPool) def get_bprop_max_pool_grad(self): """Grad definition for `MaxPool` operation.""" diff --git a/mindspore/ops/_op_impl/tbe/__init__.py b/mindspore/ops/_op_impl/tbe/__init__.py index 6067295010..27c7a0b8c4 100644 --- a/mindspore/ops/_op_impl/tbe/__init__.py +++ b/mindspore/ops/_op_impl/tbe/__init__.py @@ -277,3 +277,5 @@ from .scatter_max import _scatter_max_tbe from .scatter_min import _scatter_min_tbe from .scatter_sub import _scatter_sub_tbe from .mod import _mod_tbe +from .max_pool_grad_grad import _max_pool_grad_grad_tbe +from .max_pool_grad_grad_with_argmax import _max_pool_grad_grad_with_argmax_tbe diff --git a/mindspore/ops/_op_impl/tbe/max_pool_grad_grad.py b/mindspore/ops/_op_impl/tbe/max_pool_grad_grad.py new file mode 100644 index 0000000000..cdaceb2b0e --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/max_pool_grad_grad.py @@ -0,0 +1,41 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""MaxPoolGradGrad op""" +from mindspore.ops.op_info_register import op_info_register, TBERegOp, DataType + +max_pool_grad_grad_op_info = TBERegOp("MaxPoolGradGrad") \ + .fusion_type("OPAQUE") \ + .async_flag(False) \ + .binfile_name("max_pool_grad_grad.so") \ + .compute_cost(10) \ + .kernel_name("max_pool_grad_grad") \ + .partial_flag(True) \ + .attr("ksize", "required", "listInt", "all") \ + .attr("strides", "required", "listInt", "all") \ + .attr("padding", "required", "str", "all") \ + .attr("data_format", "optional", "str", "all") \ + .input(0, "x1", False, "required", "all") \ + .input(1, "x2", False, "required", "all") \ + .input(2, "grad", False, "required", "all") \ + .output(0, "y", False, "required", "all") \ + .dtype_format(DataType.F16_5HD, DataType.F16_5HD, DataType.F16_5HD, DataType.F16_5HD) \ + .get_op_info() + + +@op_info_register(max_pool_grad_grad_op_info) +def _max_pool_grad_grad_tbe(): + """MaxPoolGradGrad TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/max_pool_grad_grad_with_argmax.py b/mindspore/ops/_op_impl/tbe/max_pool_grad_grad_with_argmax.py new file mode 100644 index 0000000000..52a9392f3e --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/max_pool_grad_grad_with_argmax.py @@ -0,0 +1,41 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""MaxPoolGradGradWithArgmax op""" +from mindspore.ops.op_info_register import op_info_register, TBERegOp, DataType + +max_pool_grad_grad_with_argmax_op_info = TBERegOp("MaxPoolGradGradWithArgmax") \ + .fusion_type("OPAQUE") \ + .async_flag(False) \ + .binfile_name("max_pool_grad_grad_with_argmax.so") \ + .compute_cost(10) \ + .kernel_name("max_pool_grad_grad_with_argmax") \ + .partial_flag(True) \ + .attr("ksize", "required", "listInt", "all") \ + .attr("strides", "required", "listInt", "all") \ + .attr("padding", "required", "str", "all") \ + .input(0, "x", False, "required", "all") \ + .input(1, "grad", False, "required", "all") \ + .input(2, "argmax", False, "optional", "all") \ + .output(0, "y", False, "required", "all") \ + .dtype_format(DataType.F16_5HD, DataType.F16_5HD, DataType.U16_5HD, DataType.F16_5HD) \ + .dtype_format(DataType.F16_5HD, DataType.F16_5HD, DataType.I64_5HD, DataType.F16_5HD) \ + .get_op_info() + + +@op_info_register(max_pool_grad_grad_with_argmax_op_info) +def _max_pool_grad_grad_with_argmax_tbe(): + """MaxPoolGradGradWithArgmax TBE register""" + return diff --git a/mindspore/ops/operations/_grad_ops.py b/mindspore/ops/operations/_grad_ops.py index 94ba2f1bd9..8f9841b169 100644 --- a/mindspore/ops/operations/_grad_ops.py +++ b/mindspore/ops/operations/_grad_ops.py @@ -536,6 +536,51 @@ class MaxPoolGrad(_PoolGrad): return x1_dtype +class MaxPoolGradGrad(_PoolGrad): + r""" + Performs gradients of the MaxPoolGrad operation. + + Args: + ksize (Union[int, tuple[int]]): The size of kernel used to take the maximum value, + is an int number that represents height and width are both ksize, or a tuple + of two int numbers that represent height and width respectively. Default: 1. + strides (Union[int, tuple[int]]): The distance of kernel moving, an int number that represents + the height and width of movement are both strides, or a tuple of two int numbers that + represent height and width of movement respectively. Default: 1. + padding (str): The optional values for pad mode, is "same" or "valid", not case sensitive. + Default: "valid". + + - same: Adopts the way of completion. Output height and width will be the same as + the input. Total number of padding will be calculated for horizontal and vertical + direction and evenly distributed to top and bottom, left and right if possible. + Otherwise, the last extra padding will be done from the bottom and the right side. + + - valid: Adopts the way of discarding. The possibly largest height and width of output + will be return without padding. Extra pixels will be discarded. + + Inputs: + - **origin_input** (Tensor) - Tensor with data format "NCHW", data type should be float16. + - **origin_output** (Tensor) - Data type same as `origin_input`. + - **grad** (Tensor) - Data type same as `origin_input`. + + Outputs: + Tensor, With data type same as `origin_input`. + + """ + + @prim_attr_register + def __init__(self, ksize=1, strides=1, padding="VALID"): + super(MaxPoolGradGrad, self).__init__(ksize, strides, padding) + + def infer_shape(self, x1_shape, x2_shape, grad_shape): + return x1_shape + + def infer_dtype(self, x1_dtype, x2_dtype, grad_dtype): + args = {'x1_dtype': x1_dtype, 'x2_dtype': x2_dtype, 'grad_dtype': grad_dtype} + validator.check_tensor_type_same(args, [mstype.float16], self.name) + return x1_dtype + + class MaximumGrad(Primitive): """Grad for maximum.""" @@ -564,6 +609,54 @@ class MaxPoolGradWithArgmax(_PoolGrad): return grad_dtype +class MaxPoolGradGradWithArgmax(_PoolGrad): + r""" + Computes the gradients of MaxPoolGradWithArgmax. + + Args: + ksize (Union[int, tuple[int]]): The size of kernel used to take the maximum value, + is an int number that represents height and width are both ksize, or a tuple + of two int numbers that represent height and width respectively. Default: 1. + strides (Union[int, tuple[int]]): The distance of kernel moving, an int number that represents + the height and width of movement are both strides, or a tuple of two int numbers that + represent height and width of movement respectively. Default: 1. + padding (str): The optional values for pad mode, is "same" or "valid", not case sensitive. + Default: "valid". + + - same: Adopts the way of completion. Output height and width will be the same as + the input. Total number of padding will be calculated for horizontal and vertical + direction and evenly distributed to top and bottom, left and right if possible. + Otherwise, the last extra padding will be done from the bottom and the right side. + + - valid: Adopts the way of discarding. The possibly largest height and width of output + will be return without padding. Extra pixels will be discarded. + + Inputs: + - **x** (Tensor) - Tensor with data format "NCHW", data type should be float16. + - **grad** (Tensor) - Data type same as `x`. + - **argmax** (Tensor) - Data type should be uint16 or int64. + + Outputs: + Tensor, With data type same as `x`. + + """ + + @prim_attr_register + def __init__(self, ksize=1, strides=1, padding="VALID"): + self.init_prim_io_names(inputs=['x', 'grad', 'argmax'], outputs=['output']) + super(MaxPoolGradGradWithArgmax, self).__init__(ksize, strides, padding) + + def infer_shape(self, x_shape, grad_shape, argmax_shape): + if not grad_shape: + raise TypeError("The dout of MaxPoolGradGradWithArgmax should be a Tensor.") + return x_shape + + def infer_dtype(self, x_dtype, grad_dtype, argmax_dtype): + args = {'x_dtype': x_dtype, 'grad_dtype': grad_dtype} + validator.check_tensor_type_same(args, [mstype.float16], self.name) + return grad_dtype + + class MinimumGrad(Primitive): """Grad for minimum.""" diff --git a/tests/ut/python/ops/test_ops.py b/tests/ut/python/ops/test_ops.py index b3f4e50c38..7a47c8be7c 100755 --- a/tests/ut/python/ops/test_ops.py +++ b/tests/ut/python/ops/test_ops.py @@ -1509,6 +1509,20 @@ test_case_nn_ops = [ 'desc_inputs': [Tensor([0, 1, 2, 3], mstype.int32)], 'desc_bprop': [], 'skip': ['backward']}), + ('MaxPoolGradGrad', { + 'block': G.MaxPoolGradGrad(), + 'desc_inputs': [Tensor(np.random.rand(1, 1, 2, 2), mstype.float16), + Tensor(np.random.rand(1, 1, 2, 2), mstype.float16), + Tensor(np.random.rand(1, 1, 2, 2), mstype.float16)], + 'desc_bprop': [], + 'skip': ['backward']}), + ('MaxPoolGradGradWithArgmax', { + 'block': G.MaxPoolGradGradWithArgmax(), + 'desc_inputs': [Tensor(np.random.rand(1, 1, 2, 2), mstype.float16), + Tensor(np.random.rand(1, 1, 2, 2), mstype.float16), + Tensor(np.zeros((1, 1, 2, 2)), mstype.uint16)], + 'desc_bprop': [], + 'skip': ['backward']}), ] test_case_array_ops = [ From 361214fb6d489764a3203e2ee0b4ae4168204e83 Mon Sep 17 00:00:00 2001 From: zhouyuanshen Date: Wed, 1 Jul 2020 17:06:46 +0800 Subject: [PATCH 205/254] remove deprecated env variables, like ME_TBE_PLUGIN_PATH and OPTION_EXEC_EXTERN_PLUGIN_PATH --- mindspore/ccsrc/utils/context/ms_context.cc | 27 +-------------------- 1 file changed, 1 insertion(+), 26 deletions(-) diff --git a/mindspore/ccsrc/utils/context/ms_context.cc b/mindspore/ccsrc/utils/context/ms_context.cc index 3d367b90e2..0fc0006aad 100644 --- a/mindspore/ccsrc/utils/context/ms_context.cc +++ b/mindspore/ccsrc/utils/context/ms_context.cc @@ -298,19 +298,6 @@ void MsContext::GetGeOptions(std::map *ge_options) con (*ge_options)["ge.exec.profilingOptions"] = profiling_options_; } - // only not supported in ge - auto tbe_plugin_path = common::GetEnv("ME_TBE_PLUGIN_PATH"); - if (!tbe_plugin_path.empty()) { - char real_path[PATH_MAX] = {0}; - if (nullptr == realpath(tbe_plugin_path.c_str(), real_path)) { - MS_LOG(ERROR) << "Ms tbe plugin Path error, " << tbe_plugin_path; - } else { - tbe_plugin_path = real_path; - (*ge_options)["ge.TBE_plugin_path"] = tbe_plugin_path; - } - } else { - MS_LOG(ERROR) << "Set TBE plugin path failed!"; - } (*ge_options)["rank_table_file"] = ""; auto env_ddk_version = common::GetEnv("DDK_VERSION"); if (!env_ddk_version.empty()) { @@ -354,18 +341,6 @@ void MsContext::GetGeOptions(std::map *ge_options) con MS_LOG(INFO) << "Use AICPU, make sure aicpu lib is set in OPTION_EXEC_EXTERN_PLUGIN_PATH."; } - // all libs are set in same env variable "OPTION_EXEC_EXTERN_PLUGIN_PATH", such as FE, HCCL, AICPU, etc - auto load_path = common::GetEnv("OPTION_EXEC_EXTERN_PLUGIN_PATH"); - if (!load_path.empty()) { - char real_path[PATH_MAX] = {0}; - if (realpath(load_path.c_str(), real_path)) { - load_path = real_path; - (*ge_options)["ge.soLoadPath"] = load_path; - } - } else { - MS_LOG(ERROR) << "Set lib load path failed!"; - } - auto proto_lib_path = common::GetEnv("OPTION_PROTO_LIB_PATH"); if (!proto_lib_path.empty()) { char real_path[PATH_MAX] = {0}; @@ -374,7 +349,7 @@ void MsContext::GetGeOptions(std::map *ge_options) con (*ge_options)["ge.opsProtoLibPath"] = proto_lib_path; } } else { - MS_LOG(ERROR) << "Set proto lib path failed!"; + MS_LOG(WARNING) << "Set proto lib path failed!"; } // Enable auto mixed precision according to the context options From b51e499a0f244a204e648b94cf4ddf3dc0234178 Mon Sep 17 00:00:00 2001 From: leiyuning Date: Wed, 1 Jul 2020 20:32:15 +0800 Subject: [PATCH 206/254] update readme --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index a7226b0f91..25abdd6fcb 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ enrichment of the AI software/hardware application ecosystem. MindSpore Architecture -For more details please check out our [Architecture Guide](https://www.mindspore.cn/docs/en/0.5.0-beta/architecture.html). +For more details please check out our [Architecture Guide](https://www.mindspore.cn/docs/en/master/architecture.html). ### Automatic Differentiation @@ -186,7 +186,7 @@ please check out [docker](docker/README.md) repo for the details. ## Quickstart -See the [Quick Start](https://www.mindspore.cn/tutorial/en/0.5.0-beta/quick_start/quick_start.html) +See the [Quick Start](https://www.mindspore.cn/tutorial/en/master/quick_start/quick_start.html) to implement the image classification. ## Docs From c94dea6a512eddb6cbe8b591268d82d7b9aa3209 Mon Sep 17 00:00:00 2001 From: zhoufeng Date: Wed, 1 Jul 2020 21:50:33 +0800 Subject: [PATCH 207/254] Modify nested while testcase Signed-off-by: zhoufeng --- tests/st/control/test_ascend_control_sink.py | 54 ++++++++++++++++---- 1 file changed, 43 insertions(+), 11 deletions(-) diff --git a/tests/st/control/test_ascend_control_sink.py b/tests/st/control/test_ascend_control_sink.py index 2c206c9768..b38668cd25 100644 --- a/tests/st/control/test_ascend_control_sink.py +++ b/tests/st/control/test_ascend_control_sink.py @@ -102,16 +102,41 @@ class ControlIfbyIfbyIf(nn.Cell): class ControlMixedWhileIf(nn.Cell): def __init__(self): super().__init__() + self.assign = op.Assign() + self.var = Parameter(initializer(1, (1), mstype.float32), name="var") + + def construct(self, x, y, z, c2, c4): + out = self.assign(self.var, c4) + while x < c2: + y = self.assign(self.var, c4) + while y < c2 and x < c2: + if 2 * y < c2: + y = y + 2 + else: + y = y + 1 + out = out + y + z = self.assign(self.var, c4) + while z < c2: + z = z + 1 + out = out + z + x = x + 1 + out = out + x + while x < 2 * c2: + y = self.assign(self.var, c4) + x = x + 1 + while y < c2: + z = self.assign(self.var, c4) + while z < c2: + z = z + 1 + if x < c2: + y = y - 1 + else: + y = y + 1 + out = out + z + out = out + y + out = out + x + return out - def construct(self, x, y): - y = y + 4 - while x < y: - if 2 * x < y: - x = x + 1 - else: - x = x + 2 - x = x + 3 - return x @pytest.mark.level0 @pytest.mark.platform_arm_ascend_training @@ -130,6 +155,7 @@ def test_simple_if(): expect = input2 * 3 * 3 * 2 + input1 assert np.allclose(expect, output.asnumpy(), 0.0001, 0.0001) + @pytest.mark.level0 @pytest.mark.platform_arm_ascend_training @pytest.mark.platform_x86_ascend_training @@ -145,6 +171,7 @@ def test_simple_if_with_assign(): expect = input_data assert np.allclose(expect, output.asnumpy(), 0.0001, 0.0001) + @pytest.mark.level0 @pytest.mark.platform_arm_ascend_training @pytest.mark.platform_x86_ascend_training @@ -158,6 +185,7 @@ def test_if_in_if(): expect = x + y + 3 assert np.allclose(expect, output.asnumpy(), 0.0001, 0.0001) + @pytest.mark.level0 @pytest.mark.platform_arm_ascend_training @pytest.mark.platform_x86_ascend_training @@ -175,6 +203,7 @@ def test_if_by_if_by_if(): expect = input_data * 3 * 2 * 2 * 2 assert np.allclose(expect, output.asnumpy(), 0.0001, 0.0001) + @pytest.mark.level0 @pytest.mark.platform_arm_ascend_training @pytest.mark.platform_x86_ascend_training @@ -183,7 +212,10 @@ def test_mixed_while_if(): context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") x = np.array(2).astype(np.int32) y = np.array(14).astype(np.int32) + z = np.array(1).astype(np.int32) + c2 = Tensor([14], mstype.int32) + c4 = Tensor([0], mstype.int32) net = ControlMixedWhileIf() - output = net(Tensor(x), Tensor(y)) - expect = np.array(22).astype(np.int32) + output = net(Tensor(x), Tensor(y), Tensor(z), c2, c4) + expect = np.array(3318).astype(np.int32) assert np.allclose(expect, output.asnumpy(), 0.0001, 0.0001) From d0cd4cb83c17305da6e87e4eb6349a5aa3f2adb0 Mon Sep 17 00:00:00 2001 From: jonyguo Date: Wed, 1 Jul 2020 14:57:33 +0800 Subject: [PATCH 208/254] del: zhwiki & enwiki & CLUERNER2020 preprocess script --- .../nlp_to_mindrecord/CLUERNER2020/README.md | 82 - .../CLUERNER2020/create_dataset.py | 36 - .../CLUERNER2020/data/.gitignore | 1 - .../CLUERNER2020/data/README.md | 1 - .../CLUERNER2020/output/README.md | 1 - .../nlp_to_mindrecord/CLUERNER2020/run.sh | 40 - .../CLUERNER2020/run_read.sh | 17 - .../utils/nlp_to_mindrecord/enwiki/README.md | 173 - .../enwiki/create_dataset.py | 43 - .../utils/nlp_to_mindrecord/enwiki/run.sh | 133 - .../nlp_to_mindrecord/enwiki/run_read.sh | 44 - .../utils/nlp_to_mindrecord/zhwiki/README.md | 113 - .../zhwiki/create_dataset.py | 43 - .../nlp_to_mindrecord/zhwiki/data/.gitignore | 3 - .../nlp_to_mindrecord/zhwiki/data/README.md | 1 - .../nlp_to_mindrecord/zhwiki/output/README.md | 1 - .../utils/nlp_to_mindrecord/zhwiki/run.sh | 112 - .../nlp_to_mindrecord/zhwiki/run_read.sh | 34 - .../zhwiki/run_read_simple.sh | 18 - .../nlp_to_mindrecord/zhwiki/run_simple.sh | 47 - .../to_mindrecord/CLUERNER2020/README.md | 1 - .../CLUERNER2020/data_processor_seq.patch | 105 - .../patch/to_mindrecord/zhwiki/README.md | 1 - .../zhwiki/create_pretraining_data.patch | 288 - .../to_mindrecord/CLUERNER2020/.gitignore | 1 - .../to_mindrecord/CLUERNER2020/README.md | 1 - .../CLUERNER2020/data_processor_seq.py | 157 - .../to_mindrecord/CLUERNER2020/label2id.json | 43 - .../CLUERNER2020/tokenization.py | 388 - .../to_mindrecord/CLUERNER2020/vocab.txt | 21128 ---------------- third_party/to_mindrecord/zhwiki/.gitignore | 1 - third_party/to_mindrecord/zhwiki/README.md | 1 - .../zhwiki/create_pretraining_data.py | 469 - .../to_mindrecord/zhwiki/sample_text.txt | 33 - .../to_mindrecord/zhwiki/tokenization.py | 394 - third_party/to_mindrecord/zhwiki/vocab.txt | 21128 ---------------- 36 files changed, 45082 deletions(-) delete mode 100644 model_zoo/utils/nlp_to_mindrecord/CLUERNER2020/README.md delete mode 100644 model_zoo/utils/nlp_to_mindrecord/CLUERNER2020/create_dataset.py delete mode 100644 model_zoo/utils/nlp_to_mindrecord/CLUERNER2020/data/.gitignore delete mode 100644 model_zoo/utils/nlp_to_mindrecord/CLUERNER2020/data/README.md delete mode 100644 model_zoo/utils/nlp_to_mindrecord/CLUERNER2020/output/README.md delete mode 100644 model_zoo/utils/nlp_to_mindrecord/CLUERNER2020/run.sh delete mode 100644 model_zoo/utils/nlp_to_mindrecord/CLUERNER2020/run_read.sh delete mode 100644 model_zoo/utils/nlp_to_mindrecord/enwiki/README.md delete mode 100644 model_zoo/utils/nlp_to_mindrecord/enwiki/create_dataset.py delete mode 100644 model_zoo/utils/nlp_to_mindrecord/enwiki/run.sh delete mode 100644 model_zoo/utils/nlp_to_mindrecord/enwiki/run_read.sh delete mode 100644 model_zoo/utils/nlp_to_mindrecord/zhwiki/README.md delete mode 100644 model_zoo/utils/nlp_to_mindrecord/zhwiki/create_dataset.py delete mode 100644 model_zoo/utils/nlp_to_mindrecord/zhwiki/data/.gitignore delete mode 100644 model_zoo/utils/nlp_to_mindrecord/zhwiki/data/README.md delete mode 100644 model_zoo/utils/nlp_to_mindrecord/zhwiki/output/README.md delete mode 100644 model_zoo/utils/nlp_to_mindrecord/zhwiki/run.sh delete mode 100644 model_zoo/utils/nlp_to_mindrecord/zhwiki/run_read.sh delete mode 100644 model_zoo/utils/nlp_to_mindrecord/zhwiki/run_read_simple.sh delete mode 100644 model_zoo/utils/nlp_to_mindrecord/zhwiki/run_simple.sh delete mode 100644 third_party/patch/to_mindrecord/CLUERNER2020/README.md delete mode 100644 third_party/patch/to_mindrecord/CLUERNER2020/data_processor_seq.patch delete mode 100644 third_party/patch/to_mindrecord/zhwiki/README.md delete mode 100644 third_party/patch/to_mindrecord/zhwiki/create_pretraining_data.patch delete mode 100644 third_party/to_mindrecord/CLUERNER2020/.gitignore delete mode 100644 third_party/to_mindrecord/CLUERNER2020/README.md delete mode 100644 third_party/to_mindrecord/CLUERNER2020/data_processor_seq.py delete mode 100644 third_party/to_mindrecord/CLUERNER2020/label2id.json delete mode 100644 third_party/to_mindrecord/CLUERNER2020/tokenization.py delete mode 100644 third_party/to_mindrecord/CLUERNER2020/vocab.txt delete mode 100644 third_party/to_mindrecord/zhwiki/.gitignore delete mode 100644 third_party/to_mindrecord/zhwiki/README.md delete mode 100644 third_party/to_mindrecord/zhwiki/create_pretraining_data.py delete mode 100644 third_party/to_mindrecord/zhwiki/sample_text.txt delete mode 100644 third_party/to_mindrecord/zhwiki/tokenization.py delete mode 100644 third_party/to_mindrecord/zhwiki/vocab.txt diff --git a/model_zoo/utils/nlp_to_mindrecord/CLUERNER2020/README.md b/model_zoo/utils/nlp_to_mindrecord/CLUERNER2020/README.md deleted file mode 100644 index c862156a47..0000000000 --- a/model_zoo/utils/nlp_to_mindrecord/CLUERNER2020/README.md +++ /dev/null @@ -1,82 +0,0 @@ -# Guideline to Convert Training Data CLUERNER2020 to MindRecord For Bert Fine Tuning - - - -- [What does the example do](#what-does-the-example-do) -- [How to use the example to process CLUERNER2020](#how-to-use-the-example-to-process-cluerner2020) - - [Download CLUERNER2020 and unzip](#download-cluerner2020-and-unzip) - - [Generate MindRecord](#generate-mindrecord) - - [Create MindDataset By MindRecord](#create-minddataset-by-mindrecord) - - - - -## What does the example do - -This example is based on [CLUERNER2020](https://www.cluebenchmarks.com/introduce.html) training data, generating MindRecord file, and finally used for Bert Fine Tuning progress. - -1. run.sh: generate MindRecord entry script -2. run_read.py: create MindDataset by MindRecord entry script. - - create_dataset.py: use MindDataset to read MindRecord to generate dataset. - -## How to use the example to process CLUERNER2020 - -Download CLUERNER2020, convert it to MindRecord, use MindDataset to read MindRecord. - -### Download CLUERNER2020 and unzip - -1. Download the training data zip. - > [CLUERNER2020 dataset download address](https://www.cluebenchmarks.com/introduce.html) **-> 任务介绍 -> CLUENER 细粒度命名实体识别 -> cluener下载链接** - -2. Unzip the training data to dir example/nlp_to_mindrecord/CLUERNER2020/cluener_public. - ``` - unzip -d {your-mindspore}/example/nlp_to_mindrecord/CLUERNER2020/data/cluener_public cluener_public.zip - ``` - -### Generate MindRecord - -1. Run the run.sh script. - ```bash - bash run.sh - ``` - -2. Output like this: - ``` - ... - [INFO] ME(17603:139620983514944,MainProcess):2020-04-28-16:56:12.498.235 [mindspore/mindrecord/filewriter.py:313] The list of mindrecord files created are: ['data/train.mindrecord'], and the list of index files are: ['data/train.mindrecord.db'] - ... - [INFO] ME(17603,python):2020-04-28-16:56:13.400.175 [mindspore/ccsrc/mindrecord/io/shard_writer.cc:667] WriteRawData] Write 1 records successfully. - [INFO] ME(17603,python):2020-04-28-16:56:13.400.863 [mindspore/ccsrc/mindrecord/io/shard_writer.cc:667] WriteRawData] Write 1 records successfully. - [INFO] ME(17603,python):2020-04-28-16:56:13.401.534 [mindspore/ccsrc/mindrecord/io/shard_writer.cc:667] WriteRawData] Write 1 records successfully. - [INFO] ME(17603,python):2020-04-28-16:56:13.402.179 [mindspore/ccsrc/mindrecord/io/shard_writer.cc:667] WriteRawData] Write 1 records successfully. - [INFO] ME(17603,python):2020-04-28-16:56:13.402.702 [mindspore/ccsrc/mindrecord/io/shard_writer.cc:667] WriteRawData] Write 1 records successfully. - ... - [INFO] ME(17603:139620983514944,MainProcess):2020-04-28-16:56:13.431.208 [mindspore/mindrecord/filewriter.py:313] The list of mindrecord files created are: ['data/dev.mindrecord'], and the list of index files are: ['data/dev.mindrecord.db'] - ``` - -3. Generate files like this: - ```bash - $ ls output/ - dev.mindrecord dev.mindrecord.db README.md train.mindrecord train.mindrecord.db - ``` - -### Create MindDataset By MindRecord - -1. Run the run_read.sh script. - ```bash - bash run_read.sh - ``` - -2. Output like this: - ``` - ... - example 1340: input_ids: [ 101 3173 1290 4852 7676 3949 122 3299 123 126 3189 4510 8020 6381 5442 7357 2590 3636 8021 7676 3949 4294 1166 6121 3124 1277 6121 3124 7270 2135 3295 5789 3326 123 126 3189 1355 6134 1093 1325 3173 2399 6590 6791 8024 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0] - example 1340: input_mask: [1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0] - example 1340: segment_ids: [0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0] - example 1340: label_ids: [ 0 18 19 20 2 4 0 0 0 0 0 0 0 34 36 26 27 28 0 34 35 35 35 35 35 35 35 35 35 36 26 27 28 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0] - example 1341: input_ids: [ 101 1728 711 4293 3868 1168 2190 2150 3791 934 3633 3428 4638 6237 7025 8024 3297 1400 5310 3362 6206 5023 5401 1744 3297 7770 3791 7368 976 1139 1104 2137 511 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0] - example 1341: input_mask: [1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0] - example 1341: segment_ids: [0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0] - example 1341: label_ids: [ 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 18 19 19 19 19 20 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0] - ... - ``` diff --git a/model_zoo/utils/nlp_to_mindrecord/CLUERNER2020/create_dataset.py b/model_zoo/utils/nlp_to_mindrecord/CLUERNER2020/create_dataset.py deleted file mode 100644 index 616bc71028..0000000000 --- a/model_zoo/utils/nlp_to_mindrecord/CLUERNER2020/create_dataset.py +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ -"""create MindDataset by MindRecord""" -import mindspore.dataset as ds - -def create_dataset(data_file): - """create MindDataset""" - num_readers = 4 - data_set = ds.MindDataset(dataset_file=data_file, num_parallel_workers=num_readers, shuffle=True) - index = 0 - for item in data_set.create_dict_iterator(): - # print("example {}: {}".format(index, item)) - print("example {}: input_ids: {}".format(index, item['input_ids'])) - print("example {}: input_mask: {}".format(index, item['input_mask'])) - print("example {}: segment_ids: {}".format(index, item['segment_ids'])) - print("example {}: label_ids: {}".format(index, item['label_ids'])) - index += 1 - if index % 1000 == 0: - print("read rows: {}".format(index)) - print("total rows: {}".format(index)) - -if __name__ == '__main__': - create_dataset('output/train.mindrecord') - create_dataset('output/dev.mindrecord') diff --git a/model_zoo/utils/nlp_to_mindrecord/CLUERNER2020/data/.gitignore b/model_zoo/utils/nlp_to_mindrecord/CLUERNER2020/data/.gitignore deleted file mode 100644 index cbbd6256c0..0000000000 --- a/model_zoo/utils/nlp_to_mindrecord/CLUERNER2020/data/.gitignore +++ /dev/null @@ -1 +0,0 @@ -cluener_public diff --git a/model_zoo/utils/nlp_to_mindrecord/CLUERNER2020/data/README.md b/model_zoo/utils/nlp_to_mindrecord/CLUERNER2020/data/README.md deleted file mode 100644 index b54948808e..0000000000 --- a/model_zoo/utils/nlp_to_mindrecord/CLUERNER2020/data/README.md +++ /dev/null @@ -1 +0,0 @@ -## The input dataset diff --git a/model_zoo/utils/nlp_to_mindrecord/CLUERNER2020/output/README.md b/model_zoo/utils/nlp_to_mindrecord/CLUERNER2020/output/README.md deleted file mode 100644 index 7904933f43..0000000000 --- a/model_zoo/utils/nlp_to_mindrecord/CLUERNER2020/output/README.md +++ /dev/null @@ -1 +0,0 @@ -## output dir diff --git a/model_zoo/utils/nlp_to_mindrecord/CLUERNER2020/run.sh b/model_zoo/utils/nlp_to_mindrecord/CLUERNER2020/run.sh deleted file mode 100644 index 9de6f0e9fc..0000000000 --- a/model_zoo/utils/nlp_to_mindrecord/CLUERNER2020/run.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/bash -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ - -rm -f output/train.mindrecord* -rm -f output/dev.mindrecord* - -if [ ! -d "../../../../third_party/to_mindrecord/CLUERNER2020" ]; then - echo "The patch base dir ../../../../third_party/to_mindrecord/CLUERNER2020 is not exist." - exit 1 -fi - -if [ ! -f "../../../../third_party/patch/to_mindrecord/CLUERNER2020/data_processor_seq.patch" ]; then - echo "The patch file ../../../../third_party/patch/to_mindrecord/CLUERNER2020/data_processor_seq.patch is not exist." - exit 1 -fi - -# patch for data_processor_seq.py -patch -p0 -d ../../../../third_party/to_mindrecord/CLUERNER2020/ -o data_processor_seq_patched.py < ../../../../third_party/patch/to_mindrecord/CLUERNER2020/data_processor_seq.patch -if [ $? -ne 0 ]; then - echo "Patch ../../../../third_party/to_mindrecord/CLUERNER2020/data_processor_seq.py failed" - exit 1 -fi - -# use patched script -python ../../../../third_party/to_mindrecord/CLUERNER2020/data_processor_seq_patched.py \ ---vocab_file=../../../../third_party/to_mindrecord/CLUERNER2020/vocab.txt \ ---label2id_file=../../../../third_party/to_mindrecord/CLUERNER2020/label2id.json diff --git a/model_zoo/utils/nlp_to_mindrecord/CLUERNER2020/run_read.sh b/model_zoo/utils/nlp_to_mindrecord/CLUERNER2020/run_read.sh deleted file mode 100644 index 1ffe4de1cf..0000000000 --- a/model_zoo/utils/nlp_to_mindrecord/CLUERNER2020/run_read.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ - -python create_dataset.py diff --git a/model_zoo/utils/nlp_to_mindrecord/enwiki/README.md b/model_zoo/utils/nlp_to_mindrecord/enwiki/README.md deleted file mode 100644 index e92e8dbcc6..0000000000 --- a/model_zoo/utils/nlp_to_mindrecord/enwiki/README.md +++ /dev/null @@ -1,173 +0,0 @@ -# Guideline to Convert Training Data enwiki to MindRecord For Bert Pre Training - - - -- [What does the example do](#what-does-the-example-do) -- [How to use the example to process enwiki](#how-to-use-the-example-to-process-enwiki) - - [Download enwiki training data](#download-enwiki-training-data) - - [Process the enwiki](#process-the-enwiki) - - [Generate MindRecord](#generate-mindrecord) - - [Create MindDataset By MindRecord](#create-minddataset-by-mindrecord) - - - - -## What does the example do - -This example is based on [enwiki](https://dumps.wikimedia.org/enwiki) training data, generating MindRecord file, and finally used for Bert network training. - -1. run.sh: generate MindRecord entry script. -2. run_read.py: create MindDataset by MindRecord entry script. - - create_dataset.py: use MindDataset to read MindRecord to generate dataset. - -## How to use the example to process enwiki - -Download enwiki data, process it, convert it to MindRecord, use MindDataset to read MindRecord. - -### Download enwiki training data - -> [enwiki dataset download address](https://dumps.wikimedia.org/enwiki) **-> 20200501 -> enwiki-20200501-pages-articles-multistream.xml.bz2** - -### Process the enwiki - -1. Please follow the steps in [process enwiki](https://github.com/mlperf/training/tree/master/language_model/tensorflow/bert) -- All permissions of this step belong to the link address website. - -### Generate MindRecord - -1. Run the run.sh script. - ``` - bash run.sh input_dir output_dir vocab_file - ``` - - input_dir: the directory which contains files like 'part-00251-of-00500'. - - output_dir: which will store the output mindrecord files. - - vocab_file: the vocab file which you can download from other opensource project. - -2. The output like this: - ``` - ... - Begin preprocess Wed Jun 10 09:21:23 CST 2020 - Begin preprocess input file: /mnt/data/results/part-00000-of-00500 - Begin output file: part-00000-of-00500.mindrecord - Total task: 510, processing: 1 - Begin preprocess input file: /mnt/data/results/part-00001-of-00500 - Begin output file: part-00001-of-00500.mindrecord - Total task: 510, processing: 2 - Begin preprocess input file: /mnt/data/results/part-00002-of-00500 - Begin output file: part-00002-of-00500.mindrecord - Total task: 510, processing: 3 - Begin preprocess input file: /mnt/data/results/part-00003-of-00500 - Begin output file: part-00003-of-00500.mindrecord - Total task: 510, processing: 4 - Begin preprocess input file: /mnt/data/results/part-00004-of-00500 - Begin output file: part-00004-of-00500.mindrecord - Total task: 510, processing: 4 - ... - ``` - -3. Generate files like this: - ```bash - $ ls {your_output_dir}/ - part-00000-of-00500.mindrecord part-00000-of-00500.mindrecord.db part-00001-of-00500.mindrecord part-00001-of-00500.mindrecord.db part-00002-of-00500.mindrecord part-00002-of-00500.mindrecord.db ... - ``` - -### Create MindDataset By MindRecord - -1. Run the run_read.sh script. - ```bash - bash run_read.sh input_dir - ``` - - input_dir: the directory which contains mindrecord files. - -2. The output like this: - ``` - ... - example 633: input_ids: [ 101 2043 19781 4305 2140 4520 2041 1010 103 2034 2455 2002 - 7879 2003 1996 2455 1997 103 26378 4160 1012 102 7291 2001 - 1996 103 1011 2343 1997 6327 1010 3423 1998 103 4262 2005 - 1996 2118 1997 2329 3996 103 102 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0] - example 633: input_mask: [1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 - 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0] - example 633: segment_ids: [0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 - 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0] - example 633: masked_lm_positions: [ 8 17 20 25 33 41 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0] - example 633: masked_lm_ids: [ 1996 16137 1012 3580 2451 1012 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0] - example 633: masked_lm_weights: [1. 1. 1. 1. 1. 1. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. - 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. - 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. - 0. 0. 0. 0.] - example 633: next_sentence_labels: [1] - ... - ``` diff --git a/model_zoo/utils/nlp_to_mindrecord/enwiki/create_dataset.py b/model_zoo/utils/nlp_to_mindrecord/enwiki/create_dataset.py deleted file mode 100644 index d90d12b7f2..0000000000 --- a/model_zoo/utils/nlp_to_mindrecord/enwiki/create_dataset.py +++ /dev/null @@ -1,43 +0,0 @@ -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ -"""create MindDataset by MindRecord""" -import argparse -import mindspore.dataset as ds - -def create_dataset(data_file): - """create MindDataset""" - num_readers = 4 - data_set = ds.MindDataset(dataset_file=data_file, num_parallel_workers=num_readers, shuffle=True) - index = 0 - for item in data_set.create_dict_iterator(): - # print("example {}: {}".format(index, item)) - print("example {}: input_ids: {}".format(index, item['input_ids'])) - print("example {}: input_mask: {}".format(index, item['input_mask'])) - print("example {}: segment_ids: {}".format(index, item['segment_ids'])) - print("example {}: masked_lm_positions: {}".format(index, item['masked_lm_positions'])) - print("example {}: masked_lm_ids: {}".format(index, item['masked_lm_ids'])) - print("example {}: masked_lm_weights: {}".format(index, item['masked_lm_weights'])) - print("example {}: next_sentence_labels: {}".format(index, item['next_sentence_labels'])) - index += 1 - if index % 1000 == 0: - print("read rows: {}".format(index)) - print("total rows: {}".format(index)) - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.add_argument("-i", "--input_file", nargs='+', type=str, help='Input mindreord file') - args = parser.parse_args() - - create_dataset(args.input_file) diff --git a/model_zoo/utils/nlp_to_mindrecord/enwiki/run.sh b/model_zoo/utils/nlp_to_mindrecord/enwiki/run.sh deleted file mode 100644 index c36b6b9903..0000000000 --- a/model_zoo/utils/nlp_to_mindrecord/enwiki/run.sh +++ /dev/null @@ -1,133 +0,0 @@ -#!/bin/bash -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ - -if [ $# -ne 3 ]; then - echo "Usage: $0 input_dir output_dir vocab_file" - exit 1 -fi - -if [ ! -d $1 ]; then - echo "The input dir: $1 is not exist." - exit 1 -fi - -if [ ! -d $2 ]; then - echo "The output dir: $2 is not exist." - exit 1 -fi -rm -fr $2/*.mindrecord* - -if [ ! -f $3 ]; then - echo "The vocab file: $3 is not exist." - exit 1 -fi - -data_dir=$1 -output_dir=$2 -vocab_file=$3 -file_list=() -output_filename=() -file_index=0 - -function getdir() { - elements=`ls $1` - for element in ${elements[*]}; - do - dir_or_file=$1"/"$element - if [ -d $dir_or_file ]; - then - getdir $dir_or_file - else - file_list[$file_index]=$dir_or_file - echo "${dir_or_file}" | tr '/' '\n' > dir_file_list.txt # dir dir file to mapfile - mapfile parent_dir < dir_file_list.txt - rm dir_file_list.txt >/dev/null 2>&1 - tmp_output_filename=${parent_dir[${#parent_dir[@]}-1]}".mindrecord" - output_filename[$file_index]=`echo ${tmp_output_filename} | sed 's/ //g'` - file_index=`expr $file_index + 1` - fi - done -} - -getdir "${data_dir}" -# echo "The input files: "${file_list[@]} -# echo "The output files: "${output_filename[@]} - -if [ ! -d "../../../../third_party/to_mindrecord/zhwiki" ]; then - echo "The patch base dir ../../../../third_party/to_mindrecord/zhwiki is not exist." - exit 1 -fi - -if [ ! -f "../../../../third_party/patch/to_mindrecord/zhwiki/create_pretraining_data.patch" ]; then - echo "The patch file ../../../../third_party/patch/to_mindrecord/zhwiki/create_pretraining_data.patch is not exist." - exit 1 -fi - -# patch for create_pretraining_data.py -patch -p0 -d ../../../../third_party/to_mindrecord/zhwiki/ -o create_pretraining_data_patched.py < ../../../../third_party/patch/to_mindrecord/zhwiki/create_pretraining_data.patch -if [ $? -ne 0 ]; then - echo "Patch ../../../../third_party/to_mindrecord/zhwiki/create_pretraining_data.py failed" - exit 1 -fi - -# get the cpu core count -num_cpu_core=`cat /proc/cpuinfo | grep "processor" | wc -l` -avaiable_core_size=`expr $num_cpu_core / 3 \* 2` - -echo "Begin preprocess `date`" - -# using patched script to generate mindrecord -file_list_len=`expr ${#file_list[*]} - 1` -for index in $(seq 0 $file_list_len); do - echo "Begin preprocess input file: ${file_list[$index]}" - echo "Begin output file: ${output_filename[$index]}" - python ../../../../third_party/to_mindrecord/zhwiki/create_pretraining_data_patched.py \ - --input_file=${file_list[$index]} \ - --output_file=${output_dir}/${output_filename[$index]} \ - --partition_number=1 \ - --vocab_file=${vocab_file} \ - --do_lower_case=True \ - --max_seq_length=512 \ - --max_predictions_per_seq=76 \ - --masked_lm_prob=0.15 \ - --random_seed=12345 \ - --dupe_factor=10 >/tmp/${output_filename[$index]}.log 2>&1 & - process_count=`ps -ef | grep create_pretraining_data_patched | grep -v grep | wc -l` - echo "Total task: ${#file_list[*]}, processing: ${process_count}" - if [ $process_count -ge $avaiable_core_size ]; then - while [ 1 ]; do - process_num=`ps -ef | grep create_pretraining_data_patched | grep -v grep | wc -l` - if [ $process_count -gt $process_num ]; then - process_count=$process_num - break; - fi - sleep 2 - done - fi -done - -process_num=`ps -ef | grep create_pretraining_data_patched | grep -v grep | wc -l` -while [ 1 ]; do - if [ $process_num -eq 0 ]; then - break; - fi - echo "There are still ${process_num} preprocess running ..." - sleep 2 - process_num=`ps -ef | grep create_pretraining_data_patched | grep -v grep | wc -l` -done - -echo "Preprocess all the data success." -echo "End preprocess `date`" diff --git a/model_zoo/utils/nlp_to_mindrecord/enwiki/run_read.sh b/model_zoo/utils/nlp_to_mindrecord/enwiki/run_read.sh deleted file mode 100644 index 737e9375c4..0000000000 --- a/model_zoo/utils/nlp_to_mindrecord/enwiki/run_read.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/bash -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ - -if [ $# -ne 1 ]; then - echo "Usage: $0 input_dir" - exit 1 -fi - -if [ ! -d $1 ]; then - echo "The input dir: $1 is not exist." - exit 1 -fi - -file_list=() -file_index=0 - -# get all the mindrecord file from output dir -function getdir() { - elements=`ls $1/part-*.mindrecord` - for element in ${elements[*]}; - do - file_list[$file_index]=$element - file_index=`expr $file_index + 1` - done -} - -getdir $1 -echo "Get all the mindrecord files: "${file_list[*]} - -# create dataset for train -python create_dataset.py --input_file ${file_list[*]} diff --git a/model_zoo/utils/nlp_to_mindrecord/zhwiki/README.md b/model_zoo/utils/nlp_to_mindrecord/zhwiki/README.md deleted file mode 100644 index ad7afdd662..0000000000 --- a/model_zoo/utils/nlp_to_mindrecord/zhwiki/README.md +++ /dev/null @@ -1,113 +0,0 @@ -# Guideline to Convert Training Data zhwiki to MindRecord For Bert Pre Training - - - -- [What does the example do](#what-does-the-example-do) -- [Run simple test](#run-simple-test) -- [How to use the example to process zhwiki](#how-to-use-the-example-to-process-zhwiki) - - [Download zhwiki training data](#download-zhwiki-training-data) - - [Extract the zhwiki](#extract-the-zhwiki) - - [Generate MindRecord](#generate-mindrecord) - - [Create MindDataset By MindRecord](#create-minddataset-by-mindrecord) - - - - -## What does the example do - -This example is based on [zhwiki](https://dumps.wikimedia.org/zhwiki) training data, generating MindRecord file, and finally used for Bert network training. - -1. run.sh: generate MindRecord entry script. -2. run_read.py: create MindDataset by MindRecord entry script. - - create_dataset.py: use MindDataset to read MindRecord to generate dataset. - -## Run simple test - -Follow the step: - -```bash -bash run_simple.sh # generate output/simple.mindrecord* by ../../../../third_party/to_mindrecord/zhwiki/sample_text.txt -bash run_read_simple.sh # use MindDataset to read output/simple.mindrecord* -``` - -## How to use the example to process zhwiki - -Download zhwiki data, extract it, convert it to MindRecord, use MindDataset to read MindRecord. - -### Download zhwiki training data - -> [zhwiki dataset download address](https://dumps.wikimedia.org/zhwiki) **-> 20200401 -> zhwiki-20200401-pages-articles-multistream.xml.bz2** - -- put the zhwiki-20200401-pages-articles-multistream.xml.bz2 in {your-mindspore}/example/nlp_to_mindrecord/zhwiki/data directory. - -### Extract the zhwiki - -1. Download [wikiextractor](https://github.com/attardi/wikiextractor) script to {your-mindspore}/example/nlp_to_mindrecord/zhwiki/data directory. - - ``` - $ ls data/ - README.md wikiextractor zhwiki-20200401-pages-articles-multistream.xml.bz2 - ``` - -2. Extract the zhwiki. - ```python - python data/wikiextractor/WikiExtractor.py data/zhwiki-20200401-pages-articles-multistream.xml.bz2 --processes 4 --templates data/template --bytes 8M --min_text_length 0 --filter_disambig_pages --output data/extract - ``` - -3. Generate like this: - ``` - $ ls data/extract - AA AB - ``` - -### Generate MindRecord - -1. Run the run.sh script. - ``` - bash run.sh - ``` - > Caution: This process maybe slow, please wait patiently. If you do not have a machine with enough memory and cpu, it is recommended that you modify the script to generate mindrecord in step by step. - -2. The output like this: - ``` - patching file create_pretraining_data_patched.py (read from create_pretraining_data.py) - Begin preprocess input file: ./data/extract/AA/wiki_00 - Begin output file: AAwiki_00.mindrecord - Total task: 5, processing: 1 - Begin preprocess input file: ./data/extract/AA/wiki_01 - Begin output file: AAwiki_01.mindrecord - Total task: 5, processing: 2 - Begin preprocess input file: ./data/extract/AA/wiki_02 - Begin output file: AAwiki_02.mindrecord - Total task: 5, processing: 3 - Begin preprocess input file: ./data/extract/AB/wiki_02 - Begin output file: ABwiki_02.mindrecord - Total task: 5, processing: 4 - ... - ``` - -3. Generate files like this: - ```bash - $ ls output/ - AAwiki_00.mindrecord AAwiki_00.mindrecord.db AAwiki_01.mindrecord AAwiki_01.mindrecord.db AAwiki_02.mindrecord AAwiki_02.mindrecord.db ... ABwiki_00.mindrecord ABwiki_00.mindrecord.db ... - ``` - -### Create MindDataset By MindRecord - -1. Run the run_read.sh script. - ```bash - bash run_read.sh - ``` - -2. The output like this: - ``` - ... - example 74: input_ids: [ 101 8168 118 12847 8783 9977 15908 117 8256 9245 11643 8168 8847 8588 11575 8154 8228 143 8384 8376 9197 10241 103 10564 11421 8199 12268 112 161 8228 11541 9586 8436 8174 8363 9864 9702 103 103 119 103 9947 10564 103 8436 8806 11479 103 8912 119 103 103 103 12209 8303 103 8757 8824 117 8256 103 8619 8168 11541 102 11684 8196 103 8228 8847 11523 117 9059 9064 12410 8358 8181 10764 117 11167 11706 9920 148 8332 11390 8936 8205 10951 11997 103 8154 117 103 8670 10467 112 161 10951 13139 12413 117 10288 143 10425 8205 152 10795 8472 8196 103 161 12126 9172 13129 12106 8217 8174 12244 8205 143 103 8461 8277 10628 160 8221 119 102] - example 74: input_mask: [1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1] - example 74: segment_ids: [0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1] - example 74: masked_lm_positions: [ 6 22 37 38 40 43 47 50 51 52 55 60 67 76 89 92 98 109 120 0] - example 74: masked_lm_ids: [ 8118 8165 8329 8890 8554 8458 119 8850 8565 10392 8174 11467 10291 8181 8549 12718 13139 112 158 0] - example 74: masked_lm_weights: [1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 0.] - example 74: next_sentence_labels: [0] - ... - ``` diff --git a/model_zoo/utils/nlp_to_mindrecord/zhwiki/create_dataset.py b/model_zoo/utils/nlp_to_mindrecord/zhwiki/create_dataset.py deleted file mode 100644 index d90d12b7f2..0000000000 --- a/model_zoo/utils/nlp_to_mindrecord/zhwiki/create_dataset.py +++ /dev/null @@ -1,43 +0,0 @@ -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ -"""create MindDataset by MindRecord""" -import argparse -import mindspore.dataset as ds - -def create_dataset(data_file): - """create MindDataset""" - num_readers = 4 - data_set = ds.MindDataset(dataset_file=data_file, num_parallel_workers=num_readers, shuffle=True) - index = 0 - for item in data_set.create_dict_iterator(): - # print("example {}: {}".format(index, item)) - print("example {}: input_ids: {}".format(index, item['input_ids'])) - print("example {}: input_mask: {}".format(index, item['input_mask'])) - print("example {}: segment_ids: {}".format(index, item['segment_ids'])) - print("example {}: masked_lm_positions: {}".format(index, item['masked_lm_positions'])) - print("example {}: masked_lm_ids: {}".format(index, item['masked_lm_ids'])) - print("example {}: masked_lm_weights: {}".format(index, item['masked_lm_weights'])) - print("example {}: next_sentence_labels: {}".format(index, item['next_sentence_labels'])) - index += 1 - if index % 1000 == 0: - print("read rows: {}".format(index)) - print("total rows: {}".format(index)) - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.add_argument("-i", "--input_file", nargs='+', type=str, help='Input mindreord file') - args = parser.parse_args() - - create_dataset(args.input_file) diff --git a/model_zoo/utils/nlp_to_mindrecord/zhwiki/data/.gitignore b/model_zoo/utils/nlp_to_mindrecord/zhwiki/data/.gitignore deleted file mode 100644 index f15cab0c89..0000000000 --- a/model_zoo/utils/nlp_to_mindrecord/zhwiki/data/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -wikiextractor/ -zhwiki-20200401-pages-articles-multistream.xml.bz2 -extract/ diff --git a/model_zoo/utils/nlp_to_mindrecord/zhwiki/data/README.md b/model_zoo/utils/nlp_to_mindrecord/zhwiki/data/README.md deleted file mode 100644 index b54948808e..0000000000 --- a/model_zoo/utils/nlp_to_mindrecord/zhwiki/data/README.md +++ /dev/null @@ -1 +0,0 @@ -## The input dataset diff --git a/model_zoo/utils/nlp_to_mindrecord/zhwiki/output/README.md b/model_zoo/utils/nlp_to_mindrecord/zhwiki/output/README.md deleted file mode 100644 index b7cfba1b47..0000000000 --- a/model_zoo/utils/nlp_to_mindrecord/zhwiki/output/README.md +++ /dev/null @@ -1 +0,0 @@ -## Output the mindrecord diff --git a/model_zoo/utils/nlp_to_mindrecord/zhwiki/run.sh b/model_zoo/utils/nlp_to_mindrecord/zhwiki/run.sh deleted file mode 100644 index 3142056b11..0000000000 --- a/model_zoo/utils/nlp_to_mindrecord/zhwiki/run.sh +++ /dev/null @@ -1,112 +0,0 @@ -#!/bin/bash -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ - -rm -f output/*.mindrecord* - -data_dir="./data/extract" -file_list=() -output_filename=() -file_index=0 - -function getdir() { - elements=`ls $1` - for element in ${elements[*]}; - do - dir_or_file=$1"/"$element - if [ -d $dir_or_file ]; - then - getdir $dir_or_file - else - file_list[$file_index]=$dir_or_file - echo "${dir_or_file}" | tr '/' '\n' > dir_file_list.txt # dir dir file to mapfile - mapfile parent_dir < dir_file_list.txt - rm dir_file_list.txt >/dev/null 2>&1 - tmp_output_filename=${parent_dir[${#parent_dir[@]}-2]}${parent_dir[${#parent_dir[@]}-1]}".mindrecord" - output_filename[$file_index]=`echo ${tmp_output_filename} | sed 's/ //g'` - file_index=`expr $file_index + 1` - fi - done -} - -getdir "${data_dir}" -# echo "The input files: "${file_list[@]} -# echo "The output files: "${output_filename[@]} - -if [ ! -d "../../../../third_party/to_mindrecord/zhwiki" ]; then - echo "The patch base dir ../../../../third_party/to_mindrecord/zhwiki is not exist." - exit 1 -fi - -if [ ! -f "../../../../third_party/patch/to_mindrecord/zhwiki/create_pretraining_data.patch" ]; then - echo "The patch file ../../../../third_party/patch/to_mindrecord/zhwiki/create_pretraining_data.patch is not exist." - exit 1 -fi - -# patch for create_pretraining_data.py -patch -p0 -d ../../../../third_party/to_mindrecord/zhwiki/ -o create_pretraining_data_patched.py < ../../../../third_party/patch/to_mindrecord/zhwiki/create_pretraining_data.patch -if [ $? -ne 0 ]; then - echo "Patch ../../../../third_party/to_mindrecord/zhwiki/create_pretraining_data.py failed" - exit 1 -fi - -# get the cpu core count -num_cpu_core=`cat /proc/cpuinfo | grep "processor" | wc -l` -avaiable_core_size=`expr $num_cpu_core / 3 \* 2` - -echo "Begin preprocess `date`" - -# using patched script to generate mindrecord -file_list_len=`expr ${#file_list[*]} - 1` -for index in $(seq 0 $file_list_len); do - echo "Begin preprocess input file: ${file_list[$index]}" - echo "Begin output file: ${output_filename[$index]}" - python ../../../../third_party/to_mindrecord/zhwiki/create_pretraining_data_patched.py \ - --input_file=${file_list[$index]} \ - --output_file=output/${output_filename[$index]} \ - --partition_number=1 \ - --vocab_file=../../../../third_party/to_mindrecord/zhwiki/vocab.txt \ - --do_lower_case=True \ - --max_seq_length=128 \ - --max_predictions_per_seq=20 \ - --masked_lm_prob=0.15 \ - --random_seed=12345 \ - --dupe_factor=10 >/tmp/${output_filename[$index]}.log 2>&1 & # user defined - process_count=`ps -ef | grep create_pretraining_data_patched | grep -v grep | wc -l` - echo "Total task: ${#file_list[*]}, processing: ${process_count}" - if [ $process_count -ge $avaiable_core_size ]; then - while [ 1 ]; do - process_num=`ps -ef | grep create_pretraining_data_patched | grep -v grep | wc -l` - if [ $process_count -gt $process_num ]; then - process_count=$process_num - break; - fi - sleep 2 - done - fi -done - -process_num=`ps -ef | grep create_pretraining_data_patched | grep -v grep | wc -l` -while [ 1 ]; do - if [ $process_num -eq 0 ]; then - break; - fi - echo "There are still ${process_num} preprocess running ..." - sleep 2 - process_num=`ps -ef | grep create_pretraining_data_patched | grep -v grep | wc -l` -done - -echo "Preprocess all the data success." -echo "End preprocess `date`" diff --git a/model_zoo/utils/nlp_to_mindrecord/zhwiki/run_read.sh b/model_zoo/utils/nlp_to_mindrecord/zhwiki/run_read.sh deleted file mode 100644 index 3cc368457b..0000000000 --- a/model_zoo/utils/nlp_to_mindrecord/zhwiki/run_read.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ - -file_list=() -file_index=0 - -# get all the mindrecord file from output dir -function getdir() { - elements=`ls $1/[A-Z]*.mindrecord` - for element in ${elements[*]}; - do - file_list[$file_index]=$element - file_index=`expr $file_index + 1` - done -} - -getdir "./output" -echo "Get all the mindrecord files: "${file_list[*]} - -# create dataset for train -python create_dataset.py --input_file ${file_list[*]} diff --git a/model_zoo/utils/nlp_to_mindrecord/zhwiki/run_read_simple.sh b/model_zoo/utils/nlp_to_mindrecord/zhwiki/run_read_simple.sh deleted file mode 100644 index abecc20187..0000000000 --- a/model_zoo/utils/nlp_to_mindrecord/zhwiki/run_read_simple.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ - -# create dataset for train -python create_dataset.py --input_file=output/simple.mindrecord diff --git a/model_zoo/utils/nlp_to_mindrecord/zhwiki/run_simple.sh b/model_zoo/utils/nlp_to_mindrecord/zhwiki/run_simple.sh deleted file mode 100644 index a41df700b4..0000000000 --- a/model_zoo/utils/nlp_to_mindrecord/zhwiki/run_simple.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ - -rm -f output/simple.mindrecord* - -if [ ! -d "../../../../third_party/to_mindrecord/zhwiki" ]; then - echo "The patch base dir ../../../../third_party/to_mindrecord/zhwiki is not exist." - exit 1 -fi - -if [ ! -f "../../../../third_party/patch/to_mindrecord/zhwiki/create_pretraining_data.patch" ]; then - echo "The patch file ../../../../third_party/patch/to_mindrecord/zhwiki/create_pretraining_data.patch is not exist." - exit 1 -fi - -# patch for create_pretraining_data.py -patch -p0 -d ../../../../third_party/to_mindrecord/zhwiki/ -o create_pretraining_data_patched.py < ../../../../third_party/patch/to_mindrecord/zhwiki/create_pretraining_data.patch -if [ $? -ne 0 ]; then - echo "Patch ../../../../third_party/to_mindrecord/zhwiki/create_pretraining_data.py failed" - exit 1 -fi - -# using patched script to generate mindrecord -python ../../../../third_party/to_mindrecord/zhwiki/create_pretraining_data_patched.py \ ---input_file=../../../../third_party/to_mindrecord/zhwiki/sample_text.txt \ ---output_file=output/simple.mindrecord \ ---partition_number=1 \ ---vocab_file=../../../../third_party/to_mindrecord/zhwiki/vocab.txt \ ---do_lower_case=True \ ---max_seq_length=128 \ ---max_predictions_per_seq=20 \ ---masked_lm_prob=0.15 \ ---random_seed=12345 \ ---dupe_factor=10 # user defined diff --git a/third_party/patch/to_mindrecord/CLUERNER2020/README.md b/third_party/patch/to_mindrecord/CLUERNER2020/README.md deleted file mode 100644 index d0191db59a..0000000000 --- a/third_party/patch/to_mindrecord/CLUERNER2020/README.md +++ /dev/null @@ -1 +0,0 @@ -## the file is a patch which is about just change data_processor_seq.py the part of generated tfrecord to MindRecord in [CLUEbenchmark/CLUENER2020](https://github.com/CLUEbenchmark/CLUENER2020/tree/master/tf_version) diff --git a/third_party/patch/to_mindrecord/CLUERNER2020/data_processor_seq.patch b/third_party/patch/to_mindrecord/CLUERNER2020/data_processor_seq.patch deleted file mode 100644 index e24db8a537..0000000000 --- a/third_party/patch/to_mindrecord/CLUERNER2020/data_processor_seq.patch +++ /dev/null @@ -1,105 +0,0 @@ ---- data_processor_seq.py 2020-05-28 10:07:13.365947168 +0800 -+++ data_processor_seq.py 2020-05-28 10:14:33.298177130 +0800 -@@ -4,11 +4,18 @@ - @author: Cong Yu - @time: 2019-12-07 17:03 - """ -+import sys -+sys.path.append("../../../third_party/to_mindrecord/CLUERNER2020") -+ -+import argparse - import json - import tokenization - import collections --import tensorflow as tf - -+import numpy as np -+from mindspore.mindrecord import FileWriter -+ -+# pylint: skip-file - - def _truncate_seq_pair(tokens_a, tokens_b, max_length): - """Truncates a sequence pair in place to the maximum length.""" -@@ -80,11 +87,18 @@ def process_one_example(tokenizer, label - return feature - - --def prepare_tf_record_data(tokenizer, max_seq_len, label2id, path, out_path): -+def prepare_mindrecord_data(tokenizer, max_seq_len, label2id, path, out_path): - """ -- 生成训练数据， tf.record, 单标签分类模型, 随机打乱数据 -+ 生成训练数据， *.mindrecord, 单标签分类模型, 随机打乱数据 - """ -- writer = tf.python_io.TFRecordWriter(out_path) -+ writer = FileWriter(out_path) -+ -+ data_schema = {"input_ids": {"type": "int64", "shape": [-1]}, -+ "input_mask": {"type": "int64", "shape": [-1]}, -+ "segment_ids": {"type": "int64", "shape": [-1]}, -+ "label_ids": {"type": "int64", "shape": [-1]}} -+ writer.add_schema(data_schema, "CLUENER2020 schema") -+ - example_count = 0 - - for line in open(path): -@@ -113,16 +127,12 @@ def prepare_tf_record_data(tokenizer, ma - feature = process_one_example(tokenizer, label2id, list(_["text"]), labels, - max_seq_len=max_seq_len) - -- def create_int_feature(values): -- f = tf.train.Feature(int64_list=tf.train.Int64List(value=list(values))) -- return f -- - features = collections.OrderedDict() - # 序列标注任务 -- features["input_ids"] = create_int_feature(feature[0]) -- features["input_mask"] = create_int_feature(feature[1]) -- features["segment_ids"] = create_int_feature(feature[2]) -- features["label_ids"] = create_int_feature(feature[3]) -+ features["input_ids"] = np.asarray(feature[0]) -+ features["input_mask"] = np.asarray(feature[1]) -+ features["segment_ids"] = np.asarray(feature[2]) -+ features["label_ids"] = np.asarray(feature[3]) - if example_count < 5: - print("*** Example ***") - print(_["text"]) -@@ -132,8 +142,7 @@ def prepare_tf_record_data(tokenizer, ma - print("segment_ids: %s" % " ".join([str(x) for x in feature[2]])) - print("label: %s " % " ".join([str(x) for x in feature[3]])) - -- tf_example = tf.train.Example(features=tf.train.Features(feature=features)) -- writer.write(tf_example.SerializeToString()) -+ writer.write_raw_data([features]) - example_count += 1 - - # if example_count == 20: -@@ -141,17 +150,22 @@ def prepare_tf_record_data(tokenizer, ma - if example_count % 3000 == 0: - print(example_count) - print("total example:", example_count) -- writer.close() -+ writer.commit() - - - if __name__ == "__main__": -- vocab_file = "./vocab.txt" -+ parser = argparse.ArgumentParser() -+ parser.add_argument("--vocab_file", type=str, required=True, help='The vocabulary file.') -+ parser.add_argument("--label2id_file", type=str, required=True, help='The label2id.json file.') -+ args = parser.parse_args() -+ -+ vocab_file = args.vocab_file - tokenizer = tokenization.FullTokenizer(vocab_file=vocab_file) -- label2id = json.loads(open("label2id.json").read()) -+ label2id = json.loads(open(args.label2id_file).read()) - - max_seq_len = 64 - -- prepare_tf_record_data(tokenizer, max_seq_len, label2id, path="data/thuctc_train.json", -- out_path="data/train.tf_record") -- prepare_tf_record_data(tokenizer, max_seq_len, label2id, path="data/thuctc_valid.json", -- out_path="data/dev.tf_record") -+ prepare_mindrecord_data(tokenizer, max_seq_len, label2id, path="data/cluener_public/train.json", -+ out_path="output/train.mindrecord") -+ prepare_mindrecord_data(tokenizer, max_seq_len, label2id, path="data/cluener_public/dev.json", -+ out_path="output/dev.mindrecord") diff --git a/third_party/patch/to_mindrecord/zhwiki/README.md b/third_party/patch/to_mindrecord/zhwiki/README.md deleted file mode 100644 index b06360d60d..0000000000 --- a/third_party/patch/to_mindrecord/zhwiki/README.md +++ /dev/null @@ -1 +0,0 @@ -## the file is a patch which is about just change create_pretraining_data.py the part of generated tfrecord to MindRecord in [google-research/bert](https://github.com/google-research/bert) diff --git a/third_party/patch/to_mindrecord/zhwiki/create_pretraining_data.patch b/third_party/patch/to_mindrecord/zhwiki/create_pretraining_data.patch deleted file mode 100644 index 1a7b15dce2..0000000000 --- a/third_party/patch/to_mindrecord/zhwiki/create_pretraining_data.patch +++ /dev/null @@ -1,288 +0,0 @@ ---- create_pretraining_data.py 2020-05-27 17:02:14.285363720 +0800 -+++ create_pretraining_data.py 2020-05-27 17:30:52.427767841 +0800 -@@ -12,57 +12,28 @@ - # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - # See the License for the specific language governing permissions and - # limitations under the License. --"""Create masked LM/next sentence masked_lm TF examples for BERT.""" -+"""Create masked LM/next sentence masked_lm MindRecord files for BERT.""" - - from __future__ import absolute_import - from __future__ import division - from __future__ import print_function - -+import sys -+sys.path.append("../../../third_party/to_mindrecord/zhwiki") -+ -+import argparse - import collections -+import logging - import random - import tokenization --import tensorflow as tf -- --flags = tf.flags -- --FLAGS = flags.FLAGS -- --flags.DEFINE_string("input_file", None, -- "Input raw text file (or comma-separated list of files).") -- --flags.DEFINE_string( -- "output_file", None, -- "Output TF example file (or comma-separated list of files).") -- --flags.DEFINE_string("vocab_file", None, -- "The vocabulary file that the BERT model was trained on.") -- --flags.DEFINE_bool( -- "do_lower_case", True, -- "Whether to lower case the input text. Should be True for uncased " -- "models and False for cased models.") -- --flags.DEFINE_bool( -- "do_whole_word_mask", False, -- "Whether to use whole word masking rather than per-WordPiece masking.") -- --flags.DEFINE_integer("max_seq_length", 128, "Maximum sequence length.") - --flags.DEFINE_integer("max_predictions_per_seq", 20, -- "Maximum number of masked LM predictions per sequence.") -+import numpy as np -+from mindspore.mindrecord import FileWriter - --flags.DEFINE_integer("random_seed", 12345, "Random seed for data generation.") -+# pylint: skip-file - --flags.DEFINE_integer( -- "dupe_factor", 10, -- "Number of times to duplicate the input data (with different masks).") -- --flags.DEFINE_float("masked_lm_prob", 0.15, "Masked LM probability.") -- --flags.DEFINE_float( -- "short_seq_prob", 0.1, -- "Probability of creating sequences which are shorter than the " -- "maximum length.") -+logging.basicConfig(format='%(asctime)s - %(levelname)s - %(name)s - %(message)s', -+ datefmt='%m/%d/%Y %H:%M:%S', level=logging.INFO) - - - class TrainingInstance(object): -@@ -94,13 +65,19 @@ class TrainingInstance(object): - - - def write_instance_to_example_files(instances, tokenizer, max_seq_length, -- max_predictions_per_seq, output_files): -- """Create TF example files from `TrainingInstance`s.""" -- writers = [] -- for output_file in output_files: -- writers.append(tf.python_io.TFRecordWriter(output_file)) -- -- writer_index = 0 -+ max_predictions_per_seq, output_file, partition_number): -+ """Create MindRecord files from `TrainingInstance`s.""" -+ writer = FileWriter(output_file, int(partition_number)) -+ -+ data_schema = {"input_ids": {"type": "int64", "shape": [-1]}, -+ "input_mask": {"type": "int64", "shape": [-1]}, -+ "segment_ids": {"type": "int64", "shape": [-1]}, -+ "masked_lm_positions": {"type": "int64", "shape": [-1]}, -+ "masked_lm_ids": {"type": "int64", "shape": [-1]}, -+ "masked_lm_weights": {"type": "float32", "shape": [-1]}, -+ "next_sentence_labels": {"type": "int64", "shape": [-1]}, -+ } -+ writer.add_schema(data_schema, "zhwiki schema") - - total_written = 0 - for (inst_index, instance) in enumerate(instances): -@@ -130,55 +107,35 @@ def write_instance_to_example_files(inst - next_sentence_label = 1 if instance.is_random_next else 0 - - features = collections.OrderedDict() -- features["input_ids"] = create_int_feature(input_ids) -- features["input_mask"] = create_int_feature(input_mask) -- features["segment_ids"] = create_int_feature(segment_ids) -- features["masked_lm_positions"] = create_int_feature(masked_lm_positions) -- features["masked_lm_ids"] = create_int_feature(masked_lm_ids) -- features["masked_lm_weights"] = create_float_feature(masked_lm_weights) -- features["next_sentence_labels"] = create_int_feature([next_sentence_label]) -- -- tf_example = tf.train.Example(features=tf.train.Features(feature=features)) -- -- writers[writer_index].write(tf_example.SerializeToString()) -- writer_index = (writer_index + 1) % len(writers) -+ features["input_ids"] = np.asarray(input_ids, np.int64) -+ features["input_mask"] = np.asarray(input_mask, np.int64) -+ features["segment_ids"] = np.asarray(segment_ids, np.int64) -+ features["masked_lm_positions"] = np.asarray(masked_lm_positions, np.int64) -+ features["masked_lm_ids"] = np.asarray(masked_lm_ids, np.int64) -+ features["masked_lm_weights"] = np.asarray(masked_lm_weights, np.float32) -+ features["next_sentence_labels"] = np.asarray([next_sentence_label], np.int64) - - total_written += 1 - - if inst_index < 20: -- tf.logging.info("*** Example ***") -- tf.logging.info("tokens: %s" % " ".join( -+ logging.info("*** Example ***") -+ logging.info("tokens: %s" % " ".join( - [tokenization.printable_text(x) for x in instance.tokens])) - - for feature_name in features.keys(): - feature = features[feature_name] -- values = [] -- if feature.int64_list.value: -- values = feature.int64_list.value -- elif feature.float_list.value: -- values = feature.float_list.value -- tf.logging.info( -- "%s: %s" % (feature_name, " ".join([str(x) for x in values]))) -- -- for writer in writers: -- writer.close() -- -- tf.logging.info("Wrote %d total instances", total_written) -- -- --def create_int_feature(values): -- feature = tf.train.Feature(int64_list=tf.train.Int64List(value=list(values))) -- return feature -+ logging.info( -+ "%s: %s" % (feature_name, " ".join([str(x) for x in feature]))) -+ writer.write_raw_data([features]) - -+ writer.commit() - --def create_float_feature(values): -- feature = tf.train.Feature(float_list=tf.train.FloatList(value=list(values))) -- return feature -+ logging.info("Wrote %d total instances", total_written) - - - def create_training_instances(input_files, tokenizer, max_seq_length, - dupe_factor, short_seq_prob, masked_lm_prob, -- max_predictions_per_seq, rng): -+ max_predictions_per_seq, rng, do_whole_word_mask): - """Create `TrainingInstance`s from raw text.""" - all_documents = [[]] - -@@ -189,7 +146,7 @@ def create_training_instances(input_file - # (2) Blank lines between documents. Document boundaries are needed so - # that the "next sentence prediction" task doesn't span between documents. - for input_file in input_files: -- with tf.gfile.GFile(input_file, "r") as reader: -+ with open(input_file, "r") as reader: - while True: - line = tokenization.convert_to_unicode(reader.readline()) - if not line: -@@ -214,7 +171,7 @@ def create_training_instances(input_file - instances.extend( - create_instances_from_document( - all_documents, document_index, max_seq_length, short_seq_prob, -- masked_lm_prob, max_predictions_per_seq, vocab_words, rng)) -+ masked_lm_prob, max_predictions_per_seq, vocab_words, rng, do_whole_word_mask)) - - rng.shuffle(instances) - return instances -@@ -222,7 +179,7 @@ def create_training_instances(input_file - - def create_instances_from_document( - all_documents, document_index, max_seq_length, short_seq_prob, -- masked_lm_prob, max_predictions_per_seq, vocab_words, rng): -+ masked_lm_prob, max_predictions_per_seq, vocab_words, rng, do_whole_word_mask): - """Creates `TrainingInstance`s for a single document.""" - document = all_documents[document_index] - -@@ -320,7 +277,7 @@ def create_instances_from_document( - - (tokens, masked_lm_positions, - masked_lm_labels) = create_masked_lm_predictions( -- tokens, masked_lm_prob, max_predictions_per_seq, vocab_words, rng) -+ tokens, masked_lm_prob, max_predictions_per_seq, vocab_words, rng, do_whole_word_mask) - instance = TrainingInstance( - tokens=tokens, - segment_ids=segment_ids, -@@ -340,7 +297,7 @@ MaskedLmInstance = collections.namedtupl - - - def create_masked_lm_predictions(tokens, masked_lm_prob, -- max_predictions_per_seq, vocab_words, rng): -+ max_predictions_per_seq, vocab_words, rng, do_whole_word_mask): - """Creates the predictions for the masked LM objective.""" - - cand_indexes = [] -@@ -356,7 +313,7 @@ def create_masked_lm_predictions(tokens, - # Note that Whole Word Masking does *not* change the training code - # at all -- we still predict each WordPiece independently, softmaxed - # over the entire vocabulary. -- if (FLAGS.do_whole_word_mask and len(cand_indexes) >= 1 and -+ if (do_whole_word_mask and len(cand_indexes) >= 1 and - token.startswith("##")): - cand_indexes[-1].append(i) - else: -@@ -433,37 +390,42 @@ def truncate_seq_pair(tokens_a, tokens_b - trunc_tokens.pop() - - --def main(_): -- tf.logging.set_verbosity(tf.logging.INFO) -+def main(): -+ parser = argparse.ArgumentParser() -+ parser.add_argument("--input_file", type=str, required=True, help='Input raw text file (or comma-separated list of files).') -+ parser.add_argument("--output_file", type=str, required=True, help='Output MindRecord file.') -+ parser.add_argument("--partition_number", type=int, default=1, help='The MindRecord file will be split into the number of partition.') -+ parser.add_argument("--vocab_file", type=str, required=True, help='The vocabulary file than the BERT model was trained on.') -+ parser.add_argument("--do_lower_case", type=bool, default=False, help='Whether to lower case the input text. Should be True for uncased models and False for cased models.') -+ parser.add_argument("--do_whole_word_mask", type=bool, default=False, help='Whether to use whole word masking rather than per-WordPiece masking.') -+ parser.add_argument("--max_seq_length", type=int, default=128, help='Maximum sequence length.') -+ parser.add_argument("--max_predictions_per_seq", type=int, default=20, help='Maximum number of masked LM predictions per sequence.') -+ parser.add_argument("--random_seed", type=int, default=12345, help='Random seed for data generation.') -+ parser.add_argument("--dupe_factor", type=int, default=10, help='Number of times to duplicate the input data (with diffrent masks).') -+ parser.add_argument("--masked_lm_prob", type=float, default=0.15, help='Masked LM probability.') -+ parser.add_argument("--short_seq_prob", type=float, default=0.1, help='Probability of creating sequences which are shorter than the maximum length.') -+ args = parser.parse_args() - - tokenizer = tokenization.FullTokenizer( -- vocab_file=FLAGS.vocab_file, do_lower_case=FLAGS.do_lower_case) -+ vocab_file=args.vocab_file, do_lower_case=args.do_lower_case) - - input_files = [] -- for input_pattern in FLAGS.input_file.split(","): -- input_files.extend(tf.gfile.Glob(input_pattern)) -+ for input_pattern in args.input_file.split(","): -+ input_files.append(input_pattern) - -- tf.logging.info("*** Reading from input files ***") -+ logging.info("*** Reading from input files ***") - for input_file in input_files: -- tf.logging.info(" %s", input_file) -+ logging.info(" %s", input_file) - -- rng = random.Random(FLAGS.random_seed) -+ rng = random.Random(args.random_seed) - instances = create_training_instances( -- input_files, tokenizer, FLAGS.max_seq_length, FLAGS.dupe_factor, -- FLAGS.short_seq_prob, FLAGS.masked_lm_prob, FLAGS.max_predictions_per_seq, -- rng) -- -- output_files = FLAGS.output_file.split(",") -- tf.logging.info("*** Writing to output files ***") -- for output_file in output_files: -- tf.logging.info(" %s", output_file) -+ input_files, tokenizer, args.max_seq_length, args.dupe_factor, -+ args.short_seq_prob, args.masked_lm_prob, args.max_predictions_per_seq, -+ rng, args.do_whole_word_mask) - -- write_instance_to_example_files(instances, tokenizer, FLAGS.max_seq_length, -- FLAGS.max_predictions_per_seq, output_files) -+ write_instance_to_example_files(instances, tokenizer, args.max_seq_length, -+ args.max_predictions_per_seq, args.output_file, args.partition_number) - - - if __name__ == "__main__": -- flags.mark_flag_as_required("input_file") -- flags.mark_flag_as_required("output_file") -- flags.mark_flag_as_required("vocab_file") -- tf.app.run() -+ main() diff --git a/third_party/to_mindrecord/CLUERNER2020/.gitignore b/third_party/to_mindrecord/CLUERNER2020/.gitignore deleted file mode 100644 index 6c906cba1f..0000000000 --- a/third_party/to_mindrecord/CLUERNER2020/.gitignore +++ /dev/null @@ -1 +0,0 @@ -data_processor_seq_patched.py diff --git a/third_party/to_mindrecord/CLUERNER2020/README.md b/third_party/to_mindrecord/CLUERNER2020/README.md deleted file mode 100644 index 3dea3ae45a..0000000000 --- a/third_party/to_mindrecord/CLUERNER2020/README.md +++ /dev/null @@ -1 +0,0 @@ -## All the scripts here come from [CLUEbenchmark/CLUENER2020](https://github.com/CLUEbenchmark/CLUENER2020/tree/master/tf_version) diff --git a/third_party/to_mindrecord/CLUERNER2020/data_processor_seq.py b/third_party/to_mindrecord/CLUERNER2020/data_processor_seq.py deleted file mode 100644 index e917aa7ebb..0000000000 --- a/third_party/to_mindrecord/CLUERNER2020/data_processor_seq.py +++ /dev/null @@ -1,157 +0,0 @@ -#!/usr/bin/python -# coding:utf8 -""" -@author: Cong Yu -@time: 2019-12-07 17:03 -""" -import json -import tokenization -import collections -import tensorflow as tf - - -def _truncate_seq_pair(tokens_a, tokens_b, max_length): - """Truncates a sequence pair in place to the maximum length.""" - - # This is a simple heuristic which will always truncate the longer sequence - # one token at a time. This makes more sense than truncating an equal percent - # of tokens from each, since if one sequence is very short then each token - # that's truncated likely contains more information than a longer sequence. - while True: - total_length = len(tokens_a) + len(tokens_b) - if total_length <= max_length: - break - if len(tokens_a) > len(tokens_b): - tokens_a.pop() - else: - tokens_b.pop() - - -def process_one_example(tokenizer, label2id, text, label, max_seq_len=128): - # textlist = text.split(' ') - # labellist = label.split(' ') - textlist = list(text) - labellist = list(label) - tokens = [] - labels = [] - for i, word in enumerate(textlist): - token = tokenizer.tokenize(word) - tokens.extend(token) - label_1 = labellist[i] - for m in range(len(token)): - if m == 0: - labels.append(label_1) - else: - print("some unknown token...") - labels.append(labels[0]) - # tokens = tokenizer.tokenize(example.text) -2 的原因是因为序列需要加一个句首和句尾标志 - if len(tokens) >= max_seq_len - 1: - tokens = tokens[0:(max_seq_len - 2)] - labels = labels[0:(max_seq_len - 2)] - ntokens = [] - segment_ids = [] - label_ids = [] - ntokens.append("[CLS]") # 句子开始设置CLS 标志 - segment_ids.append(0) - # [CLS] [SEP] 可以为 他们构建标签，或者 统一到某个标签，反正他们是不变的，基本不参加训练 即：x-l 永远不变 - label_ids.append(0) # label2id["[CLS]"] - for i, token in enumerate(tokens): - ntokens.append(token) - segment_ids.append(0) - label_ids.append(label2id[labels[i]]) - ntokens.append("[SEP]") - segment_ids.append(0) - # append("O") or append("[SEP]") not sure! - label_ids.append(0) # label2id["[SEP]"] - input_ids = tokenizer.convert_tokens_to_ids(ntokens) - input_mask = [1] * len(input_ids) - while len(input_ids) < max_seq_len: - input_ids.append(0) - input_mask.append(0) - segment_ids.append(0) - label_ids.append(0) - ntokens.append("**NULL**") - assert len(input_ids) == max_seq_len - assert len(input_mask) == max_seq_len - assert len(segment_ids) == max_seq_len - assert len(label_ids) == max_seq_len - - feature = (input_ids, input_mask, segment_ids, label_ids) - return feature - - -def prepare_tf_record_data(tokenizer, max_seq_len, label2id, path, out_path): - """ - 生成训练数据， tf.record, 单标签分类模型, 随机打乱数据 - """ - writer = tf.python_io.TFRecordWriter(out_path) - example_count = 0 - - for line in open(path): - if not line.strip(): - continue - _ = json.loads(line.strip()) - len_ = len(_["text"]) - labels = ["O"] * len_ - for k, v in _["label"].items(): - for kk, vv in v.items(): - for vvv in vv: - span = vvv - s = span[0] - e = span[1] + 1 - # print(s, e) - if e - s == 1: - labels[s] = "S_" + k - else: - labels[s] = "B_" + k - for i in range(s + 1, e - 1): - labels[i] = "M_" + k - labels[e - 1] = "E_" + k - # print() - # feature = process_one_example(tokenizer, label2id, row[column_name_x1], row[column_name_y], - # max_seq_len=max_seq_len) - feature = process_one_example(tokenizer, label2id, list(_["text"]), labels, - max_seq_len=max_seq_len) - - def create_int_feature(values): - f = tf.train.Feature(int64_list=tf.train.Int64List(value=list(values))) - return f - - features = collections.OrderedDict() - # 序列标注任务 - features["input_ids"] = create_int_feature(feature[0]) - features["input_mask"] = create_int_feature(feature[1]) - features["segment_ids"] = create_int_feature(feature[2]) - features["label_ids"] = create_int_feature(feature[3]) - if example_count < 5: - print("*** Example ***") - print(_["text"]) - print(_["label"]) - print("input_ids: %s" % " ".join([str(x) for x in feature[0]])) - print("input_mask: %s" % " ".join([str(x) for x in feature[1]])) - print("segment_ids: %s" % " ".join([str(x) for x in feature[2]])) - print("label: %s " % " ".join([str(x) for x in feature[3]])) - - tf_example = tf.train.Example(features=tf.train.Features(feature=features)) - writer.write(tf_example.SerializeToString()) - example_count += 1 - - # if example_count == 20: - # break - if example_count % 3000 == 0: - print(example_count) - print("total example:", example_count) - writer.close() - - -if __name__ == "__main__": - vocab_file = "./vocab.txt" - tokenizer = tokenization.FullTokenizer(vocab_file=vocab_file) - label2id = json.loads(open("label2id.json").read()) - - max_seq_len = 64 - - prepare_tf_record_data(tokenizer, max_seq_len, label2id, path="data/thuctc_train.json", - out_path="data/train.tf_record") - prepare_tf_record_data(tokenizer, max_seq_len, label2id, path="data/thuctc_valid.json", - out_path="data/dev.tf_record") diff --git a/third_party/to_mindrecord/CLUERNER2020/label2id.json b/third_party/to_mindrecord/CLUERNER2020/label2id.json deleted file mode 100644 index f296bcb28f..0000000000 --- a/third_party/to_mindrecord/CLUERNER2020/label2id.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "O": 0, - "S_address": 1, - "B_address": 2, - "M_address": 3, - "E_address": 4, - "S_book": 5, - "B_book": 6, - "M_book": 7, - "E_book": 8, - "S_company": 9, - "B_company": 10, - "M_company": 11, - "E_company": 12, - "S_game": 13, - "B_game": 14, - "M_game": 15, - "E_game": 16, - "S_government": 17, - "B_government": 18, - "M_government": 19, - "E_government": 20, - "S_movie": 21, - "B_movie": 22, - "M_movie": 23, - "E_movie": 24, - "S_name": 25, - "B_name": 26, - "M_name": 27, - "E_name": 28, - "S_organization": 29, - "B_organization": 30, - "M_organization": 31, - "E_organization": 32, - "S_position": 33, - "B_position": 34, - "M_position": 35, - "E_position": 36, - "S_scene": 37, - "B_scene": 38, - "M_scene": 39, - "E_scene": 40 -} \ No newline at end of file diff --git a/third_party/to_mindrecord/CLUERNER2020/tokenization.py b/third_party/to_mindrecord/CLUERNER2020/tokenization.py deleted file mode 100644 index 856021d6a9..0000000000 --- a/third_party/to_mindrecord/CLUERNER2020/tokenization.py +++ /dev/null @@ -1,388 +0,0 @@ -"""Tokenization classes.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import collections -import re -import unicodedata -import six - -# pylint: skip-file - -def validate_case_matches_checkpoint(do_lower_case, init_checkpoint): - """Checks whether the casing config is consistent with the checkpoint name.""" - - # The casing has to be passed in by the user and there is no explicit check - # as to whether it matches the checkpoint. The casing information probably - # should have been stored in the bert_config.json file, but it's not, so - # we have to heuristically detect it to validate. - - if not init_checkpoint: - return - - m = re.match("^.*?([A-Za-z0-9_-]+)/bert_model.ckpt", init_checkpoint) - if m is None: - return - - model_name = m.group(1) - - lower_models = [ - "uncased_L-24_H-1024_A-16", "uncased_L-12_H-768_A-12", - "multilingual_L-12_H-768_A-12", "chinese_L-12_H-768_A-12" - ] - - cased_models = [ - "cased_L-12_H-768_A-12", "cased_L-24_H-1024_A-16", - "multi_cased_L-12_H-768_A-12" - ] - - is_bad_config = False - if model_name in lower_models and not do_lower_case: - is_bad_config = True - actual_flag = "False" - case_name = "lowercased" - opposite_flag = "True" - - if model_name in cased_models and do_lower_case: - is_bad_config = True - actual_flag = "True" - case_name = "cased" - opposite_flag = "False" - - if is_bad_config: - raise ValueError( - "You passed in `--do_lower_case=%s` with `--init_checkpoint=%s`. " - "However, `%s` seems to be a %s model, so you " - "should pass in `--do_lower_case=%s` so that the fine-tuning matches " - "how the model was pre-training. If this error is wrong, please " - "just comment out this check." % (actual_flag, init_checkpoint, - model_name, case_name, opposite_flag)) - - -def convert_to_unicode(text): - """Converts `text` to Unicode (if it's not already), assuming utf-8 input.""" - if six.PY3: - if isinstance(text, str): - return text - elif isinstance(text, bytes): - return text.decode("utf-8", "ignore") - else: - raise ValueError("Unsupported string type: %s" % (type(text))) - elif six.PY2: - if isinstance(text, str): - return text.decode("utf-8", "ignore") - elif isinstance(text, unicode): - return text - else: - raise ValueError("Unsupported string type: %s" % (type(text))) - else: - raise ValueError("Not running on Python2 or Python 3?") - - -def printable_text(text): - """Returns text encoded in a way suitable for print or `tf.logging`.""" - - # These functions want `str` for both Python2 and Python3, but in one case - # it's a Unicode string and in the other it's a byte string. - if six.PY3: - if isinstance(text, str): - return text - elif isinstance(text, bytes): - return text.decode("utf-8", "ignore") - else: - raise ValueError("Unsupported string type: %s" % (type(text))) - elif six.PY2: - if isinstance(text, str): - return text - elif isinstance(text, unicode): - return text.encode("utf-8") - else: - raise ValueError("Unsupported string type: %s" % (type(text))) - else: - raise ValueError("Not running on Python2 or Python 3?") - - -def load_vocab(vocab_file): - """Loads a vocabulary file into a dictionary.""" - vocab = collections.OrderedDict() - index = 0 - with open(vocab_file, "r") as reader: - while True: - token = convert_to_unicode(reader.readline()) - if not token: - break - token = token.strip() - vocab[token] = index - index += 1 - return vocab - - -def convert_by_vocab(vocab, items): - """Converts a sequence of [tokens|ids] using the vocab.""" - output = [] - for item in items: - if item in vocab: - output.append(vocab[item]) - else: - output.append(vocab['[UNK]']) - return output - - -def convert_tokens_to_ids(vocab, tokens): - return convert_by_vocab(vocab, tokens) - - -def convert_ids_to_tokens(inv_vocab, ids): - return convert_by_vocab(inv_vocab, ids) - - -def whitespace_tokenize(text): - """Runs basic whitespace cleaning and splitting on a piece of text.""" - text = text.strip() - if not text: - return [] - tokens = text.split() - return tokens - - -class FullTokenizer(object): - """Runs end-to-end tokenziation.""" - - def __init__(self, vocab_file, do_lower_case=True): - self.vocab = load_vocab(vocab_file) - self.inv_vocab = {v: k for k, v in self.vocab.items()} - self.basic_tokenizer = BasicTokenizer(do_lower_case=do_lower_case) - self.wordpiece_tokenizer = WordpieceTokenizer(vocab=self.vocab) - - def tokenize(self, text): - split_tokens = [] - for token in self.basic_tokenizer.tokenize(text): - for sub_token in self.wordpiece_tokenizer.tokenize(token): - split_tokens.append(sub_token) - - return split_tokens - - def convert_tokens_to_ids(self, tokens): - return convert_by_vocab(self.vocab, tokens) - - def convert_ids_to_tokens(self, ids): - return convert_by_vocab(self.inv_vocab, ids) - - -class BasicTokenizer(object): - """Runs basic tokenization (punctuation splitting, lower casing, etc.).""" - - def __init__(self, do_lower_case=True): - """Constructs a BasicTokenizer. - - Args: - do_lower_case: Whether to lower case the input. - """ - self.do_lower_case = do_lower_case - - def tokenize(self, text): - """Tokenizes a piece of text.""" - text = convert_to_unicode(text) - text = self._clean_text(text) - - # This was added on November 1st, 2018 for the multilingual and Chinese - # models. This is also applied to the English models now, but it doesn't - # matter since the English models were not trained on any Chinese data - # and generally don't have any Chinese data in them (there are Chinese - # characters in the vocabulary because Wikipedia does have some Chinese - # words in the English Wikipedia.). - text = self._tokenize_chinese_chars(text) - - orig_tokens = whitespace_tokenize(text) - split_tokens = [] - for token in orig_tokens: - if self.do_lower_case: - token = token.lower() - token = self._run_strip_accents(token) - split_tokens.extend(self._run_split_on_punc(token)) - - output_tokens = whitespace_tokenize(" ".join(split_tokens)) - return output_tokens - - def _run_strip_accents(self, text): - """Strips accents from a piece of text.""" - text = unicodedata.normalize("NFD", text) - output = [] - for char in text: - cat = unicodedata.category(char) - if cat == "Mn": - continue - output.append(char) - return "".join(output) - - def _run_split_on_punc(self, text): - """Splits punctuation on a piece of text.""" - chars = list(text) - i = 0 - start_new_word = True - output = [] - while i < len(chars): - char = chars[i] - if _is_punctuation(char): - output.append([char]) - start_new_word = True - else: - if start_new_word: - output.append([]) - start_new_word = False - output[-1].append(char) - i += 1 - - return ["".join(x) for x in output] - - def _tokenize_chinese_chars(self, text): - """Adds whitespace around any CJK character.""" - output = [] - for char in text: - cp = ord(char) - if self._is_chinese_char(cp): - output.append(" ") - output.append(char) - output.append(" ") - else: - output.append(char) - return "".join(output) - - def _is_chinese_char(self, cp): - """Checks whether CP is the codepoint of a CJK character.""" - # This defines a "chinese character" as anything in the CJK Unicode block: - # https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block) - # - # Note that the CJK Unicode block is NOT all Japanese and Korean characters, - # despite its name. The modern Korean Hangul alphabet is a different block, - # as is Japanese Hiragana and Katakana. Those alphabets are used to write - # space-separated words, so they are not treated specially and handled - # like the all of the other languages. - if ((cp >= 0x4E00 and cp <= 0x9FFF) or # - (cp >= 0x3400 and cp <= 0x4DBF) or # - (cp >= 0x20000 and cp <= 0x2A6DF) or # - (cp >= 0x2A700 and cp <= 0x2B73F) or # - (cp >= 0x2B740 and cp <= 0x2B81F) or # - (cp >= 0x2B820 and cp <= 0x2CEAF) or - (cp >= 0xF900 and cp <= 0xFAFF) or # - (cp >= 0x2F800 and cp <= 0x2FA1F)): # - return True - - return False - - def _clean_text(self, text): - """Performs invalid character removal and whitespace cleanup on text.""" - output = [] - for char in text: - cp = ord(char) - if cp == 0 or cp == 0xfffd or _is_control(char): - continue - if _is_whitespace(char): - output.append(" ") - else: - output.append(char) - return "".join(output) - - -class WordpieceTokenizer(object): - """Runs WordPiece tokenziation.""" - - def __init__(self, vocab, unk_token="[UNK]", max_input_chars_per_word=200): - self.vocab = vocab - self.unk_token = unk_token - self.max_input_chars_per_word = max_input_chars_per_word - - def tokenize(self, text): - """Tokenizes a piece of text into its word pieces. - - This uses a greedy longest-match-first algorithm to perform tokenization - using the given vocabulary. - - For example: - input = "unaffable" - output = ["un", "##aff", "##able"] - - Args: - text: A single token or whitespace separated tokens. This should have - already been passed through `BasicTokenizer. - - Returns: - A list of wordpiece tokens. - """ - - text = convert_to_unicode(text) - - output_tokens = [] - for token in whitespace_tokenize(text): - chars = list(token) - if len(chars) > self.max_input_chars_per_word: - output_tokens.append(self.unk_token) - continue - - is_bad = False - start = 0 - sub_tokens = [] - while start < len(chars): - end = len(chars) - cur_substr = None - while start < end: - substr = "".join(chars[start:end]) - if start > 0: - substr = "##" + substr - if substr in self.vocab: - cur_substr = substr - break - end -= 1 - if cur_substr is None: - is_bad = True - break - sub_tokens.append(cur_substr) - start = end - - if is_bad: - output_tokens.append(self.unk_token) - else: - output_tokens.extend(sub_tokens) - return output_tokens - - -def _is_whitespace(char): - """Checks whether `chars` is a whitespace character.""" - # \t, \n, and \r are technically contorl characters but we treat them - # as whitespace since they are generally considered as such. - if char == " " or char == "\t" or char == "\n" or char == "\r": - return True - cat = unicodedata.category(char) - if cat == "Zs": - return True - return False - - -def _is_control(char): - """Checks whether `chars` is a control character.""" - # These are technically control characters but we count them as whitespace - # characters. - if char == "\t" or char == "\n" or char == "\r": - return False - cat = unicodedata.category(char) - if cat.startswith("C"): - return True - return False - - -def _is_punctuation(char): - """Checks whether `chars` is a punctuation character.""" - cp = ord(char) - # We treat all non-letter/number ASCII as punctuation. - # Characters such as "^", "$", and "`" are not in the Unicode - # Punctuation class but we treat them as punctuation anyways, for - # consistency. - if ((cp >= 33 and cp <= 47) or (cp >= 58 and cp <= 64) or - (cp >= 91 and cp <= 96) or (cp >= 123 and cp <= 126)): - return True - cat = unicodedata.category(char) - if cat.startswith("P"): - return True - return False diff --git a/third_party/to_mindrecord/CLUERNER2020/vocab.txt b/third_party/to_mindrecord/CLUERNER2020/vocab.txt deleted file mode 100644 index ca4f978103..0000000000 --- a/third_party/to_mindrecord/CLUERNER2020/vocab.txt +++ /dev/null @@ -1,21128 +0,0 @@ -[PAD] -[unused1] -[unused2] -[unused3] -[unused4] -[unused5] -[unused6] -[unused7] -[unused8] -[unused9] -[unused10] -[unused11] -[unused12] -[unused13] -[unused14] -[unused15] -[unused16] -[unused17] -[unused18] -[unused19] -[unused20] -[unused21] -[unused22] -[unused23] -[unused24] -[unused25] -[unused26] -[unused27] -[unused28] -[unused29] -[unused30] -[unused31] -[unused32] -[unused33] -[unused34] -[unused35] -[unused36] -[unused37] -[unused38] -[unused39] -[unused40] -[unused41] -[unused42] -[unused43] -[unused44] -[unused45] -[unused46] -[unused47] -[unused48] -[unused49] -[unused50] -[unused51] -[unused52] -[unused53] -[unused54] -[unused55] -[unused56] -[unused57] -[unused58] -[unused59] -[unused60] -[unused61] -[unused62] -[unused63] -[unused64] -[unused65] -[unused66] -[unused67] -[unused68] -[unused69] -[unused70] -[unused71] -[unused72] -[unused73] -[unused74] -[unused75] -[unused76] -[unused77] -[unused78] -[unused79] -[unused80] -[unused81] -[unused82] -[unused83] -[unused84] -[unused85] -[unused86] -[unused87] -[unused88] -[unused89] -[unused90] -[unused91] -[unused92] -[unused93] -[unused94] -[unused95] -[unused96] -[unused97] -[unused98] -[unused99] -[UNK] -[CLS] -[SEP] -[MASK] - - -! -" -# -$ -% -& -' -( -) -* -+ -, -- -. -/ -0 -1 -2 -3 -4 -5 -6 -7 -8 -9 -: -; -< -= -> -? -@ -[ -\ -] -^ -_ -a -b -c -d -e -f -g -h -i -j -k -l -m -n -o -p -q -r -s -t -u -v -w -x -y -z -{ -| -} -~ -£ -¤ -¥ -§ -© -« -® -° -± -² -³ -µ -· -¹ -º -» -¼ -× -ß -æ -÷ -ø -đ -ŋ -ɔ -ə -ɡ -ʰ -ˇ -ˈ -ˊ -ˋ -ˍ -ː -˙ -˚ -ˢ -α -β -γ -δ -ε -η -θ -ι -κ -λ -μ -ν -ο -π -ρ -ς -σ -τ -υ -φ -χ -ψ -ω -а -б -в -г -д -е -ж -з -и -к -л -м -н -о -п -р -с -т -у -ф -х -ц -ч -ш -ы -ь -я -і -ا -ب -ة -ت -د -ر -س -ع -ل -م -ن -ه -و -ي -۩ -ก -ง -น -ม -ย -ร -อ -า -เ -๑ -་ -ღ -ᄀ -ᄁ -ᄂ -ᄃ -ᄅ -ᄆ -ᄇ -ᄈ -ᄉ -ᄋ -ᄌ -ᄎ -ᄏ -ᄐ -ᄑ -ᄒ -ᅡ -ᅢ -ᅣ -ᅥ -ᅦ -ᅧ -ᅨ -ᅩ -ᅪ -ᅬ -ᅭ -ᅮ -ᅯ -ᅲ -ᅳ -ᅴ -ᅵ -ᆨ -ᆫ -ᆯ -ᆷ -ᆸ -ᆺ -ᆻ -ᆼ -ᗜ -ᵃ -ᵉ -ᵍ -ᵏ -ᵐ -ᵒ -ᵘ -‖ -„ -† -• -‥ -‧ -
 -‰ -′ -″ -‹ -› -※ -‿ -⁄ -ⁱ -⁺ -ⁿ -₁ -₂ -₃ -₄ -€ -℃ -№ -™ -ⅰ -ⅱ -ⅲ -ⅳ -ⅴ -← -↑ -→ -↓ -↔ -↗ -↘ -⇒ -∀ -− -∕ -∙ -√ -∞ -∟ -∠ -∣ -∥ -∩ -∮ -∶ -∼ -∽ -≈ -≒ -≡ -≤ -≥ -≦ -≧ -≪ -≫ -⊙ -⋅ -⋈ -⋯ -⌒ -① -② -③ -④ -⑤ -⑥ -⑦ -⑧ -⑨ -⑩ -⑴ -⑵ -⑶ -⑷ -⑸ -⒈ -⒉ -⒊ -⒋ -ⓒ -ⓔ -ⓘ -─ -━ -│ -┃ -┅ -┆ -┊ -┌ -└ -├ -┣ -═ -║ -╚ -╞ -╠ -╭ -╮ -╯ -╰ -╱ -╳ -▂ -▃ -▅ -▇ -█ -▉ -▋ -▌ -▍ -▎ -■ -□ -▪ -▫ -▬ -▲ -△ -▶ -► -▼ -▽ -◆ -◇ -○ -◎ -● -◕ -◠ -◢ -◤ -☀ -★ -☆ -☕ -☞ -☺ -☼ -♀ -♂ -♠ -♡ -♣ -♥ -♦ -♪ -♫ -♬ -✈ -✔ -✕ -✖ -✦ -✨ -✪ -✰ -✿ -❀ -❤ -➜ -➤ -⦿ -、 -。 -〃 -々 -〇 -〈 -〉 -《 -》 -「 -」 -『 -』 -【 -】 -〓 -〔 -〕 -〖 -〗 -〜 -〝 -〞 -ぁ -あ -ぃ -い -う -ぇ -え -お -か -き -く -け -こ -さ -し -す -せ -そ -た -ち -っ -つ -て -と -な -に -ぬ -ね -の -は -ひ -ふ -へ -ほ -ま -み -む -め -も -ゃ -や -ゅ -ゆ -ょ -よ -ら -り -る -れ -ろ -わ -を -ん -゜ -ゝ -ァ -ア -ィ -イ -ゥ -ウ -ェ -エ -ォ -オ -カ -キ -ク -ケ -コ -サ -シ -ス -セ -ソ -タ -チ -ッ -ツ -テ -ト -ナ -ニ -ヌ -ネ -ノ -ハ -ヒ -フ -ヘ -ホ -マ -ミ -ム -メ -モ -ャ -ヤ -ュ -ユ -ョ -ヨ -ラ -リ -ル -レ -ロ -ワ -ヲ -ン -ヶ -・ -ー -ヽ -ㄅ -ㄆ -ㄇ -ㄉ -ㄋ -ㄌ -ㄍ -ㄎ -ㄏ -ㄒ -ㄚ -ㄛ -ㄞ -ㄟ -ㄢ -ㄤ -ㄥ -ㄧ -ㄨ -ㆍ -㈦ -㊣ -㎡ -㗎 -一 -丁 -七 -万 -丈 -三 -上 -下 -不 -与 -丐 -丑 -专 -且 -丕 -世 -丘 -丙 -业 -丛 -东 -丝 -丞 -丟 -両 -丢 -两 -严 -並 -丧 -丨 -个 -丫 -中 -丰 -串 -临 -丶 -丸 -丹 -为 -主 -丼 -丽 -举 -丿 -乂 -乃 -久 -么 -义 -之 -乌 -乍 -乎 -乏 -乐 -乒 -乓 -乔 -乖 -乗 -乘 -乙 -乜 -九 -乞 -也 -习 -乡 -书 -乩 -买 -乱 -乳 -乾 -亀 -亂 -了 -予 -争 -事 -二 -于 -亏 -云 -互 -五 -井 -亘 -亙 -亚 -些 -亜 -亞 -亟 -亡 -亢 -交 -亥 -亦 -产 -亨 -亩 -享 -京 -亭 -亮 -亲 -亳 -亵 -人 -亿 -什 -仁 -仃 -仄 -仅 -仆 -仇 -今 -介 -仍 -从 -仏 -仑 -仓 -仔 -仕 -他 -仗 -付 -仙 -仝 -仞 -仟 -代 -令 -以 -仨 -仪 -们 -仮 -仰 -仲 -件 -价 -任 -份 -仿 -企 -伉 -伊 -伍 -伎 -伏 -伐 -休 -伕 -众 -优 -伙 -会 -伝 -伞 -伟 -传 -伢 -伤 -伦 -伪 -伫 -伯 -估 -伴 -伶 -伸 -伺 -似 -伽 -佃 -但 -佇 -佈 -位 -低 -住 -佐 -佑 -体 -佔 -何 -佗 -佘 -余 -佚 -佛 -作 -佝 -佞 -佟 -你 -佢 -佣 -佤 -佥 -佩 -佬 -佯 -佰 -佳 -併 -佶 -佻 -佼 -使 -侃 -侄 -來 -侈 -例 -侍 -侏 -侑 -侖 -侗 -供 -依 -侠 -価 -侣 -侥 -侦 -侧 -侨 -侬 -侮 -侯 -侵 -侶 -侷 -便 -係 -促 -俄 -俊 -俎 -俏 -俐 -俑 -俗 -俘 -俚 -保 -俞 -俟 -俠 -信 -俨 -俩 -俪 -俬 -俭 -修 -俯 -俱 -俳 -俸 -俺 -俾 -倆 -倉 -個 -倌 -倍 -倏 -們 -倒 -倔 -倖 -倘 -候 -倚 -倜 -借 -倡 -値 -倦 -倩 -倪 -倫 -倬 -倭 -倶 -债 -值 -倾 -偃 -假 -偈 -偉 -偌 -偎 -偏 -偕 -做 -停 -健 -側 -偵 -偶 -偷 -偻 -偽 -偿 -傀 -傅 -傍 -傑 -傘 -備 -傚 -傢 -傣 -傥 -储 -傩 -催 -傭 -傲 -傳 -債 -傷 -傻 -傾 -僅 -働 -像 -僑 -僕 -僖 -僚 -僥 -僧 -僭 -僮 -僱 -僵 -價 -僻 -儀 -儂 -億 -儆 -儉 -儋 -儒 -儕 -儘 -償 -儡 -優 -儲 -儷 -儼 -儿 -兀 -允 -元 -兄 -充 -兆 -兇 -先 -光 -克 -兌 -免 -児 -兑 -兒 -兔 -兖 -党 -兜 -兢 -入 -內 -全 -兩 -八 -公 -六 -兮 -兰 -共 -兲 -关 -兴 -兵 -其 -具 -典 -兹 -养 -兼 -兽 -冀 -内 -円 -冇 -冈 -冉 -冊 -册 -再 -冏 -冒 -冕 -冗 -写 -军 -农 -冠 -冢 -冤 -冥 -冨 -冪 -冬 -冯 -冰 -冲 -决 -况 -冶 -冷 -冻 -冼 -冽 -冾 -净 -凄 -准 -凇 -凈 -凉 -凋 -凌 -凍 -减 -凑 -凛 -凜 -凝 -几 -凡 -凤 -処 -凪 -凭 -凯 -凰 -凱 -凳 -凶 -凸 -凹 -出 -击 -函 -凿 -刀 -刁 -刃 -分 -切 -刈 -刊 -刍 -刎 -刑 -划 -列 -刘 -则 -刚 -创 -初 -删 -判 -別 -刨 -利 -刪 -别 -刮 -到 -制 -刷 -券 -刹 -刺 -刻 -刽 -剁 -剂 -剃 -則 -剉 -削 -剋 -剌 -前 -剎 -剐 -剑 -剔 -剖 -剛 -剜 -剝 -剣 -剤 -剥 -剧 -剩 -剪 -副 -割 -創 -剷 -剽 -剿 -劃 -劇 -劈 -劉 -劊 -劍 -劏 -劑 -力 -劝 -办 -功 -加 -务 -劣 -动 -助 -努 -劫 -劭 -励 -劲 -劳 -労 -劵 -効 -劾 -势 -勁 -勃 -勇 -勉 -勋 -勐 -勒 -動 -勖 -勘 -務 -勛 -勝 -勞 -募 -勢 -勤 -勧 -勳 -勵 -勸 -勺 -勻 -勾 -勿 -匀 -包 -匆 -匈 -匍 -匐 -匕 -化 -北 -匙 -匝 -匠 -匡 -匣 -匪 -匮 -匯 -匱 -匹 -区 -医 -匾 -匿 -區 -十 -千 -卅 -升 -午 -卉 -半 -卍 -华 -协 -卑 -卒 -卓 -協 -单 -卖 -南 -単 -博 -卜 -卞 -卟 -占 -卡 -卢 -卤 -卦 -卧 -卫 -卮 -卯 -印 -危 -即 -却 -卵 -卷 -卸 -卻 -卿 -厂 -厄 -厅 -历 -厉 -压 -厌 -厕 -厘 -厚 -厝 -原 -厢 -厥 -厦 -厨 -厩 -厭 -厮 -厲 -厳 -去 -县 -叁 -参 -參 -又 -叉 -及 -友 -双 -反 -収 -发 -叔 -取 -受 -变 -叙 -叛 -叟 -叠 -叡 -叢 -口 -古 -句 -另 -叨 -叩 -只 -叫 -召 -叭 -叮 -可 -台 -叱 -史 -右 -叵 -叶 -号 -司 -叹 -叻 -叼 -叽 -吁 -吃 -各 -吆 -合 -吉 -吊 -吋 -同 -名 -后 -吏 -吐 -向 -吒 -吓 -吕 -吖 -吗 -君 -吝 -吞 -吟 -吠 -吡 -否 -吧 -吨 -吩 -含 -听 -吭 -吮 -启 -吱 -吳 -吴 -吵 -吶 -吸 -吹 -吻 -吼 -吽 -吾 -呀 -呂 -呃 -呆 -呈 -告 -呋 -呎 -呐 -呓 -呕 -呗 -员 -呛 -呜 -呢 -呤 -呦 -周 -呱 -呲 -味 -呵 -呷 -呸 -呻 -呼 -命 -咀 -咁 -咂 -咄 -咆 -咋 -和 -咎 -咏 -咐 -咒 -咔 -咕 -咖 -咗 -咘 -咙 -咚 -咛 -咣 -咤 -咦 -咧 -咨 -咩 -咪 -咫 -咬 -咭 -咯 -咱 -咲 -咳 -咸 -咻 -咽 -咿 -哀 -品 -哂 -哄 -哆 -哇 -哈 -哉 -哋 -哌 -响 -哎 -哏 -哐 -哑 -哒 -哔 -哗 -哟 -員 -哥 -哦 -哧 -哨 -哩 -哪 -哭 -哮 -哲 -哺 -哼 -哽 -唁 -唄 -唆 -唇 -唉 -唏 -唐 -唑 -唔 -唠 -唤 -唧 -唬 -售 -唯 -唰 -唱 -唳 -唷 -唸 -唾 -啃 -啄 -商 -啉 -啊 -問 -啓 -啕 -啖 -啜 -啞 -啟 -啡 -啤 -啥 -啦 -啧 -啪 -啫 -啬 -啮 -啰 -啱 -啲 -啵 -啶 -啷 -啸 -啻 -啼 -啾 -喀 -喂 -喃 -善 -喆 -喇 -喉 -喊 -喋 -喎 -喏 -喔 -喘 -喙 -喚 -喜 -喝 -喟 -喧 -喪 -喫 -喬 -單 -喰 -喱 -喲 -喳 -喵 -営 -喷 -喹 -喺 -喻 -喽 -嗅 -嗆 -嗇 -嗎 -嗑 -嗒 -嗓 -嗔 -嗖 -嗚 -嗜 -嗝 -嗟 -嗡 -嗣 -嗤 -嗦 -嗨 -嗪 -嗬 -嗯 -嗰 -嗲 -嗳 -嗶 -嗷 -嗽 -嘀 -嘅 -嘆 -嘈 -嘉 -嘌 -嘍 -嘎 -嘔 -嘖 -嘗 -嘘 -嘚 -嘛 -嘜 -嘞 -嘟 -嘢 -嘣 -嘤 -嘧 -嘩 -嘭 -嘮 -嘯 -嘰 -嘱 -嘲 -嘴 -嘶 -嘸 -嘹 -嘻 -嘿 -噁 -噌 -噎 -噓 -噔 -噗 -噙 -噜 -噠 -噢 -噤 -器 -噩 -噪 -噬 -噱 -噴 -噶 -噸 -噹 -噻 -噼 -嚀 -嚇 -嚎 -嚏 -嚐 -嚓 -嚕 -嚟 -嚣 -嚥 -嚨 -嚮 -嚴 -嚷 -嚼 -囂 -囉 -囊 -囍 -囑 -囔 -囗 -囚 -四 -囝 -回 -囟 -因 -囡 -团 -団 -囤 -囧 -囪 -囫 -园 -困 -囱 -囲 -図 -围 -囹 -固 -国 -图 -囿 -圃 -圄 -圆 -圈 -國 -圍 -圏 -園 -圓 -圖 -團 -圜 -土 -圣 -圧 -在 -圩 -圭 -地 -圳 -场 -圻 -圾 -址 -坂 -均 -坊 -坍 -坎 -坏 -坐 -坑 -块 -坚 -坛 -坝 -坞 -坟 -坠 -坡 -坤 -坦 -坨 -坪 -坯 -坳 -坵 -坷 -垂 -垃 -垄 -型 -垒 -垚 -垛 -垠 -垢 -垣 -垦 -垩 -垫 -垭 -垮 -垵 -埂 -埃 -埋 -城 -埔 -埕 -埗 -域 -埠 -埤 -埵 -執 -埸 -培 -基 -埼 -堀 -堂 -堃 -堅 -堆 -堇 -堑 -堕 -堙 -堡 -堤 -堪 -堯 -堰 -報 -場 -堵 -堺 -堿 -塊 -塌 -塑 -塔 -塗 -塘 -塚 -塞 -塢 -塩 -填 -塬 -塭 -塵 -塾 -墀 -境 -墅 -墉 -墊 -墒 -墓 -増 -墘 -墙 -墜 -增 -墟 -墨 -墩 -墮 -墳 -墻 -墾 -壁 -壅 -壆 -壇 -壊 -壑 -壓 -壕 -壘 -壞 -壟 -壢 -壤 -壩 -士 -壬 -壮 -壯 -声 -売 -壳 -壶 -壹 -壺 -壽 -处 -备 -変 -复 -夏 -夔 -夕 -外 -夙 -多 -夜 -够 -夠 -夢 -夥 -大 -天 -太 -夫 -夭 -央 -夯 -失 -头 -夷 -夸 -夹 -夺 -夾 -奂 -奄 -奇 -奈 -奉 -奋 -奎 -奏 -奐 -契 -奔 -奕 -奖 -套 -奘 -奚 -奠 -奢 -奥 -奧 -奪 -奬 -奮 -女 -奴 -奶 -奸 -她 -好 -如 -妃 -妄 -妆 -妇 -妈 -妊 -妍 -妒 -妓 -妖 -妘 -妙 -妝 -妞 -妣 -妤 -妥 -妨 -妩 -妪 -妮 -妲 -妳 -妹 -妻 -妾 -姆 -姉 -姊 -始 -姍 -姐 -姑 -姒 -姓 -委 -姗 -姚 -姜 -姝 -姣 -姥 -姦 -姨 -姪 -姫 -姬 -姹 -姻 -姿 -威 -娃 -娄 -娅 -娆 -娇 -娉 -娑 -娓 -娘 -娛 -娜 -娟 -娠 -娣 -娥 -娩 -娱 -娲 -娴 -娶 -娼 -婀 -婁 -婆 -婉 -婊 -婕 -婚 -婢 -婦 -婧 -婪 -婭 -婴 -婵 -婶 -婷 -婺 -婿 -媒 -媚 -媛 -媞 -媧 -媲 -媳 -媽 -媾 -嫁 -嫂 -嫉 -嫌 -嫑 -嫔 -嫖 -嫘 -嫚 -嫡 -嫣 -嫦 -嫩 -嫲 -嫵 -嫻 -嬅 -嬉 -嬌 -嬗 -嬛 -嬢 -嬤 -嬪 -嬰 -嬴 -嬷 -嬸 -嬿 -孀 -孃 -子 -孑 -孔 -孕 -孖 -字 -存 -孙 -孚 -孛 -孜 -孝 -孟 -孢 -季 -孤 -学 -孩 -孪 -孫 -孬 -孰 -孱 -孳 -孵 -學 -孺 -孽 -孿 -宁 -它 -宅 -宇 -守 -安 -宋 -完 -宏 -宓 -宕 -宗 -官 -宙 -定 -宛 -宜 -宝 -实 -実 -宠 -审 -客 -宣 -室 -宥 -宦 -宪 -宫 -宮 -宰 -害 -宴 -宵 -家 -宸 -容 -宽 -宾 -宿 -寂 -寄 -寅 -密 -寇 -富 -寐 -寒 -寓 -寛 -寝 -寞 -察 -寡 -寢 -寥 -實 -寧 -寨 -審 -寫 -寬 -寮 -寰 -寵 -寶 -寸 -对 -寺 -寻 -导 -対 -寿 -封 -専 -射 -将 -將 -專 -尉 -尊 -尋 -對 -導 -小 -少 -尔 -尕 -尖 -尘 -尚 -尝 -尤 -尧 -尬 -就 -尴 -尷 -尸 -尹 -尺 -尻 -尼 -尽 -尾 -尿 -局 -屁 -层 -屄 -居 -屆 -屈 -屉 -届 -屋 -屌 -屍 -屎 -屏 -屐 -屑 -展 -屜 -属 -屠 -屡 -屢 -層 -履 -屬 -屯 -山 -屹 -屿 -岀 -岁 -岂 -岌 -岐 -岑 -岔 -岖 -岗 -岘 -岙 -岚 -岛 -岡 -岩 -岫 -岬 -岭 -岱 -岳 -岷 -岸 -峇 -峋 -峒 -峙 -峡 -峤 -峥 -峦 -峨 -峪 -峭 -峯 -峰 -峴 -島 -峻 -峽 -崁 -崂 -崆 -崇 -崎 -崑 -崔 -崖 -崗 -崙 -崛 -崧 -崩 -崭 -崴 -崽 -嵇 -嵊 -嵋 -嵌 -嵐 -嵘 -嵩 -嵬 -嵯 -嶂 -嶄 -嶇 -嶋 -嶙 -嶺 -嶼 -嶽 -巅 -巍 -巒 -巔 -巖 -川 -州 -巡 -巢 -工 -左 -巧 -巨 -巩 -巫 -差 -己 -已 -巳 -巴 -巷 -巻 -巽 -巾 -巿 -币 -市 -布 -帅 -帆 -师 -希 -帐 -帑 -帕 -帖 -帘 -帚 -帛 -帜 -帝 -帥 -带 -帧 -師 -席 -帮 -帯 -帰 -帳 -帶 -帷 -常 -帼 -帽 -幀 -幂 -幄 -幅 -幌 -幔 -幕 -幟 -幡 -幢 -幣 -幫 -干 -平 -年 -并 -幸 -幹 -幺 -幻 -幼 -幽 -幾 -广 -庁 -広 -庄 -庆 -庇 -床 -序 -庐 -库 -应 -底 -庖 -店 -庙 -庚 -府 -庞 -废 -庠 -度 -座 -庫 -庭 -庵 -庶 -康 -庸 -庹 -庾 -廁 -廂 -廃 -廈 -廉 -廊 -廓 -廖 -廚 -廝 -廟 -廠 -廢 -廣 -廬 -廳 -延 -廷 -建 -廿 -开 -弁 -异 -弃 -弄 -弈 -弊 -弋 -式 -弑 -弒 -弓 -弔 -引 -弗 -弘 -弛 -弟 -张 -弥 -弦 -弧 -弩 -弭 -弯 -弱 -張 -強 -弹 -强 -弼 -弾 -彅 -彆 -彈 -彌 -彎 -归 -当 -录 -彗 -彙 -彝 -形 -彤 -彥 -彦 -彧 -彩 -彪 -彫 -彬 -彭 -彰 -影 -彷 -役 -彻 -彼 -彿 -往 -征 -径 -待 -徇 -很 -徉 -徊 -律 -後 -徐 -徑 -徒 -従 -徕 -得 -徘 -徙 -徜 -從 -徠 -御 -徨 -復 -循 -徬 -微 -徳 -徴 -徵 -德 -徹 -徼 -徽 -心 -必 -忆 -忌 -忍 -忏 -忐 -忑 -忒 -忖 -志 -忘 -忙 -応 -忠 -忡 -忤 -忧 -忪 -快 -忱 -念 -忻 -忽 -忿 -怀 -态 -怂 -怅 -怆 -怎 -怏 -怒 -怔 -怕 -怖 -怙 -怜 -思 -怠 -怡 -急 -怦 -性 -怨 -怪 -怯 -怵 -总 -怼 -恁 -恃 -恆 -恋 -恍 -恐 -恒 -恕 -恙 -恚 -恢 -恣 -恤 -恥 -恨 -恩 -恪 -恫 -恬 -恭 -息 -恰 -恳 -恵 -恶 -恸 -恺 -恻 -恼 -恿 -悄 -悅 -悉 -悌 -悍 -悔 -悖 -悚 -悟 -悠 -患 -悦 -您 -悩 -悪 -悬 -悯 -悱 -悲 -悴 -悵 -悶 -悸 -悻 -悼 -悽 -情 -惆 -惇 -惊 -惋 -惑 -惕 -惘 -惚 -惜 -惟 -惠 -惡 -惦 -惧 -惨 -惩 -惫 -惬 -惭 -惮 -惯 -惰 -惱 -想 -惴 -惶 -惹 -惺 -愁 -愆 -愈 -愉 -愍 -意 -愕 -愚 -愛 -愜 -感 -愣 -愤 -愧 -愫 -愷 -愿 -慄 -慈 -態 -慌 -慎 -慑 -慕 -慘 -慚 -慟 -慢 -慣 -慧 -慨 -慫 -慮 -慰 -慳 -慵 -慶 -慷 -慾 -憂 -憊 -憋 -憎 -憐 -憑 -憔 -憚 -憤 -憧 -憨 -憩 -憫 -憬 -憲 -憶 -憾 -懂 -懇 -懈 -應 -懊 -懋 -懑 -懒 -懦 -懲 -懵 -懶 -懷 -懸 -懺 -懼 -懾 -懿 -戀 -戈 -戊 -戌 -戍 -戎 -戏 -成 -我 -戒 -戕 -或 -战 -戚 -戛 -戟 -戡 -戦 -截 -戬 -戮 -戰 -戲 -戳 -戴 -戶 -户 -戸 -戻 -戾 -房 -所 -扁 -扇 -扈 -扉 -手 -才 -扎 -扑 -扒 -打 -扔 -払 -托 -扛 -扣 -扦 -执 -扩 -扪 -扫 -扬 -扭 -扮 -扯 -扰 -扱 -扳 -扶 -批 -扼 -找 -承 -技 -抄 -抉 -把 -抑 -抒 -抓 -投 -抖 -抗 -折 -抚 -抛 -抜 -択 -抟 -抠 -抡 -抢 -护 -报 -抨 -披 -抬 -抱 -抵 -抹 -押 -抽 -抿 -拂 -拄 -担 -拆 -拇 -拈 -拉 -拋 -拌 -拍 -拎 -拐 -拒 -拓 -拔 -拖 -拗 -拘 -拙 -拚 -招 -拜 -拟 -拡 -拢 -拣 -拥 -拦 -拧 -拨 -择 -括 -拭 -拮 -拯 -拱 -拳 -拴 -拷 -拼 -拽 -拾 -拿 -持 -挂 -指 -挈 -按 -挎 -挑 -挖 -挙 -挚 -挛 -挝 -挞 -挟 -挠 -挡 -挣 -挤 -挥 -挨 -挪 -挫 -振 -挲 -挹 -挺 -挽 -挾 -捂 -捅 -捆 -捉 -捋 -捌 -捍 -捎 -捏 -捐 -捕 -捞 -损 -捡 -换 -捣 -捧 -捨 -捩 -据 -捱 -捲 -捶 -捷 -捺 -捻 -掀 -掂 -掃 -掇 -授 -掉 -掌 -掏 -掐 -排 -掖 -掘 -掙 -掛 -掠 -採 -探 -掣 -接 -控 -推 -掩 -措 -掬 -掰 -掲 -掳 -掴 -掷 -掸 -掺 -揀 -揃 -揄 -揆 -揉 -揍 -描 -提 -插 -揖 -揚 -換 -握 -揣 -揩 -揪 -揭 -揮 -援 -揶 -揸 -揹 -揽 -搀 -搁 -搂 -搅 -損 -搏 -搐 -搓 -搔 -搖 -搗 -搜 -搞 -搡 -搪 -搬 -搭 -搵 -搶 -携 -搽 -摀 -摁 -摄 -摆 -摇 -摈 -摊 -摒 -摔 -摘 -摞 -摟 -摧 -摩 -摯 -摳 -摸 -摹 -摺 -摻 -撂 -撃 -撅 -撇 -撈 -撐 -撑 -撒 -撓 -撕 -撚 -撞 -撤 -撥 -撩 -撫 -撬 -播 -撮 -撰 -撲 -撵 -撷 -撸 -撻 -撼 -撿 -擀 -擁 -擂 -擄 -擅 -擇 -擊 -擋 -操 -擎 -擒 -擔 -擘 -據 -擞 -擠 -擡 -擢 -擦 -擬 -擰 -擱 -擲 -擴 -擷 -擺 -擼 -擾 -攀 -攏 -攒 -攔 -攘 -攙 -攜 -攝 -攞 -攢 -攣 -攤 -攥 -攪 -攫 -攬 -支 -收 -攸 -改 -攻 -放 -政 -故 -效 -敌 -敍 -敎 -敏 -救 -敕 -敖 -敗 -敘 -教 -敛 -敝 -敞 -敢 -散 -敦 -敬 -数 -敲 -整 -敵 -敷 -數 -斂 -斃 -文 -斋 -斌 -斎 -斐 -斑 -斓 -斗 -料 -斛 -斜 -斟 -斡 -斤 -斥 -斧 -斩 -斫 -斬 -断 -斯 -新 -斷 -方 -於 -施 -旁 -旃 -旅 -旋 -旌 -旎 -族 -旖 -旗 -无 -既 -日 -旦 -旧 -旨 -早 -旬 -旭 -旮 -旱 -时 -旷 -旺 -旻 -昀 -昂 -昆 -昇 -昉 -昊 -昌 -明 -昏 -易 -昔 -昕 -昙 -星 -映 -春 -昧 -昨 -昭 -是 -昱 -昴 -昵 -昶 -昼 -显 -晁 -時 -晃 -晉 -晋 -晌 -晏 -晒 -晓 -晔 -晕 -晖 -晗 -晚 -晝 -晞 -晟 -晤 -晦 -晨 -晩 -普 -景 -晰 -晴 -晶 -晷 -智 -晾 -暂 -暄 -暇 -暈 -暉 -暌 -暐 -暑 -暖 -暗 -暝 -暢 -暧 -暨 -暫 -暮 -暱 -暴 -暸 -暹 -曄 -曆 -曇 -曉 -曖 -曙 -曜 -曝 -曠 -曦 -曬 -曰 -曲 -曳 -更 -書 -曹 -曼 -曾 -替 -最 -會 -月 -有 -朋 -服 -朐 -朔 -朕 -朗 -望 -朝 -期 -朦 -朧 -木 -未 -末 -本 -札 -朮 -术 -朱 -朴 -朵 -机 -朽 -杀 -杂 -权 -杆 -杈 -杉 -李 -杏 -材 -村 -杓 -杖 -杜 -杞 -束 -杠 -条 -来 -杨 -杭 -杯 -杰 -東 -杳 -杵 -杷 -杼 -松 -板 -极 -构 -枇 -枉 -枋 -析 -枕 -林 -枚 -果 -枝 -枢 -枣 -枪 -枫 -枭 -枯 -枰 -枱 -枳 -架 -枷 -枸 -柄 -柏 -某 -柑 -柒 -染 -柔 -柘 -柚 -柜 -柞 -柠 -柢 -查 -柩 -柬 -柯 -柱 -柳 -柴 -柵 -査 -柿 -栀 -栃 -栄 -栅 -标 -栈 -栉 -栋 -栎 -栏 -树 -栓 -栖 -栗 -校 -栩 -株 -样 -核 -根 -格 -栽 -栾 -桀 -桁 -桂 -桃 -桅 -框 -案 -桉 -桌 -桎 -桐 -桑 -桓 -桔 -桜 -桠 -桡 -桢 -档 -桥 -桦 -桧 -桨 -桩 -桶 -桿 -梁 -梅 -梆 -梏 -梓 -梗 -條 -梟 -梢 -梦 -梧 -梨 -梭 -梯 -械 -梳 -梵 -梶 -检 -棂 -棄 -棉 -棋 -棍 -棒 -棕 -棗 -棘 -棚 -棟 -棠 -棣 -棧 -森 -棱 -棲 -棵 -棹 -棺 -椁 -椅 -椋 -植 -椎 -椒 -検 -椪 -椭 -椰 -椹 -椽 -椿 -楂 -楊 -楓 -楔 -楚 -楝 -楞 -楠 -楣 -楨 -楫 -業 -楮 -極 -楷 -楸 -楹 -楼 -楽 -概 -榄 -榆 -榈 -榉 -榔 -榕 -榖 -榛 -榜 -榨 -榫 -榭 -榮 -榱 -榴 -榷 -榻 -槁 -槃 -構 -槌 -槍 -槎 -槐 -槓 -様 -槛 -槟 -槤 -槭 -槲 -槳 -槻 -槽 -槿 -樁 -樂 -樊 -樑 -樓 -標 -樞 -樟 -模 -樣 -権 -横 -樫 -樯 -樱 -樵 -樸 -樹 -樺 -樽 -樾 -橄 -橇 -橋 -橐 -橘 -橙 -機 -橡 -橢 -橫 -橱 -橹 -橼 -檀 -檄 -檎 -檐 -檔 -檗 -檜 -檢 -檬 -檯 -檳 -檸 -檻 -櫃 -櫚 -櫛 -櫥 -櫸 -櫻 -欄 -權 -欒 -欖 -欠 -次 -欢 -欣 -欧 -欲 -欸 -欺 -欽 -款 -歆 -歇 -歉 -歌 -歎 -歐 -歓 -歙 -歛 -歡 -止 -正 -此 -步 -武 -歧 -歩 -歪 -歯 -歲 -歳 -歴 -歷 -歸 -歹 -死 -歼 -殁 -殃 -殆 -殇 -殉 -殊 -残 -殒 -殓 -殖 -殘 -殞 -殡 -殤 -殭 -殯 -殲 -殴 -段 -殷 -殺 -殼 -殿 -毀 -毁 -毂 -毅 -毆 -毋 -母 -毎 -每 -毒 -毓 -比 -毕 -毗 -毘 -毙 -毛 -毡 -毫 -毯 -毽 -氈 -氏 -氐 -民 -氓 -气 -氖 -気 -氙 -氛 -氟 -氡 -氢 -氣 -氤 -氦 -氧 -氨 -氪 -氫 -氮 -氯 -氰 -氲 -水 -氷 -永 -氹 -氾 -汀 -汁 -求 -汆 -汇 -汉 -汎 -汐 -汕 -汗 -汙 -汛 -汝 -汞 -江 -池 -污 -汤 -汨 -汩 -汪 -汰 -汲 -汴 -汶 -汹 -決 -汽 -汾 -沁 -沂 -沃 -沅 -沈 -沉 -沌 -沏 -沐 -沒 -沓 -沖 -沙 -沛 -沟 -没 -沢 -沣 -沥 -沦 -沧 -沪 -沫 -沭 -沮 -沱 -河 -沸 -油 -治 -沼 -沽 -沾 -沿 -況 -泄 -泉 -泊 -泌 -泓 -法 -泗 -泛 -泞 -泠 -泡 -波 -泣 -泥 -注 -泪 -泫 -泮 -泯 -泰 -泱 -泳 -泵 -泷 -泸 -泻 -泼 -泽 -泾 -洁 -洄 -洋 -洒 -洗 -洙 -洛 -洞 -津 -洩 -洪 -洮 -洱 -洲 -洵 -洶 -洸 -洹 -活 -洼 -洽 -派 -流 -浃 -浄 -浅 -浆 -浇 -浊 -测 -济 -浏 -浑 -浒 -浓 -浔 -浙 -浚 -浜 -浣 -浦 -浩 -浪 -浬 -浮 -浯 -浴 -海 -浸 -涂 -涅 -涇 -消 -涉 -涌 -涎 -涓 -涔 -涕 -涙 -涛 -涝 -涞 -涟 -涠 -涡 -涣 -涤 -润 -涧 -涨 -涩 -涪 -涮 -涯 -液 -涵 -涸 -涼 -涿 -淀 -淄 -淅 -淆 -淇 -淋 -淌 -淑 -淒 -淖 -淘 -淙 -淚 -淞 -淡 -淤 -淦 -淨 -淩 -淪 -淫 -淬 -淮 -深 -淳 -淵 -混 -淹 -淺 -添 -淼 -清 -済 -渉 -渊 -渋 -渍 -渎 -渐 -渔 -渗 -渙 -渚 -減 -渝 -渠 -渡 -渣 -渤 -渥 -渦 -温 -測 -渭 -港 -渲 -渴 -游 -渺 -渾 -湃 -湄 -湊 -湍 -湖 -湘 -湛 -湟 -湧 -湫 -湮 -湯 -湳 -湾 -湿 -満 -溃 -溅 -溉 -溏 -源 -準 -溜 -溝 -溟 -溢 -溥 -溧 -溪 -溫 -溯 -溱 -溴 -溶 -溺 -溼 -滁 -滂 -滄 -滅 -滇 -滋 -滌 -滑 -滓 -滔 -滕 -滙 -滚 -滝 -滞 -滟 -满 -滢 -滤 -滥 -滦 -滨 -滩 -滬 -滯 -滲 -滴 -滷 -滸 -滾 -滿 -漁 -漂 -漆 -漉 -漏 -漓 -演 -漕 -漠 -漢 -漣 -漩 -漪 -漫 -漬 -漯 -漱 -漲 -漳 -漸 -漾 -漿 -潆 -潇 -潋 -潍 -潑 -潔 -潘 -潛 -潜 -潞 -潟 -潢 -潤 -潦 -潧 -潭 -潮 -潰 -潴 -潸 -潺 -潼 -澀 -澄 -澆 -澈 -澍 -澎 -澗 -澜 -澡 -澤 -澧 -澱 -澳 -澹 -激 -濁 -濂 -濃 -濑 -濒 -濕 -濘 -濛 -濟 -濠 -濡 -濤 -濫 -濬 -濮 -濯 -濱 -濺 -濾 -瀅 -瀆 -瀉 -瀋 -瀏 -瀑 -瀕 -瀘 -瀚 -瀛 -瀝 -瀞 -瀟 -瀧 -瀨 -瀬 -瀰 -瀾 -灌 -灏 -灑 -灘 -灝 -灞 -灣 -火 -灬 -灭 -灯 -灰 -灵 -灶 -灸 -灼 -災 -灾 -灿 -炀 -炁 -炅 -炉 -炊 -炎 -炒 -炔 -炕 -炖 -炙 -炜 -炫 -炬 -炭 -炮 -炯 -炳 -炷 -炸 -点 -為 -炼 -炽 -烁 -烂 -烃 -烈 -烊 -烏 -烘 -烙 -烛 -烟 -烤 -烦 -烧 -烨 -烩 -烫 -烬 -热 -烯 -烷 -烹 -烽 -焉 -焊 -焕 -焖 -焗 -焘 -焙 -焚 -焜 -無 -焦 -焯 -焰 -焱 -然 -焼 -煅 -煉 -煊 -煌 -煎 -煒 -煖 -煙 -煜 -煞 -煤 -煥 -煦 -照 -煨 -煩 -煮 -煲 -煸 -煽 -熄 -熊 -熏 -熒 -熔 -熙 -熟 -熠 -熨 -熬 -熱 -熵 -熹 -熾 -燁 -燃 -燄 -燈 -燉 -燊 -燎 -燒 -燔 -燕 -燙 -燜 -營 -燥 -燦 -燧 -燭 -燮 -燴 -燻 -燼 -燿 -爆 -爍 -爐 -爛 -爪 -爬 -爭 -爰 -爱 -爲 -爵 -父 -爷 -爸 -爹 -爺 -爻 -爽 -爾 -牆 -片 -版 -牌 -牍 -牒 -牙 -牛 -牝 -牟 -牠 -牡 -牢 -牦 -牧 -物 -牯 -牲 -牴 -牵 -特 -牺 -牽 -犀 -犁 -犄 -犊 -犍 -犒 -犢 -犧 -犬 -犯 -状 -犷 -犸 -犹 -狀 -狂 -狄 -狈 -狎 -狐 -狒 -狗 -狙 -狞 -狠 -狡 -狩 -独 -狭 -狮 -狰 -狱 -狸 -狹 -狼 -狽 -猎 -猕 -猖 -猗 -猙 -猛 -猜 -猝 -猥 -猩 -猪 -猫 -猬 -献 -猴 -猶 -猷 -猾 -猿 -獄 -獅 -獎 -獐 -獒 -獗 -獠 -獣 -獨 -獭 -獰 -獲 -獵 -獷 -獸 -獺 -獻 -獼 -獾 -玄 -率 -玉 -王 -玑 -玖 -玛 -玟 -玠 -玥 -玩 -玫 -玮 -环 -现 -玲 -玳 -玷 -玺 -玻 -珀 -珂 -珅 -珈 -珉 -珊 -珍 -珏 -珐 -珑 -珙 -珞 -珠 -珣 -珥 -珩 -珪 -班 -珮 -珲 -珺 -現 -球 -琅 -理 -琇 -琉 -琊 -琍 -琏 -琐 -琛 -琢 -琥 -琦 -琨 -琪 -琬 -琮 -琰 -琲 -琳 -琴 -琵 -琶 -琺 -琼 -瑀 -瑁 -瑄 -瑋 -瑕 -瑗 -瑙 -瑚 -瑛 -瑜 -瑞 -瑟 -瑠 -瑣 -瑤 -瑩 -瑪 -瑯 -瑰 -瑶 -瑾 -璀 -璁 -璃 -璇 -璉 -璋 -璎 -璐 -璜 -璞 -璟 -璧 -璨 -環 -璽 -璿 -瓊 -瓏 -瓒 -瓜 -瓢 -瓣 -瓤 -瓦 -瓮 -瓯 -瓴 -瓶 -瓷 -甄 -甌 -甕 -甘 -甙 -甚 -甜 -生 -產 -産 -甥 -甦 -用 -甩 -甫 -甬 -甭 -甯 -田 -由 -甲 -申 -电 -男 -甸 -町 -画 -甾 -畀 -畅 -界 -畏 -畑 -畔 -留 -畜 -畝 -畢 -略 -畦 -番 -畫 -異 -畲 -畳 -畴 -當 -畸 -畹 -畿 -疆 -疇 -疊 -疏 -疑 -疔 -疖 -疗 -疙 -疚 -疝 -疟 -疡 -疣 -疤 -疥 -疫 -疮 -疯 -疱 -疲 -疳 -疵 -疸 -疹 -疼 -疽 -疾 -痂 -病 -症 -痈 -痉 -痊 -痍 -痒 -痔 -痕 -痘 -痙 -痛 -痞 -痠 -痢 -痣 -痤 -痧 -痨 -痪 -痫 -痰 -痱 -痴 -痹 -痺 -痼 -痿 -瘀 -瘁 -瘋 -瘍 -瘓 -瘘 -瘙 -瘟 -瘠 -瘡 -瘢 -瘤 -瘦 -瘧 -瘩 -瘪 -瘫 -瘴 -瘸 -瘾 -療 -癇 -癌 -癒 -癖 -癜 -癞 -癡 -癢 -癣 -癥 -癫 -癬 -癮 -癱 -癲 -癸 -発 -登 -發 -白 -百 -皂 -的 -皆 -皇 -皈 -皋 -皎 -皑 -皓 -皖 -皙 -皚 -皮 -皰 -皱 -皴 -皺 -皿 -盂 -盃 -盅 -盆 -盈 -益 -盎 -盏 -盐 -监 -盒 -盔 -盖 -盗 -盘 -盛 -盜 -盞 -盟 -盡 -監 -盤 -盥 -盧 -盪 -目 -盯 -盱 -盲 -直 -相 -盹 -盼 -盾 -省 -眈 -眉 -看 -県 -眙 -眞 -真 -眠 -眦 -眨 -眩 -眯 -眶 -眷 -眸 -眺 -眼 -眾 -着 -睁 -睇 -睏 -睐 -睑 -睛 -睜 -睞 -睡 -睢 -督 -睥 -睦 -睨 -睪 -睫 -睬 -睹 -睽 -睾 -睿 -瞄 -瞅 -瞇 -瞋 -瞌 -瞎 -瞑 -瞒 -瞓 -瞞 -瞟 -瞠 -瞥 -瞧 -瞩 -瞪 -瞬 -瞭 -瞰 -瞳 -瞻 -瞼 -瞿 -矇 -矍 -矗 -矚 -矛 -矜 -矢 -矣 -知 -矩 -矫 -短 -矮 -矯 -石 -矶 -矽 -矾 -矿 -码 -砂 -砌 -砍 -砒 -研 -砖 -砗 -砚 -砝 -砣 -砥 -砧 -砭 -砰 -砲 -破 -砷 -砸 -砺 -砼 -砾 -础 -硅 -硐 -硒 -硕 -硝 -硫 -硬 -确 -硯 -硼 -碁 -碇 -碉 -碌 -碍 -碎 -碑 -碓 -碗 -碘 -碚 -碛 -碟 -碣 -碧 -碩 -碰 -碱 -碳 -碴 -確 -碼 -碾 -磁 -磅 -磊 -磋 -磐 -磕 -磚 -磡 -磨 -磬 -磯 -磲 -磷 -磺 -礁 -礎 -礙 -礡 -礦 -礪 -礫 -礴 -示 -礼 -社 -祀 -祁 -祂 -祇 -祈 -祉 -祎 -祐 -祕 -祖 -祗 -祚 -祛 -祜 -祝 -神 -祟 -祠 -祢 -祥 -票 -祭 -祯 -祷 -祸 -祺 -祿 -禀 -禁 -禄 -禅 -禍 -禎 -福 -禛 -禦 -禧 -禪 -禮 -禱 -禹 -禺 -离 -禽 -禾 -禿 -秀 -私 -秃 -秆 -秉 -秋 -种 -科 -秒 -秘 -租 -秣 -秤 -秦 -秧 -秩 -秭 -积 -称 -秸 -移 -秽 -稀 -稅 -程 -稍 -税 -稔 -稗 -稚 -稜 -稞 -稟 -稠 -稣 -種 -稱 -稲 -稳 -稷 -稹 -稻 -稼 -稽 -稿 -穀 -穂 -穆 -穌 -積 -穎 -穗 -穢 -穩 -穫 -穴 -究 -穷 -穹 -空 -穿 -突 -窃 -窄 -窈 -窍 -窑 -窒 -窓 -窕 -窖 -窗 -窘 -窜 -窝 -窟 -窠 -窥 -窦 -窨 -窩 -窪 -窮 -窯 -窺 -窿 -竄 -竅 -竇 -竊 -立 -竖 -站 -竜 -竞 -竟 -章 -竣 -童 -竭 -端 -競 -竹 -竺 -竽 -竿 -笃 -笆 -笈 -笋 -笏 -笑 -笔 -笙 -笛 -笞 -笠 -符 -笨 -第 -笹 -笺 -笼 -筆 -等 -筊 -筋 -筍 -筏 -筐 -筑 -筒 -答 -策 -筛 -筝 -筠 -筱 -筲 -筵 -筷 -筹 -签 -简 -箇 -箋 -箍 -箏 -箐 -箔 -箕 -算 -箝 -管 -箩 -箫 -箭 -箱 -箴 -箸 -節 -篁 -範 -篆 -篇 -築 -篑 -篓 -篙 -篝 -篠 -篡 -篤 -篩 -篪 -篮 -篱 -篷 -簇 -簌 -簍 -簡 -簦 -簧 -簪 -簫 -簷 -簸 -簽 -簾 -簿 -籁 -籃 -籌 -籍 -籐 -籟 -籠 -籤 -籬 -籮 -籲 -米 -类 -籼 -籽 -粄 -粉 -粑 -粒 -粕 -粗 -粘 -粟 -粤 -粥 -粧 -粪 -粮 -粱 -粲 -粳 -粵 -粹 -粼 -粽 -精 -粿 -糅 -糊 -糍 -糕 -糖 -糗 -糙 -糜 -糞 -糟 -糠 -糧 -糬 -糯 -糰 -糸 -系 -糾 -紀 -紂 -約 -紅 -紉 -紊 -紋 -納 -紐 -紓 -純 -紗 -紘 -紙 -級 -紛 -紜 -素 -紡 -索 -紧 -紫 -紮 -累 -細 -紳 -紹 -紺 -終 -絃 -組 -絆 -経 -結 -絕 -絞 -絡 -絢 -給 -絨 -絮 -統 -絲 -絳 -絵 -絶 -絹 -綁 -綏 -綑 -經 -継 -続 -綜 -綠 -綢 -綦 -綫 -綬 -維 -綱 -網 -綴 -綵 -綸 -綺 -綻 -綽 -綾 -綿 -緊 -緋 -総 -緑 -緒 -緘 -線 -緝 -緞 -締 -緣 -編 -緩 -緬 -緯 -練 -緹 -緻 -縁 -縄 -縈 -縛 -縝 -縣 -縫 -縮 -縱 -縴 -縷 -總 -績 -繁 -繃 -繆 -繇 -繋 -織 -繕 -繚 -繞 -繡 -繩 -繪 -繫 -繭 -繳 -繹 -繼 -繽 -纂 -續 -纍 -纏 -纓 -纔 -纖 -纜 -纠 -红 -纣 -纤 -约 -级 -纨 -纪 -纫 -纬 -纭 -纯 -纰 -纱 -纲 -纳 -纵 -纶 -纷 -纸 -纹 -纺 -纽 -纾 -线 -绀 -练 -组 -绅 -细 -织 -终 -绊 -绍 -绎 -经 -绑 -绒 -结 -绔 -绕 -绘 -给 -绚 -绛 -络 -绝 -绞 -统 -绡 -绢 -绣 -绥 -绦 -继 -绩 -绪 -绫 -续 -绮 -绯 -绰 -绳 -维 -绵 -绶 -绷 -绸 -绻 -综 -绽 -绾 -绿 -缀 -缄 -缅 -缆 -缇 -缈 -缉 -缎 -缓 -缔 -缕 -编 -缘 -缙 -缚 -缜 -缝 -缠 -缢 -缤 -缥 -缨 -缩 -缪 -缭 -缮 -缰 -缱 -缴 -缸 -缺 -缽 -罂 -罄 -罌 -罐 -网 -罔 -罕 -罗 -罚 -罡 -罢 -罩 -罪 -置 -罰 -署 -罵 -罷 -罹 -羁 -羅 -羈 -羊 -羌 -美 -羔 -羚 -羞 -羟 -羡 -羣 -群 -羥 -羧 -羨 -義 -羯 -羲 -羸 -羹 -羽 -羿 -翁 -翅 -翊 -翌 -翎 -習 -翔 -翘 -翟 -翠 -翡 -翦 -翩 -翰 -翱 -翳 -翹 -翻 -翼 -耀 -老 -考 -耄 -者 -耆 -耋 -而 -耍 -耐 -耒 -耕 -耗 -耘 -耙 -耦 -耨 -耳 -耶 -耷 -耸 -耻 -耽 -耿 -聂 -聆 -聊 -聋 -职 -聒 -联 -聖 -聘 -聚 -聞 -聪 -聯 -聰 -聲 -聳 -聴 -聶 -職 -聽 -聾 -聿 -肃 -肄 -肅 -肆 -肇 -肉 -肋 -肌 -肏 -肓 -肖 -肘 -肚 -肛 -肝 -肠 -股 -肢 -肤 -肥 -肩 -肪 -肮 -肯 -肱 -育 -肴 -肺 -肽 -肾 -肿 -胀 -胁 -胃 -胄 -胆 -背 -胍 -胎 -胖 -胚 -胛 -胜 -胝 -胞 -胡 -胤 -胥 -胧 -胫 -胭 -胯 -胰 -胱 -胳 -胴 -胶 -胸 -胺 -能 -脂 -脅 -脆 -脇 -脈 -脉 -脊 -脍 -脏 -脐 -脑 -脓 -脖 -脘 -脚 -脛 -脣 -脩 -脫 -脯 -脱 -脲 -脳 -脸 -脹 -脾 -腆 -腈 -腊 -腋 -腌 -腎 -腐 -腑 -腓 -腔 -腕 -腥 -腦 -腩 -腫 -腭 -腮 -腰 -腱 -腳 -腴 -腸 -腹 -腺 -腻 -腼 -腾 -腿 -膀 -膈 -膊 -膏 -膑 -膘 -膚 -膛 -膜 -膝 -膠 -膦 -膨 -膩 -膳 -膺 -膻 -膽 -膾 -膿 -臀 -臂 -臃 -臆 -臉 -臊 -臍 -臓 -臘 -臟 -臣 -臥 -臧 -臨 -自 -臬 -臭 -至 -致 -臺 -臻 -臼 -臾 -舀 -舂 -舅 -舆 -與 -興 -舉 -舊 -舌 -舍 -舎 -舐 -舒 -舔 -舖 -舗 -舛 -舜 -舞 -舟 -航 -舫 -般 -舰 -舱 -舵 -舶 -舷 -舸 -船 -舺 -舾 -艇 -艋 -艘 -艙 -艦 -艮 -良 -艰 -艱 -色 -艳 -艷 -艹 -艺 -艾 -节 -芃 -芈 -芊 -芋 -芍 -芎 -芒 -芙 -芜 -芝 -芡 -芥 -芦 -芩 -芪 -芫 -芬 -芭 -芮 -芯 -花 -芳 -芷 -芸 -芹 -芻 -芽 -芾 -苁 -苄 -苇 -苋 -苍 -苏 -苑 -苒 -苓 -苔 -苕 -苗 -苛 -苜 -苞 -苟 -苡 -苣 -若 -苦 -苫 -苯 -英 -苷 -苹 -苻 -茁 -茂 -范 -茄 -茅 -茉 -茎 -茏 -茗 -茜 -茧 -茨 -茫 -茬 -茭 -茯 -茱 -茲 -茴 -茵 -茶 -茸 -茹 -茼 -荀 -荃 -荆 -草 -荊 -荏 -荐 -荒 -荔 -荖 -荘 -荚 -荞 -荟 -荠 -荡 -荣 -荤 -荥 -荧 -荨 -荪 -荫 -药 -荳 -荷 -荸 -荻 -荼 -荽 -莅 -莆 -莉 -莊 -莎 -莒 -莓 -莖 -莘 -莞 -莠 -莢 -莧 -莪 -莫 -莱 -莲 -莴 -获 -莹 -莺 -莽 -莿 -菀 -菁 -菅 -菇 -菈 -菊 -菌 -菏 -菓 -菖 -菘 -菜 -菟 -菠 -菡 -菩 -華 -菱 -菲 -菸 -菽 -萁 -萃 -萄 -萊 -萋 -萌 -萍 -萎 -萘 -萝 -萤 -营 -萦 -萧 -萨 -萩 -萬 -萱 -萵 -萸 -萼 -落 -葆 -葉 -著 -葚 -葛 -葡 -董 -葦 -葩 -葫 -葬 -葭 -葯 -葱 -葳 -葵 -葷 -葺 -蒂 -蒋 -蒐 -蒔 -蒙 -蒜 -蒞 -蒟 -蒡 -蒨 -蒲 -蒸 -蒹 -蒻 -蒼 -蒿 -蓁 -蓄 -蓆 -蓉 -蓋 -蓑 -蓓 -蓖 -蓝 -蓟 -蓦 -蓬 -蓮 -蓼 -蓿 -蔑 -蔓 -蔔 -蔗 -蔘 -蔚 -蔡 -蔣 -蔥 -蔫 -蔬 -蔭 -蔵 -蔷 -蔺 -蔻 -蔼 -蔽 -蕁 -蕃 -蕈 -蕉 -蕊 -蕎 -蕙 -蕤 -蕨 -蕩 -蕪 -蕭 -蕲 -蕴 -蕻 -蕾 -薄 -薅 -薇 -薈 -薊 -薏 -薑 -薔 -薙 -薛 -薦 -薨 -薩 -薪 -薬 -薯 -薰 -薹 -藉 -藍 -藏 -藐 -藓 -藕 -藜 -藝 -藤 -藥 -藩 -藹 -藻 -藿 -蘆 -蘇 -蘊 -蘋 -蘑 -蘚 -蘭 -蘸 -蘼 -蘿 -虎 -虏 -虐 -虑 -虔 -處 -虚 -虛 -虜 -虞 -號 -虢 -虧 -虫 -虬 -虱 -虹 -虻 -虽 -虾 -蚀 -蚁 -蚂 -蚊 -蚌 -蚓 -蚕 -蚜 -蚝 -蚣 -蚤 -蚩 -蚪 -蚯 -蚱 -蚵 -蛀 -蛆 -蛇 -蛊 -蛋 -蛎 -蛐 -蛔 -蛙 -蛛 -蛟 -蛤 -蛭 -蛮 -蛰 -蛳 -蛹 -蛻 -蛾 -蜀 -蜂 -蜃 -蜆 -蜇 -蜈 -蜊 -蜍 -蜒 -蜓 -蜕 -蜗 -蜘 -蜚 -蜜 -蜡 -蜢 -蜥 -蜱 -蜴 -蜷 -蜻 -蜿 -蝇 -蝈 -蝉 -蝌 -蝎 -蝕 -蝗 -蝙 -蝟 -蝠 -蝦 -蝨 -蝴 -蝶 -蝸 -蝼 -螂 -螃 -融 -螞 -螢 -螨 -螯 -螳 -螺 -蟀 -蟄 -蟆 -蟋 -蟎 -蟑 -蟒 -蟠 -蟬 -蟲 -蟹 -蟻 -蟾 -蠅 -蠍 -蠔 -蠕 -蠛 -蠟 -蠡 -蠢 -蠣 -蠱 -蠶 -蠹 -蠻 -血 -衄 -衅 -衆 -行 -衍 -術 -衔 -街 -衙 -衛 -衝 -衞 -衡 -衢 -衣 -补 -表 -衩 -衫 -衬 -衮 -衰 -衲 -衷 -衹 -衾 -衿 -袁 -袂 -袄 -袅 -袈 -袋 -袍 -袒 -袖 -袜 -袞 -袤 -袪 -被 -袭 -袱 -裁 -裂 -装 -裆 -裊 -裏 -裔 -裕 -裘 -裙 -補 -裝 -裟 -裡 -裤 -裨 -裱 -裳 -裴 -裸 -裹 -製 -裾 -褂 -複 -褐 -褒 -褓 -褔 -褚 -褥 -褪 -褫 -褲 -褶 -褻 -襁 -襄 -襟 -襠 -襪 -襬 -襯 -襲 -西 -要 -覃 -覆 -覇 -見 -規 -覓 -視 -覚 -覦 -覧 -親 -覬 -観 -覷 -覺 -覽 -觀 -见 -观 -规 -觅 -视 -览 -觉 -觊 -觎 -觐 -觑 -角 -觞 -解 -觥 -触 -觸 -言 -訂 -計 -訊 -討 -訓 -訕 -訖 -託 -記 -訛 -訝 -訟 -訣 -訥 -訪 -設 -許 -訳 -訴 -訶 -診 -註 -証 -詆 -詐 -詔 -評 -詛 -詞 -詠 -詡 -詢 -詣 -試 -詩 -詫 -詬 -詭 -詮 -詰 -話 -該 -詳 -詹 -詼 -誅 -誇 -誉 -誌 -認 -誓 -誕 -誘 -語 -誠 -誡 -誣 -誤 -誥 -誦 -誨 -說 -説 -読 -誰 -課 -誹 -誼 -調 -諄 -談 -請 -諏 -諒 -論 -諗 -諜 -諡 -諦 -諧 -諫 -諭 -諮 -諱 -諳 -諷 -諸 -諺 -諾 -謀 -謁 -謂 -謄 -謊 -謎 -謐 -謔 -謗 -謙 -講 -謝 -謠 -謨 -謬 -謹 -謾 -譁 -證 -譎 -譏 -識 -譙 -譚 -譜 -警 -譬 -譯 -議 -譲 -譴 -護 -譽 -讀 -變 -讓 -讚 -讞 -计 -订 -认 -讥 -讧 -讨 -让 -讪 -讫 -训 -议 -讯 -记 -讲 -讳 -讴 -讶 -讷 -许 -讹 -论 -讼 -讽 -设 -访 -诀 -证 -诃 -评 -诅 -识 -诈 -诉 -诊 -诋 -词 -诏 -译 -试 -诗 -诘 -诙 -诚 -诛 -话 -诞 -诟 -诠 -诡 -询 -诣 -诤 -该 -详 -诧 -诩 -诫 -诬 -语 -误 -诰 -诱 -诲 -说 -诵 -诶 -请 -诸 -诺 -读 -诽 -课 -诿 -谀 -谁 -调 -谄 -谅 -谆 -谈 -谊 -谋 -谌 -谍 -谎 -谏 -谐 -谑 -谒 -谓 -谔 -谕 -谗 -谘 -谙 -谚 -谛 -谜 -谟 -谢 -谣 -谤 -谥 -谦 -谧 -谨 -谩 -谪 -谬 -谭 -谯 -谱 -谲 -谴 -谶 -谷 -豁 -豆 -豇 -豈 -豉 -豊 -豌 -豎 -豐 -豔 -豚 -象 -豢 -豪 -豫 -豬 -豹 -豺 -貂 -貅 -貌 -貓 -貔 -貘 -貝 -貞 -負 -財 -貢 -貧 -貨 -販 -貪 -貫 -責 -貯 -貰 -貳 -貴 -貶 -買 -貸 -費 -貼 -貽 -貿 -賀 -賁 -賂 -賃 -賄 -資 -賈 -賊 -賑 -賓 -賜 -賞 -賠 -賡 -賢 -賣 -賤 -賦 -質 -賬 -賭 -賴 -賺 -購 -賽 -贅 -贈 -贊 -贍 -贏 -贓 -贖 -贛 -贝 -贞 -负 -贡 -财 -责 -贤 -败 -账 -货 -质 -贩 -贪 -贫 -贬 -购 -贮 -贯 -贰 -贱 -贲 -贴 -贵 -贷 -贸 -费 -贺 -贻 -贼 -贾 -贿 -赁 -赂 -赃 -资 -赅 -赈 -赊 -赋 -赌 -赎 -赏 -赐 -赓 -赔 -赖 -赘 -赚 -赛 -赝 -赞 -赠 -赡 -赢 -赣 -赤 -赦 -赧 -赫 -赭 -走 -赳 -赴 -赵 -赶 -起 -趁 -超 -越 -趋 -趕 -趙 -趟 -趣 -趨 -足 -趴 -趵 -趸 -趺 -趾 -跃 -跄 -跆 -跋 -跌 -跎 -跑 -跖 -跚 -跛 -距 -跟 -跡 -跤 -跨 -跩 -跪 -路 -跳 -践 -跷 -跹 -跺 -跻 -踉 -踊 -踌 -踏 -踐 -踝 -踞 -踟 -踢 -踩 -踪 -踮 -踱 -踴 -踵 -踹 -蹂 -蹄 -蹇 -蹈 -蹉 -蹊 -蹋 -蹑 -蹒 -蹙 -蹟 -蹣 -蹤 -蹦 -蹩 -蹬 -蹭 -蹲 -蹴 -蹶 -蹺 -蹼 -蹿 -躁 -躇 -躉 -躊 -躋 -躍 -躏 -躪 -身 -躬 -躯 -躲 -躺 -軀 -車 -軋 -軌 -軍 -軒 -軟 -転 -軸 -軼 -軽 -軾 -較 -載 -輒 -輓 -輔 -輕 -輛 -輝 -輟 -輩 -輪 -輯 -輸 -輻 -輾 -輿 -轄 -轅 -轆 -轉 -轍 -轎 -轟 -车 -轧 -轨 -轩 -转 -轭 -轮 -软 -轰 -轲 -轴 -轶 -轻 -轼 -载 -轿 -较 -辄 -辅 -辆 -辇 -辈 -辉 -辊 -辍 -辐 -辑 -输 -辕 -辖 -辗 -辘 -辙 -辛 -辜 -辞 -辟 -辣 -辦 -辨 -辩 -辫 -辭 -辮 -辯 -辰 -辱 -農 -边 -辺 -辻 -込 -辽 -达 -迁 -迂 -迄 -迅 -过 -迈 -迎 -运 -近 -返 -还 -这 -进 -远 -违 -连 -迟 -迢 -迤 -迥 -迦 -迩 -迪 -迫 -迭 -述 -迴 -迷 -迸 -迹 -迺 -追 -退 -送 -适 -逃 -逅 -逆 -选 -逊 -逍 -透 -逐 -递 -途 -逕 -逗 -這 -通 -逛 -逝 -逞 -速 -造 -逢 -連 -逮 -週 -進 -逵 -逶 -逸 -逻 -逼 -逾 -遁 -遂 -遅 -遇 -遊 -運 -遍 -過 -遏 -遐 -遑 -遒 -道 -達 -違 -遗 -遙 -遛 -遜 -遞 -遠 -遢 -遣 -遥 -遨 -適 -遭 -遮 -遲 -遴 -遵 -遶 -遷 -選 -遺 -遼 -遽 -避 -邀 -邁 -邂 -邃 -還 -邇 -邈 -邊 -邋 -邏 -邑 -邓 -邕 -邛 -邝 -邢 -那 -邦 -邨 -邪 -邬 -邮 -邯 -邰 -邱 -邳 -邵 -邸 -邹 -邺 -邻 -郁 -郅 -郊 -郎 -郑 -郜 -郝 -郡 -郢 -郤 -郦 -郧 -部 -郫 -郭 -郴 -郵 -郷 -郸 -都 -鄂 -鄉 -鄒 -鄔 -鄙 -鄞 -鄢 -鄧 -鄭 -鄰 -鄱 -鄲 -鄺 -酉 -酊 -酋 -酌 -配 -酐 -酒 -酗 -酚 -酝 -酢 -酣 -酥 -酩 -酪 -酬 -酮 -酯 -酰 -酱 -酵 -酶 -酷 -酸 -酿 -醃 -醇 -醉 -醋 -醍 -醐 -醒 -醚 -醛 -醜 -醞 -醣 -醪 -醫 -醬 -醮 -醯 -醴 -醺 -釀 -釁 -采 -釉 -释 -釋 -里 -重 -野 -量 -釐 -金 -釗 -釘 -釜 -針 -釣 -釦 -釧 -釵 -鈀 -鈉 -鈍 -鈎 -鈔 -鈕 -鈞 -鈣 -鈦 -鈪 -鈴 -鈺 -鈾 -鉀 -鉄 -鉅 -鉉 -鉑 -鉗 -鉚 -鉛 -鉤 -鉴 -鉻 -銀 -銃 -銅 -銑 -銓 -銖 -銘 -銜 -銬 -銭 -銮 -銳 -銷 -銹 -鋁 -鋅 -鋒 -鋤 -鋪 -鋰 -鋸 -鋼 -錄 -錐 -錘 -錚 -錠 -錢 -錦 -錨 -錫 -錮 -錯 -録 -錳 -錶 -鍊 -鍋 -鍍 -鍛 -鍥 -鍰 -鍵 -鍺 -鍾 -鎂 -鎊 -鎌 -鎏 -鎔 -鎖 -鎗 -鎚 -鎧 -鎬 -鎮 -鎳 -鏈 -鏖 -鏗 -鏘 -鏞 -鏟 -鏡 -鏢 -鏤 -鏽 -鐘 -鐮 -鐲 -鐳 -鐵 -鐸 -鐺 -鑄 -鑊 -鑑 -鑒 -鑣 -鑫 -鑰 -鑲 -鑼 -鑽 -鑾 -鑿 -针 -钉 -钊 -钎 -钏 -钒 -钓 -钗 -钙 -钛 -钜 -钝 -钞 -钟 -钠 -钡 -钢 -钣 -钤 -钥 -钦 -钧 -钨 -钩 -钮 -钯 -钰 -钱 -钳 -钴 -钵 -钺 -钻 -钼 -钾 -钿 -铀 -铁 -铂 -铃 -铄 -铅 -铆 -铉 -铎 -铐 -铛 -铜 -铝 -铠 -铡 -铢 -铣 -铤 -铨 -铩 -铬 -铭 -铮 -铰 -铲 -铵 -银 -铸 -铺 -链 -铿 -销 -锁 -锂 -锄 -锅 -锆 -锈 -锉 -锋 -锌 -锏 -锐 -锑 -错 -锚 -锟 -锡 -锢 -锣 -锤 -锥 -锦 -锭 -键 -锯 -锰 -锲 -锵 -锹 -锺 -锻 -镀 -镁 -镂 -镇 -镉 -镌 -镍 -镐 -镑 -镕 -镖 -镗 -镛 -镜 -镣 -镭 -镯 -镰 -镳 -镶 -長 -长 -門 -閃 -閉 -開 -閎 -閏 -閑 -閒 -間 -閔 -閘 -閡 -関 -閣 -閥 -閨 -閩 -閱 -閲 -閹 -閻 -閾 -闆 -闇 -闊 -闌 -闍 -闔 -闕 -闖 -闘 -關 -闡 -闢 -门 -闪 -闫 -闭 -问 -闯 -闰 -闲 -间 -闵 -闷 -闸 -闹 -闺 -闻 -闽 -闾 -阀 -阁 -阂 -阅 -阆 -阇 -阈 -阉 -阎 -阐 -阑 -阔 -阕 -阖 -阙 -阚 -阜 -队 -阡 -阪 -阮 -阱 -防 -阳 -阴 -阵 -阶 -阻 -阿 -陀 -陂 -附 -际 -陆 -陇 -陈 -陋 -陌 -降 -限 -陕 -陛 -陝 -陞 -陟 -陡 -院 -陣 -除 -陨 -险 -陪 -陰 -陲 -陳 -陵 -陶 -陷 -陸 -険 -陽 -隅 -隆 -隈 -隊 -隋 -隍 -階 -随 -隐 -隔 -隕 -隘 -隙 -際 -障 -隠 -隣 -隧 -隨 -險 -隱 -隴 -隶 -隸 -隻 -隼 -隽 -难 -雀 -雁 -雄 -雅 -集 -雇 -雉 -雋 -雌 -雍 -雎 -雏 -雑 -雒 -雕 -雖 -雙 -雛 -雜 -雞 -離 -難 -雨 -雪 -雯 -雰 -雲 -雳 -零 -雷 -雹 -電 -雾 -需 -霁 -霄 -霆 -震 -霈 -霉 -霊 -霍 -霎 -霏 -霑 -霓 -霖 -霜 -霞 -霧 -霭 -霰 -露 -霸 -霹 -霽 -霾 -靂 -靄 -靈 -青 -靓 -靖 -静 -靚 -靛 -靜 -非 -靠 -靡 -面 -靥 -靦 -革 -靳 -靴 -靶 -靼 -鞅 -鞋 -鞍 -鞏 -鞑 -鞘 -鞠 -鞣 -鞦 -鞭 -韆 -韋 -韌 -韓 -韜 -韦 -韧 -韩 -韬 -韭 -音 -韵 -韶 -韻 -響 -頁 -頂 -頃 -項 -順 -須 -頌 -預 -頑 -頒 -頓 -頗 -領 -頜 -頡 -頤 -頫 -頭 -頰 -頷 -頸 -頹 -頻 -頼 -顆 -題 -額 -顎 -顏 -顔 -願 -顛 -類 -顧 -顫 -顯 -顱 -顴 -页 -顶 -顷 -项 -顺 -须 -顼 -顽 -顾 -顿 -颁 -颂 -预 -颅 -领 -颇 -颈 -颉 -颊 -颌 -颍 -颐 -频 -颓 -颔 -颖 -颗 -题 -颚 -颛 -颜 -额 -颞 -颠 -颡 -颢 -颤 -颦 -颧 -風 -颯 -颱 -颳 -颶 -颼 -飄 -飆 -风 -飒 -飓 -飕 -飘 -飙 -飚 -飛 -飞 -食 -飢 -飨 -飩 -飪 -飯 -飲 -飼 -飽 -飾 -餃 -餅 -餉 -養 -餌 -餐 -餒 -餓 -餘 -餚 -餛 -餞 -餡 -館 -餮 -餵 -餾 -饅 -饈 -饋 -饌 -饍 -饑 -饒 -饕 -饗 -饞 -饥 -饨 -饪 -饬 -饭 -饮 -饯 -饰 -饱 -饲 -饴 -饵 -饶 -饷 -饺 -饼 -饽 -饿 -馀 -馁 -馄 -馅 -馆 -馈 -馋 -馍 -馏 -馒 -馔 -首 -馗 -香 -馥 -馨 -馬 -馭 -馮 -馳 -馴 -駁 -駄 -駅 -駆 -駐 -駒 -駕 -駛 -駝 -駭 -駱 -駿 -騁 -騎 -騏 -験 -騙 -騨 -騰 -騷 -驀 -驅 -驊 -驍 -驒 -驕 -驗 -驚 -驛 -驟 -驢 -驥 -马 -驭 -驮 -驯 -驰 -驱 -驳 -驴 -驶 -驷 -驸 -驹 -驻 -驼 -驾 -驿 -骁 -骂 -骄 -骅 -骆 -骇 -骈 -骊 -骋 -验 -骏 -骐 -骑 -骗 -骚 -骛 -骜 -骞 -骠 -骡 -骤 -骥 -骧 -骨 -骯 -骰 -骶 -骷 -骸 -骼 -髂 -髅 -髋 -髏 -髒 -髓 -體 -髖 -高 -髦 -髪 -髮 -髯 -髻 -鬃 -鬆 -鬍 -鬓 -鬚 -鬟 -鬢 -鬣 -鬥 -鬧 -鬱 -鬼 -魁 -魂 -魄 -魅 -魇 -魍 -魏 -魔 -魘 -魚 -魯 -魷 -鮑 -鮨 -鮪 -鮭 -鮮 -鯉 -鯊 -鯖 -鯛 -鯨 -鯰 -鯽 -鰍 -鰓 -鰭 -鰲 -鰻 -鰾 -鱈 -鱉 -鱔 -鱗 -鱷 -鱸 -鱼 -鱿 -鲁 -鲈 -鲍 -鲑 -鲛 -鲜 -鲟 -鲢 -鲤 -鲨 -鲫 -鲱 -鲲 -鲶 -鲷 -鲸 -鳃 -鳄 -鳅 -鳌 -鳍 -鳕 -鳖 -鳗 -鳝 -鳞 -鳥 -鳩 -鳳 -鳴 -鳶 -鴉 -鴕 -鴛 -鴦 -鴨 -鴻 -鴿 -鵑 -鵜 -鵝 -鵡 -鵬 -鵰 -鵲 -鶘 -鶩 -鶯 -鶴 -鷗 -鷲 -鷹 -鷺 -鸚 -鸞 -鸟 -鸠 -鸡 -鸢 -鸣 -鸥 -鸦 -鸨 -鸪 -鸭 -鸯 -鸳 -鸵 -鸽 -鸾 -鸿 -鹂 -鹃 -鹄 -鹅 -鹈 -鹉 -鹊 -鹌 -鹏 -鹑 -鹕 -鹘 -鹜 -鹞 -鹤 -鹦 -鹧 -鹫 -鹭 -鹰 -鹳 -鹵 -鹹 -鹼 -鹽 -鹿 -麂 -麋 -麒 -麓 -麗 -麝 -麟 -麥 -麦 -麩 -麴 -麵 -麸 -麺 -麻 -麼 -麽 -麾 -黃 -黄 -黍 -黎 -黏 -黑 -黒 -黔 -默 -黛 -黜 -黝 -點 -黠 -黨 -黯 -黴 -鼋 -鼎 -鼐 -鼓 -鼠 -鼬 -鼹 -鼻 -鼾 -齁 -齊 -齋 -齐 -齒 -齡 -齢 -齣 -齦 -齿 -龄 -龅 -龈 -龊 -龋 -龌 -龍 -龐 -龔 -龕 -龙 -龚 -龛 -龜 -龟 -︰ -︱ -︶ -︿ -﹁ -﹂ -﹍ -﹏ -﹐ -﹑ -﹒ -﹔ -﹕ -﹖ -﹗ -﹙ -﹚ -﹝ -﹞ -﹡ -﹣ -！ -＂ -＃ -＄ -％ -＆ -＇ -（ -） -＊ -＋ -， -－ -． -／ -０ -１ -２ -３ -４ -５ -６ -７ -８ -９ -： -； -＜ -＝ -＞ -？ -＠ -［ -＼ -］ -＾ -＿ -｀ -ａ -ｂ -ｃ -ｄ -ｅ -ｆ -ｇ -ｈ -ｉ -ｊ -ｋ -ｌ -ｍ -ｎ -ｏ -ｐ -ｑ -ｒ -ｓ -ｔ -ｕ -ｖ -ｗ -ｘ -ｙ -ｚ -｛ -｜ -｝ -～ -｡ -｢ -｣ -､ -･ -ｯ -ｰ -ｲ -ｸ -ｼ -ｽ -ﾄ -ﾉ -ﾌ -ﾗ -ﾙ -ﾝ -ﾞ -ﾟ -￣ -￥ -👍 -🔥 -😂 -😎 -... -yam -10 -2017 -12 -11 -2016 -20 -30 -15 -06 -lofter -##s -2015 -by -16 -14 -18 -13 -24 -17 -2014 -21 -##0 -22 -19 -25 -23 -com -100 -00 -05 -2013 -##a -03 -09 -08 -28 -##2 -50 -01 -04 -##1 -27 -02 -2012 -##3 -26 -##e -07 -##8 -##5 -##6 -##4 -##9 -##7 -29 -2011 -40 -##t -2010 -##o -##d -##i -2009 -##n -app -www -the -##m -31 -##c -##l -##y -##r -##g -2008 -60 -http -200 -qq -##p -80 -##f -google -pixnet -90 -cookies -tripadvisor -500 -##er -##k -35 -##h -facebook -2007 -2000 -70 -##b -of -##x -##u -45 -300 -iphone -32 -1000 -2006 -48 -ip -36 -in -38 -3d -##w -##ing -55 -ctrip -##on -##v -33 -##の -to -34 -400 -id -2005 -it -37 -windows -llc -top -99 -42 -39 -000 -led -at -##an -41 -51 -52 -46 -49 -43 -53 -44 -##z -android -58 -and -59 -2004 -56 -vr -##か -5000 -2003 -47 -blogthis -twitter -54 -##le -150 -ok -2018 -57 -75 -cn -no -ios -##in -##mm -##00 -800 -on -te -3000 -65 -2001 -360 -95 -ig -lv -120 -##ng -##を -##us -##に -pc -てす -── -600 -##te -85 -2002 -88 -##ed -html -ncc -wifi -email -64 -blog -is -##10 -##て -mail -online -##al -dvd -##ic -studio -##は -##℃ -##ia -##と -line -vip -72 -##q -98 -##ce -##en -for -##is -##ra -##es -##j -usb -net -cp -1999 -asia -4g -##cm -diy -new -3c -##お -ta -66 -language -vs -apple -tw -86 -web -##ne -ipad -62 -you -##re -101 -68 -##tion -ps -de -bt -pony -atm -##2017 -1998 -67 -##ch -ceo -##or -go -##na -av -pro -cafe -96 -pinterest -97 -63 -pixstyleme3c -##ta -more -said -##2016 -1997 -mp3 -700 -##ll -nba -jun -##20 -92 -tv -1995 -pm -61 -76 -nbsp -250 -##ie -linux -##ma -cd -110 -hd -##17 -78 -##ion -77 -6000 -am -##th -##st -94 -##se -##et -69 -180 -gdp -my -105 -81 -abc -89 -flash -79 -one -93 -1990 -1996 -##ck -gps -##も -##ly -web885 -106 -2020 -91 -##ge -4000 -1500 -xd -boss -isbn -1994 -org -##ry -me -love -##11 -0fork -73 -##12 -3g -##ter -##ar -71 -82 -##la -hotel -130 -1970 -pk -83 -87 -140 -ie -##os -##30 -##el -74 -##50 -seo -cpu -##ml -p2p -84 -may -##る -sun -tue -internet -cc -posted -youtube -##at -##ン -##man -ii -##ル -##15 -abs -nt -pdf -yahoo -ago -1980 -##it -news -mac -104 -##てす -##me -##り -java -1992 -spa -##de -##nt -hk -all -plus -la -1993 -##mb -##16 -##ve -west -##da -160 -air -##い -##ps -から -##to -1989 -logo -htc -php -https -fi -momo -##son -sat -##ke -##80 -ebd -suv -wi -day -apk -##88 -##um -mv -galaxy -wiki -or -brake -##ス -1200 -する -this -1991 -mon -##こ -❤2017 -po -##ない -javascript -life -home -june -##ss -system -900 -##ー -##０ -pp -1988 -world -fb -4k -br -##as -ic -ai -leonardo -safari -##60 -live -free -xx -wed -win7 -kiehl -##co -lg -o2o -##go -us -235 -1949 -mm -しい -vfm -kanye -##90 -##2015 -##id -jr -##ey -123 -rss -##sa -##ro -##am -##no -thu -fri -350 -##sh -##ki -103 -comments -name -##のて -##pe -##ine -max -1987 -8000 -uber -##mi -##ton -wordpress -office -1986 -1985 -##ment -107 -bd -win10 -##ld -##li -gmail -bb -dior -##rs -##ri -##rd -##ます -up -cad -##® -dr -して -read -##21 -をお -##io -##99 -url -1984 -pvc -paypal -show -policy -##40 -##ty -##18 -with -##★ -##01 -txt -102 -##ba -dna -from -post -mini -ar -taiwan -john -##ga -privacy -agoda -##13 -##ny -word -##24 -##22 -##by -##ur -##hz -1982 -##ang -265 -cookie -netscape -108 -##ka -##～ -##ad -house -share -note -ibm -code -hello -nike -sim -survey -##016 -1979 -1950 -wikia -##32 -##017 -5g -cbc -##tor -##kg -1983 -##rt -##14 -campaign -store -2500 -os -##ct -##ts -##° -170 -api -##ns -365 -excel -##な -##ao -##ら -##し -～～ -##nd -university -163 -には -518 -##70 -##ya -##il -##25 -pierre -ipo -0020 -897 -##23 -hotels -##ian -のお -125 -years -6606 -##ers -##26 -high -##day -time -##ay -bug -##line -##く -##す -##be -xp -talk2yam -yamservice -10000 -coco -##dy -sony -##ies -1978 -microsoft -david -people -##ha -1960 -instagram -intel -その -##ot -iso -1981 -##va -115 -##mo -##land -xxx -man -co -ltxsw -##ation -baby -220 -##pa -##ol -1945 -7000 -tag -450 -##ue -msn -##31 -oppo -##ト -##ca -control -##om -st -chrome -##ure -##ん -be -##き -lol -##19 -した -##bo -240 -lady -##100 -##way -##から -4600 -##ko -##do -##un -4s -corporation -168 -##ni -herme -##28 -ｃｐ -978 -##up -##06 -ui -##ds -ppt -admin -three -します -bbc -re -128 -##48 -ca -##015 -##35 -hp -##ee -tpp -##た -##ive -×× -root -##cc -##ました -##ble -##ity -adobe -park -114 -et -oled -city -##ex -##ler -##ap -china -##book -20000 -view -##ice -global -##km -your -hong -##mg -out -##ms -ng -ebay -##29 -menu -ubuntu -##cy -rom -##view -open -ktv -do -server -##lo -if -english -##ね -##５ -##oo -1600 -##02 -step1 -kong -club -135 -july -inc -1976 -mr -hi -##net -touch -##ls -##ii -michael -lcd -##05 -##33 -phone -james -step2 -1300 -ios9 -##box -dc -##２ -##ley -samsung -111 -280 -pokemon -css -##ent -##les -いいえ -##１ -s8 -atom -play -bmw -##said -sa -etf -ctrl -♥yoyo♥ -##55 -2025 -##2014 -##66 -adidas -amazon -1958 -##ber -##ner -visa -##77 -##der -1800 -connectivity -##hi -firefox -109 -118 -hr -so -style -mark -pop -ol -skip -1975 -as -##27 -##ir -##61 -190 -mba -##う -##ai -le -##ver -1900 -cafe2017 -lte -super -113 -129 -##ron -amd -like -##☆ -are -##ster -we -##sk -paul -data -international -##ft -longchamp -ssd -good -##ート -##ti -reply -##my -↓↓↓ -apr -star -##ker -source -136 -js -112 -get -force -photo -##one -126 -##2013 -##ow -link -bbs -1972 -goods -##lin -python -119 -##ip -game -##ics -##ません -blue -##● -520 -##45 -page -itunes -##03 -1955 -260 -1968 -gt -gif -618 -##ff -##47 -group -くたさい -about -bar -ganji -##nce -music -lee -not -1977 -1971 -1973 -##per -an -faq -comment -##って -days -##ock -116 -##bs -1974 -1969 -v1 -player -1956 -xbox -sql -fm -f1 -139 -##ah -210 -##lv -##mp -##000 -melody -1957 -##３ -550 -17life -199 -1966 -xml -market -##au -##71 -999 -##04 -what -gl -##95 -##age -tips -##68 -book -##ting -mysql -can -1959 -230 -##ung -wonderland -watch -10℃ -##ction -9000 -mar -mobile -1946 -1962 -article -##db -part -▲top -party -って -1967 -1964 -1948 -##07 -##ore -##op -この -dj -##78 -##38 -010 -main -225 -1965 -##ong -art -320 -ad -134 -020 -##73 -117 -pm2 -japan -228 -##08 -ts -1963 -##ica -der -sm -##36 -2019 -##wa -ct -##７ -##や -##64 -1937 -homemesh -search -##85 -##れは -##tv -##di -macbook -##９ -##くたさい -service -##♥ -type -った -750 -##ier -##si -##75 -##います -##ok -best -##ット -goris -lock -##った -cf -3m -big -##ut -ftp -carol -##vi -１０ -1961 -happy -sd -##ac -122 -anti -pe -cnn -iii -1920 -138 -##ラ -1940 -esp -jan -tags -##98 -##51 -august -vol -##86 -154 -##™ -##fs -##れ -##sion -design -ac -##ム -press -jordan -ppp -that -key -check -##６ -##tt -##㎡ -1080p -##lt -power -##42 -1952 -##bc -vivi -##ック -he -133 -121 -jpg -##rry -201 -175 -3500 -1947 -nb -##ted -##rn -しています -1954 -usd -##t00 -master -##ンク -001 -model -##58 -al -##09 -1953 -##34 -ram -goo -ても -##ui -127 -1930 -red -##ary -rpg -item -##pm -##41 -270 -##za -project -##2012 -hot -td -blogabstract -##ger -##62 -650 -##44 -gr2 -##します -##ｍ -black -electronic -nfc -year -asus -また -html5 -cindy -##hd -m3 -132 -esc -##od -booking -##53 -fed -tvb -##81 -##ina -mit -165 -##いる -chan -192 -distribution -next -になる -peter -bios -steam -cm -1941 -にも -pk10 -##ix -##65 -##91 -dec -nasa -##ana -icecat -00z -b1 -will -##46 -li -se -##ji -##み -##ard -oct -##ain -jp -##ze -##bi -cio -##56 -smart -h5 -##39 -##port -curve -vpn -##nm -##dia -utc -##あり -12345678910 -##52 -rmvb -chanel -a4 -miss -##and -##im -media -who -##63 -she -girl -5s -124 -vera -##して -class -vivo -king -##フ -##ei -national -ab -1951 -5cm -888 -145 -ipod -ap -1100 -5mm -211 -ms -2756 -##69 -mp4 -msci -##po -##89 -131 -mg -index -380 -##bit -##out -##zz -##97 -##67 -158 -apec -##８ -photoshop -opec -￥799 -ては -##96 -##tes -##ast -2g -○○ -##ール -￥2899 -##ling -##よ -##ory -1938 -##ical -kitty -content -##43 -step3 -##cn -win8 -155 -vc -1400 -iphone7 -robert -##した -tcl -137 -beauty -##87 -en -dollars -##ys -##oc -step -pay -yy -a1 -##2011 -##lly -##ks -##♪ -1939 -188 -download -1944 -sep -exe -ph -います -school -gb -center -pr -street -##board -uv -##37 -##lan -winrar -##que -##ua -##com -1942 -1936 -480 -gpu -##４ -ettoday -fu -tom -##54 -##ren -##via -149 -##72 -b2b -144 -##79 -##tch -rose -arm -mb -##49 -##ial -##nn -nvidia -step4 -mvp -00㎡ -york -156 -##イ -how -cpi -591 -2765 -gov -kg -joe -##xx -mandy -pa -##ser -copyright -fashion -1935 -don -##け -ecu -##ist -##art -erp -wap -have -##lm -talk -##ek -##ning -##if -ch -##ite -video -1943 -cs -san -iot -look -##84 -##2010 -##ku -october -##ux -trump -##hs -##ide -box -141 -first -##ins -april -##ight -##83 -185 -angel -protected -aa -151 -162 -x1 -m2 -##fe -##× -##ho -size -143 -min -ofo -fun -gomaji -ex -hdmi -food -dns -march -chris -kevin -##のか -##lla -##pp -##ec -ag -ems -6s -720p -##rm -##ham -off -##92 -asp -team -fandom -ed -299 -▌♥ -##ell -info -されています -##82 -sina -4066 -161 -##able -##ctor -330 -399 -315 -dll -rights -ltd -idc -jul -3kg -1927 -142 -ma -surface -##76 -##ク -～～～ -304 -mall -eps -146 -green -##59 -map -space -donald -v2 -sodu -##light -1931 -148 -1700 -まて -310 -reserved -htm -##han -##57 -2d -178 -mod -##ise -##tions -152 -ti -##shi -doc -1933 -icp -055 -wang -##ram -shopping -aug -##pi -##well -now -wam -b2 -からお -##hu -236 -1928 -##gb -266 -f2 -##93 -153 -mix -##ef -##uan -bwl -##plus -##res -core -##ess -tea -5℃ -hktvmall -nhk -##ate -list -##ese -301 -feb -4m -inn -ての -nov -159 -12345 -daniel -##ci -pass -##bet -##nk -coffee -202 -ssl -airbnb -##ute -fbi -woshipm -skype -ea -cg -sp -##fc -##www -yes -edge -alt -007 -##94 -fpga -##ght -##gs -iso9001 -さい -##ile -##wood -##uo -image -lin -icon -american -##em -1932 -set -says -##king -##tive -blogger -##74 -なと -256 -147 -##ox -##zy -##red -##ium -##lf -nokia -claire -##リ -##ding -november -lohas -##500 -##tic -##マ -##cs -##ある -##che -##ire -##gy -##ult -db -january -win -##カ -166 -road -ptt -##ま -##つ -198 -##fa -##mer -anna -pchome -はい -udn -ef -420 -##time -##tte -2030 -##ア -g20 -white -かかります -1929 -308 -garden -eleven -di -##おります -chen -309b -777 -172 -young -cosplay -ちてない -4500 -bat -##123 -##tra -##ては -kindle -npc -steve -etc -##ern -##｜ -call -xperia -ces -travel -sk -s7 -##ous -1934 -##int -みいたたけます -183 -edu -file -cho -qr -##car -##our -186 -##ant -##ｄ -eric -1914 -rends -##jo -##する -mastercard -##2000 -kb -##min -290 -##ino -vista -##ris -##ud -jack -2400 -##set -169 -pos -1912 -##her -##ou -taipei -しく -205 -beta -##ませんか -232 -##fi -express -255 -body -##ill -aphojoy -user -december -meiki -##ick -tweet -richard -##av -##ᆫ -iphone6 -##dd -ちてすか -views -##mark -321 -pd -##００ -times -##▲ -level -##ash -10g -point -5l -##ome -208 -koreanmall -##ak -george -q2 -206 -wma -tcp -##200 -スタッフ -full -mlb -##lle -##watch -tm -run -179 -911 -smith -business -##und -1919 -color -##tal -222 -171 -##less -moon -4399 -##rl -update -pcb -shop -499 -157 -little -なし -end -##mhz -van -dsp -easy -660 -##house -##key -history -##ｏ -oh -##001 -##hy -##web -oem -let -was -##2009 -##gg -review -##wan -182 -##°c -203 -uc -title -##val -united -233 -2021 -##ons -doi -trivago -overdope -sbs -##ance -##ち -grand -special -573032185 -imf -216 -wx17house -##so -##ーム -audi -##he -london -william -##rp -##ake -science -beach -cfa -amp -ps4 -880 -##800 -##link -##hp -crm -ferragamo -bell -make -##eng -195 -under -zh -photos -2300 -##style -##ント -via -176 -da -##gi -company -i7 -##ray -thomas -370 -ufo -i5 -##max -plc -ben -back -research -8g -173 -mike -##pc -##ッフ -september -189 -##ace -vps -february -167 -pantos -wp -lisa -1921 -★★ -jquery -night -long -offer -##berg -##news -1911 -##いて -ray -fks -wto -せます -over -164 -340 -##all -##rus -1924 -##888 -##works -blogtitle -loftpermalink -##→ -187 -martin -test -ling -km -##め -15000 -fda -v3 -##ja -##ロ -ｗedding -かある -outlet -family -##ea -をこ -##top -story -##ness -salvatore -##lu -204 -swift -215 -room -している -oracle -##ul -1925 -sam -b2c -week -pi -rock -##のは -##ａ -##けと -##ean -##300 -##gle -cctv -after -chinese -##back -powered -x2 -##tan -1918 -##nes -##イン -canon -only -181 -##zi -##las -say -##oe -184 -##sd -221 -##bot -##world -##zo -sky -made -top100 -just -1926 -pmi -802 -234 -gap -##vr -177 -les -174 -▲topoct -ball -vogue -vi -ing -ofweek -cos -##list -##ort -▲topmay -##なら -##lon -として -last -##tc -##of -##bus -##gen -real -eva -##コ -a3 -nas -##lie -##ria -##coin -##bt -▲topapr -his -212 -cat -nata -vive -health -⋯⋯ -drive -sir -▲topmar -du -cup -##カー -##ook -##よう -##sy -alex -msg -tour -しました -3ce -##word -193 -ebooks -r8 -block -318 -##より -2200 -nice -pvp -207 -months -1905 -rewards -##ther -1917 -0800 -##xi -##チ -##sc -micro -850 -gg -blogfp -op -1922 -daily -m1 -264 -true -##bb -ml -##tar -##のお -##ky -anthony -196 -253 -##yo -state -218 -##ara -##aa -##rc -##tz -##ston -より -gear -##eo -##ade -ge -see -1923 -##win -##ura -ss -heart -##den -##ita -down -##sm -el -png -2100 -610 -rakuten -whatsapp -bay -dream -add -##use -680 -311 -pad -gucci -mpv -##ode -##fo -island -▲topjun -##▼ -223 -jason -214 -chicago -##❤ -しの -##hone -io -##れる -##ことか -sogo -be2 -##ology -990 -cloud -vcd -##con -2～3 -##ford -##joy -##kb -##こさいます -##rade -but -##ach -docker -##ful -rfid -ul -##ase -hit -ford -##star -580 -##○ -１１ -a2 -sdk -reading -edited -##are -cmos -##mc -238 -siri -light -##ella -##ため -bloomberg -##read -pizza -##ison -jimmy -##vm -college -node -journal -ba -18k -##play -245 -##cer -２０ -magic -##yu -191 -jump -288 -tt -##ings -asr -##lia -3200 -step5 -network -##cd -mc -いします -1234 -pixstyleme -273 -##600 -2800 -money -★★★★★ -1280 -１２ -430 -bl -みの -act -##tus -tokyo -##rial -##life -emba -##ae -saas -tcs -##rk -##wang -summer -##sp -ko -##ving -390 -premium -##その -netflix -##ヒ -uk -mt -##lton -right -frank -two -209 -える -##ple -##cal -021 -##んな -##sen -##ville -hold -nexus -dd -##ius -てお -##mah -##なく -tila -zero -820 -ce -##tin -resort -##ws -charles -old -p10 -5d -report -##360 -##ru -##には -bus -vans -lt -##est -pv -##レ -links -rebecca -##ツ -##dm -azure -##365 -きな -limited -bit -4gb -##mon -1910 -moto -##eam -213 -1913 -var -eos -なとの -226 -blogspot -された -699 -e3 -dos -dm -fc -##ments -##ik -##kw -boy -##bin -##ata -960 -er -##せ -219 -##vin -##tu -##ula -194 -##∥ -station -##ろ -##ature -835 -files -zara -hdr -top10 -nature -950 -magazine -s6 -marriott -##シ -avira -case -##っと -tab -##ran -tony -##home -oculus -im -##ral -jean -saint -cry -307 -rosie -##force -##ini -ice -##bert -のある -##nder -##mber -pet -2600 -##◆ -plurk -▲topdec -##sis -00kg -▲topnov -720 -##ence -tim -##ω -##nc -##ても -##name -log -ips -great -ikea -malaysia -unix -##イト -3600 -##ncy -##nie -12000 -akb48 -##ye -##oid -404 -##chi -##いた -oa -xuehai -##1000 -##orm -##rf -275 -さん -##ware -##リー -980 -ho -##pro -text -##era -560 -bob -227 -##ub -##2008 -8891 -scp -avi -##zen -2022 -mi -wu -museum -qvod -apache -lake -jcb -▲topaug -★★★ -ni -##hr -hill -302 -ne -weibo -490 -ruby -##ーシ -##ヶ -##row -4d -▲topjul -iv -##ish -github -306 -mate -312 -##スト -##lot -##ane -andrew -のハイト -##tina -t1 -rf -ed2k -##vel -##900 -way -final -りの -ns -5a -705 -197 -##メ -sweet -bytes -##ene -▲topjan -231 -##cker -##2007 -##px -100g -topapp -229 -helpapp -rs -low -14k -g4g -care -630 -ldquo -あり -##fork -leave -rm -edition -##gan -##zon -##qq -▲topsep -##google -##ism -gold -224 -explorer -##zer -toyota -category -select -visual -##labels -restaurant -##md -posts -s1 -##ico -もっと -angelababy -123456 -217 -sports -s3 -mbc -1915 -してくたさい -shell -x86 -candy -##new -kbs -face -xl -470 -##here -4a -swissinfo -v8 -▲topfeb -dram -##ual -##vice -3a -##wer -sport -q1 -ios10 -public -int -card -##ｃ -ep -au -rt -##れた -1080 -bill -##mll -kim -３０ -460 -wan -##uk -##ミ -x3 -298 -0t -scott -##ming -239 -e5 -##3d -h7n9 -worldcat -brown -##あります -##vo -##led -##580 -##ax -249 -410 -##ert -paris -##～6 -polo -925 -##lr -599 -##ナ -capital -##hing -bank -cv -1g -##chat -##ｓ -##たい -adc -##ule -2m -##ｅ -digital -hotmail -268 -##pad -870 -bbq -quot -##ring -before -wali -##まて -mcu -2k -2b -という -costco -316 -north -333 -switch -##city -##ｐ -philips -##mann -management -panasonic -##cl -##vd -##ping -##rge -alice -##lk -##ましょう -css3 -##ney -vision -alpha -##ular -##400 -##tter -lz -にお -##ありません -mode -gre -1916 -pci -##tm -237 -1～2 -##yan -##そ -について -##let -##キ -work -war -coach -ah -mary -##ᅵ -huang -##pt -a8 -pt -follow -##berry -1895 -##ew -a5 -ghost -##ション -##wn -##og -south -##code -girls -##rid -action -villa -git -r11 -table -games -##cket -error -##anonymoussaid -##ag -here -##ame -##gc -qa -##■ -##lis -gmp -##gin -vmalife -##cher -yu -wedding -##tis -demo -dragon -530 -soho -social -bye -##rant -river -orz -acer -325 -##↑ -##ース -##ats -261 -del -##ven -440 -ups -##ように -##ター -305 -value -macd -yougou -##dn -661 -##ano -ll -##urt -##rent -continue -script -##wen -##ect -paper -263 -319 -shift -##chel -##フト -##cat -258 -x5 -fox -243 -##さん -car -aaa -##blog -loading -##yn -##tp -kuso -799 -si -sns -イカせるテンマ -ヒンクテンマ3 -rmb -vdc -forest -central -prime -help -ultra -##rmb -##ような -241 -square -688 -##しい -のないフロクに -##field -##reen -##ors -##ju -c1 -start -510 -##air -##map -cdn -##wo -cba -stephen -m8 -100km -##get -opera -##base -##ood -vsa -com™ -##aw -##ail -251 -なのて -count -t2 -##ᅡ -##een -2700 -hop -##gp -vsc -tree -##eg -##ose -816 -285 -##ories -##shop -alphago -v4 -1909 -simon -##ᆼ -fluke62max -zip -スホンサー -##sta -louis -cr -bas -##～10 -bc -##yer -hadoop -##ube -##wi -1906 -0755 -hola -##low -place -centre -5v -d3 -##fer -252 -##750 -##media -281 -540 -0l -exchange -262 -series -##ハー -##san -eb -##bank -##ｋ -q3 -##nge -##mail -take -##lp -259 -1888 -client -east -cache -event -vincent -##ールを -きを -##nse -sui -855 -adchoice -##и -##stry -##なたの -246 -##zone -ga -apps -sea -##ab -248 -cisco -##タ -##rner -kymco -##care -dha -##pu -##yi -minkoff -royal -p1 -への -annie -269 -collection -kpi -playstation -257 -になります -866 -bh -##bar -queen -505 -radio -1904 -andy -armani -##xy -manager -iherb -##ery -##share -spring -raid -johnson -1908 -##ob -volvo -hall -##ball -v6 -our -taylor -##hk -bi -242 -##cp -kate -bo -water -technology -##rie -サイトは -277 -##ona -##sl -hpv -303 -gtx -hip -rdquo -jayz -stone -##lex -##rum -namespace -##やり -620 -##ale -##atic -des -##erson -##ql -##ves -##type -enter -##この -##てきます -d2 -##168 -##mix -##bian -との -a9 -jj -ky -##lc -access -movie -##hc -リストに -tower -##ration -##mit -ます -##nch -ua -tel -prefix -##o2 -1907 -##point -1901 -ott -～10 -##http -##ury -baidu -##ink -member -##logy -bigbang -nownews -##js -##shot -##tb -##こと -247 -eba -##tics -##lus -ける -v5 -spark -##ama -there -##ions -god -##lls -##down -hiv -##ress -burberry -day2 -##kv -◆◆ -jeff -related -film -edit -joseph -283 -##ark -cx -32gb -order -g9 -30000 -##ans -##tty -s5 -##bee -かあります -thread -xr -buy -sh -005 -land -spotify -mx -##ari -276 -##verse -×email -sf -why -##ことて -244 -7headlines -nego -sunny -dom -exo -401 -666 -positioning -fit -rgb -##tton -278 -kiss -alexa -adam -lp -みリストを -##ｇ -mp -##ties -##llow -amy -##du -np -002 -institute -271 -##rth -##lar -2345 -590 -##des -sidebar -１５ -imax -site -##cky -##kit -##ime -##009 -season -323 -##fun -##ンター -##ひ -gogoro -a7 -pu -lily -fire -twd600 -##ッセーシを -いて -##vis -30ml -##cture -##をお -information -##オ -close -friday -##くれる -yi -nick -てすか -##tta -##tel -6500 -##lock -cbd -economy -254 -かお -267 -tinker -double -375 -8gb -voice -##app -oops -channel -today -985 -##right -raw -xyz -##＋ -jim -edm -##cent -7500 -supreme -814 -ds -##its -##asia -dropbox -##てすか -##tti -books -272 -100ml -##tle -##ller -##ken -##more -##boy -sex -309 -##dom -t3 -##ider -##なります -##unch -1903 -810 -feel -5500 -##かった -##put -により -s2 -mo -##gh -men -ka -amoled -div -##tr -##n1 -port -howard -##tags -ken -dnf -##nus -adsense -##а -ide -##へ -buff -thunder -##town -##ique -has -##body -auto -pin -##erry -tee -てした -295 -number -##the -##013 -object -psp -cool -udnbkk -16gb -##mic -miui -##tro -most -r2 -##alk -##nity -1880 -±0 -##いました -428 -s4 -law -version -##oa -n1 -sgs -docomo -##tf -##ack -henry -fc2 -##ded -##sco -##014 -##rite -286 -0mm -linkedin -##ada -##now -wii -##ndy -ucbug -##◎ -sputniknews -legalminer -##ika -##xp -2gb -##bu -q10 -oo -b6 -come -##rman -cheese -ming -maker -##gm -nikon -##fig -ppi -kelly -##ります -jchere -てきます -ted -md -003 -fgo -tech -##tto -dan -soc -##gl -##len -hair -earth -640 -521 -img -##pper -##a1 -##てきる -##ロク -acca -##ition -##ference -suite -##ig -outlook -##mond -##cation -398 -##pr -279 -101vip -358 -##999 -282 -64gb -3800 -345 -airport -##over -284 -##おり -jones -##ith -lab -##su -##いるのて -co2 -town -piece -##llo -no1 -vmware -24h -##qi -focus -reader -##admin -##ora -tb -false -##log -1898 -know -lan -838 -##ces -f4 -##ume -motel -stop -##oper -na -flickr -netcomponents -##af -##─ -pose -williams -local -##ound -##cg -##site -##iko -いお -274 -5m -gsm -con -##ath -1902 -friends -##hip -cell -317 -##rey -780 -cream -##cks -012 -##dp -facebooktwitterpinterestgoogle -sso -324 -shtml -song -swiss -##mw -##キンク -lumia -xdd -string -tiffany -522 -marc -られた -insee -russell -sc -dell -##ations -ｏｋ -camera -289 -##vs -##flow -##late -classic -287 -##nter -stay -g1 -mtv -512 -##ever -##lab -##nger -qe -sata -ryan -d1 -50ml -cms -##cing -su -292 -3300 -editor -296 -##nap -security -sunday -association -##ens -##700 -##bra -acg -##かり -sofascore -とは -mkv -##ign -jonathan -gary -build -labels -##oto -tesla -moba -qi -gohappy -general -ajax -1024 -##かる -サイト -society -##test -##urs -wps -fedora -##ich -mozilla -328 -##480 -##dr -usa -urn -##lina -##ｒ -grace -##die -##try -##ader -1250 -##なり -elle -570 -##chen -##ᆯ -price -##ten -uhz -##ough -eq -##hen -states -push -session -balance -wow -506 -##cus -##py -when -##ward -##ep -34e -wong -library -prada -##サイト -##cle -running -##ree -313 -ck -date -q4 -##ctive -##ool -##＞ -mk -##ira -##163 -388 -die -secret -rq -dota -buffet -は１ヶ -e6 -##ez -pan -368 -ha -##card -##cha -2a -##さ -alan -day3 -eye -f3 -##end -france -keep -adi -rna -tvbs -##ala -solo -nova -##え -##tail -##ょう -support -##ries -##なる -##ved -base -copy -iis -fps -##ways -hero -hgih -profile -fish -mu -ssh -entertainment -chang -##wd -click -cake -##ond -pre -##tom -kic -pixel -##ov -##fl -product -6a -##pd -dear -##gate -es -yumi -audio -##² -##sky -echo -bin -where -##ture -329 -##ape -find -sap -isis -##なと -nand -##101 -##load -##ream -band -a6 -525 -never -##post -festival -50cm -##we -555 -guide -314 -zenfone -##ike -335 -gd -forum -jessica -strong -alexander -##ould -software -allen -##ious -program -360° -else -lohasthree -##gar -することかてきます -please -##れます -rc -##ggle -##ric -bim -50000 -##own -eclipse -355 -brian -3ds -##side -061 -361 -##other -##ける -##tech -##ator -485 -engine -##ged -##ｔ -plaza -##fit -cia -ngo -westbrook -shi -tbs -50mm -##みませんか -sci -291 -reuters -##ily -contextlink -##hn -af -##cil -bridge -very -##cel -1890 -cambridge -##ize -15g -##aid -##data -790 -frm -##head -award -butler -##sun -meta -##mar -america -ps3 -puma -pmid -##すか -lc -670 -kitchen -##lic -オーフン5 -きなしソフトサーヒス -そして -day1 -future -★★★★ -##text -##page -##rris -pm1 -##ket -fans -##っています -1001 -christian -bot -kids -trackback -##hai -c3 -display -##hl -n2 -1896 -idea -さんも -##sent -airmail -##ug -##men -pwm -けます -028 -##lution -369 -852 -awards -schemas -354 -asics -wikipedia -font -##tional -##vy -c2 -293 -##れている -##dget -##ein -っている -contact -pepper -スキル -339 -##～5 -294 -##uel -##ument -730 -##hang -みてす -q5 -##sue -rain -##ndi -wei -swatch -##cept -わせ -331 -popular -##ste -##tag -p2 -501 -trc -1899 -##west -##live -justin -honda -ping -messenger -##rap -v9 -543 -##とは -unity -appqq -はすへて -025 -leo -##tone -##テ -##ass -uniqlo -##010 -502 -her -jane -memory -moneydj -##tical -human -12306 -していると -##m2 -coc -miacare -##mn -tmt -##core -vim -kk -##may -fan -target -use -too -338 -435 -2050 -867 -737 -fast -##2c -services -##ope -omega -energy -##わ -pinkoi -1a -##なから -##rain -jackson -##ement -##シャンルの -374 -366 -そんな -p9 -rd -##ᆨ -1111 -##tier -##vic -zone -##│ -385 -690 -dl -isofix -cpa -m4 -322 -kimi -めて -davis -##lay -lulu -##uck -050 -weeks -qs -##hop -920 -##ｎ -ae -##ear -～5 -eia -405 -##fly -korea -jpeg -boost -##ship -small -##リア -1860 -eur -297 -425 -valley -##iel -simple -##ude -rn -k2 -##ena -されます -non -patrick -しているから -##ナー -feed -5757 -30g -process -well -qqmei -##thing -they -aws -lu -pink -##ters -##kin -または -board -##vertisement -wine -##ien -unicode -##dge -r1 -359 -##tant -いを -##twitter -##3c -cool1 -される -##れて -##ｌ -isp -##012 -standard -45㎡2 -402 -##150 -matt -##fu -326 -##iner -googlemsn -pixnetfacebookyahoo -##ラン -x7 -886 -##uce -メーカー -sao -##ev -##きました -##file -9678 -403 -xddd -shirt -6l -##rio -##hat -3mm -givenchy -ya -bang -##lio -monday -crystal -ロクイン -##abc -336 -head -890 -ubuntuforumwikilinuxpastechat -##vc -##～20 -##rity -cnc -7866 -ipv6 -null -1897 -##ost -yang -imsean -tiger -##fet -##ンス -352 -##＝ -dji -327 -ji -maria -##come -##んて -foundation -3100 -##beth -##なった -1m -601 -active -##aft -##don -3p -sr -349 -emma -##khz -living -415 -353 -1889 -341 -709 -457 -sas -x6 -##face -pptv -x4 -##mate -han -sophie -##jing -337 -fifa -##mand -other -sale -inwedding -##gn -てきちゃいます -##mmy -##pmlast -bad -nana -nbc -してみてくたさいね -なとはお -##wu -##かあります -##あ -note7 -single -##340 -せからこ -してくたさい♪この -しにはとんとんワークケートを -するとあなたにもっとマッチした -ならワークケートへ -もみつかっちゃうかも -ワークケートの -##bel -window -##dio -##ht -union -age -382 -１４ -##ivity -##ｙ -コメント -domain -neo -##isa -##lter -5k -f5 -steven -##cts -powerpoint -tft -self -g2 -ft -##テル -zol -##act -mwc -381 -343 -もう -nbapop -408 -てある -eds -ace -##room -previous -author -tomtom -il -##ets -hu -financial -☆☆☆ -っています -bp -5t -chi -1gb -##hg -fairmont -cross -008 -gay -h2 -function -##けて -356 -also -1b -625 -##ータ -##raph -1894 -3～5 -##ils -i3 -334 -avenue -##host -による -##bon -##tsu -message -navigation -50g -fintech -h6 -##ことを -8cm -##ject -##vas -##firm -credit -##wf -xxxx -form -##nor -##space -huawei -plan -json -sbl -##dc -machine -921 -392 -wish -##120 -##sol -windows7 -edward -##ために -development -washington -##nsis -lo -818 -##sio -##ym -##bor -planet -##～8 -##wt -ieee -gpa -##めて -camp -ann -gm -##tw -##oka -connect -##rss -##work -##atus -wall -chicken -soul -2mm -##times -fa -##ather -##cord -009 -##eep -hitachi -gui -harry -##pan -e1 -disney -##press -##ーション -wind -386 -frigidaire -##tl -liu -hsu -332 -basic -von -ev -いた -てきる -スホンサーサイト -learning -##ull -expedia -archives -change -##wei -santa -cut -ins -6gb -turbo -brand -cf1 -508 -004 -return -747 -##rip -h1 -##nis -##をこ -128gb -##にお -3t -application -しており -emc -rx -##oon -384 -quick -412 -15058 -wilson -wing -chapter -##bug -beyond -##cms -##dar -##oh -zoom -e2 -trip -sb -##nba -rcep -342 -aspx -ci -080 -gc -gnu -める -##count -advanced -dance -dv -##url -##ging -367 -8591 -am09 -shadow -battle -346 -##ｉ -##cia -##という -emily -##のてす -##tation -host -ff -techorz -sars -##mini -##mporary -##ering -nc -4200 -798 -##next -cma -##mbps -##gas -##ift -##dot -##ィ -455 -##～17 -amana -##りの -426 -##ros -ir -00㎡1 -##eet -##ible -##↓ -710 -ˋ▽ˊ -##aka -dcs -iq -##ｖ -l1 -##lor -maggie -##011 -##iu -588 -##～1 -830 -##gt -1tb -articles -create -##burg -##iki -database -fantasy -##rex -##cam -dlc -dean -##you -hard -path -gaming -victoria -maps -cb -##lee -##itor -overchicstoretvhome -systems -##xt -416 -p3 -sarah -760 -##nan -407 -486 -x9 -install -second -626 -##ann -##ph -##rcle -##nic -860 -##nar -ec -##とう -768 -metro -chocolate -##rian -～4 -##table -##しています -skin -##sn -395 -mountain -##0mm -inparadise -6m -7x24 -ib -4800 -##jia -eeworld -creative -g5 -g3 -357 -parker -ecfa -village -からの -18000 -sylvia -サーヒス -hbl -##ques -##onsored -##x2 -##きます -##v4 -##tein -ie6 -383 -##stack -389 -ver -##ads -##baby -sound -bbe -##110 -##lone -##uid -ads -022 -gundam -351 -thinkpad -006 -scrum -match -##ave -mems -##470 -##oy -##なりました -##talk -glass -lamigo -span -##eme -job -##a5 -jay -wade -kde -498 -##lace -ocean -tvg -##covery -##r3 -##ners -##rea -junior -think -##aine -cover -##ision -##sia -↓↓ -##bow -msi -413 -458 -406 -##love -711 -801 -soft -z2 -##pl -456 -1840 -mobil -mind -##uy -427 -nginx -##oi -めた -##rr -6221 -##mple -##sson -##ーシてす -371 -##nts -91tv -comhd -crv3000 -##uard -1868 -397 -deep -lost -field -gallery -##bia -rate -spf -redis -traction -930 -icloud -011 -なら -fe -jose -372 -##tory -into -sohu -fx -899 -379 -kicstart2 -##hia -すく -##～3 -##sit -ra -２４ -##walk -##xure -500g -##pact -pacific -xa -natural -carlo -##250 -##walker -1850 -##can -cto -gigi -516 -##サー -pen -##hoo -ob -matlab -##ｂ -##yy -13913459 -##iti -mango -##bbs -sense -c5 -oxford -##ニア -walker -jennifer -##ola -course -##bre -701 -##pus -##rder -lucky -075 -##ぁ -ivy -なお -##nia -sotheby -side -##ugh -joy -##orage -##ush -##bat -##dt -364 -r9 -##2d -##gio -511 -country -wear -##lax -##～7 -##moon -393 -seven -study -411 -348 -lonzo -8k -##ェ -evolution -##イフ -##kk -gs -kd -##レス -arduino -344 -b12 -##lux -arpg -##rdon -cook -##x5 -dark -five -##als -##ida -とても -sign -362 -##ちの -something -20mm -##nda -387 -##posted -fresh -tf -1870 -422 -cam -##mine -##skip -##form -##ssion -education -394 -##tee -dyson -stage -##jie -want -##night -epson -pack -あります -##ppy -テリヘル -##█ -wd -##eh -##rence -left -##lvin -golden -mhz -discovery -##trix -##n2 -loft -##uch -##dra -##sse -speed -～1 -1mdb -sorry -welcome -##urn -wave -gaga -##lmer -teddy -##160 -トラックハック -せよ -611 -##f2016 -378 -rp -##sha -rar -##あなたに -##きた -840 -holiday -##ュー -373 -074 -##vg -##nos -##rail -gartner -gi -6p -##dium -kit -488 -b3 -eco -##ろう -20g -sean -##stone -autocad -nu -##np -f16 -write -029 -m5 -##ias -images -atp -##dk -fsm -504 -1350 -ve -52kb -##xxx -##のに -##cake -414 -unit -lim -ru -1v -##ification -published -angela -16g -analytics -ak -##ｑ -##nel -gmt -##icon -again -##₂ -##bby -ios11 -445 -かこさいます -waze -いてす -##ハ -9985 -##ust -##ティー -framework -##007 -iptv -delete -52sykb -cl -wwdc -027 -30cm -##fw -##ての -1389 -##xon -brandt -##ses -##dragon -tc -vetements -anne -monte -modern -official -##へて -##ere -##nne -##oud -もちろん -５０ -etnews -##a2 -##graphy -421 -863 -##ちゃん -444 -##rtex -##てお -l2 -##gma -mount -ccd -たと -archive -morning -tan -ddos -e7 -##ホ -day4 -##ウ -gis -453 -its -495 -factory -bruce -pg -##ito -ってくたさい -guest -cdma -##lling -536 -n3 -しかし -3～4 -mega -eyes -ro -１３ -women -dac -church -##jun -singapore -##facebook -6991 -starbucks -##tos -##stin -##shine -zen -##mu -tina -20℃ -1893 -##たけて -503 -465 -request -##gence -qt -##っ -1886 -347 -363 -q7 -##zzi -diary -##tore -409 -##ead -468 -cst -##osa -canada -agent -va -##jiang -##ちは -##ーク -##lam -sg -##nix -##sday -##よって -g6 -##master -bing -##zl -charlie -１６ -8mm -nb40 -##ーン -thai -##ルフ -ln284ct -##itz -##2f -bonnie -##food -##lent -originals -##stro -##lts -418 -∟∣ -##bscribe -children -ntd -yesstyle -##かも -hmv -##tment -d5 -2cm -arts -sms -##pn -##я -##いい -topios9 -539 -lifestyle -virtual -##ague -xz -##deo -muji -024 -unt -##nnis -##ᅩ -faq1 -1884 -396 -##ette -fly -64㎡ -はしめまして -441 -curry -##pop -のこ -release -##← -##◆◆ -##cast -073 -ありな -500ml -##ews -5c -##stle -ios7 -##ima -787 -dog -lenovo -##r4 -roger -013 -cbs -vornado -100m -417 -##desk -##クok -##ald -1867 -9595 -2900 -##van -oil -##ｘ -some -break -common -##jy -##lines -g7 -twice -419 -ella -nano -belle -にこ -##mes -##self -##note -jb -##ことかてきます -benz -##との -##ova -451 -save -##wing -##ますのて -kai -りは -##hua -##rect -rainer -##unge -448 -##0m -adsl -##かな -guestname -##uma -##kins -##zu -tokichoi -##price -county -##med -##mus -rmk -391 -address -vm -えて -openload -##group -##hin -##iginal -amg -urban -##oz -jobs -emi -##public -beautiful -##sch -album -##dden -##bell -jerry -works -hostel -miller -##drive -##rmin -##１０ -376 -boot -828 -##370 -##fx -##cm～ -1885 -##nome -##ctionary -##oman -##lish -##cr -##hm -433 -##how -432 -francis -xi -c919 -b5 -evernote -##uc -vga -##3000 -coupe -##urg -##cca -##uality -019 -6g -れる -multi -##また -##ett -em -hey -##ani -##tax -##rma -inside -than -740 -leonnhurt -##jin -ict -れた -bird -notes -200mm -くの -##dical -##lli -result -442 -iu -ee -438 -smap -gopro -##last -yin -pure -998 -32g -けた -5kg -##dan -##rame -mama -##oot -bean -marketing -##hur -2l -bella -sync -xuite -##ground -515 -discuz -##getrelax -##ince -##bay -##5s -cj -##イス -gmat -apt -##pass -jing -##rix -c4 -rich -##とても -niusnews -##ello -bag -770 -##eting -##mobile -１８ -culture -015 -##のてすか -377 -1020 -area -##ience -616 -details -gp -universal -silver -dit -はお -private -ddd -u11 -kanshu -##ified -fung -##nny -dx -##520 -tai -475 -023 -##fr -##lean -3s -##pin -429 -##rin -25000 -ly -rick -##bility -usb3 -banner -##baru -##gion -metal -dt -vdf -1871 -karl -qualcomm -bear -1010 -oldid -ian -jo -##tors -population -##ernel -1882 -mmorpg -##mv -##bike -603 -##© -ww -friend -##ager -exhibition -##del -##pods -fpx -structure -##free -##tings -kl -##rley -##copyright -##mma -california -3400 -orange -yoga -4l -canmake -honey -##anda -##コメント -595 -nikkie -##ルハイト -dhl -publishing -##mall -##gnet -20cm -513 -##クセス -##┅ -e88 -970 -##dog -fishbase -##! -##" -### -##$ -##% -##& -##' -##( -##) -##* -##+ -##, -##- -##. -##/ -##: -##; -##< -##= -##> -##? -##@ -##[ -##\ -##] -##^ -##_ -##{ -##| -##} -##~ -##£ -##¤ -##¥ -##§ -##« -##± -##³ -##µ -##· -##¹ -##º -##» -##¼ -##ß -##æ -##÷ -##ø -##đ -##ŋ -##ɔ -##ə -##ɡ -##ʰ -##ˇ -##ˈ -##ˊ -##ˋ -##ˍ -##ː -##˙ -##˚ -##ˢ -##α -##β -##γ -##δ -##ε -##η -##θ -##ι -##κ -##λ -##μ -##ν -##ο -##π -##ρ -##ς -##σ -##τ -##υ -##φ -##χ -##ψ -##б -##в -##г -##д -##е -##ж -##з -##к -##л -##м -##н -##о -##п -##р -##с -##т -##у -##ф -##х -##ц -##ч -##ш -##ы -##ь -##і -##ا -##ب -##ة -##ت -##د -##ر -##س -##ع -##ل -##م -##ن -##ه -##و -##ي -##۩ -##ก -##ง -##น -##ม -##ย -##ร -##อ -##า -##เ -##๑ -##་ -##ღ -##ᄀ -##ᄁ -##ᄂ -##ᄃ -##ᄅ -##ᄆ -##ᄇ -##ᄈ -##ᄉ -##ᄋ -##ᄌ -##ᄎ -##ᄏ -##ᄐ -##ᄑ -##ᄒ -##ᅢ -##ᅣ -##ᅥ -##ᅦ -##ᅧ -##ᅨ -##ᅪ -##ᅬ -##ᅭ -##ᅮ -##ᅯ -##ᅲ -##ᅳ -##ᅴ -##ᆷ -##ᆸ -##ᆺ -##ᆻ -##ᗜ -##ᵃ -##ᵉ -##ᵍ -##ᵏ -##ᵐ -##ᵒ -##ᵘ -##‖ -##„ -##† -##• -##‥ -##‧ -##
 -##‰ -##′ -##″ -##‹ -##› -##※ -##‿ -##⁄ -##ⁱ -##⁺ -##ⁿ -##₁ -##₃ -##₄ -##€ -##№ -##ⅰ -##ⅱ -##ⅲ -##ⅳ -##ⅴ -##↔ -##↗ -##↘ -##⇒ -##∀ -##− -##∕ -##∙ -##√ -##∞ -##∟ -##∠ -##∣ -##∩ -##∮ -##∶ -##∼ -##∽ -##≈ -##≒ -##≡ -##≤ -##≥ -##≦ -##≧ -##≪ -##≫ -##⊙ -##⋅ -##⋈ -##⋯ -##⌒ -##① -##② -##③ -##④ -##⑤ -##⑥ -##⑦ -##⑧ -##⑨ -##⑩ -##⑴ -##⑵ -##⑶ -##⑷ -##⑸ -##⒈ -##⒉ -##⒊ -##⒋ -##ⓒ -##ⓔ -##ⓘ -##━ -##┃ -##┆ -##┊ -##┌ -##└ -##├ -##┣ -##═ -##║ -##╚ -##╞ -##╠ -##╭ -##╮ -##╯ -##╰ -##╱ -##╳ -##▂ -##▃ -##▅ -##▇ -##▉ -##▋ -##▌ -##▍ -##▎ -##□ -##▪ -##▫ -##▬ -##△ -##▶ -##► -##▽ -##◇ -##◕ -##◠ -##◢ -##◤ -##☀ -##☕ -##☞ -##☺ -##☼ -##♀ -##♂ -##♠ -##♡ -##♣ -##♦ -##♫ -##♬ -##✈ -##✔ -##✕ -##✖ -##✦ -##✨ -##✪ -##✰ -##✿ -##❀ -##➜ -##➤ -##⦿ -##、 -##。 -##〃 -##々 -##〇 -##〈 -##〉 -##《 -##》 -##「 -##」 -##『 -##』 -##【 -##】 -##〓 -##〔 -##〕 -##〖 -##〗 -##〜 -##〝 -##〞 -##ぃ -##ぇ -##ぬ -##ふ -##ほ -##む -##ゃ -##ゅ -##ゆ -##ょ -##゜ -##ゝ -##ァ -##ゥ -##エ -##ォ -##ケ -##サ -##セ -##ソ -##ッ -##ニ -##ヌ -##ネ -##ノ -##ヘ -##モ -##ャ -##ヤ -##ュ -##ユ -##ョ -##ヨ -##ワ -##ヲ -##・ -##ヽ -##ㄅ -##ㄆ -##ㄇ -##ㄉ -##ㄋ -##ㄌ -##ㄍ -##ㄎ -##ㄏ -##ㄒ -##ㄚ -##ㄛ -##ㄞ -##ㄟ -##ㄢ -##ㄤ -##ㄥ -##ㄧ -##ㄨ -##ㆍ -##㈦ -##㊣ -##㗎 -##一 -##丁 -##七 -##万 -##丈 -##三 -##上 -##下 -##不 -##与 -##丐 -##丑 -##专 -##且 -##丕 -##世 -##丘 -##丙 -##业 -##丛 -##东 -##丝 -##丞 -##丟 -##両 -##丢 -##两 -##严 -##並 -##丧 -##丨 -##个 -##丫 -##中 -##丰 -##串 -##临 -##丶 -##丸 -##丹 -##为 -##主 -##丼 -##丽 -##举 -##丿 -##乂 -##乃 -##久 -##么 -##义 -##之 -##乌 -##乍 -##乎 -##乏 -##乐 -##乒 -##乓 -##乔 -##乖 -##乗 -##乘 -##乙 -##乜 -##九 -##乞 -##也 -##习 -##乡 -##书 -##乩 -##买 -##乱 -##乳 -##乾 -##亀 -##亂 -##了 -##予 -##争 -##事 -##二 -##于 -##亏 -##云 -##互 -##五 -##井 -##亘 -##亙 -##亚 -##些 -##亜 -##亞 -##亟 -##亡 -##亢 -##交 -##亥 -##亦 -##产 -##亨 -##亩 -##享 -##京 -##亭 -##亮 -##亲 -##亳 -##亵 -##人 -##亿 -##什 -##仁 -##仃 -##仄 -##仅 -##仆 -##仇 -##今 -##介 -##仍 -##从 -##仏 -##仑 -##仓 -##仔 -##仕 -##他 -##仗 -##付 -##仙 -##仝 -##仞 -##仟 -##代 -##令 -##以 -##仨 -##仪 -##们 -##仮 -##仰 -##仲 -##件 -##价 -##任 -##份 -##仿 -##企 -##伉 -##伊 -##伍 -##伎 -##伏 -##伐 -##休 -##伕 -##众 -##优 -##伙 -##会 -##伝 -##伞 -##伟 -##传 -##伢 -##伤 -##伦 -##伪 -##伫 -##伯 -##估 -##伴 -##伶 -##伸 -##伺 -##似 -##伽 -##佃 -##但 -##佇 -##佈 -##位 -##低 -##住 -##佐 -##佑 -##体 -##佔 -##何 -##佗 -##佘 -##余 -##佚 -##佛 -##作 -##佝 -##佞 -##佟 -##你 -##佢 -##佣 -##佤 -##佥 -##佩 -##佬 -##佯 -##佰 -##佳 -##併 -##佶 -##佻 -##佼 -##使 -##侃 -##侄 -##來 -##侈 -##例 -##侍 -##侏 -##侑 -##侖 -##侗 -##供 -##依 -##侠 -##価 -##侣 -##侥 -##侦 -##侧 -##侨 -##侬 -##侮 -##侯 -##侵 -##侶 -##侷 -##便 -##係 -##促 -##俄 -##俊 -##俎 -##俏 -##俐 -##俑 -##俗 -##俘 -##俚 -##保 -##俞 -##俟 -##俠 -##信 -##俨 -##俩 -##俪 -##俬 -##俭 -##修 -##俯 -##俱 -##俳 -##俸 -##俺 -##俾 -##倆 -##倉 -##個 -##倌 -##倍 -##倏 -##們 -##倒 -##倔 -##倖 -##倘 -##候 -##倚 -##倜 -##借 -##倡 -##値 -##倦 -##倩 -##倪 -##倫 -##倬 -##倭 -##倶 -##债 -##值 -##倾 -##偃 -##假 -##偈 -##偉 -##偌 -##偎 -##偏 -##偕 -##做 -##停 -##健 -##側 -##偵 -##偶 -##偷 -##偻 -##偽 -##偿 -##傀 -##傅 -##傍 -##傑 -##傘 -##備 -##傚 -##傢 -##傣 -##傥 -##储 -##傩 -##催 -##傭 -##傲 -##傳 -##債 -##傷 -##傻 -##傾 -##僅 -##働 -##像 -##僑 -##僕 -##僖 -##僚 -##僥 -##僧 -##僭 -##僮 -##僱 -##僵 -##價 -##僻 -##儀 -##儂 -##億 -##儆 -##儉 -##儋 -##儒 -##儕 -##儘 -##償 -##儡 -##優 -##儲 -##儷 -##儼 -##儿 -##兀 -##允 -##元 -##兄 -##充 -##兆 -##兇 -##先 -##光 -##克 -##兌 -##免 -##児 -##兑 -##兒 -##兔 -##兖 -##党 -##兜 -##兢 -##入 -##內 -##全 -##兩 -##八 -##公 -##六 -##兮 -##兰 -##共 -##兲 -##关 -##兴 -##兵 -##其 -##具 -##典 -##兹 -##养 -##兼 -##兽 -##冀 -##内 -##円 -##冇 -##冈 -##冉 -##冊 -##册 -##再 -##冏 -##冒 -##冕 -##冗 -##写 -##军 -##农 -##冠 -##冢 -##冤 -##冥 -##冨 -##冪 -##冬 -##冯 -##冰 -##冲 -##决 -##况 -##冶 -##冷 -##冻 -##冼 -##冽 -##冾 -##净 -##凄 -##准 -##凇 -##凈 -##凉 -##凋 -##凌 -##凍 -##减 -##凑 -##凛 -##凜 -##凝 -##几 -##凡 -##凤 -##処 -##凪 -##凭 -##凯 -##凰 -##凱 -##凳 -##凶 -##凸 -##凹 -##出 -##击 -##函 -##凿 -##刀 -##刁 -##刃 -##分 -##切 -##刈 -##刊 -##刍 -##刎 -##刑 -##划 -##列 -##刘 -##则 -##刚 -##创 -##初 -##删 -##判 -##別 -##刨 -##利 -##刪 -##别 -##刮 -##到 -##制 -##刷 -##券 -##刹 -##刺 -##刻 -##刽 -##剁 -##剂 -##剃 -##則 -##剉 -##削 -##剋 -##剌 -##前 -##剎 -##剐 -##剑 -##剔 -##剖 -##剛 -##剜 -##剝 -##剣 -##剤 -##剥 -##剧 -##剩 -##剪 -##副 -##割 -##創 -##剷 -##剽 -##剿 -##劃 -##劇 -##劈 -##劉 -##劊 -##劍 -##劏 -##劑 -##力 -##劝 -##办 -##功 -##加 -##务 -##劣 -##动 -##助 -##努 -##劫 -##劭 -##励 -##劲 -##劳 -##労 -##劵 -##効 -##劾 -##势 -##勁 -##勃 -##勇 -##勉 -##勋 -##勐 -##勒 -##動 -##勖 -##勘 -##務 -##勛 -##勝 -##勞 -##募 -##勢 -##勤 -##勧 -##勳 -##勵 -##勸 -##勺 -##勻 -##勾 -##勿 -##匀 -##包 -##匆 -##匈 -##匍 -##匐 -##匕 -##化 -##北 -##匙 -##匝 -##匠 -##匡 -##匣 -##匪 -##匮 -##匯 -##匱 -##匹 -##区 -##医 -##匾 -##匿 -##區 -##十 -##千 -##卅 -##升 -##午 -##卉 -##半 -##卍 -##华 -##协 -##卑 -##卒 -##卓 -##協 -##单 -##卖 -##南 -##単 -##博 -##卜 -##卞 -##卟 -##占 -##卡 -##卢 -##卤 -##卦 -##卧 -##卫 -##卮 -##卯 -##印 -##危 -##即 -##却 -##卵 -##卷 -##卸 -##卻 -##卿 -##厂 -##厄 -##厅 -##历 -##厉 -##压 -##厌 -##厕 -##厘 -##厚 -##厝 -##原 -##厢 -##厥 -##厦 -##厨 -##厩 -##厭 -##厮 -##厲 -##厳 -##去 -##县 -##叁 -##参 -##參 -##又 -##叉 -##及 -##友 -##双 -##反 -##収 -##发 -##叔 -##取 -##受 -##变 -##叙 -##叛 -##叟 -##叠 -##叡 -##叢 -##口 -##古 -##句 -##另 -##叨 -##叩 -##只 -##叫 -##召 -##叭 -##叮 -##可 -##台 -##叱 -##史 -##右 -##叵 -##叶 -##号 -##司 -##叹 -##叻 -##叼 -##叽 -##吁 -##吃 -##各 -##吆 -##合 -##吉 -##吊 -##吋 -##同 -##名 -##后 -##吏 -##吐 -##向 -##吒 -##吓 -##吕 -##吖 -##吗 -##君 -##吝 -##吞 -##吟 -##吠 -##吡 -##否 -##吧 -##吨 -##吩 -##含 -##听 -##吭 -##吮 -##启 -##吱 -##吳 -##吴 -##吵 -##吶 -##吸 -##吹 -##吻 -##吼 -##吽 -##吾 -##呀 -##呂 -##呃 -##呆 -##呈 -##告 -##呋 -##呎 -##呐 -##呓 -##呕 -##呗 -##员 -##呛 -##呜 -##呢 -##呤 -##呦 -##周 -##呱 -##呲 -##味 -##呵 -##呷 -##呸 -##呻 -##呼 -##命 -##咀 -##咁 -##咂 -##咄 -##咆 -##咋 -##和 -##咎 -##咏 -##咐 -##咒 -##咔 -##咕 -##咖 -##咗 -##咘 -##咙 -##咚 -##咛 -##咣 -##咤 -##咦 -##咧 -##咨 -##咩 -##咪 -##咫 -##咬 -##咭 -##咯 -##咱 -##咲 -##咳 -##咸 -##咻 -##咽 -##咿 -##哀 -##品 -##哂 -##哄 -##哆 -##哇 -##哈 -##哉 -##哋 -##哌 -##响 -##哎 -##哏 -##哐 -##哑 -##哒 -##哔 -##哗 -##哟 -##員 -##哥 -##哦 -##哧 -##哨 -##哩 -##哪 -##哭 -##哮 -##哲 -##哺 -##哼 -##哽 -##唁 -##唄 -##唆 -##唇 -##唉 -##唏 -##唐 -##唑 -##唔 -##唠 -##唤 -##唧 -##唬 -##售 -##唯 -##唰 -##唱 -##唳 -##唷 -##唸 -##唾 -##啃 -##啄 -##商 -##啉 -##啊 -##問 -##啓 -##啕 -##啖 -##啜 -##啞 -##啟 -##啡 -##啤 -##啥 -##啦 -##啧 -##啪 -##啫 -##啬 -##啮 -##啰 -##啱 -##啲 -##啵 -##啶 -##啷 -##啸 -##啻 -##啼 -##啾 -##喀 -##喂 -##喃 -##善 -##喆 -##喇 -##喉 -##喊 -##喋 -##喎 -##喏 -##喔 -##喘 -##喙 -##喚 -##喜 -##喝 -##喟 -##喧 -##喪 -##喫 -##喬 -##單 -##喰 -##喱 -##喲 -##喳 -##喵 -##営 -##喷 -##喹 -##喺 -##喻 -##喽 -##嗅 -##嗆 -##嗇 -##嗎 -##嗑 -##嗒 -##嗓 -##嗔 -##嗖 -##嗚 -##嗜 -##嗝 -##嗟 -##嗡 -##嗣 -##嗤 -##嗦 -##嗨 -##嗪 -##嗬 -##嗯 -##嗰 -##嗲 -##嗳 -##嗶 -##嗷 -##嗽 -##嘀 -##嘅 -##嘆 -##嘈 -##嘉 -##嘌 -##嘍 -##嘎 -##嘔 -##嘖 -##嘗 -##嘘 -##嘚 -##嘛 -##嘜 -##嘞 -##嘟 -##嘢 -##嘣 -##嘤 -##嘧 -##嘩 -##嘭 -##嘮 -##嘯 -##嘰 -##嘱 -##嘲 -##嘴 -##嘶 -##嘸 -##嘹 -##嘻 -##嘿 -##噁 -##噌 -##噎 -##噓 -##噔 -##噗 -##噙 -##噜 -##噠 -##噢 -##噤 -##器 -##噩 -##噪 -##噬 -##噱 -##噴 -##噶 -##噸 -##噹 -##噻 -##噼 -##嚀 -##嚇 -##嚎 -##嚏 -##嚐 -##嚓 -##嚕 -##嚟 -##嚣 -##嚥 -##嚨 -##嚮 -##嚴 -##嚷 -##嚼 -##囂 -##囉 -##囊 -##囍 -##囑 -##囔 -##囗 -##囚 -##四 -##囝 -##回 -##囟 -##因 -##囡 -##团 -##団 -##囤 -##囧 -##囪 -##囫 -##园 -##困 -##囱 -##囲 -##図 -##围 -##囹 -##固 -##国 -##图 -##囿 -##圃 -##圄 -##圆 -##圈 -##國 -##圍 -##圏 -##園 -##圓 -##圖 -##團 -##圜 -##土 -##圣 -##圧 -##在 -##圩 -##圭 -##地 -##圳 -##场 -##圻 -##圾 -##址 -##坂 -##均 -##坊 -##坍 -##坎 -##坏 -##坐 -##坑 -##块 -##坚 -##坛 -##坝 -##坞 -##坟 -##坠 -##坡 -##坤 -##坦 -##坨 -##坪 -##坯 -##坳 -##坵 -##坷 -##垂 -##垃 -##垄 -##型 -##垒 -##垚 -##垛 -##垠 -##垢 -##垣 -##垦 -##垩 -##垫 -##垭 -##垮 -##垵 -##埂 -##埃 -##埋 -##城 -##埔 -##埕 -##埗 -##域 -##埠 -##埤 -##埵 -##執 -##埸 -##培 -##基 -##埼 -##堀 -##堂 -##堃 -##堅 -##堆 -##堇 -##堑 -##堕 -##堙 -##堡 -##堤 -##堪 -##堯 -##堰 -##報 -##場 -##堵 -##堺 -##堿 -##塊 -##塌 -##塑 -##塔 -##塗 -##塘 -##塚 -##塞 -##塢 -##塩 -##填 -##塬 -##塭 -##塵 -##塾 -##墀 -##境 -##墅 -##墉 -##墊 -##墒 -##墓 -##増 -##墘 -##墙 -##墜 -##增 -##墟 -##墨 -##墩 -##墮 -##墳 -##墻 -##墾 -##壁 -##壅 -##壆 -##壇 -##壊 -##壑 -##壓 -##壕 -##壘 -##壞 -##壟 -##壢 -##壤 -##壩 -##士 -##壬 -##壮 -##壯 -##声 -##売 -##壳 -##壶 -##壹 -##壺 -##壽 -##处 -##备 -##変 -##复 -##夏 -##夔 -##夕 -##外 -##夙 -##多 -##夜 -##够 -##夠 -##夢 -##夥 -##大 -##天 -##太 -##夫 -##夭 -##央 -##夯 -##失 -##头 -##夷 -##夸 -##夹 -##夺 -##夾 -##奂 -##奄 -##奇 -##奈 -##奉 -##奋 -##奎 -##奏 -##奐 -##契 -##奔 -##奕 -##奖 -##套 -##奘 -##奚 -##奠 -##奢 -##奥 -##奧 -##奪 -##奬 -##奮 -##女 -##奴 -##奶 -##奸 -##她 -##好 -##如 -##妃 -##妄 -##妆 -##妇 -##妈 -##妊 -##妍 -##妒 -##妓 -##妖 -##妘 -##妙 -##妝 -##妞 -##妣 -##妤 -##妥 -##妨 -##妩 -##妪 -##妮 -##妲 -##妳 -##妹 -##妻 -##妾 -##姆 -##姉 -##姊 -##始 -##姍 -##姐 -##姑 -##姒 -##姓 -##委 -##姗 -##姚 -##姜 -##姝 -##姣 -##姥 -##姦 -##姨 -##姪 -##姫 -##姬 -##姹 -##姻 -##姿 -##威 -##娃 -##娄 -##娅 -##娆 -##娇 -##娉 -##娑 -##娓 -##娘 -##娛 -##娜 -##娟 -##娠 -##娣 -##娥 -##娩 -##娱 -##娲 -##娴 -##娶 -##娼 -##婀 -##婁 -##婆 -##婉 -##婊 -##婕 -##婚 -##婢 -##婦 -##婧 -##婪 -##婭 -##婴 -##婵 -##婶 -##婷 -##婺 -##婿 -##媒 -##媚 -##媛 -##媞 -##媧 -##媲 -##媳 -##媽 -##媾 -##嫁 -##嫂 -##嫉 -##嫌 -##嫑 -##嫔 -##嫖 -##嫘 -##嫚 -##嫡 -##嫣 -##嫦 -##嫩 -##嫲 -##嫵 -##嫻 -##嬅 -##嬉 -##嬌 -##嬗 -##嬛 -##嬢 -##嬤 -##嬪 -##嬰 -##嬴 -##嬷 -##嬸 -##嬿 -##孀 -##孃 -##子 -##孑 -##孔 -##孕 -##孖 -##字 -##存 -##孙 -##孚 -##孛 -##孜 -##孝 -##孟 -##孢 -##季 -##孤 -##学 -##孩 -##孪 -##孫 -##孬 -##孰 -##孱 -##孳 -##孵 -##學 -##孺 -##孽 -##孿 -##宁 -##它 -##宅 -##宇 -##守 -##安 -##宋 -##完 -##宏 -##宓 -##宕 -##宗 -##官 -##宙 -##定 -##宛 -##宜 -##宝 -##实 -##実 -##宠 -##审 -##客 -##宣 -##室 -##宥 -##宦 -##宪 -##宫 -##宮 -##宰 -##害 -##宴 -##宵 -##家 -##宸 -##容 -##宽 -##宾 -##宿 -##寂 -##寄 -##寅 -##密 -##寇 -##富 -##寐 -##寒 -##寓 -##寛 -##寝 -##寞 -##察 -##寡 -##寢 -##寥 -##實 -##寧 -##寨 -##審 -##寫 -##寬 -##寮 -##寰 -##寵 -##寶 -##寸 -##对 -##寺 -##寻 -##导 -##対 -##寿 -##封 -##専 -##射 -##将 -##將 -##專 -##尉 -##尊 -##尋 -##對 -##導 -##小 -##少 -##尔 -##尕 -##尖 -##尘 -##尚 -##尝 -##尤 -##尧 -##尬 -##就 -##尴 -##尷 -##尸 -##尹 -##尺 -##尻 -##尼 -##尽 -##尾 -##尿 -##局 -##屁 -##层 -##屄 -##居 -##屆 -##屈 -##屉 -##届 -##屋 -##屌 -##屍 -##屎 -##屏 -##屐 -##屑 -##展 -##屜 -##属 -##屠 -##屡 -##屢 -##層 -##履 -##屬 -##屯 -##山 -##屹 -##屿 -##岀 -##岁 -##岂 -##岌 -##岐 -##岑 -##岔 -##岖 -##岗 -##岘 -##岙 -##岚 -##岛 -##岡 -##岩 -##岫 -##岬 -##岭 -##岱 -##岳 -##岷 -##岸 -##峇 -##峋 -##峒 -##峙 -##峡 -##峤 -##峥 -##峦 -##峨 -##峪 -##峭 -##峯 -##峰 -##峴 -##島 -##峻 -##峽 -##崁 -##崂 -##崆 -##崇 -##崎 -##崑 -##崔 -##崖 -##崗 -##崙 -##崛 -##崧 -##崩 -##崭 -##崴 -##崽 -##嵇 -##嵊 -##嵋 -##嵌 -##嵐 -##嵘 -##嵩 -##嵬 -##嵯 -##嶂 -##嶄 -##嶇 -##嶋 -##嶙 -##嶺 -##嶼 -##嶽 -##巅 -##巍 -##巒 -##巔 -##巖 -##川 -##州 -##巡 -##巢 -##工 -##左 -##巧 -##巨 -##巩 -##巫 -##差 -##己 -##已 -##巳 -##巴 -##巷 -##巻 -##巽 -##巾 -##巿 -##币 -##市 -##布 -##帅 -##帆 -##师 -##希 -##帐 -##帑 -##帕 -##帖 -##帘 -##帚 -##帛 -##帜 -##帝 -##帥 -##带 -##帧 -##師 -##席 -##帮 -##帯 -##帰 -##帳 -##帶 -##帷 -##常 -##帼 -##帽 -##幀 -##幂 -##幄 -##幅 -##幌 -##幔 -##幕 -##幟 -##幡 -##幢 -##幣 -##幫 -##干 -##平 -##年 -##并 -##幸 -##幹 -##幺 -##幻 -##幼 -##幽 -##幾 -##广 -##庁 -##広 -##庄 -##庆 -##庇 -##床 -##序 -##庐 -##库 -##应 -##底 -##庖 -##店 -##庙 -##庚 -##府 -##庞 -##废 -##庠 -##度 -##座 -##庫 -##庭 -##庵 -##庶 -##康 -##庸 -##庹 -##庾 -##廁 -##廂 -##廃 -##廈 -##廉 -##廊 -##廓 -##廖 -##廚 -##廝 -##廟 -##廠 -##廢 -##廣 -##廬 -##廳 -##延 -##廷 -##建 -##廿 -##开 -##弁 -##异 -##弃 -##弄 -##弈 -##弊 -##弋 -##式 -##弑 -##弒 -##弓 -##弔 -##引 -##弗 -##弘 -##弛 -##弟 -##张 -##弥 -##弦 -##弧 -##弩 -##弭 -##弯 -##弱 -##張 -##強 -##弹 -##强 -##弼 -##弾 -##彅 -##彆 -##彈 -##彌 -##彎 -##归 -##当 -##录 -##彗 -##彙 -##彝 -##形 -##彤 -##彥 -##彦 -##彧 -##彩 -##彪 -##彫 -##彬 -##彭 -##彰 -##影 -##彷 -##役 -##彻 -##彼 -##彿 -##往 -##征 -##径 -##待 -##徇 -##很 -##徉 -##徊 -##律 -##後 -##徐 -##徑 -##徒 -##従 -##徕 -##得 -##徘 -##徙 -##徜 -##從 -##徠 -##御 -##徨 -##復 -##循 -##徬 -##微 -##徳 -##徴 -##徵 -##德 -##徹 -##徼 -##徽 -##心 -##必 -##忆 -##忌 -##忍 -##忏 -##忐 -##忑 -##忒 -##忖 -##志 -##忘 -##忙 -##応 -##忠 -##忡 -##忤 -##忧 -##忪 -##快 -##忱 -##念 -##忻 -##忽 -##忿 -##怀 -##态 -##怂 -##怅 -##怆 -##怎 -##怏 -##怒 -##怔 -##怕 -##怖 -##怙 -##怜 -##思 -##怠 -##怡 -##急 -##怦 -##性 -##怨 -##怪 -##怯 -##怵 -##总 -##怼 -##恁 -##恃 -##恆 -##恋 -##恍 -##恐 -##恒 -##恕 -##恙 -##恚 -##恢 -##恣 -##恤 -##恥 -##恨 -##恩 -##恪 -##恫 -##恬 -##恭 -##息 -##恰 -##恳 -##恵 -##恶 -##恸 -##恺 -##恻 -##恼 -##恿 -##悄 -##悅 -##悉 -##悌 -##悍 -##悔 -##悖 -##悚 -##悟 -##悠 -##患 -##悦 -##您 -##悩 -##悪 -##悬 -##悯 -##悱 -##悲 -##悴 -##悵 -##悶 -##悸 -##悻 -##悼 -##悽 -##情 -##惆 -##惇 -##惊 -##惋 -##惑 -##惕 -##惘 -##惚 -##惜 -##惟 -##惠 -##惡 -##惦 -##惧 -##惨 -##惩 -##惫 -##惬 -##惭 -##惮 -##惯 -##惰 -##惱 -##想 -##惴 -##惶 -##惹 -##惺 -##愁 -##愆 -##愈 -##愉 -##愍 -##意 -##愕 -##愚 -##愛 -##愜 -##感 -##愣 -##愤 -##愧 -##愫 -##愷 -##愿 -##慄 -##慈 -##態 -##慌 -##慎 -##慑 -##慕 -##慘 -##慚 -##慟 -##慢 -##慣 -##慧 -##慨 -##慫 -##慮 -##慰 -##慳 -##慵 -##慶 -##慷 -##慾 -##憂 -##憊 -##憋 -##憎 -##憐 -##憑 -##憔 -##憚 -##憤 -##憧 -##憨 -##憩 -##憫 -##憬 -##憲 -##憶 -##憾 -##懂 -##懇 -##懈 -##應 -##懊 -##懋 -##懑 -##懒 -##懦 -##懲 -##懵 -##懶 -##懷 -##懸 -##懺 -##懼 -##懾 -##懿 -##戀 -##戈 -##戊 -##戌 -##戍 -##戎 -##戏 -##成 -##我 -##戒 -##戕 -##或 -##战 -##戚 -##戛 -##戟 -##戡 -##戦 -##截 -##戬 -##戮 -##戰 -##戲 -##戳 -##戴 -##戶 -##户 -##戸 -##戻 -##戾 -##房 -##所 -##扁 -##扇 -##扈 -##扉 -##手 -##才 -##扎 -##扑 -##扒 -##打 -##扔 -##払 -##托 -##扛 -##扣 -##扦 -##执 -##扩 -##扪 -##扫 -##扬 -##扭 -##扮 -##扯 -##扰 -##扱 -##扳 -##扶 -##批 -##扼 -##找 -##承 -##技 -##抄 -##抉 -##把 -##抑 -##抒 -##抓 -##投 -##抖 -##抗 -##折 -##抚 -##抛 -##抜 -##択 -##抟 -##抠 -##抡 -##抢 -##护 -##报 -##抨 -##披 -##抬 -##抱 -##抵 -##抹 -##押 -##抽 -##抿 -##拂 -##拄 -##担 -##拆 -##拇 -##拈 -##拉 -##拋 -##拌 -##拍 -##拎 -##拐 -##拒 -##拓 -##拔 -##拖 -##拗 -##拘 -##拙 -##拚 -##招 -##拜 -##拟 -##拡 -##拢 -##拣 -##拥 -##拦 -##拧 -##拨 -##择 -##括 -##拭 -##拮 -##拯 -##拱 -##拳 -##拴 -##拷 -##拼 -##拽 -##拾 -##拿 -##持 -##挂 -##指 -##挈 -##按 -##挎 -##挑 -##挖 -##挙 -##挚 -##挛 -##挝 -##挞 -##挟 -##挠 -##挡 -##挣 -##挤 -##挥 -##挨 -##挪 -##挫 -##振 -##挲 -##挹 -##挺 -##挽 -##挾 -##捂 -##捅 -##捆 -##捉 -##捋 -##捌 -##捍 -##捎 -##捏 -##捐 -##捕 -##捞 -##损 -##捡 -##换 -##捣 -##捧 -##捨 -##捩 -##据 -##捱 -##捲 -##捶 -##捷 -##捺 -##捻 -##掀 -##掂 -##掃 -##掇 -##授 -##掉 -##掌 -##掏 -##掐 -##排 -##掖 -##掘 -##掙 -##掛 -##掠 -##採 -##探 -##掣 -##接 -##控 -##推 -##掩 -##措 -##掬 -##掰 -##掲 -##掳 -##掴 -##掷 -##掸 -##掺 -##揀 -##揃 -##揄 -##揆 -##揉 -##揍 -##描 -##提 -##插 -##揖 -##揚 -##換 -##握 -##揣 -##揩 -##揪 -##揭 -##揮 -##援 -##揶 -##揸 -##揹 -##揽 -##搀 -##搁 -##搂 -##搅 -##損 -##搏 -##搐 -##搓 -##搔 -##搖 -##搗 -##搜 -##搞 -##搡 -##搪 -##搬 -##搭 -##搵 -##搶 -##携 -##搽 -##摀 -##摁 -##摄 -##摆 -##摇 -##摈 -##摊 -##摒 -##摔 -##摘 -##摞 -##摟 -##摧 -##摩 -##摯 -##摳 -##摸 -##摹 -##摺 -##摻 -##撂 -##撃 -##撅 -##撇 -##撈 -##撐 -##撑 -##撒 -##撓 -##撕 -##撚 -##撞 -##撤 -##撥 -##撩 -##撫 -##撬 -##播 -##撮 -##撰 -##撲 -##撵 -##撷 -##撸 -##撻 -##撼 -##撿 -##擀 -##擁 -##擂 -##擄 -##擅 -##擇 -##擊 -##擋 -##操 -##擎 -##擒 -##擔 -##擘 -##據 -##擞 -##擠 -##擡 -##擢 -##擦 -##擬 -##擰 -##擱 -##擲 -##擴 -##擷 -##擺 -##擼 -##擾 -##攀 -##攏 -##攒 -##攔 -##攘 -##攙 -##攜 -##攝 -##攞 -##攢 -##攣 -##攤 -##攥 -##攪 -##攫 -##攬 -##支 -##收 -##攸 -##改 -##攻 -##放 -##政 -##故 -##效 -##敌 -##敍 -##敎 -##敏 -##救 -##敕 -##敖 -##敗 -##敘 -##教 -##敛 -##敝 -##敞 -##敢 -##散 -##敦 -##敬 -##数 -##敲 -##整 -##敵 -##敷 -##數 -##斂 -##斃 -##文 -##斋 -##斌 -##斎 -##斐 -##斑 -##斓 -##斗 -##料 -##斛 -##斜 -##斟 -##斡 -##斤 -##斥 -##斧 -##斩 -##斫 -##斬 -##断 -##斯 -##新 -##斷 -##方 -##於 -##施 -##旁 -##旃 -##旅 -##旋 -##旌 -##旎 -##族 -##旖 -##旗 -##无 -##既 -##日 -##旦 -##旧 -##旨 -##早 -##旬 -##旭 -##旮 -##旱 -##时 -##旷 -##旺 -##旻 -##昀 -##昂 -##昆 -##昇 -##昉 -##昊 -##昌 -##明 -##昏 -##易 -##昔 -##昕 -##昙 -##星 -##映 -##春 -##昧 -##昨 -##昭 -##是 -##昱 -##昴 -##昵 -##昶 -##昼 -##显 -##晁 -##時 -##晃 -##晉 -##晋 -##晌 -##晏 -##晒 -##晓 -##晔 -##晕 -##晖 -##晗 -##晚 -##晝 -##晞 -##晟 -##晤 -##晦 -##晨 -##晩 -##普 -##景 -##晰 -##晴 -##晶 -##晷 -##智 -##晾 -##暂 -##暄 -##暇 -##暈 -##暉 -##暌 -##暐 -##暑 -##暖 -##暗 -##暝 -##暢 -##暧 -##暨 -##暫 -##暮 -##暱 -##暴 -##暸 -##暹 -##曄 -##曆 -##曇 -##曉 -##曖 -##曙 -##曜 -##曝 -##曠 -##曦 -##曬 -##曰 -##曲 -##曳 -##更 -##書 -##曹 -##曼 -##曾 -##替 -##最 -##會 -##月 -##有 -##朋 -##服 -##朐 -##朔 -##朕 -##朗 -##望 -##朝 -##期 -##朦 -##朧 -##木 -##未 -##末 -##本 -##札 -##朮 -##术 -##朱 -##朴 -##朵 -##机 -##朽 -##杀 -##杂 -##权 -##杆 -##杈 -##杉 -##李 -##杏 -##材 -##村 -##杓 -##杖 -##杜 -##杞 -##束 -##杠 -##条 -##来 -##杨 -##杭 -##杯 -##杰 -##東 -##杳 -##杵 -##杷 -##杼 -##松 -##板 -##极 -##构 -##枇 -##枉 -##枋 -##析 -##枕 -##林 -##枚 -##果 -##枝 -##枢 -##枣 -##枪 -##枫 -##枭 -##枯 -##枰 -##枱 -##枳 -##架 -##枷 -##枸 -##柄 -##柏 -##某 -##柑 -##柒 -##染 -##柔 -##柘 -##柚 -##柜 -##柞 -##柠 -##柢 -##查 -##柩 -##柬 -##柯 -##柱 -##柳 -##柴 -##柵 -##査 -##柿 -##栀 -##栃 -##栄 -##栅 -##标 -##栈 -##栉 -##栋 -##栎 -##栏 -##树 -##栓 -##栖 -##栗 -##校 -##栩 -##株 -##样 -##核 -##根 -##格 -##栽 -##栾 -##桀 -##桁 -##桂 -##桃 -##桅 -##框 -##案 -##桉 -##桌 -##桎 -##桐 -##桑 -##桓 -##桔 -##桜 -##桠 -##桡 -##桢 -##档 -##桥 -##桦 -##桧 -##桨 -##桩 -##桶 -##桿 -##梁 -##梅 -##梆 -##梏 -##梓 -##梗 -##條 -##梟 -##梢 -##梦 -##梧 -##梨 -##梭 -##梯 -##械 -##梳 -##梵 -##梶 -##检 -##棂 -##棄 -##棉 -##棋 -##棍 -##棒 -##棕 -##棗 -##棘 -##棚 -##棟 -##棠 -##棣 -##棧 -##森 -##棱 -##棲 -##棵 -##棹 -##棺 -##椁 -##椅 -##椋 -##植 -##椎 -##椒 -##検 -##椪 -##椭 -##椰 -##椹 -##椽 -##椿 -##楂 -##楊 -##楓 -##楔 -##楚 -##楝 -##楞 -##楠 -##楣 -##楨 -##楫 -##業 -##楮 -##極 -##楷 -##楸 -##楹 -##楼 -##楽 -##概 -##榄 -##榆 -##榈 -##榉 -##榔 -##榕 -##榖 -##榛 -##榜 -##榨 -##榫 -##榭 -##榮 -##榱 -##榴 -##榷 -##榻 -##槁 -##槃 -##構 -##槌 -##槍 -##槎 -##槐 -##槓 -##様 -##槛 -##槟 -##槤 -##槭 -##槲 -##槳 -##槻 -##槽 -##槿 -##樁 -##樂 -##樊 -##樑 -##樓 -##標 -##樞 -##樟 -##模 -##樣 -##権 -##横 -##樫 -##樯 -##樱 -##樵 -##樸 -##樹 -##樺 -##樽 -##樾 -##橄 -##橇 -##橋 -##橐 -##橘 -##橙 -##機 -##橡 -##橢 -##橫 -##橱 -##橹 -##橼 -##檀 -##檄 -##檎 -##檐 -##檔 -##檗 -##檜 -##檢 -##檬 -##檯 -##檳 -##檸 -##檻 -##櫃 -##櫚 -##櫛 -##櫥 -##櫸 -##櫻 -##欄 -##權 -##欒 -##欖 -##欠 -##次 -##欢 -##欣 -##欧 -##欲 -##欸 -##欺 -##欽 -##款 -##歆 -##歇 -##歉 -##歌 -##歎 -##歐 -##歓 -##歙 -##歛 -##歡 -##止 -##正 -##此 -##步 -##武 -##歧 -##歩 -##歪 -##歯 -##歲 -##歳 -##歴 -##歷 -##歸 -##歹 -##死 -##歼 -##殁 -##殃 -##殆 -##殇 -##殉 -##殊 -##残 -##殒 -##殓 -##殖 -##殘 -##殞 -##殡 -##殤 -##殭 -##殯 -##殲 -##殴 -##段 -##殷 -##殺 -##殼 -##殿 -##毀 -##毁 -##毂 -##毅 -##毆 -##毋 -##母 -##毎 -##每 -##毒 -##毓 -##比 -##毕 -##毗 -##毘 -##毙 -##毛 -##毡 -##毫 -##毯 -##毽 -##氈 -##氏 -##氐 -##民 -##氓 -##气 -##氖 -##気 -##氙 -##氛 -##氟 -##氡 -##氢 -##氣 -##氤 -##氦 -##氧 -##氨 -##氪 -##氫 -##氮 -##氯 -##氰 -##氲 -##水 -##氷 -##永 -##氹 -##氾 -##汀 -##汁 -##求 -##汆 -##汇 -##汉 -##汎 -##汐 -##汕 -##汗 -##汙 -##汛 -##汝 -##汞 -##江 -##池 -##污 -##汤 -##汨 -##汩 -##汪 -##汰 -##汲 -##汴 -##汶 -##汹 -##決 -##汽 -##汾 -##沁 -##沂 -##沃 -##沅 -##沈 -##沉 -##沌 -##沏 -##沐 -##沒 -##沓 -##沖 -##沙 -##沛 -##沟 -##没 -##沢 -##沣 -##沥 -##沦 -##沧 -##沪 -##沫 -##沭 -##沮 -##沱 -##河 -##沸 -##油 -##治 -##沼 -##沽 -##沾 -##沿 -##況 -##泄 -##泉 -##泊 -##泌 -##泓 -##法 -##泗 -##泛 -##泞 -##泠 -##泡 -##波 -##泣 -##泥 -##注 -##泪 -##泫 -##泮 -##泯 -##泰 -##泱 -##泳 -##泵 -##泷 -##泸 -##泻 -##泼 -##泽 -##泾 -##洁 -##洄 -##洋 -##洒 -##洗 -##洙 -##洛 -##洞 -##津 -##洩 -##洪 -##洮 -##洱 -##洲 -##洵 -##洶 -##洸 -##洹 -##活 -##洼 -##洽 -##派 -##流 -##浃 -##浄 -##浅 -##浆 -##浇 -##浊 -##测 -##济 -##浏 -##浑 -##浒 -##浓 -##浔 -##浙 -##浚 -##浜 -##浣 -##浦 -##浩 -##浪 -##浬 -##浮 -##浯 -##浴 -##海 -##浸 -##涂 -##涅 -##涇 -##消 -##涉 -##涌 -##涎 -##涓 -##涔 -##涕 -##涙 -##涛 -##涝 -##涞 -##涟 -##涠 -##涡 -##涣 -##涤 -##润 -##涧 -##涨 -##涩 -##涪 -##涮 -##涯 -##液 -##涵 -##涸 -##涼 -##涿 -##淀 -##淄 -##淅 -##淆 -##淇 -##淋 -##淌 -##淑 -##淒 -##淖 -##淘 -##淙 -##淚 -##淞 -##淡 -##淤 -##淦 -##淨 -##淩 -##淪 -##淫 -##淬 -##淮 -##深 -##淳 -##淵 -##混 -##淹 -##淺 -##添 -##淼 -##清 -##済 -##渉 -##渊 -##渋 -##渍 -##渎 -##渐 -##渔 -##渗 -##渙 -##渚 -##減 -##渝 -##渠 -##渡 -##渣 -##渤 -##渥 -##渦 -##温 -##測 -##渭 -##港 -##渲 -##渴 -##游 -##渺 -##渾 -##湃 -##湄 -##湊 -##湍 -##湖 -##湘 -##湛 -##湟 -##湧 -##湫 -##湮 -##湯 -##湳 -##湾 -##湿 -##満 -##溃 -##溅 -##溉 -##溏 -##源 -##準 -##溜 -##溝 -##溟 -##溢 -##溥 -##溧 -##溪 -##溫 -##溯 -##溱 -##溴 -##溶 -##溺 -##溼 -##滁 -##滂 -##滄 -##滅 -##滇 -##滋 -##滌 -##滑 -##滓 -##滔 -##滕 -##滙 -##滚 -##滝 -##滞 -##滟 -##满 -##滢 -##滤 -##滥 -##滦 -##滨 -##滩 -##滬 -##滯 -##滲 -##滴 -##滷 -##滸 -##滾 -##滿 -##漁 -##漂 -##漆 -##漉 -##漏 -##漓 -##演 -##漕 -##漠 -##漢 -##漣 -##漩 -##漪 -##漫 -##漬 -##漯 -##漱 -##漲 -##漳 -##漸 -##漾 -##漿 -##潆 -##潇 -##潋 -##潍 -##潑 -##潔 -##潘 -##潛 -##潜 -##潞 -##潟 -##潢 -##潤 -##潦 -##潧 -##潭 -##潮 -##潰 -##潴 -##潸 -##潺 -##潼 -##澀 -##澄 -##澆 -##澈 -##澍 -##澎 -##澗 -##澜 -##澡 -##澤 -##澧 -##澱 -##澳 -##澹 -##激 -##濁 -##濂 -##濃 -##濑 -##濒 -##濕 -##濘 -##濛 -##濟 -##濠 -##濡 -##濤 -##濫 -##濬 -##濮 -##濯 -##濱 -##濺 -##濾 -##瀅 -##瀆 -##瀉 -##瀋 -##瀏 -##瀑 -##瀕 -##瀘 -##瀚 -##瀛 -##瀝 -##瀞 -##瀟 -##瀧 -##瀨 -##瀬 -##瀰 -##瀾 -##灌 -##灏 -##灑 -##灘 -##灝 -##灞 -##灣 -##火 -##灬 -##灭 -##灯 -##灰 -##灵 -##灶 -##灸 -##灼 -##災 -##灾 -##灿 -##炀 -##炁 -##炅 -##炉 -##炊 -##炎 -##炒 -##炔 -##炕 -##炖 -##炙 -##炜 -##炫 -##炬 -##炭 -##炮 -##炯 -##炳 -##炷 -##炸 -##点 -##為 -##炼 -##炽 -##烁 -##烂 -##烃 -##烈 -##烊 -##烏 -##烘 -##烙 -##烛 -##烟 -##烤 -##烦 -##烧 -##烨 -##烩 -##烫 -##烬 -##热 -##烯 -##烷 -##烹 -##烽 -##焉 -##焊 -##焕 -##焖 -##焗 -##焘 -##焙 -##焚 -##焜 -##無 -##焦 -##焯 -##焰 -##焱 -##然 -##焼 -##煅 -##煉 -##煊 -##煌 -##煎 -##煒 -##煖 -##煙 -##煜 -##煞 -##煤 -##煥 -##煦 -##照 -##煨 -##煩 -##煮 -##煲 -##煸 -##煽 -##熄 -##熊 -##熏 -##熒 -##熔 -##熙 -##熟 -##熠 -##熨 -##熬 -##熱 -##熵 -##熹 -##熾 -##燁 -##燃 -##燄 -##燈 -##燉 -##燊 -##燎 -##燒 -##燔 -##燕 -##燙 -##燜 -##營 -##燥 -##燦 -##燧 -##燭 -##燮 -##燴 -##燻 -##燼 -##燿 -##爆 -##爍 -##爐 -##爛 -##爪 -##爬 -##爭 -##爰 -##爱 -##爲 -##爵 -##父 -##爷 -##爸 -##爹 -##爺 -##爻 -##爽 -##爾 -##牆 -##片 -##版 -##牌 -##牍 -##牒 -##牙 -##牛 -##牝 -##牟 -##牠 -##牡 -##牢 -##牦 -##牧 -##物 -##牯 -##牲 -##牴 -##牵 -##特 -##牺 -##牽 -##犀 -##犁 -##犄 -##犊 -##犍 -##犒 -##犢 -##犧 -##犬 -##犯 -##状 -##犷 -##犸 -##犹 -##狀 -##狂 -##狄 -##狈 -##狎 -##狐 -##狒 -##狗 -##狙 -##狞 -##狠 -##狡 -##狩 -##独 -##狭 -##狮 -##狰 -##狱 -##狸 -##狹 -##狼 -##狽 -##猎 -##猕 -##猖 -##猗 -##猙 -##猛 -##猜 -##猝 -##猥 -##猩 -##猪 -##猫 -##猬 -##献 -##猴 -##猶 -##猷 -##猾 -##猿 -##獄 -##獅 -##獎 -##獐 -##獒 -##獗 -##獠 -##獣 -##獨 -##獭 -##獰 -##獲 -##獵 -##獷 -##獸 -##獺 -##獻 -##獼 -##獾 -##玄 -##率 -##玉 -##王 -##玑 -##玖 -##玛 -##玟 -##玠 -##玥 -##玩 -##玫 -##玮 -##环 -##现 -##玲 -##玳 -##玷 -##玺 -##玻 -##珀 -##珂 -##珅 -##珈 -##珉 -##珊 -##珍 -##珏 -##珐 -##珑 -##珙 -##珞 -##珠 -##珣 -##珥 -##珩 -##珪 -##班 -##珮 -##珲 -##珺 -##現 -##球 -##琅 -##理 -##琇 -##琉 -##琊 -##琍 -##琏 -##琐 -##琛 -##琢 -##琥 -##琦 -##琨 -##琪 -##琬 -##琮 -##琰 -##琲 -##琳 -##琴 -##琵 -##琶 -##琺 -##琼 -##瑀 -##瑁 -##瑄 -##瑋 -##瑕 -##瑗 -##瑙 -##瑚 -##瑛 -##瑜 -##瑞 -##瑟 -##瑠 -##瑣 -##瑤 -##瑩 -##瑪 -##瑯 -##瑰 -##瑶 -##瑾 -##璀 -##璁 -##璃 -##璇 -##璉 -##璋 -##璎 -##璐 -##璜 -##璞 -##璟 -##璧 -##璨 -##環 -##璽 -##璿 -##瓊 -##瓏 -##瓒 -##瓜 -##瓢 -##瓣 -##瓤 -##瓦 -##瓮 -##瓯 -##瓴 -##瓶 -##瓷 -##甄 -##甌 -##甕 -##甘 -##甙 -##甚 -##甜 -##生 -##產 -##産 -##甥 -##甦 -##用 -##甩 -##甫 -##甬 -##甭 -##甯 -##田 -##由 -##甲 -##申 -##电 -##男 -##甸 -##町 -##画 -##甾 -##畀 -##畅 -##界 -##畏 -##畑 -##畔 -##留 -##畜 -##畝 -##畢 -##略 -##畦 -##番 -##畫 -##異 -##畲 -##畳 -##畴 -##當 -##畸 -##畹 -##畿 -##疆 -##疇 -##疊 -##疏 -##疑 -##疔 -##疖 -##疗 -##疙 -##疚 -##疝 -##疟 -##疡 -##疣 -##疤 -##疥 -##疫 -##疮 -##疯 -##疱 -##疲 -##疳 -##疵 -##疸 -##疹 -##疼 -##疽 -##疾 -##痂 -##病 -##症 -##痈 -##痉 -##痊 -##痍 -##痒 -##痔 -##痕 -##痘 -##痙 -##痛 -##痞 -##痠 -##痢 -##痣 -##痤 -##痧 -##痨 -##痪 -##痫 -##痰 -##痱 -##痴 -##痹 -##痺 -##痼 -##痿 -##瘀 -##瘁 -##瘋 -##瘍 -##瘓 -##瘘 -##瘙 -##瘟 -##瘠 -##瘡 -##瘢 -##瘤 -##瘦 -##瘧 -##瘩 -##瘪 -##瘫 -##瘴 -##瘸 -##瘾 -##療 -##癇 -##癌 -##癒 -##癖 -##癜 -##癞 -##癡 -##癢 -##癣 -##癥 -##癫 -##癬 -##癮 -##癱 -##癲 -##癸 -##発 -##登 -##發 -##白 -##百 -##皂 -##的 -##皆 -##皇 -##皈 -##皋 -##皎 -##皑 -##皓 -##皖 -##皙 -##皚 -##皮 -##皰 -##皱 -##皴 -##皺 -##皿 -##盂 -##盃 -##盅 -##盆 -##盈 -##益 -##盎 -##盏 -##盐 -##监 -##盒 -##盔 -##盖 -##盗 -##盘 -##盛 -##盜 -##盞 -##盟 -##盡 -##監 -##盤 -##盥 -##盧 -##盪 -##目 -##盯 -##盱 -##盲 -##直 -##相 -##盹 -##盼 -##盾 -##省 -##眈 -##眉 -##看 -##県 -##眙 -##眞 -##真 -##眠 -##眦 -##眨 -##眩 -##眯 -##眶 -##眷 -##眸 -##眺 -##眼 -##眾 -##着 -##睁 -##睇 -##睏 -##睐 -##睑 -##睛 -##睜 -##睞 -##睡 -##睢 -##督 -##睥 -##睦 -##睨 -##睪 -##睫 -##睬 -##睹 -##睽 -##睾 -##睿 -##瞄 -##瞅 -##瞇 -##瞋 -##瞌 -##瞎 -##瞑 -##瞒 -##瞓 -##瞞 -##瞟 -##瞠 -##瞥 -##瞧 -##瞩 -##瞪 -##瞬 -##瞭 -##瞰 -##瞳 -##瞻 -##瞼 -##瞿 -##矇 -##矍 -##矗 -##矚 -##矛 -##矜 -##矢 -##矣 -##知 -##矩 -##矫 -##短 -##矮 -##矯 -##石 -##矶 -##矽 -##矾 -##矿 -##码 -##砂 -##砌 -##砍 -##砒 -##研 -##砖 -##砗 -##砚 -##砝 -##砣 -##砥 -##砧 -##砭 -##砰 -##砲 -##破 -##砷 -##砸 -##砺 -##砼 -##砾 -##础 -##硅 -##硐 -##硒 -##硕 -##硝 -##硫 -##硬 -##确 -##硯 -##硼 -##碁 -##碇 -##碉 -##碌 -##碍 -##碎 -##碑 -##碓 -##碗 -##碘 -##碚 -##碛 -##碟 -##碣 -##碧 -##碩 -##碰 -##碱 -##碳 -##碴 -##確 -##碼 -##碾 -##磁 -##磅 -##磊 -##磋 -##磐 -##磕 -##磚 -##磡 -##磨 -##磬 -##磯 -##磲 -##磷 -##磺 -##礁 -##礎 -##礙 -##礡 -##礦 -##礪 -##礫 -##礴 -##示 -##礼 -##社 -##祀 -##祁 -##祂 -##祇 -##祈 -##祉 -##祎 -##祐 -##祕 -##祖 -##祗 -##祚 -##祛 -##祜 -##祝 -##神 -##祟 -##祠 -##祢 -##祥 -##票 -##祭 -##祯 -##祷 -##祸 -##祺 -##祿 -##禀 -##禁 -##禄 -##禅 -##禍 -##禎 -##福 -##禛 -##禦 -##禧 -##禪 -##禮 -##禱 -##禹 -##禺 -##离 -##禽 -##禾 -##禿 -##秀 -##私 -##秃 -##秆 -##秉 -##秋 -##种 -##科 -##秒 -##秘 -##租 -##秣 -##秤 -##秦 -##秧 -##秩 -##秭 -##积 -##称 -##秸 -##移 -##秽 -##稀 -##稅 -##程 -##稍 -##税 -##稔 -##稗 -##稚 -##稜 -##稞 -##稟 -##稠 -##稣 -##種 -##稱 -##稲 -##稳 -##稷 -##稹 -##稻 -##稼 -##稽 -##稿 -##穀 -##穂 -##穆 -##穌 -##積 -##穎 -##穗 -##穢 -##穩 -##穫 -##穴 -##究 -##穷 -##穹 -##空 -##穿 -##突 -##窃 -##窄 -##窈 -##窍 -##窑 -##窒 -##窓 -##窕 -##窖 -##窗 -##窘 -##窜 -##窝 -##窟 -##窠 -##窥 -##窦 -##窨 -##窩 -##窪 -##窮 -##窯 -##窺 -##窿 -##竄 -##竅 -##竇 -##竊 -##立 -##竖 -##站 -##竜 -##竞 -##竟 -##章 -##竣 -##童 -##竭 -##端 -##競 -##竹 -##竺 -##竽 -##竿 -##笃 -##笆 -##笈 -##笋 -##笏 -##笑 -##笔 -##笙 -##笛 -##笞 -##笠 -##符 -##笨 -##第 -##笹 -##笺 -##笼 -##筆 -##等 -##筊 -##筋 -##筍 -##筏 -##筐 -##筑 -##筒 -##答 -##策 -##筛 -##筝 -##筠 -##筱 -##筲 -##筵 -##筷 -##筹 -##签 -##简 -##箇 -##箋 -##箍 -##箏 -##箐 -##箔 -##箕 -##算 -##箝 -##管 -##箩 -##箫 -##箭 -##箱 -##箴 -##箸 -##節 -##篁 -##範 -##篆 -##篇 -##築 -##篑 -##篓 -##篙 -##篝 -##篠 -##篡 -##篤 -##篩 -##篪 -##篮 -##篱 -##篷 -##簇 -##簌 -##簍 -##簡 -##簦 -##簧 -##簪 -##簫 -##簷 -##簸 -##簽 -##簾 -##簿 -##籁 -##籃 -##籌 -##籍 -##籐 -##籟 -##籠 -##籤 -##籬 -##籮 -##籲 -##米 -##类 -##籼 -##籽 -##粄 -##粉 -##粑 -##粒 -##粕 -##粗 -##粘 -##粟 -##粤 -##粥 -##粧 -##粪 -##粮 -##粱 -##粲 -##粳 -##粵 -##粹 -##粼 -##粽 -##精 -##粿 -##糅 -##糊 -##糍 -##糕 -##糖 -##糗 -##糙 -##糜 -##糞 -##糟 -##糠 -##糧 -##糬 -##糯 -##糰 -##糸 -##系 -##糾 -##紀 -##紂 -##約 -##紅 -##紉 -##紊 -##紋 -##納 -##紐 -##紓 -##純 -##紗 -##紘 -##紙 -##級 -##紛 -##紜 -##素 -##紡 -##索 -##紧 -##紫 -##紮 -##累 -##細 -##紳 -##紹 -##紺 -##終 -##絃 -##組 -##絆 -##経 -##結 -##絕 -##絞 -##絡 -##絢 -##給 -##絨 -##絮 -##統 -##絲 -##絳 -##絵 -##絶 -##絹 -##綁 -##綏 -##綑 -##經 -##継 -##続 -##綜 -##綠 -##綢 -##綦 -##綫 -##綬 -##維 -##綱 -##網 -##綴 -##綵 -##綸 -##綺 -##綻 -##綽 -##綾 -##綿 -##緊 -##緋 -##総 -##緑 -##緒 -##緘 -##線 -##緝 -##緞 -##締 -##緣 -##編 -##緩 -##緬 -##緯 -##練 -##緹 -##緻 -##縁 -##縄 -##縈 -##縛 -##縝 -##縣 -##縫 -##縮 -##縱 -##縴 -##縷 -##總 -##績 -##繁 -##繃 -##繆 -##繇 -##繋 -##織 -##繕 -##繚 -##繞 -##繡 -##繩 -##繪 -##繫 -##繭 -##繳 -##繹 -##繼 -##繽 -##纂 -##續 -##纍 -##纏 -##纓 -##纔 -##纖 -##纜 -##纠 -##红 -##纣 -##纤 -##约 -##级 -##纨 -##纪 -##纫 -##纬 -##纭 -##纯 -##纰 -##纱 -##纲 -##纳 -##纵 -##纶 -##纷 -##纸 -##纹 -##纺 -##纽 -##纾 -##线 -##绀 -##练 -##组 -##绅 -##细 -##织 -##终 -##绊 -##绍 -##绎 -##经 -##绑 -##绒 -##结 -##绔 -##绕 -##绘 -##给 -##绚 -##绛 -##络 -##绝 -##绞 -##统 -##绡 -##绢 -##绣 -##绥 -##绦 -##继 -##绩 -##绪 -##绫 -##续 -##绮 -##绯 -##绰 -##绳 -##维 -##绵 -##绶 -##绷 -##绸 -##绻 -##综 -##绽 -##绾 -##绿 -##缀 -##缄 -##缅 -##缆 -##缇 -##缈 -##缉 -##缎 -##缓 -##缔 -##缕 -##编 -##缘 -##缙 -##缚 -##缜 -##缝 -##缠 -##缢 -##缤 -##缥 -##缨 -##缩 -##缪 -##缭 -##缮 -##缰 -##缱 -##缴 -##缸 -##缺 -##缽 -##罂 -##罄 -##罌 -##罐 -##网 -##罔 -##罕 -##罗 -##罚 -##罡 -##罢 -##罩 -##罪 -##置 -##罰 -##署 -##罵 -##罷 -##罹 -##羁 -##羅 -##羈 -##羊 -##羌 -##美 -##羔 -##羚 -##羞 -##羟 -##羡 -##羣 -##群 -##羥 -##羧 -##羨 -##義 -##羯 -##羲 -##羸 -##羹 -##羽 -##羿 -##翁 -##翅 -##翊 -##翌 -##翎 -##習 -##翔 -##翘 -##翟 -##翠 -##翡 -##翦 -##翩 -##翰 -##翱 -##翳 -##翹 -##翻 -##翼 -##耀 -##老 -##考 -##耄 -##者 -##耆 -##耋 -##而 -##耍 -##耐 -##耒 -##耕 -##耗 -##耘 -##耙 -##耦 -##耨 -##耳 -##耶 -##耷 -##耸 -##耻 -##耽 -##耿 -##聂 -##聆 -##聊 -##聋 -##职 -##聒 -##联 -##聖 -##聘 -##聚 -##聞 -##聪 -##聯 -##聰 -##聲 -##聳 -##聴 -##聶 -##職 -##聽 -##聾 -##聿 -##肃 -##肄 -##肅 -##肆 -##肇 -##肉 -##肋 -##肌 -##肏 -##肓 -##肖 -##肘 -##肚 -##肛 -##肝 -##肠 -##股 -##肢 -##肤 -##肥 -##肩 -##肪 -##肮 -##肯 -##肱 -##育 -##肴 -##肺 -##肽 -##肾 -##肿 -##胀 -##胁 -##胃 -##胄 -##胆 -##背 -##胍 -##胎 -##胖 -##胚 -##胛 -##胜 -##胝 -##胞 -##胡 -##胤 -##胥 -##胧 -##胫 -##胭 -##胯 -##胰 -##胱 -##胳 -##胴 -##胶 -##胸 -##胺 -##能 -##脂 -##脅 -##脆 -##脇 -##脈 -##脉 -##脊 -##脍 -##脏 -##脐 -##脑 -##脓 -##脖 -##脘 -##脚 -##脛 -##脣 -##脩 -##脫 -##脯 -##脱 -##脲 -##脳 -##脸 -##脹 -##脾 -##腆 -##腈 -##腊 -##腋 -##腌 -##腎 -##腐 -##腑 -##腓 -##腔 -##腕 -##腥 -##腦 -##腩 -##腫 -##腭 -##腮 -##腰 -##腱 -##腳 -##腴 -##腸 -##腹 -##腺 -##腻 -##腼 -##腾 -##腿 -##膀 -##膈 -##膊 -##膏 -##膑 -##膘 -##膚 -##膛 -##膜 -##膝 -##膠 -##膦 -##膨 -##膩 -##膳 -##膺 -##膻 -##膽 -##膾 -##膿 -##臀 -##臂 -##臃 -##臆 -##臉 -##臊 -##臍 -##臓 -##臘 -##臟 -##臣 -##臥 -##臧 -##臨 -##自 -##臬 -##臭 -##至 -##致 -##臺 -##臻 -##臼 -##臾 -##舀 -##舂 -##舅 -##舆 -##與 -##興 -##舉 -##舊 -##舌 -##舍 -##舎 -##舐 -##舒 -##舔 -##舖 -##舗 -##舛 -##舜 -##舞 -##舟 -##航 -##舫 -##般 -##舰 -##舱 -##舵 -##舶 -##舷 -##舸 -##船 -##舺 -##舾 -##艇 -##艋 -##艘 -##艙 -##艦 -##艮 -##良 -##艰 -##艱 -##色 -##艳 -##艷 -##艹 -##艺 -##艾 -##节 -##芃 -##芈 -##芊 -##芋 -##芍 -##芎 -##芒 -##芙 -##芜 -##芝 -##芡 -##芥 -##芦 -##芩 -##芪 -##芫 -##芬 -##芭 -##芮 -##芯 -##花 -##芳 -##芷 -##芸 -##芹 -##芻 -##芽 -##芾 -##苁 -##苄 -##苇 -##苋 -##苍 -##苏 -##苑 -##苒 -##苓 -##苔 -##苕 -##苗 -##苛 -##苜 -##苞 -##苟 -##苡 -##苣 -##若 -##苦 -##苫 -##苯 -##英 -##苷 -##苹 -##苻 -##茁 -##茂 -##范 -##茄 -##茅 -##茉 -##茎 -##茏 -##茗 -##茜 -##茧 -##茨 -##茫 -##茬 -##茭 -##茯 -##茱 -##茲 -##茴 -##茵 -##茶 -##茸 -##茹 -##茼 -##荀 -##荃 -##荆 -##草 -##荊 -##荏 -##荐 -##荒 -##荔 -##荖 -##荘 -##荚 -##荞 -##荟 -##荠 -##荡 -##荣 -##荤 -##荥 -##荧 -##荨 -##荪 -##荫 -##药 -##荳 -##荷 -##荸 -##荻 -##荼 -##荽 -##莅 -##莆 -##莉 -##莊 -##莎 -##莒 -##莓 -##莖 -##莘 -##莞 -##莠 -##莢 -##莧 -##莪 -##莫 -##莱 -##莲 -##莴 -##获 -##莹 -##莺 -##莽 -##莿 -##菀 -##菁 -##菅 -##菇 -##菈 -##菊 -##菌 -##菏 -##菓 -##菖 -##菘 -##菜 -##菟 -##菠 -##菡 -##菩 -##華 -##菱 -##菲 -##菸 -##菽 -##萁 -##萃 -##萄 -##萊 -##萋 -##萌 -##萍 -##萎 -##萘 -##萝 -##萤 -##营 -##萦 -##萧 -##萨 -##萩 -##萬 -##萱 -##萵 -##萸 -##萼 -##落 -##葆 -##葉 -##著 -##葚 -##葛 -##葡 -##董 -##葦 -##葩 -##葫 -##葬 -##葭 -##葯 -##葱 -##葳 -##葵 -##葷 -##葺 -##蒂 -##蒋 -##蒐 -##蒔 -##蒙 -##蒜 -##蒞 -##蒟 -##蒡 -##蒨 -##蒲 -##蒸 -##蒹 -##蒻 -##蒼 -##蒿 -##蓁 -##蓄 -##蓆 -##蓉 -##蓋 -##蓑 -##蓓 -##蓖 -##蓝 -##蓟 -##蓦 -##蓬 -##蓮 -##蓼 -##蓿 -##蔑 -##蔓 -##蔔 -##蔗 -##蔘 -##蔚 -##蔡 -##蔣 -##蔥 -##蔫 -##蔬 -##蔭 -##蔵 -##蔷 -##蔺 -##蔻 -##蔼 -##蔽 -##蕁 -##蕃 -##蕈 -##蕉 -##蕊 -##蕎 -##蕙 -##蕤 -##蕨 -##蕩 -##蕪 -##蕭 -##蕲 -##蕴 -##蕻 -##蕾 -##薄 -##薅 -##薇 -##薈 -##薊 -##薏 -##薑 -##薔 -##薙 -##薛 -##薦 -##薨 -##薩 -##薪 -##薬 -##薯 -##薰 -##薹 -##藉 -##藍 -##藏 -##藐 -##藓 -##藕 -##藜 -##藝 -##藤 -##藥 -##藩 -##藹 -##藻 -##藿 -##蘆 -##蘇 -##蘊 -##蘋 -##蘑 -##蘚 -##蘭 -##蘸 -##蘼 -##蘿 -##虎 -##虏 -##虐 -##虑 -##虔 -##處 -##虚 -##虛 -##虜 -##虞 -##號 -##虢 -##虧 -##虫 -##虬 -##虱 -##虹 -##虻 -##虽 -##虾 -##蚀 -##蚁 -##蚂 -##蚊 -##蚌 -##蚓 -##蚕 -##蚜 -##蚝 -##蚣 -##蚤 -##蚩 -##蚪 -##蚯 -##蚱 -##蚵 -##蛀 -##蛆 -##蛇 -##蛊 -##蛋 -##蛎 -##蛐 -##蛔 -##蛙 -##蛛 -##蛟 -##蛤 -##蛭 -##蛮 -##蛰 -##蛳 -##蛹 -##蛻 -##蛾 -##蜀 -##蜂 -##蜃 -##蜆 -##蜇 -##蜈 -##蜊 -##蜍 -##蜒 -##蜓 -##蜕 -##蜗 -##蜘 -##蜚 -##蜜 -##蜡 -##蜢 -##蜥 -##蜱 -##蜴 -##蜷 -##蜻 -##蜿 -##蝇 -##蝈 -##蝉 -##蝌 -##蝎 -##蝕 -##蝗 -##蝙 -##蝟 -##蝠 -##蝦 -##蝨 -##蝴 -##蝶 -##蝸 -##蝼 -##螂 -##螃 -##融 -##螞 -##螢 -##螨 -##螯 -##螳 -##螺 -##蟀 -##蟄 -##蟆 -##蟋 -##蟎 -##蟑 -##蟒 -##蟠 -##蟬 -##蟲 -##蟹 -##蟻 -##蟾 -##蠅 -##蠍 -##蠔 -##蠕 -##蠛 -##蠟 -##蠡 -##蠢 -##蠣 -##蠱 -##蠶 -##蠹 -##蠻 -##血 -##衄 -##衅 -##衆 -##行 -##衍 -##術 -##衔 -##街 -##衙 -##衛 -##衝 -##衞 -##衡 -##衢 -##衣 -##补 -##表 -##衩 -##衫 -##衬 -##衮 -##衰 -##衲 -##衷 -##衹 -##衾 -##衿 -##袁 -##袂 -##袄 -##袅 -##袈 -##袋 -##袍 -##袒 -##袖 -##袜 -##袞 -##袤 -##袪 -##被 -##袭 -##袱 -##裁 -##裂 -##装 -##裆 -##裊 -##裏 -##裔 -##裕 -##裘 -##裙 -##補 -##裝 -##裟 -##裡 -##裤 -##裨 -##裱 -##裳 -##裴 -##裸 -##裹 -##製 -##裾 -##褂 -##複 -##褐 -##褒 -##褓 -##褔 -##褚 -##褥 -##褪 -##褫 -##褲 -##褶 -##褻 -##襁 -##襄 -##襟 -##襠 -##襪 -##襬 -##襯 -##襲 -##西 -##要 -##覃 -##覆 -##覇 -##見 -##規 -##覓 -##視 -##覚 -##覦 -##覧 -##親 -##覬 -##観 -##覷 -##覺 -##覽 -##觀 -##见 -##观 -##规 -##觅 -##视 -##览 -##觉 -##觊 -##觎 -##觐 -##觑 -##角 -##觞 -##解 -##觥 -##触 -##觸 -##言 -##訂 -##計 -##訊 -##討 -##訓 -##訕 -##訖 -##託 -##記 -##訛 -##訝 -##訟 -##訣 -##訥 -##訪 -##設 -##許 -##訳 -##訴 -##訶 -##診 -##註 -##証 -##詆 -##詐 -##詔 -##評 -##詛 -##詞 -##詠 -##詡 -##詢 -##詣 -##試 -##詩 -##詫 -##詬 -##詭 -##詮 -##詰 -##話 -##該 -##詳 -##詹 -##詼 -##誅 -##誇 -##誉 -##誌 -##認 -##誓 -##誕 -##誘 -##語 -##誠 -##誡 -##誣 -##誤 -##誥 -##誦 -##誨 -##說 -##説 -##読 -##誰 -##課 -##誹 -##誼 -##調 -##諄 -##談 -##請 -##諏 -##諒 -##論 -##諗 -##諜 -##諡 -##諦 -##諧 -##諫 -##諭 -##諮 -##諱 -##諳 -##諷 -##諸 -##諺 -##諾 -##謀 -##謁 -##謂 -##謄 -##謊 -##謎 -##謐 -##謔 -##謗 -##謙 -##講 -##謝 -##謠 -##謨 -##謬 -##謹 -##謾 -##譁 -##證 -##譎 -##譏 -##識 -##譙 -##譚 -##譜 -##警 -##譬 -##譯 -##議 -##譲 -##譴 -##護 -##譽 -##讀 -##變 -##讓 -##讚 -##讞 -##计 -##订 -##认 -##讥 -##讧 -##讨 -##让 -##讪 -##讫 -##训 -##议 -##讯 -##记 -##讲 -##讳 -##讴 -##讶 -##讷 -##许 -##讹 -##论 -##讼 -##讽 -##设 -##访 -##诀 -##证 -##诃 -##评 -##诅 -##识 -##诈 -##诉 -##诊 -##诋 -##词 -##诏 -##译 -##试 -##诗 -##诘 -##诙 -##诚 -##诛 -##话 -##诞 -##诟 -##诠 -##诡 -##询 -##诣 -##诤 -##该 -##详 -##诧 -##诩 -##诫 -##诬 -##语 -##误 -##诰 -##诱 -##诲 -##说 -##诵 -##诶 -##请 -##诸 -##诺 -##读 -##诽 -##课 -##诿 -##谀 -##谁 -##调 -##谄 -##谅 -##谆 -##谈 -##谊 -##谋 -##谌 -##谍 -##谎 -##谏 -##谐 -##谑 -##谒 -##谓 -##谔 -##谕 -##谗 -##谘 -##谙 -##谚 -##谛 -##谜 -##谟 -##谢 -##谣 -##谤 -##谥 -##谦 -##谧 -##谨 -##谩 -##谪 -##谬 -##谭 -##谯 -##谱 -##谲 -##谴 -##谶 -##谷 -##豁 -##豆 -##豇 -##豈 -##豉 -##豊 -##豌 -##豎 -##豐 -##豔 -##豚 -##象 -##豢 -##豪 -##豫 -##豬 -##豹 -##豺 -##貂 -##貅 -##貌 -##貓 -##貔 -##貘 -##貝 -##貞 -##負 -##財 -##貢 -##貧 -##貨 -##販 -##貪 -##貫 -##責 -##貯 -##貰 -##貳 -##貴 -##貶 -##買 -##貸 -##費 -##貼 -##貽 -##貿 -##賀 -##賁 -##賂 -##賃 -##賄 -##資 -##賈 -##賊 -##賑 -##賓 -##賜 -##賞 -##賠 -##賡 -##賢 -##賣 -##賤 -##賦 -##質 -##賬 -##賭 -##賴 -##賺 -##購 -##賽 -##贅 -##贈 -##贊 -##贍 -##贏 -##贓 -##贖 -##贛 -##贝 -##贞 -##负 -##贡 -##财 -##责 -##贤 -##败 -##账 -##货 -##质 -##贩 -##贪 -##贫 -##贬 -##购 -##贮 -##贯 -##贰 -##贱 -##贲 -##贴 -##贵 -##贷 -##贸 -##费 -##贺 -##贻 -##贼 -##贾 -##贿 -##赁 -##赂 -##赃 -##资 -##赅 -##赈 -##赊 -##赋 -##赌 -##赎 -##赏 -##赐 -##赓 -##赔 -##赖 -##赘 -##赚 -##赛 -##赝 -##赞 -##赠 -##赡 -##赢 -##赣 -##赤 -##赦 -##赧 -##赫 -##赭 -##走 -##赳 -##赴 -##赵 -##赶 -##起 -##趁 -##超 -##越 -##趋 -##趕 -##趙 -##趟 -##趣 -##趨 -##足 -##趴 -##趵 -##趸 -##趺 -##趾 -##跃 -##跄 -##跆 -##跋 -##跌 -##跎 -##跑 -##跖 -##跚 -##跛 -##距 -##跟 -##跡 -##跤 -##跨 -##跩 -##跪 -##路 -##跳 -##践 -##跷 -##跹 -##跺 -##跻 -##踉 -##踊 -##踌 -##踏 -##踐 -##踝 -##踞 -##踟 -##踢 -##踩 -##踪 -##踮 -##踱 -##踴 -##踵 -##踹 -##蹂 -##蹄 -##蹇 -##蹈 -##蹉 -##蹊 -##蹋 -##蹑 -##蹒 -##蹙 -##蹟 -##蹣 -##蹤 -##蹦 -##蹩 -##蹬 -##蹭 -##蹲 -##蹴 -##蹶 -##蹺 -##蹼 -##蹿 -##躁 -##躇 -##躉 -##躊 -##躋 -##躍 -##躏 -##躪 -##身 -##躬 -##躯 -##躲 -##躺 -##軀 -##車 -##軋 -##軌 -##軍 -##軒 -##軟 -##転 -##軸 -##軼 -##軽 -##軾 -##較 -##載 -##輒 -##輓 -##輔 -##輕 -##輛 -##輝 -##輟 -##輩 -##輪 -##輯 -##輸 -##輻 -##輾 -##輿 -##轄 -##轅 -##轆 -##轉 -##轍 -##轎 -##轟 -##车 -##轧 -##轨 -##轩 -##转 -##轭 -##轮 -##软 -##轰 -##轲 -##轴 -##轶 -##轻 -##轼 -##载 -##轿 -##较 -##辄 -##辅 -##辆 -##辇 -##辈 -##辉 -##辊 -##辍 -##辐 -##辑 -##输 -##辕 -##辖 -##辗 -##辘 -##辙 -##辛 -##辜 -##辞 -##辟 -##辣 -##辦 -##辨 -##辩 -##辫 -##辭 -##辮 -##辯 -##辰 -##辱 -##農 -##边 -##辺 -##辻 -##込 -##辽 -##达 -##迁 -##迂 -##迄 -##迅 -##过 -##迈 -##迎 -##运 -##近 -##返 -##还 -##这 -##进 -##远 -##违 -##连 -##迟 -##迢 -##迤 -##迥 -##迦 -##迩 -##迪 -##迫 -##迭 -##述 -##迴 -##迷 -##迸 -##迹 -##迺 -##追 -##退 -##送 -##适 -##逃 -##逅 -##逆 -##选 -##逊 -##逍 -##透 -##逐 -##递 -##途 -##逕 -##逗 -##這 -##通 -##逛 -##逝 -##逞 -##速 -##造 -##逢 -##連 -##逮 -##週 -##進 -##逵 -##逶 -##逸 -##逻 -##逼 -##逾 -##遁 -##遂 -##遅 -##遇 -##遊 -##運 -##遍 -##過 -##遏 -##遐 -##遑 -##遒 -##道 -##達 -##違 -##遗 -##遙 -##遛 -##遜 -##遞 -##遠 -##遢 -##遣 -##遥 -##遨 -##適 -##遭 -##遮 -##遲 -##遴 -##遵 -##遶 -##遷 -##選 -##遺 -##遼 -##遽 -##避 -##邀 -##邁 -##邂 -##邃 -##還 -##邇 -##邈 -##邊 -##邋 -##邏 -##邑 -##邓 -##邕 -##邛 -##邝 -##邢 -##那 -##邦 -##邨 -##邪 -##邬 -##邮 -##邯 -##邰 -##邱 -##邳 -##邵 -##邸 -##邹 -##邺 -##邻 -##郁 -##郅 -##郊 -##郎 -##郑 -##郜 -##郝 -##郡 -##郢 -##郤 -##郦 -##郧 -##部 -##郫 -##郭 -##郴 -##郵 -##郷 -##郸 -##都 -##鄂 -##鄉 -##鄒 -##鄔 -##鄙 -##鄞 -##鄢 -##鄧 -##鄭 -##鄰 -##鄱 -##鄲 -##鄺 -##酉 -##酊 -##酋 -##酌 -##配 -##酐 -##酒 -##酗 -##酚 -##酝 -##酢 -##酣 -##酥 -##酩 -##酪 -##酬 -##酮 -##酯 -##酰 -##酱 -##酵 -##酶 -##酷 -##酸 -##酿 -##醃 -##醇 -##醉 -##醋 -##醍 -##醐 -##醒 -##醚 -##醛 -##醜 -##醞 -##醣 -##醪 -##醫 -##醬 -##醮 -##醯 -##醴 -##醺 -##釀 -##釁 -##采 -##釉 -##释 -##釋 -##里 -##重 -##野 -##量 -##釐 -##金 -##釗 -##釘 -##釜 -##針 -##釣 -##釦 -##釧 -##釵 -##鈀 -##鈉 -##鈍 -##鈎 -##鈔 -##鈕 -##鈞 -##鈣 -##鈦 -##鈪 -##鈴 -##鈺 -##鈾 -##鉀 -##鉄 -##鉅 -##鉉 -##鉑 -##鉗 -##鉚 -##鉛 -##鉤 -##鉴 -##鉻 -##銀 -##銃 -##銅 -##銑 -##銓 -##銖 -##銘 -##銜 -##銬 -##銭 -##銮 -##銳 -##銷 -##銹 -##鋁 -##鋅 -##鋒 -##鋤 -##鋪 -##鋰 -##鋸 -##鋼 -##錄 -##錐 -##錘 -##錚 -##錠 -##錢 -##錦 -##錨 -##錫 -##錮 -##錯 -##録 -##錳 -##錶 -##鍊 -##鍋 -##鍍 -##鍛 -##鍥 -##鍰 -##鍵 -##鍺 -##鍾 -##鎂 -##鎊 -##鎌 -##鎏 -##鎔 -##鎖 -##鎗 -##鎚 -##鎧 -##鎬 -##鎮 -##鎳 -##鏈 -##鏖 -##鏗 -##鏘 -##鏞 -##鏟 -##鏡 -##鏢 -##鏤 -##鏽 -##鐘 -##鐮 -##鐲 -##鐳 -##鐵 -##鐸 -##鐺 -##鑄 -##鑊 -##鑑 -##鑒 -##鑣 -##鑫 -##鑰 -##鑲 -##鑼 -##鑽 -##鑾 -##鑿 -##针 -##钉 -##钊 -##钎 -##钏 -##钒 -##钓 -##钗 -##钙 -##钛 -##钜 -##钝 -##钞 -##钟 -##钠 -##钡 -##钢 -##钣 -##钤 -##钥 -##钦 -##钧 -##钨 -##钩 -##钮 -##钯 -##钰 -##钱 -##钳 -##钴 -##钵 -##钺 -##钻 -##钼 -##钾 -##钿 -##铀 -##铁 -##铂 -##铃 -##铄 -##铅 -##铆 -##铉 -##铎 -##铐 -##铛 -##铜 -##铝 -##铠 -##铡 -##铢 -##铣 -##铤 -##铨 -##铩 -##铬 -##铭 -##铮 -##铰 -##铲 -##铵 -##银 -##铸 -##铺 -##链 -##铿 -##销 -##锁 -##锂 -##锄 -##锅 -##锆 -##锈 -##锉 -##锋 -##锌 -##锏 -##锐 -##锑 -##错 -##锚 -##锟 -##锡 -##锢 -##锣 -##锤 -##锥 -##锦 -##锭 -##键 -##锯 -##锰 -##锲 -##锵 -##锹 -##锺 -##锻 -##镀 -##镁 -##镂 -##镇 -##镉 -##镌 -##镍 -##镐 -##镑 -##镕 -##镖 -##镗 -##镛 -##镜 -##镣 -##镭 -##镯 -##镰 -##镳 -##镶 -##長 -##长 -##門 -##閃 -##閉 -##開 -##閎 -##閏 -##閑 -##閒 -##間 -##閔 -##閘 -##閡 -##関 -##閣 -##閥 -##閨 -##閩 -##閱 -##閲 -##閹 -##閻 -##閾 -##闆 -##闇 -##闊 -##闌 -##闍 -##闔 -##闕 -##闖 -##闘 -##關 -##闡 -##闢 -##门 -##闪 -##闫 -##闭 -##问 -##闯 -##闰 -##闲 -##间 -##闵 -##闷 -##闸 -##闹 -##闺 -##闻 -##闽 -##闾 -##阀 -##阁 -##阂 -##阅 -##阆 -##阇 -##阈 -##阉 -##阎 -##阐 -##阑 -##阔 -##阕 -##阖 -##阙 -##阚 -##阜 -##队 -##阡 -##阪 -##阮 -##阱 -##防 -##阳 -##阴 -##阵 -##阶 -##阻 -##阿 -##陀 -##陂 -##附 -##际 -##陆 -##陇 -##陈 -##陋 -##陌 -##降 -##限 -##陕 -##陛 -##陝 -##陞 -##陟 -##陡 -##院 -##陣 -##除 -##陨 -##险 -##陪 -##陰 -##陲 -##陳 -##陵 -##陶 -##陷 -##陸 -##険 -##陽 -##隅 -##隆 -##隈 -##隊 -##隋 -##隍 -##階 -##随 -##隐 -##隔 -##隕 -##隘 -##隙 -##際 -##障 -##隠 -##隣 -##隧 -##隨 -##險 -##隱 -##隴 -##隶 -##隸 -##隻 -##隼 -##隽 -##难 -##雀 -##雁 -##雄 -##雅 -##集 -##雇 -##雉 -##雋 -##雌 -##雍 -##雎 -##雏 -##雑 -##雒 -##雕 -##雖 -##雙 -##雛 -##雜 -##雞 -##離 -##難 -##雨 -##雪 -##雯 -##雰 -##雲 -##雳 -##零 -##雷 -##雹 -##電 -##雾 -##需 -##霁 -##霄 -##霆 -##震 -##霈 -##霉 -##霊 -##霍 -##霎 -##霏 -##霑 -##霓 -##霖 -##霜 -##霞 -##霧 -##霭 -##霰 -##露 -##霸 -##霹 -##霽 -##霾 -##靂 -##靄 -##靈 -##青 -##靓 -##靖 -##静 -##靚 -##靛 -##靜 -##非 -##靠 -##靡 -##面 -##靥 -##靦 -##革 -##靳 -##靴 -##靶 -##靼 -##鞅 -##鞋 -##鞍 -##鞏 -##鞑 -##鞘 -##鞠 -##鞣 -##鞦 -##鞭 -##韆 -##韋 -##韌 -##韓 -##韜 -##韦 -##韧 -##韩 -##韬 -##韭 -##音 -##韵 -##韶 -##韻 -##響 -##頁 -##頂 -##頃 -##項 -##順 -##須 -##頌 -##預 -##頑 -##頒 -##頓 -##頗 -##領 -##頜 -##頡 -##頤 -##頫 -##頭 -##頰 -##頷 -##頸 -##頹 -##頻 -##頼 -##顆 -##題 -##額 -##顎 -##顏 -##顔 -##願 -##顛 -##類 -##顧 -##顫 -##顯 -##顱 -##顴 -##页 -##顶 -##顷 -##项 -##顺 -##须 -##顼 -##顽 -##顾 -##顿 -##颁 -##颂 -##预 -##颅 -##领 -##颇 -##颈 -##颉 -##颊 -##颌 -##颍 -##颐 -##频 -##颓 -##颔 -##颖 -##颗 -##题 -##颚 -##颛 -##颜 -##额 -##颞 -##颠 -##颡 -##颢 -##颤 -##颦 -##颧 -##風 -##颯 -##颱 -##颳 -##颶 -##颼 -##飄 -##飆 -##风 -##飒 -##飓 -##飕 -##飘 -##飙 -##飚 -##飛 -##飞 -##食 -##飢 -##飨 -##飩 -##飪 -##飯 -##飲 -##飼 -##飽 -##飾 -##餃 -##餅 -##餉 -##養 -##餌 -##餐 -##餒 -##餓 -##餘 -##餚 -##餛 -##餞 -##餡 -##館 -##餮 -##餵 -##餾 -##饅 -##饈 -##饋 -##饌 -##饍 -##饑 -##饒 -##饕 -##饗 -##饞 -##饥 -##饨 -##饪 -##饬 -##饭 -##饮 -##饯 -##饰 -##饱 -##饲 -##饴 -##饵 -##饶 -##饷 -##饺 -##饼 -##饽 -##饿 -##馀 -##馁 -##馄 -##馅 -##馆 -##馈 -##馋 -##馍 -##馏 -##馒 -##馔 -##首 -##馗 -##香 -##馥 -##馨 -##馬 -##馭 -##馮 -##馳 -##馴 -##駁 -##駄 -##駅 -##駆 -##駐 -##駒 -##駕 -##駛 -##駝 -##駭 -##駱 -##駿 -##騁 -##騎 -##騏 -##験 -##騙 -##騨 -##騰 -##騷 -##驀 -##驅 -##驊 -##驍 -##驒 -##驕 -##驗 -##驚 -##驛 -##驟 -##驢 -##驥 -##马 -##驭 -##驮 -##驯 -##驰 -##驱 -##驳 -##驴 -##驶 -##驷 -##驸 -##驹 -##驻 -##驼 -##驾 -##驿 -##骁 -##骂 -##骄 -##骅 -##骆 -##骇 -##骈 -##骊 -##骋 -##验 -##骏 -##骐 -##骑 -##骗 -##骚 -##骛 -##骜 -##骞 -##骠 -##骡 -##骤 -##骥 -##骧 -##骨 -##骯 -##骰 -##骶 -##骷 -##骸 -##骼 -##髂 -##髅 -##髋 -##髏 -##髒 -##髓 -##體 -##髖 -##高 -##髦 -##髪 -##髮 -##髯 -##髻 -##鬃 -##鬆 -##鬍 -##鬓 -##鬚 -##鬟 -##鬢 -##鬣 -##鬥 -##鬧 -##鬱 -##鬼 -##魁 -##魂 -##魄 -##魅 -##魇 -##魍 -##魏 -##魔 -##魘 -##魚 -##魯 -##魷 -##鮑 -##鮨 -##鮪 -##鮭 -##鮮 -##鯉 -##鯊 -##鯖 -##鯛 -##鯨 -##鯰 -##鯽 -##鰍 -##鰓 -##鰭 -##鰲 -##鰻 -##鰾 -##鱈 -##鱉 -##鱔 -##鱗 -##鱷 -##鱸 -##鱼 -##鱿 -##鲁 -##鲈 -##鲍 -##鲑 -##鲛 -##鲜 -##鲟 -##鲢 -##鲤 -##鲨 -##鲫 -##鲱 -##鲲 -##鲶 -##鲷 -##鲸 -##鳃 -##鳄 -##鳅 -##鳌 -##鳍 -##鳕 -##鳖 -##鳗 -##鳝 -##鳞 -##鳥 -##鳩 -##鳳 -##鳴 -##鳶 -##鴉 -##鴕 -##鴛 -##鴦 -##鴨 -##鴻 -##鴿 -##鵑 -##鵜 -##鵝 -##鵡 -##鵬 -##鵰 -##鵲 -##鶘 -##鶩 -##鶯 -##鶴 -##鷗 -##鷲 -##鷹 -##鷺 -##鸚 -##鸞 -##鸟 -##鸠 -##鸡 -##鸢 -##鸣 -##鸥 -##鸦 -##鸨 -##鸪 -##鸭 -##鸯 -##鸳 -##鸵 -##鸽 -##鸾 -##鸿 -##鹂 -##鹃 -##鹄 -##鹅 -##鹈 -##鹉 -##鹊 -##鹌 -##鹏 -##鹑 -##鹕 -##鹘 -##鹜 -##鹞 -##鹤 -##鹦 -##鹧 -##鹫 -##鹭 -##鹰 -##鹳 -##鹵 -##鹹 -##鹼 -##鹽 -##鹿 -##麂 -##麋 -##麒 -##麓 -##麗 -##麝 -##麟 -##麥 -##麦 -##麩 -##麴 -##麵 -##麸 -##麺 -##麻 -##麼 -##麽 -##麾 -##黃 -##黄 -##黍 -##黎 -##黏 -##黑 -##黒 -##黔 -##默 -##黛 -##黜 -##黝 -##點 -##黠 -##黨 -##黯 -##黴 -##鼋 -##鼎 -##鼐 -##鼓 -##鼠 -##鼬 -##鼹 -##鼻 -##鼾 -##齁 -##齊 -##齋 -##齐 -##齒 -##齡 -##齢 -##齣 -##齦 -##齿 -##龄 -##龅 -##龈 -##龊 -##龋 -##龌 -##龍 -##龐 -##龔 -##龕 -##龙 -##龚 -##龛 -##龜 -##龟 -##︰ -##︱ -##︶ -##︿ -##﹁ -##﹂ -##﹍ -##﹏ -##﹐ -##﹑ -##﹒ -##﹔ -##﹕ -##﹖ -##﹗ -##﹙ -##﹚ -##﹝ -##﹞ -##﹡ -##﹣ -##！ -##＂ -##＃ -##＄ -##％ -##＆ -##＇ -##（ -##） -##＊ -##， -##－ -##． -##／ -##： -##； -##＜ -##？ -##＠ -##［ -##＼ -##］ -##＾ -##＿ -##｀ -##ｆ -##ｈ -##ｊ -##ｕ -##ｗ -##ｚ -##｛ -##｝ -##｡ -##｢ -##｣ -##､ -##･ -##ｯ -##ｰ -##ｲ -##ｸ -##ｼ -##ｽ -##ﾄ -##ﾉ -##ﾌ -##ﾗ -##ﾙ -##ﾝ -##ﾞ -##ﾟ -##￣ -##￥ -##👍 -##🔥 -##😂 -##😎 diff --git a/third_party/to_mindrecord/zhwiki/.gitignore b/third_party/to_mindrecord/zhwiki/.gitignore deleted file mode 100644 index 38ab986b46..0000000000 --- a/third_party/to_mindrecord/zhwiki/.gitignore +++ /dev/null @@ -1 +0,0 @@ -create_pretraining_data_patched.py diff --git a/third_party/to_mindrecord/zhwiki/README.md b/third_party/to_mindrecord/zhwiki/README.md deleted file mode 100644 index 6199240d7c..0000000000 --- a/third_party/to_mindrecord/zhwiki/README.md +++ /dev/null @@ -1 +0,0 @@ -## All the scripts here come from [google-research/bert](https://github.com/google-research/bert) diff --git a/third_party/to_mindrecord/zhwiki/create_pretraining_data.py b/third_party/to_mindrecord/zhwiki/create_pretraining_data.py deleted file mode 100644 index 5340d96ae3..0000000000 --- a/third_party/to_mindrecord/zhwiki/create_pretraining_data.py +++ /dev/null @@ -1,469 +0,0 @@ -# coding=utf-8 -# Copyright 2018 The Google AI Language Team Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Create masked LM/next sentence masked_lm TF examples for BERT.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import collections -import random -import tokenization -import tensorflow as tf - -flags = tf.flags - -FLAGS = flags.FLAGS - -flags.DEFINE_string("input_file", None, - "Input raw text file (or comma-separated list of files).") - -flags.DEFINE_string( - "output_file", None, - "Output TF example file (or comma-separated list of files).") - -flags.DEFINE_string("vocab_file", None, - "The vocabulary file that the BERT model was trained on.") - -flags.DEFINE_bool( - "do_lower_case", True, - "Whether to lower case the input text. Should be True for uncased " - "models and False for cased models.") - -flags.DEFINE_bool( - "do_whole_word_mask", False, - "Whether to use whole word masking rather than per-WordPiece masking.") - -flags.DEFINE_integer("max_seq_length", 128, "Maximum sequence length.") - -flags.DEFINE_integer("max_predictions_per_seq", 20, - "Maximum number of masked LM predictions per sequence.") - -flags.DEFINE_integer("random_seed", 12345, "Random seed for data generation.") - -flags.DEFINE_integer( - "dupe_factor", 10, - "Number of times to duplicate the input data (with different masks).") - -flags.DEFINE_float("masked_lm_prob", 0.15, "Masked LM probability.") - -flags.DEFINE_float( - "short_seq_prob", 0.1, - "Probability of creating sequences which are shorter than the " - "maximum length.") - - -class TrainingInstance(object): - """A single training instance (sentence pair).""" - - def __init__(self, tokens, segment_ids, masked_lm_positions, masked_lm_labels, - is_random_next): - self.tokens = tokens - self.segment_ids = segment_ids - self.is_random_next = is_random_next - self.masked_lm_positions = masked_lm_positions - self.masked_lm_labels = masked_lm_labels - - def __str__(self): - s = "" - s += "tokens: %s\n" % (" ".join( - [tokenization.printable_text(x) for x in self.tokens])) - s += "segment_ids: %s\n" % (" ".join([str(x) for x in self.segment_ids])) - s += "is_random_next: %s\n" % self.is_random_next - s += "masked_lm_positions: %s\n" % (" ".join( - [str(x) for x in self.masked_lm_positions])) - s += "masked_lm_labels: %s\n" % (" ".join( - [tokenization.printable_text(x) for x in self.masked_lm_labels])) - s += "\n" - return s - - def __repr__(self): - return self.__str__() - - -def write_instance_to_example_files(instances, tokenizer, max_seq_length, - max_predictions_per_seq, output_files): - """Create TF example files from `TrainingInstance`s.""" - writers = [] - for output_file in output_files: - writers.append(tf.python_io.TFRecordWriter(output_file)) - - writer_index = 0 - - total_written = 0 - for (inst_index, instance) in enumerate(instances): - input_ids = tokenizer.convert_tokens_to_ids(instance.tokens) - input_mask = [1] * len(input_ids) - segment_ids = list(instance.segment_ids) - assert len(input_ids) <= max_seq_length - - while len(input_ids) < max_seq_length: - input_ids.append(0) - input_mask.append(0) - segment_ids.append(0) - - assert len(input_ids) == max_seq_length - assert len(input_mask) == max_seq_length - assert len(segment_ids) == max_seq_length - - masked_lm_positions = list(instance.masked_lm_positions) - masked_lm_ids = tokenizer.convert_tokens_to_ids(instance.masked_lm_labels) - masked_lm_weights = [1.0] * len(masked_lm_ids) - - while len(masked_lm_positions) < max_predictions_per_seq: - masked_lm_positions.append(0) - masked_lm_ids.append(0) - masked_lm_weights.append(0.0) - - next_sentence_label = 1 if instance.is_random_next else 0 - - features = collections.OrderedDict() - features["input_ids"] = create_int_feature(input_ids) - features["input_mask"] = create_int_feature(input_mask) - features["segment_ids"] = create_int_feature(segment_ids) - features["masked_lm_positions"] = create_int_feature(masked_lm_positions) - features["masked_lm_ids"] = create_int_feature(masked_lm_ids) - features["masked_lm_weights"] = create_float_feature(masked_lm_weights) - features["next_sentence_labels"] = create_int_feature([next_sentence_label]) - - tf_example = tf.train.Example(features=tf.train.Features(feature=features)) - - writers[writer_index].write(tf_example.SerializeToString()) - writer_index = (writer_index + 1) % len(writers) - - total_written += 1 - - if inst_index < 20: - tf.logging.info("*** Example ***") - tf.logging.info("tokens: %s" % " ".join( - [tokenization.printable_text(x) for x in instance.tokens])) - - for feature_name in features.keys(): - feature = features[feature_name] - values = [] - if feature.int64_list.value: - values = feature.int64_list.value - elif feature.float_list.value: - values = feature.float_list.value - tf.logging.info( - "%s: %s" % (feature_name, " ".join([str(x) for x in values]))) - - for writer in writers: - writer.close() - - tf.logging.info("Wrote %d total instances", total_written) - - -def create_int_feature(values): - feature = tf.train.Feature(int64_list=tf.train.Int64List(value=list(values))) - return feature - - -def create_float_feature(values): - feature = tf.train.Feature(float_list=tf.train.FloatList(value=list(values))) - return feature - - -def create_training_instances(input_files, tokenizer, max_seq_length, - dupe_factor, short_seq_prob, masked_lm_prob, - max_predictions_per_seq, rng): - """Create `TrainingInstance`s from raw text.""" - all_documents = [[]] - - # Input file format: - # (1) One sentence per line. These should ideally be actual sentences, not - # entire paragraphs or arbitrary spans of text. (Because we use the - # sentence boundaries for the "next sentence prediction" task). - # (2) Blank lines between documents. Document boundaries are needed so - # that the "next sentence prediction" task doesn't span between documents. - for input_file in input_files: - with tf.gfile.GFile(input_file, "r") as reader: - while True: - line = tokenization.convert_to_unicode(reader.readline()) - if not line: - break - line = line.strip() - - # Empty lines are used as document delimiters - if not line: - all_documents.append([]) - tokens = tokenizer.tokenize(line) - if tokens: - all_documents[-1].append(tokens) - - # Remove empty documents - all_documents = [x for x in all_documents if x] - rng.shuffle(all_documents) - - vocab_words = list(tokenizer.vocab.keys()) - instances = [] - for _ in range(dupe_factor): - for document_index in range(len(all_documents)): - instances.extend( - create_instances_from_document( - all_documents, document_index, max_seq_length, short_seq_prob, - masked_lm_prob, max_predictions_per_seq, vocab_words, rng)) - - rng.shuffle(instances) - return instances - - -def create_instances_from_document( - all_documents, document_index, max_seq_length, short_seq_prob, - masked_lm_prob, max_predictions_per_seq, vocab_words, rng): - """Creates `TrainingInstance`s for a single document.""" - document = all_documents[document_index] - - # Account for [CLS], [SEP], [SEP] - max_num_tokens = max_seq_length - 3 - - # We *usually* want to fill up the entire sequence since we are padding - # to `max_seq_length` anyways, so short sequences are generally wasted - # computation. However, we *sometimes* - # (i.e., short_seq_prob == 0.1 == 10% of the time) want to use shorter - # sequences to minimize the mismatch between pre-training and fine-tuning. - # The `target_seq_length` is just a rough target however, whereas - # `max_seq_length` is a hard limit. - target_seq_length = max_num_tokens - if rng.random() < short_seq_prob: - target_seq_length = rng.randint(2, max_num_tokens) - - # We DON'T just concatenate all of the tokens from a document into a long - # sequence and choose an arbitrary split point because this would make the - # next sentence prediction task too easy. Instead, we split the input into - # segments "A" and "B" based on the actual "sentences" provided by the user - # input. - instances = [] - current_chunk = [] - current_length = 0 - i = 0 - while i < len(document): - segment = document[i] - current_chunk.append(segment) - current_length += len(segment) - if i == len(document) - 1 or current_length >= target_seq_length: - if current_chunk: - # `a_end` is how many segments from `current_chunk` go into the `A` - # (first) sentence. - a_end = 1 - if len(current_chunk) >= 2: - a_end = rng.randint(1, len(current_chunk) - 1) - - tokens_a = [] - for j in range(a_end): - tokens_a.extend(current_chunk[j]) - - tokens_b = [] - # Random next - is_random_next = False - if len(current_chunk) == 1 or rng.random() < 0.5: - is_random_next = True - target_b_length = target_seq_length - len(tokens_a) - - # This should rarely go for more than one iteration for large - # corpora. However, just to be careful, we try to make sure that - # the random document is not the same as the document - # we're processing. - for _ in range(10): - random_document_index = rng.randint(0, len(all_documents) - 1) - if random_document_index != document_index: - break - - random_document = all_documents[random_document_index] - random_start = rng.randint(0, len(random_document) - 1) - for j in range(random_start, len(random_document)): - tokens_b.extend(random_document[j]) - if len(tokens_b) >= target_b_length: - break - # We didn't actually use these segments so we "put them back" so - # they don't go to waste. - num_unused_segments = len(current_chunk) - a_end - i -= num_unused_segments - # Actual next - else: - is_random_next = False - for j in range(a_end, len(current_chunk)): - tokens_b.extend(current_chunk[j]) - truncate_seq_pair(tokens_a, tokens_b, max_num_tokens, rng) - - assert len(tokens_a) >= 1 - assert len(tokens_b) >= 1 - - tokens = [] - segment_ids = [] - tokens.append("[CLS]") - segment_ids.append(0) - for token in tokens_a: - tokens.append(token) - segment_ids.append(0) - - tokens.append("[SEP]") - segment_ids.append(0) - - for token in tokens_b: - tokens.append(token) - segment_ids.append(1) - tokens.append("[SEP]") - segment_ids.append(1) - - (tokens, masked_lm_positions, - masked_lm_labels) = create_masked_lm_predictions( - tokens, masked_lm_prob, max_predictions_per_seq, vocab_words, rng) - instance = TrainingInstance( - tokens=tokens, - segment_ids=segment_ids, - is_random_next=is_random_next, - masked_lm_positions=masked_lm_positions, - masked_lm_labels=masked_lm_labels) - instances.append(instance) - current_chunk = [] - current_length = 0 - i += 1 - - return instances - - -MaskedLmInstance = collections.namedtuple("MaskedLmInstance", - ["index", "label"]) - - -def create_masked_lm_predictions(tokens, masked_lm_prob, - max_predictions_per_seq, vocab_words, rng): - """Creates the predictions for the masked LM objective.""" - - cand_indexes = [] - for (i, token) in enumerate(tokens): - if token == "[CLS]" or token == "[SEP]": - continue - # Whole Word Masking means that if we mask all of the wordpieces - # corresponding to an original word. When a word has been split into - # WordPieces, the first token does not have any marker and any subsequence - # tokens are prefixed with ##. So whenever we see the ## token, we - # append it to the previous set of word indexes. - # - # Note that Whole Word Masking does *not* change the training code - # at all -- we still predict each WordPiece independently, softmaxed - # over the entire vocabulary. - if (FLAGS.do_whole_word_mask and len(cand_indexes) >= 1 and - token.startswith("##")): - cand_indexes[-1].append(i) - else: - cand_indexes.append([i]) - - rng.shuffle(cand_indexes) - - output_tokens = list(tokens) - - num_to_predict = min(max_predictions_per_seq, - max(1, int(round(len(tokens) * masked_lm_prob)))) - - masked_lms = [] - covered_indexes = set() - for index_set in cand_indexes: - if len(masked_lms) >= num_to_predict: - break - # If adding a whole-word mask would exceed the maximum number of - # predictions, then just skip this candidate. - if len(masked_lms) + len(index_set) > num_to_predict: - continue - is_any_index_covered = False - for index in index_set: - if index in covered_indexes: - is_any_index_covered = True - break - if is_any_index_covered: - continue - for index in index_set: - covered_indexes.add(index) - - masked_token = None - # 80% of the time, replace with [MASK] - if rng.random() < 0.8: - masked_token = "[MASK]" - else: - # 10% of the time, keep original - if rng.random() < 0.5: - masked_token = tokens[index] - # 10% of the time, replace with random word - else: - masked_token = vocab_words[rng.randint(0, len(vocab_words) - 1)] - - output_tokens[index] = masked_token - - masked_lms.append(MaskedLmInstance(index=index, label=tokens[index])) - assert len(masked_lms) <= num_to_predict - masked_lms = sorted(masked_lms, key=lambda x: x.index) - - masked_lm_positions = [] - masked_lm_labels = [] - for p in masked_lms: - masked_lm_positions.append(p.index) - masked_lm_labels.append(p.label) - - return (output_tokens, masked_lm_positions, masked_lm_labels) - - -def truncate_seq_pair(tokens_a, tokens_b, max_num_tokens, rng): - """Truncates a pair of sequences to a maximum sequence length.""" - while True: - total_length = len(tokens_a) + len(tokens_b) - if total_length <= max_num_tokens: - break - - trunc_tokens = tokens_a if len(tokens_a) > len(tokens_b) else tokens_b - assert len(trunc_tokens) >= 1 - - # We want to sometimes truncate from the front and sometimes from the - # back to add more randomness and avoid biases. - if rng.random() < 0.5: - del trunc_tokens[0] - else: - trunc_tokens.pop() - - -def main(_): - tf.logging.set_verbosity(tf.logging.INFO) - - tokenizer = tokenization.FullTokenizer( - vocab_file=FLAGS.vocab_file, do_lower_case=FLAGS.do_lower_case) - - input_files = [] - for input_pattern in FLAGS.input_file.split(","): - input_files.extend(tf.gfile.Glob(input_pattern)) - - tf.logging.info("*** Reading from input files ***") - for input_file in input_files: - tf.logging.info(" %s", input_file) - - rng = random.Random(FLAGS.random_seed) - instances = create_training_instances( - input_files, tokenizer, FLAGS.max_seq_length, FLAGS.dupe_factor, - FLAGS.short_seq_prob, FLAGS.masked_lm_prob, FLAGS.max_predictions_per_seq, - rng) - - output_files = FLAGS.output_file.split(",") - tf.logging.info("*** Writing to output files ***") - for output_file in output_files: - tf.logging.info(" %s", output_file) - - write_instance_to_example_files(instances, tokenizer, FLAGS.max_seq_length, - FLAGS.max_predictions_per_seq, output_files) - - -if __name__ == "__main__": - flags.mark_flag_as_required("input_file") - flags.mark_flag_as_required("output_file") - flags.mark_flag_as_required("vocab_file") - tf.app.run() diff --git a/third_party/to_mindrecord/zhwiki/sample_text.txt b/third_party/to_mindrecord/zhwiki/sample_text.txt deleted file mode 100644 index a42812060c..0000000000 --- a/third_party/to_mindrecord/zhwiki/sample_text.txt +++ /dev/null @@ -1,33 +0,0 @@ -This text is included to make sure Unicode is handled properly: 力加勝北区ᴵᴺᵀᵃছজটডণত -Text should be one-sentence-per-line, with empty lines between documents. -This sample text is public domain and was randomly selected from Project Guttenberg. - -The rain had only ceased with the gray streaks of morning at Blazing Star, and the settlement awoke to a moral sense of cleanliness, and the finding of forgotten knives, tin cups, and smaller camp utensils, where the heavy showers had washed away the debris and dust heaps before the cabin doors. -Indeed, it was recorded in Blazing Star that a fortunate early riser had once picked up on the highway a solid chunk of gold quartz which the rain had freed from its incumbering soil, and washed into immediate and glittering popularity. -Possibly this may have been the reason why early risers in that locality, during the rainy season, adopted a thoughtful habit of body, and seldom lifted their eyes to the rifted or india-ink washed skies above them. -"Cass" Beard had risen early that morning, but not with a view to discovery. -A leak in his cabin roof,--quite consistent with his careless, improvident habits,--had roused him at 4 A. M., with a flooded "bunk" and wet blankets. -The chips from his wood pile refused to kindle a fire to dry his bed-clothes, and he had recourse to a more provident neighbor's to supply the deficiency. -This was nearly opposite. -Mr. Cassius crossed the highway, and stopped suddenly. -Something glittered in the nearest red pool before him. -Gold, surely! -But, wonderful to relate, not an irregular, shapeless fragment of crude ore, fresh from Nature's crucible, but a bit of jeweler's handicraft in the form of a plain gold ring. -Looking at it more attentively, he saw that it bore the inscription, "May to Cass." -Like most of his fellow gold-seekers, Cass was superstitious. - -The fountain of classic wisdom, Hypatia herself. -As the ancient sage--the name is unimportant to a monk--pumped water nightly that he might study by day, so I, the guardian of cloaks and parasols, at the sacred doors of her lecture-room, imbibe celestial knowledge. -From my youth I felt in me a soul above the matter-entangled herd. -She revealed to me the glorious fact, that I am a spark of Divinity itself. -A fallen star, I am, sir!' continued he, pensively, stroking his lean stomach--'a fallen star!--fallen, if the dignity of philosophy will allow of the simile, among the hogs of the lower world--indeed, even into the hog-bucket itself. Well, after all, I will show you the way to the Archbishop's. -There is a philosophic pleasure in opening one's treasures to the modest young. -Perhaps you will assist me by carrying this basket of fruit?' And the little man jumped up, put his basket on Philammon's head, and trotted off up a neighbouring street. -Philammon followed, half contemptuous, half wondering at what this philosophy might be, which could feed the self-conceit of anything so abject as his ragged little apish guide; -but the novel roar and whirl of the street, the perpetual stream of busy faces, the line of curricles, palanquins, laden asses, camels, elephants, which met and passed him, and squeezed him up steps and into doorways, as they threaded their way through the great Moon-gate into the ample street beyond, drove everything from his mind but wondering curiosity, and a vague, helpless dread of that great living wilderness, more terrible than any dead wilderness of sand which he had left behind. -Already he longed for the repose, the silence of the Laura--for faces which knew him and smiled upon him; but it was too late to turn back now. -His guide held on for more than a mile up the great main street, crossed in the centre of the city, at right angles, by one equally magnificent, at each end of which, miles away, appeared, dim and distant over the heads of the living stream of passengers, the yellow sand-hills of the desert; -while at the end of the vista in front of them gleamed the blue harbour, through a network of countless masts. -At last they reached the quay at the opposite end of the street; -and there burst on Philammon's astonished eyes a vast semicircle of blue sea, ringed with palaces and towers. -He stopped involuntarily; and his little guide stopped also, and looked askance at the young monk, to watch the effect which that grand panorama should produce on him. diff --git a/third_party/to_mindrecord/zhwiki/tokenization.py b/third_party/to_mindrecord/zhwiki/tokenization.py deleted file mode 100644 index 50e9445a19..0000000000 --- a/third_party/to_mindrecord/zhwiki/tokenization.py +++ /dev/null @@ -1,394 +0,0 @@ -# coding=utf-8 -# Copyright 2018 The Google AI Language Team Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Tokenization classes.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import collections -import re -import unicodedata -import six - -# pylint: skip-file - -def validate_case_matches_checkpoint(do_lower_case, init_checkpoint): - """Checks whether the casing config is consistent with the checkpoint name.""" - - # The casing has to be passed in by the user and there is no explicit check - # as to whether it matches the checkpoint. The casing information probably - # should have been stored in the bert_config.json file, but it's not, so - # we have to heuristically detect it to validate. - - if not init_checkpoint: - return - - m = re.match("^.*?([A-Za-z0-9_-]+)/bert_model.ckpt", init_checkpoint) - if m is None: - return - - model_name = m.group(1) - - lower_models = [ - "uncased_L-24_H-1024_A-16", "uncased_L-12_H-768_A-12", - "multilingual_L-12_H-768_A-12", "chinese_L-12_H-768_A-12" - ] - - cased_models = [ - "cased_L-12_H-768_A-12", "cased_L-24_H-1024_A-16", - "multi_cased_L-12_H-768_A-12" - ] - - is_bad_config = False - if model_name in lower_models and not do_lower_case: - is_bad_config = True - actual_flag = "False" - case_name = "lowercased" - opposite_flag = "True" - - if model_name in cased_models and do_lower_case: - is_bad_config = True - actual_flag = "True" - case_name = "cased" - opposite_flag = "False" - - if is_bad_config: - raise ValueError( - "You passed in `--do_lower_case=%s` with `--init_checkpoint=%s`. " - "However, `%s` seems to be a %s model, so you " - "should pass in `--do_lower_case=%s` so that the fine-tuning matches " - "how the model was pre-training. If this error is wrong, please " - "just comment out this check." % (actual_flag, init_checkpoint, - model_name, case_name, opposite_flag)) - - -def convert_to_unicode(text): - """Converts `text` to Unicode (if it's not already), assuming utf-8 input.""" - if six.PY3: - if isinstance(text, str): - return text - elif isinstance(text, bytes): - return text.decode("utf-8", "ignore") - else: - raise ValueError("Unsupported string type: %s" % (type(text))) - elif six.PY2: - if isinstance(text, str): - return text.decode("utf-8", "ignore") - elif isinstance(text, unicode): - return text - else: - raise ValueError("Unsupported string type: %s" % (type(text))) - else: - raise ValueError("Not running on Python2 or Python 3?") - - -def printable_text(text): - """Returns text encoded in a way suitable for print or `tf.logging`.""" - - # These functions want `str` for both Python2 and Python3, but in one case - # it's a Unicode string and in the other it's a byte string. - if six.PY3: - if isinstance(text, str): - return text - elif isinstance(text, bytes): - return text.decode("utf-8", "ignore") - else: - raise ValueError("Unsupported string type: %s" % (type(text))) - elif six.PY2: - if isinstance(text, str): - return text - elif isinstance(text, unicode): - return text.encode("utf-8") - else: - raise ValueError("Unsupported string type: %s" % (type(text))) - else: - raise ValueError("Not running on Python2 or Python 3?") - - -def load_vocab(vocab_file): - """Loads a vocabulary file into a dictionary.""" - vocab = collections.OrderedDict() - index = 0 - with open(vocab_file, "r") as reader: - while True: - token = convert_to_unicode(reader.readline()) - if not token: - break - token = token.strip() - vocab[token] = index - index += 1 - return vocab - - -def convert_by_vocab(vocab, items): - """Converts a sequence of [tokens|ids] using the vocab.""" - output = [] - for item in items: - output.append(vocab[item]) - return output - - -def convert_tokens_to_ids(vocab, tokens): - return convert_by_vocab(vocab, tokens) - - -def convert_ids_to_tokens(inv_vocab, ids): - return convert_by_vocab(inv_vocab, ids) - - -def whitespace_tokenize(text): - """Runs basic whitespace cleaning and splitting on a piece of text.""" - text = text.strip() - if not text: - return [] - tokens = text.split() - return tokens - - -class FullTokenizer(object): - """Runs end-to-end tokenziation.""" - - def __init__(self, vocab_file, do_lower_case=True): - self.vocab = load_vocab(vocab_file) - self.inv_vocab = {v: k for k, v in self.vocab.items()} - self.basic_tokenizer = BasicTokenizer(do_lower_case=do_lower_case) - self.wordpiece_tokenizer = WordpieceTokenizer(vocab=self.vocab) - - def tokenize(self, text): - split_tokens = [] - for token in self.basic_tokenizer.tokenize(text): - for sub_token in self.wordpiece_tokenizer.tokenize(token): - split_tokens.append(sub_token) - - return split_tokens - - def convert_tokens_to_ids(self, tokens): - return convert_by_vocab(self.vocab, tokens) - - def convert_ids_to_tokens(self, ids): - return convert_by_vocab(self.inv_vocab, ids) - - -class BasicTokenizer(object): - """Runs basic tokenization (punctuation splitting, lower casing, etc.).""" - - def __init__(self, do_lower_case=True): - """Constructs a BasicTokenizer. - Args: - do_lower_case: Whether to lower case the input. - """ - self.do_lower_case = do_lower_case - - def tokenize(self, text): - """Tokenizes a piece of text.""" - text = convert_to_unicode(text) - text = self._clean_text(text) - - # This was added on November 1st, 2018 for the multilingual and Chinese - # models. This is also applied to the English models now, but it doesn't - # matter since the English models were not trained on any Chinese data - # and generally don't have any Chinese data in them (there are Chinese - # characters in the vocabulary because Wikipedia does have some Chinese - # words in the English Wikipedia.). - text = self._tokenize_chinese_chars(text) - - orig_tokens = whitespace_tokenize(text) - split_tokens = [] - for token in orig_tokens: - if self.do_lower_case: - token = token.lower() - token = self._run_strip_accents(token) - split_tokens.extend(self._run_split_on_punc(token)) - - output_tokens = whitespace_tokenize(" ".join(split_tokens)) - return output_tokens - - def _run_strip_accents(self, text): - """Strips accents from a piece of text.""" - text = unicodedata.normalize("NFD", text) - output = [] - for char in text: - cat = unicodedata.category(char) - if cat == "Mn": - continue - output.append(char) - return "".join(output) - - def _run_split_on_punc(self, text): - """Splits punctuation on a piece of text.""" - chars = list(text) - i = 0 - start_new_word = True - output = [] - while i < len(chars): - char = chars[i] - if _is_punctuation(char): - output.append([char]) - start_new_word = True - else: - if start_new_word: - output.append([]) - start_new_word = False - output[-1].append(char) - i += 1 - - return ["".join(x) for x in output] - - def _tokenize_chinese_chars(self, text): - """Adds whitespace around any CJK character.""" - output = [] - for char in text: - cp = ord(char) - if self._is_chinese_char(cp): - output.append(" ") - output.append(char) - output.append(" ") - else: - output.append(char) - return "".join(output) - - def _is_chinese_char(self, cp): - """Checks whether CP is the codepoint of a CJK character.""" - # This defines a "chinese character" as anything in the CJK Unicode block: - # https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block) - # - # Note that the CJK Unicode block is NOT all Japanese and Korean characters, - # despite its name. The modern Korean Hangul alphabet is a different block, - # as is Japanese Hiragana and Katakana. Those alphabets are used to write - # space-separated words, so they are not treated specially and handled - # like the all of the other languages. - if ((cp >= 0x4E00 and cp <= 0x9FFF) or # - (cp >= 0x3400 and cp <= 0x4DBF) or # - (cp >= 0x20000 and cp <= 0x2A6DF) or # - (cp >= 0x2A700 and cp <= 0x2B73F) or # - (cp >= 0x2B740 and cp <= 0x2B81F) or # - (cp >= 0x2B820 and cp <= 0x2CEAF) or - (cp >= 0xF900 and cp <= 0xFAFF) or # - (cp >= 0x2F800 and cp <= 0x2FA1F)): # - return True - - return False - - def _clean_text(self, text): - """Performs invalid character removal and whitespace cleanup on text.""" - output = [] - for char in text: - cp = ord(char) - if cp == 0 or cp == 0xfffd or _is_control(char): - continue - if _is_whitespace(char): - output.append(" ") - else: - output.append(char) - return "".join(output) - - -class WordpieceTokenizer(object): - """Runs WordPiece tokenziation.""" - - def __init__(self, vocab, unk_token="[UNK]", max_input_chars_per_word=200): - self.vocab = vocab - self.unk_token = unk_token - self.max_input_chars_per_word = max_input_chars_per_word - - def tokenize(self, text): - """Tokenizes a piece of text into its word pieces. - This uses a greedy longest-match-first algorithm to perform tokenization - using the given vocabulary. - For example: - input = "unaffable" - output = ["un", "##aff", "##able"] - Args: - text: A single token or whitespace separated tokens. This should have - already been passed through `BasicTokenizer. - Returns: - A list of wordpiece tokens. - """ - - text = convert_to_unicode(text) - - output_tokens = [] - for token in whitespace_tokenize(text): - chars = list(token) - if len(chars) > self.max_input_chars_per_word: - output_tokens.append(self.unk_token) - continue - - is_bad = False - start = 0 - sub_tokens = [] - while start < len(chars): - end = len(chars) - cur_substr = None - while start < end: - substr = "".join(chars[start:end]) - if start > 0: - substr = "##" + substr - if substr in self.vocab: - cur_substr = substr - break - end -= 1 - if cur_substr is None: - is_bad = True - break - sub_tokens.append(cur_substr) - start = end - - if is_bad: - output_tokens.append(self.unk_token) - else: - output_tokens.extend(sub_tokens) - return output_tokens - - -def _is_whitespace(char): - """Checks whether `chars` is a whitespace character.""" - # \t, \n, and \r are technically contorl characters but we treat them - # as whitespace since they are generally considered as such. - if char == " " or char == "\t" or char == "\n" or char == "\r": - return True - cat = unicodedata.category(char) - if cat == "Zs": - return True - return False - - -def _is_control(char): - """Checks whether `chars` is a control character.""" - # These are technically control characters but we count them as whitespace - # characters. - if char == "\t" or char == "\n" or char == "\r": - return False - cat = unicodedata.category(char) - if cat in ("Cc", "Cf"): - return True - return False - - -def _is_punctuation(char): - """Checks whether `chars` is a punctuation character.""" - cp = ord(char) - # We treat all non-letter/number ASCII as punctuation. - # Characters such as "^", "$", and "`" are not in the Unicode - # Punctuation class but we treat them as punctuation anyways, for - # consistency. - if ((cp >= 33 and cp <= 47) or (cp >= 58 and cp <= 64) or - (cp >= 91 and cp <= 96) or (cp >= 123 and cp <= 126)): - return True - cat = unicodedata.category(char) - if cat.startswith("P"): - return True - return False diff --git a/third_party/to_mindrecord/zhwiki/vocab.txt b/third_party/to_mindrecord/zhwiki/vocab.txt deleted file mode 100644 index ca4f978103..0000000000 --- a/third_party/to_mindrecord/zhwiki/vocab.txt +++ /dev/null @@ -1,21128 +0,0 @@ -[PAD] -[unused1] -[unused2] -[unused3] -[unused4] -[unused5] -[unused6] -[unused7] -[unused8] -[unused9] -[unused10] -[unused11] -[unused12] -[unused13] -[unused14] -[unused15] -[unused16] -[unused17] -[unused18] -[unused19] -[unused20] -[unused21] -[unused22] -[unused23] -[unused24] -[unused25] -[unused26] -[unused27] -[unused28] -[unused29] -[unused30] -[unused31] -[unused32] -[unused33] -[unused34] -[unused35] -[unused36] -[unused37] -[unused38] -[unused39] -[unused40] -[unused41] -[unused42] -[unused43] -[unused44] -[unused45] -[unused46] -[unused47] -[unused48] -[unused49] -[unused50] -[unused51] -[unused52] -[unused53] -[unused54] -[unused55] -[unused56] -[unused57] -[unused58] -[unused59] -[unused60] -[unused61] -[unused62] -[unused63] -[unused64] -[unused65] -[unused66] -[unused67] -[unused68] -[unused69] -[unused70] -[unused71] -[unused72] -[unused73] -[unused74] -[unused75] -[unused76] -[unused77] -[unused78] -[unused79] -[unused80] -[unused81] -[unused82] -[unused83] -[unused84] -[unused85] -[unused86] -[unused87] -[unused88] -[unused89] -[unused90] -[unused91] -[unused92] -[unused93] -[unused94] -[unused95] -[unused96] -[unused97] -[unused98] -[unused99] -[UNK] -[CLS] -[SEP] -[MASK] - - -! -" -# -$ -% -& -' -( -) -* -+ -, -- -. -/ -0 -1 -2 -3 -4 -5 -6 -7 -8 -9 -: -; -< -= -> -? -@ -[ -\ -] -^ -_ -a -b -c -d -e -f -g -h -i -j -k -l -m -n -o -p -q -r -s -t -u -v -w -x -y -z -{ -| -} -~ -£ -¤ -¥ -§ -© -« -® -° -± -² -³ -µ -· -¹ -º -» -¼ -× -ß -æ -÷ -ø -đ -ŋ -ɔ -ə -ɡ -ʰ -ˇ -ˈ -ˊ -ˋ -ˍ -ː -˙ -˚ -ˢ -α -β -γ -δ -ε -η -θ -ι -κ -λ -μ -ν -ο -π -ρ -ς -σ -τ -υ -φ -χ -ψ -ω -а -б -в -г -д -е -ж -з -и -к -л -м -н -о -п -р -с -т -у -ф -х -ц -ч -ш -ы -ь -я -і -ا -ب -ة -ت -د -ر -س -ع -ل -م -ن -ه -و -ي -۩ -ก -ง -น -ม -ย -ร -อ -า -เ -๑ -་ -ღ -ᄀ -ᄁ -ᄂ -ᄃ -ᄅ -ᄆ -ᄇ -ᄈ -ᄉ -ᄋ -ᄌ -ᄎ -ᄏ -ᄐ -ᄑ -ᄒ -ᅡ -ᅢ -ᅣ -ᅥ -ᅦ -ᅧ -ᅨ -ᅩ -ᅪ -ᅬ -ᅭ -ᅮ -ᅯ -ᅲ -ᅳ -ᅴ -ᅵ -ᆨ -ᆫ -ᆯ -ᆷ -ᆸ -ᆺ -ᆻ -ᆼ -ᗜ -ᵃ -ᵉ -ᵍ -ᵏ -ᵐ -ᵒ -ᵘ -‖ -„ -† -• -‥ -‧ -
 -‰ -′ -″ -‹ -› -※ -‿ -⁄ -ⁱ -⁺ -ⁿ -₁ -₂ -₃ -₄ -€ -℃ -№ -™ -ⅰ -ⅱ -ⅲ -ⅳ -ⅴ -← -↑ -→ -↓ -↔ -↗ -↘ -⇒ -∀ -− -∕ -∙ -√ -∞ -∟ -∠ -∣ -∥ -∩ -∮ -∶ -∼ -∽ -≈ -≒ -≡ -≤ -≥ -≦ -≧ -≪ -≫ -⊙ -⋅ -⋈ -⋯ -⌒ -① -② -③ -④ -⑤ -⑥ -⑦ -⑧ -⑨ -⑩ -⑴ -⑵ -⑶ -⑷ -⑸ -⒈ -⒉ -⒊ -⒋ -ⓒ -ⓔ -ⓘ -─ -━ -│ -┃ -┅ -┆ -┊ -┌ -└ -├ -┣ -═ -║ -╚ -╞ -╠ -╭ -╮ -╯ -╰ -╱ -╳ -▂ -▃ -▅ -▇ -█ -▉ -▋ -▌ -▍ -▎ -■ -□ -▪ -▫ -▬ -▲ -△ -▶ -► -▼ -▽ -◆ -◇ -○ -◎ -● -◕ -◠ -◢ -◤ -☀ -★ -☆ -☕ -☞ -☺ -☼ -♀ -♂ -♠ -♡ -♣ -♥ -♦ -♪ -♫ -♬ -✈ -✔ -✕ -✖ -✦ -✨ -✪ -✰ -✿ -❀ -❤ -➜ -➤ -⦿ -、 -。 -〃 -々 -〇 -〈 -〉 -《 -》 -「 -」 -『 -』 -【 -】 -〓 -〔 -〕 -〖 -〗 -〜 -〝 -〞 -ぁ -あ -ぃ -い -う -ぇ -え -お -か -き -く -け -こ -さ -し -す -せ -そ -た -ち -っ -つ -て -と -な -に -ぬ -ね -の -は -ひ -ふ -へ -ほ -ま -み -む -め -も -ゃ -や -ゅ -ゆ -ょ -よ -ら -り -る -れ -ろ -わ -を -ん -゜ -ゝ -ァ -ア -ィ -イ -ゥ -ウ -ェ -エ -ォ -オ -カ -キ -ク -ケ -コ -サ -シ -ス -セ -ソ -タ -チ -ッ -ツ -テ -ト -ナ -ニ -ヌ -ネ -ノ -ハ -ヒ -フ -ヘ -ホ -マ -ミ -ム -メ -モ -ャ -ヤ -ュ -ユ -ョ -ヨ -ラ -リ -ル -レ -ロ -ワ -ヲ -ン -ヶ -・ -ー -ヽ -ㄅ -ㄆ -ㄇ -ㄉ -ㄋ -ㄌ -ㄍ -ㄎ -ㄏ -ㄒ -ㄚ -ㄛ -ㄞ -ㄟ -ㄢ -ㄤ -ㄥ -ㄧ -ㄨ -ㆍ -㈦ -㊣ -㎡ -㗎 -一 -丁 -七 -万 -丈 -三 -上 -下 -不 -与 -丐 -丑 -专 -且 -丕 -世 -丘 -丙 -业 -丛 -东 -丝 -丞 -丟 -両 -丢 -两 -严 -並 -丧 -丨 -个 -丫 -中 -丰 -串 -临 -丶 -丸 -丹 -为 -主 -丼 -丽 -举 -丿 -乂 -乃 -久 -么 -义 -之 -乌 -乍 -乎 -乏 -乐 -乒 -乓 -乔 -乖 -乗 -乘 -乙 -乜 -九 -乞 -也 -习 -乡 -书 -乩 -买 -乱 -乳 -乾 -亀 -亂 -了 -予 -争 -事 -二 -于 -亏 -云 -互 -五 -井 -亘 -亙 -亚 -些 -亜 -亞 -亟 -亡 -亢 -交 -亥 -亦 -产 -亨 -亩 -享 -京 -亭 -亮 -亲 -亳 -亵 -人 -亿 -什 -仁 -仃 -仄 -仅 -仆 -仇 -今 -介 -仍 -从 -仏 -仑 -仓 -仔 -仕 -他 -仗 -付 -仙 -仝 -仞 -仟 -代 -令 -以 -仨 -仪 -们 -仮 -仰 -仲 -件 -价 -任 -份 -仿 -企 -伉 -伊 -伍 -伎 -伏 -伐 -休 -伕 -众 -优 -伙 -会 -伝 -伞 -伟 -传 -伢 -伤 -伦 -伪 -伫 -伯 -估 -伴 -伶 -伸 -伺 -似 -伽 -佃 -但 -佇 -佈 -位 -低 -住 -佐 -佑 -体 -佔 -何 -佗 -佘 -余 -佚 -佛 -作 -佝 -佞 -佟 -你 -佢 -佣 -佤 -佥 -佩 -佬 -佯 -佰 -佳 -併 -佶 -佻 -佼 -使 -侃 -侄 -來 -侈 -例 -侍 -侏 -侑 -侖 -侗 -供 -依 -侠 -価 -侣 -侥 -侦 -侧 -侨 -侬 -侮 -侯 -侵 -侶 -侷 -便 -係 -促 -俄 -俊 -俎 -俏 -俐 -俑 -俗 -俘 -俚 -保 -俞 -俟 -俠 -信 -俨 -俩 -俪 -俬 -俭 -修 -俯 -俱 -俳 -俸 -俺 -俾 -倆 -倉 -個 -倌 -倍 -倏 -們 -倒 -倔 -倖 -倘 -候 -倚 -倜 -借 -倡 -値 -倦 -倩 -倪 -倫 -倬 -倭 -倶 -债 -值 -倾 -偃 -假 -偈 -偉 -偌 -偎 -偏 -偕 -做 -停 -健 -側 -偵 -偶 -偷 -偻 -偽 -偿 -傀 -傅 -傍 -傑 -傘 -備 -傚 -傢 -傣 -傥 -储 -傩 -催 -傭 -傲 -傳 -債 -傷 -傻 -傾 -僅 -働 -像 -僑 -僕 -僖 -僚 -僥 -僧 -僭 -僮 -僱 -僵 -價 -僻 -儀 -儂 -億 -儆 -儉 -儋 -儒 -儕 -儘 -償 -儡 -優 -儲 -儷 -儼 -儿 -兀 -允 -元 -兄 -充 -兆 -兇 -先 -光 -克 -兌 -免 -児 -兑 -兒 -兔 -兖 -党 -兜 -兢 -入 -內 -全 -兩 -八 -公 -六 -兮 -兰 -共 -兲 -关 -兴 -兵 -其 -具 -典 -兹 -养 -兼 -兽 -冀 -内 -円 -冇 -冈 -冉 -冊 -册 -再 -冏 -冒 -冕 -冗 -写 -军 -农 -冠 -冢 -冤 -冥 -冨 -冪 -冬 -冯 -冰 -冲 -决 -况 -冶 -冷 -冻 -冼 -冽 -冾 -净 -凄 -准 -凇 -凈 -凉 -凋 -凌 -凍 -减 -凑 -凛 -凜 -凝 -几 -凡 -凤 -処 -凪 -凭 -凯 -凰 -凱 -凳 -凶 -凸 -凹 -出 -击 -函 -凿 -刀 -刁 -刃 -分 -切 -刈 -刊 -刍 -刎 -刑 -划 -列 -刘 -则 -刚 -创 -初 -删 -判 -別 -刨 -利 -刪 -别 -刮 -到 -制 -刷 -券 -刹 -刺 -刻 -刽 -剁 -剂 -剃 -則 -剉 -削 -剋 -剌 -前 -剎 -剐 -剑 -剔 -剖 -剛 -剜 -剝 -剣 -剤 -剥 -剧 -剩 -剪 -副 -割 -創 -剷 -剽 -剿 -劃 -劇 -劈 -劉 -劊 -劍 -劏 -劑 -力 -劝 -办 -功 -加 -务 -劣 -动 -助 -努 -劫 -劭 -励 -劲 -劳 -労 -劵 -効 -劾 -势 -勁 -勃 -勇 -勉 -勋 -勐 -勒 -動 -勖 -勘 -務 -勛 -勝 -勞 -募 -勢 -勤 -勧 -勳 -勵 -勸 -勺 -勻 -勾 -勿 -匀 -包 -匆 -匈 -匍 -匐 -匕 -化 -北 -匙 -匝 -匠 -匡 -匣 -匪 -匮 -匯 -匱 -匹 -区 -医 -匾 -匿 -區 -十 -千 -卅 -升 -午 -卉 -半 -卍 -华 -协 -卑 -卒 -卓 -協 -单 -卖 -南 -単 -博 -卜 -卞 -卟 -占 -卡 -卢 -卤 -卦 -卧 -卫 -卮 -卯 -印 -危 -即 -却 -卵 -卷 -卸 -卻 -卿 -厂 -厄 -厅 -历 -厉 -压 -厌 -厕 -厘 -厚 -厝 -原 -厢 -厥 -厦 -厨 -厩 -厭 -厮 -厲 -厳 -去 -县 -叁 -参 -參 -又 -叉 -及 -友 -双 -反 -収 -发 -叔 -取 -受 -变 -叙 -叛 -叟 -叠 -叡 -叢 -口 -古 -句 -另 -叨 -叩 -只 -叫 -召 -叭 -叮 -可 -台 -叱 -史 -右 -叵 -叶 -号 -司 -叹 -叻 -叼 -叽 -吁 -吃 -各 -吆 -合 -吉 -吊 -吋 -同 -名 -后 -吏 -吐 -向 -吒 -吓 -吕 -吖 -吗 -君 -吝 -吞 -吟 -吠 -吡 -否 -吧 -吨 -吩 -含 -听 -吭 -吮 -启 -吱 -吳 -吴 -吵 -吶 -吸 -吹 -吻 -吼 -吽 -吾 -呀 -呂 -呃 -呆 -呈 -告 -呋 -呎 -呐 -呓 -呕 -呗 -员 -呛 -呜 -呢 -呤 -呦 -周 -呱 -呲 -味 -呵 -呷 -呸 -呻 -呼 -命 -咀 -咁 -咂 -咄 -咆 -咋 -和 -咎 -咏 -咐 -咒 -咔 -咕 -咖 -咗 -咘 -咙 -咚 -咛 -咣 -咤 -咦 -咧 -咨 -咩 -咪 -咫 -咬 -咭 -咯 -咱 -咲 -咳 -咸 -咻 -咽 -咿 -哀 -品 -哂 -哄 -哆 -哇 -哈 -哉 -哋 -哌 -响 -哎 -哏 -哐 -哑 -哒 -哔 -哗 -哟 -員 -哥 -哦 -哧 -哨 -哩 -哪 -哭 -哮 -哲 -哺 -哼 -哽 -唁 -唄 -唆 -唇 -唉 -唏 -唐 -唑 -唔 -唠 -唤 -唧 -唬 -售 -唯 -唰 -唱 -唳 -唷 -唸 -唾 -啃 -啄 -商 -啉 -啊 -問 -啓 -啕 -啖 -啜 -啞 -啟 -啡 -啤 -啥 -啦 -啧 -啪 -啫 -啬 -啮 -啰 -啱 -啲 -啵 -啶 -啷 -啸 -啻 -啼 -啾 -喀 -喂 -喃 -善 -喆 -喇 -喉 -喊 -喋 -喎 -喏 -喔 -喘 -喙 -喚 -喜 -喝 -喟 -喧 -喪 -喫 -喬 -單 -喰 -喱 -喲 -喳 -喵 -営 -喷 -喹 -喺 -喻 -喽 -嗅 -嗆 -嗇 -嗎 -嗑 -嗒 -嗓 -嗔 -嗖 -嗚 -嗜 -嗝 -嗟 -嗡 -嗣 -嗤 -嗦 -嗨 -嗪 -嗬 -嗯 -嗰 -嗲 -嗳 -嗶 -嗷 -嗽 -嘀 -嘅 -嘆 -嘈 -嘉 -嘌 -嘍 -嘎 -嘔 -嘖 -嘗 -嘘 -嘚 -嘛 -嘜 -嘞 -嘟 -嘢 -嘣 -嘤 -嘧 -嘩 -嘭 -嘮 -嘯 -嘰 -嘱 -嘲 -嘴 -嘶 -嘸 -嘹 -嘻 -嘿 -噁 -噌 -噎 -噓 -噔 -噗 -噙 -噜 -噠 -噢 -噤 -器 -噩 -噪 -噬 -噱 -噴 -噶 -噸 -噹 -噻 -噼 -嚀 -嚇 -嚎 -嚏 -嚐 -嚓 -嚕 -嚟 -嚣 -嚥 -嚨 -嚮 -嚴 -嚷 -嚼 -囂 -囉 -囊 -囍 -囑 -囔 -囗 -囚 -四 -囝 -回 -囟 -因 -囡 -团 -団 -囤 -囧 -囪 -囫 -园 -困 -囱 -囲 -図 -围 -囹 -固 -国 -图 -囿 -圃 -圄 -圆 -圈 -國 -圍 -圏 -園 -圓 -圖 -團 -圜 -土 -圣 -圧 -在 -圩 -圭 -地 -圳 -场 -圻 -圾 -址 -坂 -均 -坊 -坍 -坎 -坏 -坐 -坑 -块 -坚 -坛 -坝 -坞 -坟 -坠 -坡 -坤 -坦 -坨 -坪 -坯 -坳 -坵 -坷 -垂 -垃 -垄 -型 -垒 -垚 -垛 -垠 -垢 -垣 -垦 -垩 -垫 -垭 -垮 -垵 -埂 -埃 -埋 -城 -埔 -埕 -埗 -域 -埠 -埤 -埵 -執 -埸 -培 -基 -埼 -堀 -堂 -堃 -堅 -堆 -堇 -堑 -堕 -堙 -堡 -堤 -堪 -堯 -堰 -報 -場 -堵 -堺 -堿 -塊 -塌 -塑 -塔 -塗 -塘 -塚 -塞 -塢 -塩 -填 -塬 -塭 -塵 -塾 -墀 -境 -墅 -墉 -墊 -墒 -墓 -増 -墘 -墙 -墜 -增 -墟 -墨 -墩 -墮 -墳 -墻 -墾 -壁 -壅 -壆 -壇 -壊 -壑 -壓 -壕 -壘 -壞 -壟 -壢 -壤 -壩 -士 -壬 -壮 -壯 -声 -売 -壳 -壶 -壹 -壺 -壽 -处 -备 -変 -复 -夏 -夔 -夕 -外 -夙 -多 -夜 -够 -夠 -夢 -夥 -大 -天 -太 -夫 -夭 -央 -夯 -失 -头 -夷 -夸 -夹 -夺 -夾 -奂 -奄 -奇 -奈 -奉 -奋 -奎 -奏 -奐 -契 -奔 -奕 -奖 -套 -奘 -奚 -奠 -奢 -奥 -奧 -奪 -奬 -奮 -女 -奴 -奶 -奸 -她 -好 -如 -妃 -妄 -妆 -妇 -妈 -妊 -妍 -妒 -妓 -妖 -妘 -妙 -妝 -妞 -妣 -妤 -妥 -妨 -妩 -妪 -妮 -妲 -妳 -妹 -妻 -妾 -姆 -姉 -姊 -始 -姍 -姐 -姑 -姒 -姓 -委 -姗 -姚 -姜 -姝 -姣 -姥 -姦 -姨 -姪 -姫 -姬 -姹 -姻 -姿 -威 -娃 -娄 -娅 -娆 -娇 -娉 -娑 -娓 -娘 -娛 -娜 -娟 -娠 -娣 -娥 -娩 -娱 -娲 -娴 -娶 -娼 -婀 -婁 -婆 -婉 -婊 -婕 -婚 -婢 -婦 -婧 -婪 -婭 -婴 -婵 -婶 -婷 -婺 -婿 -媒 -媚 -媛 -媞 -媧 -媲 -媳 -媽 -媾 -嫁 -嫂 -嫉 -嫌 -嫑 -嫔 -嫖 -嫘 -嫚 -嫡 -嫣 -嫦 -嫩 -嫲 -嫵 -嫻 -嬅 -嬉 -嬌 -嬗 -嬛 -嬢 -嬤 -嬪 -嬰 -嬴 -嬷 -嬸 -嬿 -孀 -孃 -子 -孑 -孔 -孕 -孖 -字 -存 -孙 -孚 -孛 -孜 -孝 -孟 -孢 -季 -孤 -学 -孩 -孪 -孫 -孬 -孰 -孱 -孳 -孵 -學 -孺 -孽 -孿 -宁 -它 -宅 -宇 -守 -安 -宋 -完 -宏 -宓 -宕 -宗 -官 -宙 -定 -宛 -宜 -宝 -实 -実 -宠 -审 -客 -宣 -室 -宥 -宦 -宪 -宫 -宮 -宰 -害 -宴 -宵 -家 -宸 -容 -宽 -宾 -宿 -寂 -寄 -寅 -密 -寇 -富 -寐 -寒 -寓 -寛 -寝 -寞 -察 -寡 -寢 -寥 -實 -寧 -寨 -審 -寫 -寬 -寮 -寰 -寵 -寶 -寸 -对 -寺 -寻 -导 -対 -寿 -封 -専 -射 -将 -將 -專 -尉 -尊 -尋 -對 -導 -小 -少 -尔 -尕 -尖 -尘 -尚 -尝 -尤 -尧 -尬 -就 -尴 -尷 -尸 -尹 -尺 -尻 -尼 -尽 -尾 -尿 -局 -屁 -层 -屄 -居 -屆 -屈 -屉 -届 -屋 -屌 -屍 -屎 -屏 -屐 -屑 -展 -屜 -属 -屠 -屡 -屢 -層 -履 -屬 -屯 -山 -屹 -屿 -岀 -岁 -岂 -岌 -岐 -岑 -岔 -岖 -岗 -岘 -岙 -岚 -岛 -岡 -岩 -岫 -岬 -岭 -岱 -岳 -岷 -岸 -峇 -峋 -峒 -峙 -峡 -峤 -峥 -峦 -峨 -峪 -峭 -峯 -峰 -峴 -島 -峻 -峽 -崁 -崂 -崆 -崇 -崎 -崑 -崔 -崖 -崗 -崙 -崛 -崧 -崩 -崭 -崴 -崽 -嵇 -嵊 -嵋 -嵌 -嵐 -嵘 -嵩 -嵬 -嵯 -嶂 -嶄 -嶇 -嶋 -嶙 -嶺 -嶼 -嶽 -巅 -巍 -巒 -巔 -巖 -川 -州 -巡 -巢 -工 -左 -巧 -巨 -巩 -巫 -差 -己 -已 -巳 -巴 -巷 -巻 -巽 -巾 -巿 -币 -市 -布 -帅 -帆 -师 -希 -帐 -帑 -帕 -帖 -帘 -帚 -帛 -帜 -帝 -帥 -带 -帧 -師 -席 -帮 -帯 -帰 -帳 -帶 -帷 -常 -帼 -帽 -幀 -幂 -幄 -幅 -幌 -幔 -幕 -幟 -幡 -幢 -幣 -幫 -干 -平 -年 -并 -幸 -幹 -幺 -幻 -幼 -幽 -幾 -广 -庁 -広 -庄 -庆 -庇 -床 -序 -庐 -库 -应 -底 -庖 -店 -庙 -庚 -府 -庞 -废 -庠 -度 -座 -庫 -庭 -庵 -庶 -康 -庸 -庹 -庾 -廁 -廂 -廃 -廈 -廉 -廊 -廓 -廖 -廚 -廝 -廟 -廠 -廢 -廣 -廬 -廳 -延 -廷 -建 -廿 -开 -弁 -异 -弃 -弄 -弈 -弊 -弋 -式 -弑 -弒 -弓 -弔 -引 -弗 -弘 -弛 -弟 -张 -弥 -弦 -弧 -弩 -弭 -弯 -弱 -張 -強 -弹 -强 -弼 -弾 -彅 -彆 -彈 -彌 -彎 -归 -当 -录 -彗 -彙 -彝 -形 -彤 -彥 -彦 -彧 -彩 -彪 -彫 -彬 -彭 -彰 -影 -彷 -役 -彻 -彼 -彿 -往 -征 -径 -待 -徇 -很 -徉 -徊 -律 -後 -徐 -徑 -徒 -従 -徕 -得 -徘 -徙 -徜 -從 -徠 -御 -徨 -復 -循 -徬 -微 -徳 -徴 -徵 -德 -徹 -徼 -徽 -心 -必 -忆 -忌 -忍 -忏 -忐 -忑 -忒 -忖 -志 -忘 -忙 -応 -忠 -忡 -忤 -忧 -忪 -快 -忱 -念 -忻 -忽 -忿 -怀 -态 -怂 -怅 -怆 -怎 -怏 -怒 -怔 -怕 -怖 -怙 -怜 -思 -怠 -怡 -急 -怦 -性 -怨 -怪 -怯 -怵 -总 -怼 -恁 -恃 -恆 -恋 -恍 -恐 -恒 -恕 -恙 -恚 -恢 -恣 -恤 -恥 -恨 -恩 -恪 -恫 -恬 -恭 -息 -恰 -恳 -恵 -恶 -恸 -恺 -恻 -恼 -恿 -悄 -悅 -悉 -悌 -悍 -悔 -悖 -悚 -悟 -悠 -患 -悦 -您 -悩 -悪 -悬 -悯 -悱 -悲 -悴 -悵 -悶 -悸 -悻 -悼 -悽 -情 -惆 -惇 -惊 -惋 -惑 -惕 -惘 -惚 -惜 -惟 -惠 -惡 -惦 -惧 -惨 -惩 -惫 -惬 -惭 -惮 -惯 -惰 -惱 -想 -惴 -惶 -惹 -惺 -愁 -愆 -愈 -愉 -愍 -意 -愕 -愚 -愛 -愜 -感 -愣 -愤 -愧 -愫 -愷 -愿 -慄 -慈 -態 -慌 -慎 -慑 -慕 -慘 -慚 -慟 -慢 -慣 -慧 -慨 -慫 -慮 -慰 -慳 -慵 -慶 -慷 -慾 -憂 -憊 -憋 -憎 -憐 -憑 -憔 -憚 -憤 -憧 -憨 -憩 -憫 -憬 -憲 -憶 -憾 -懂 -懇 -懈 -應 -懊 -懋 -懑 -懒 -懦 -懲 -懵 -懶 -懷 -懸 -懺 -懼 -懾 -懿 -戀 -戈 -戊 -戌 -戍 -戎 -戏 -成 -我 -戒 -戕 -或 -战 -戚 -戛 -戟 -戡 -戦 -截 -戬 -戮 -戰 -戲 -戳 -戴 -戶 -户 -戸 -戻 -戾 -房 -所 -扁 -扇 -扈 -扉 -手 -才 -扎 -扑 -扒 -打 -扔 -払 -托 -扛 -扣 -扦 -执 -扩 -扪 -扫 -扬 -扭 -扮 -扯 -扰 -扱 -扳 -扶 -批 -扼 -找 -承 -技 -抄 -抉 -把 -抑 -抒 -抓 -投 -抖 -抗 -折 -抚 -抛 -抜 -択 -抟 -抠 -抡 -抢 -护 -报 -抨 -披 -抬 -抱 -抵 -抹 -押 -抽 -抿 -拂 -拄 -担 -拆 -拇 -拈 -拉 -拋 -拌 -拍 -拎 -拐 -拒 -拓 -拔 -拖 -拗 -拘 -拙 -拚 -招 -拜 -拟 -拡 -拢 -拣 -拥 -拦 -拧 -拨 -择 -括 -拭 -拮 -拯 -拱 -拳 -拴 -拷 -拼 -拽 -拾 -拿 -持 -挂 -指 -挈 -按 -挎 -挑 -挖 -挙 -挚 -挛 -挝 -挞 -挟 -挠 -挡 -挣 -挤 -挥 -挨 -挪 -挫 -振 -挲 -挹 -挺 -挽 -挾 -捂 -捅 -捆 -捉 -捋 -捌 -捍 -捎 -捏 -捐 -捕 -捞 -损 -捡 -换 -捣 -捧 -捨 -捩 -据 -捱 -捲 -捶 -捷 -捺 -捻 -掀 -掂 -掃 -掇 -授 -掉 -掌 -掏 -掐 -排 -掖 -掘 -掙 -掛 -掠 -採 -探 -掣 -接 -控 -推 -掩 -措 -掬 -掰 -掲 -掳 -掴 -掷 -掸 -掺 -揀 -揃 -揄 -揆 -揉 -揍 -描 -提 -插 -揖 -揚 -換 -握 -揣 -揩 -揪 -揭 -揮 -援 -揶 -揸 -揹 -揽 -搀 -搁 -搂 -搅 -損 -搏 -搐 -搓 -搔 -搖 -搗 -搜 -搞 -搡 -搪 -搬 -搭 -搵 -搶 -携 -搽 -摀 -摁 -摄 -摆 -摇 -摈 -摊 -摒 -摔 -摘 -摞 -摟 -摧 -摩 -摯 -摳 -摸 -摹 -摺 -摻 -撂 -撃 -撅 -撇 -撈 -撐 -撑 -撒 -撓 -撕 -撚 -撞 -撤 -撥 -撩 -撫 -撬 -播 -撮 -撰 -撲 -撵 -撷 -撸 -撻 -撼 -撿 -擀 -擁 -擂 -擄 -擅 -擇 -擊 -擋 -操 -擎 -擒 -擔 -擘 -據 -擞 -擠 -擡 -擢 -擦 -擬 -擰 -擱 -擲 -擴 -擷 -擺 -擼 -擾 -攀 -攏 -攒 -攔 -攘 -攙 -攜 -攝 -攞 -攢 -攣 -攤 -攥 -攪 -攫 -攬 -支 -收 -攸 -改 -攻 -放 -政 -故 -效 -敌 -敍 -敎 -敏 -救 -敕 -敖 -敗 -敘 -教 -敛 -敝 -敞 -敢 -散 -敦 -敬 -数 -敲 -整 -敵 -敷 -數 -斂 -斃 -文 -斋 -斌 -斎 -斐 -斑 -斓 -斗 -料 -斛 -斜 -斟 -斡 -斤 -斥 -斧 -斩 -斫 -斬 -断 -斯 -新 -斷 -方 -於 -施 -旁 -旃 -旅 -旋 -旌 -旎 -族 -旖 -旗 -无 -既 -日 -旦 -旧 -旨 -早 -旬 -旭 -旮 -旱 -时 -旷 -旺 -旻 -昀 -昂 -昆 -昇 -昉 -昊 -昌 -明 -昏 -易 -昔 -昕 -昙 -星 -映 -春 -昧 -昨 -昭 -是 -昱 -昴 -昵 -昶 -昼 -显 -晁 -時 -晃 -晉 -晋 -晌 -晏 -晒 -晓 -晔 -晕 -晖 -晗 -晚 -晝 -晞 -晟 -晤 -晦 -晨 -晩 -普 -景 -晰 -晴 -晶 -晷 -智 -晾 -暂 -暄 -暇 -暈 -暉 -暌 -暐 -暑 -暖 -暗 -暝 -暢 -暧 -暨 -暫 -暮 -暱 -暴 -暸 -暹 -曄 -曆 -曇 -曉 -曖 -曙 -曜 -曝 -曠 -曦 -曬 -曰 -曲 -曳 -更 -書 -曹 -曼 -曾 -替 -最 -會 -月 -有 -朋 -服 -朐 -朔 -朕 -朗 -望 -朝 -期 -朦 -朧 -木 -未 -末 -本 -札 -朮 -术 -朱 -朴 -朵 -机 -朽 -杀 -杂 -权 -杆 -杈 -杉 -李 -杏 -材 -村 -杓 -杖 -杜 -杞 -束 -杠 -条 -来 -杨 -杭 -杯 -杰 -東 -杳 -杵 -杷 -杼 -松 -板 -极 -构 -枇 -枉 -枋 -析 -枕 -林 -枚 -果 -枝 -枢 -枣 -枪 -枫 -枭 -枯 -枰 -枱 -枳 -架 -枷 -枸 -柄 -柏 -某 -柑 -柒 -染 -柔 -柘 -柚 -柜 -柞 -柠 -柢 -查 -柩 -柬 -柯 -柱 -柳 -柴 -柵 -査 -柿 -栀 -栃 -栄 -栅 -标 -栈 -栉 -栋 -栎 -栏 -树 -栓 -栖 -栗 -校 -栩 -株 -样 -核 -根 -格 -栽 -栾 -桀 -桁 -桂 -桃 -桅 -框 -案 -桉 -桌 -桎 -桐 -桑 -桓 -桔 -桜 -桠 -桡 -桢 -档 -桥 -桦 -桧 -桨 -桩 -桶 -桿 -梁 -梅 -梆 -梏 -梓 -梗 -條 -梟 -梢 -梦 -梧 -梨 -梭 -梯 -械 -梳 -梵 -梶 -检 -棂 -棄 -棉 -棋 -棍 -棒 -棕 -棗 -棘 -棚 -棟 -棠 -棣 -棧 -森 -棱 -棲 -棵 -棹 -棺 -椁 -椅 -椋 -植 -椎 -椒 -検 -椪 -椭 -椰 -椹 -椽 -椿 -楂 -楊 -楓 -楔 -楚 -楝 -楞 -楠 -楣 -楨 -楫 -業 -楮 -極 -楷 -楸 -楹 -楼 -楽 -概 -榄 -榆 -榈 -榉 -榔 -榕 -榖 -榛 -榜 -榨 -榫 -榭 -榮 -榱 -榴 -榷 -榻 -槁 -槃 -構 -槌 -槍 -槎 -槐 -槓 -様 -槛 -槟 -槤 -槭 -槲 -槳 -槻 -槽 -槿 -樁 -樂 -樊 -樑 -樓 -標 -樞 -樟 -模 -樣 -権 -横 -樫 -樯 -樱 -樵 -樸 -樹 -樺 -樽 -樾 -橄 -橇 -橋 -橐 -橘 -橙 -機 -橡 -橢 -橫 -橱 -橹 -橼 -檀 -檄 -檎 -檐 -檔 -檗 -檜 -檢 -檬 -檯 -檳 -檸 -檻 -櫃 -櫚 -櫛 -櫥 -櫸 -櫻 -欄 -權 -欒 -欖 -欠 -次 -欢 -欣 -欧 -欲 -欸 -欺 -欽 -款 -歆 -歇 -歉 -歌 -歎 -歐 -歓 -歙 -歛 -歡 -止 -正 -此 -步 -武 -歧 -歩 -歪 -歯 -歲 -歳 -歴 -歷 -歸 -歹 -死 -歼 -殁 -殃 -殆 -殇 -殉 -殊 -残 -殒 -殓 -殖 -殘 -殞 -殡 -殤 -殭 -殯 -殲 -殴 -段 -殷 -殺 -殼 -殿 -毀 -毁 -毂 -毅 -毆 -毋 -母 -毎 -每 -毒 -毓 -比 -毕 -毗 -毘 -毙 -毛 -毡 -毫 -毯 -毽 -氈 -氏 -氐 -民 -氓 -气 -氖 -気 -氙 -氛 -氟 -氡 -氢 -氣 -氤 -氦 -氧 -氨 -氪 -氫 -氮 -氯 -氰 -氲 -水 -氷 -永 -氹 -氾 -汀 -汁 -求 -汆 -汇 -汉 -汎 -汐 -汕 -汗 -汙 -汛 -汝 -汞 -江 -池 -污 -汤 -汨 -汩 -汪 -汰 -汲 -汴 -汶 -汹 -決 -汽 -汾 -沁 -沂 -沃 -沅 -沈 -沉 -沌 -沏 -沐 -沒 -沓 -沖 -沙 -沛 -沟 -没 -沢 -沣 -沥 -沦 -沧 -沪 -沫 -沭 -沮 -沱 -河 -沸 -油 -治 -沼 -沽 -沾 -沿 -況 -泄 -泉 -泊 -泌 -泓 -法 -泗 -泛 -泞 -泠 -泡 -波 -泣 -泥 -注 -泪 -泫 -泮 -泯 -泰 -泱 -泳 -泵 -泷 -泸 -泻 -泼 -泽 -泾 -洁 -洄 -洋 -洒 -洗 -洙 -洛 -洞 -津 -洩 -洪 -洮 -洱 -洲 -洵 -洶 -洸 -洹 -活 -洼 -洽 -派 -流 -浃 -浄 -浅 -浆 -浇 -浊 -测 -济 -浏 -浑 -浒 -浓 -浔 -浙 -浚 -浜 -浣 -浦 -浩 -浪 -浬 -浮 -浯 -浴 -海 -浸 -涂 -涅 -涇 -消 -涉 -涌 -涎 -涓 -涔 -涕 -涙 -涛 -涝 -涞 -涟 -涠 -涡 -涣 -涤 -润 -涧 -涨 -涩 -涪 -涮 -涯 -液 -涵 -涸 -涼 -涿 -淀 -淄 -淅 -淆 -淇 -淋 -淌 -淑 -淒 -淖 -淘 -淙 -淚 -淞 -淡 -淤 -淦 -淨 -淩 -淪 -淫 -淬 -淮 -深 -淳 -淵 -混 -淹 -淺 -添 -淼 -清 -済 -渉 -渊 -渋 -渍 -渎 -渐 -渔 -渗 -渙 -渚 -減 -渝 -渠 -渡 -渣 -渤 -渥 -渦 -温 -測 -渭 -港 -渲 -渴 -游 -渺 -渾 -湃 -湄 -湊 -湍 -湖 -湘 -湛 -湟 -湧 -湫 -湮 -湯 -湳 -湾 -湿 -満 -溃 -溅 -溉 -溏 -源 -準 -溜 -溝 -溟 -溢 -溥 -溧 -溪 -溫 -溯 -溱 -溴 -溶 -溺 -溼 -滁 -滂 -滄 -滅 -滇 -滋 -滌 -滑 -滓 -滔 -滕 -滙 -滚 -滝 -滞 -滟 -满 -滢 -滤 -滥 -滦 -滨 -滩 -滬 -滯 -滲 -滴 -滷 -滸 -滾 -滿 -漁 -漂 -漆 -漉 -漏 -漓 -演 -漕 -漠 -漢 -漣 -漩 -漪 -漫 -漬 -漯 -漱 -漲 -漳 -漸 -漾 -漿 -潆 -潇 -潋 -潍 -潑 -潔 -潘 -潛 -潜 -潞 -潟 -潢 -潤 -潦 -潧 -潭 -潮 -潰 -潴 -潸 -潺 -潼 -澀 -澄 -澆 -澈 -澍 -澎 -澗 -澜 -澡 -澤 -澧 -澱 -澳 -澹 -激 -濁 -濂 -濃 -濑 -濒 -濕 -濘 -濛 -濟 -濠 -濡 -濤 -濫 -濬 -濮 -濯 -濱 -濺 -濾 -瀅 -瀆 -瀉 -瀋 -瀏 -瀑 -瀕 -瀘 -瀚 -瀛 -瀝 -瀞 -瀟 -瀧 -瀨 -瀬 -瀰 -瀾 -灌 -灏 -灑 -灘 -灝 -灞 -灣 -火 -灬 -灭 -灯 -灰 -灵 -灶 -灸 -灼 -災 -灾 -灿 -炀 -炁 -炅 -炉 -炊 -炎 -炒 -炔 -炕 -炖 -炙 -炜 -炫 -炬 -炭 -炮 -炯 -炳 -炷 -炸 -点 -為 -炼 -炽 -烁 -烂 -烃 -烈 -烊 -烏 -烘 -烙 -烛 -烟 -烤 -烦 -烧 -烨 -烩 -烫 -烬 -热 -烯 -烷 -烹 -烽 -焉 -焊 -焕 -焖 -焗 -焘 -焙 -焚 -焜 -無 -焦 -焯 -焰 -焱 -然 -焼 -煅 -煉 -煊 -煌 -煎 -煒 -煖 -煙 -煜 -煞 -煤 -煥 -煦 -照 -煨 -煩 -煮 -煲 -煸 -煽 -熄 -熊 -熏 -熒 -熔 -熙 -熟 -熠 -熨 -熬 -熱 -熵 -熹 -熾 -燁 -燃 -燄 -燈 -燉 -燊 -燎 -燒 -燔 -燕 -燙 -燜 -營 -燥 -燦 -燧 -燭 -燮 -燴 -燻 -燼 -燿 -爆 -爍 -爐 -爛 -爪 -爬 -爭 -爰 -爱 -爲 -爵 -父 -爷 -爸 -爹 -爺 -爻 -爽 -爾 -牆 -片 -版 -牌 -牍 -牒 -牙 -牛 -牝 -牟 -牠 -牡 -牢 -牦 -牧 -物 -牯 -牲 -牴 -牵 -特 -牺 -牽 -犀 -犁 -犄 -犊 -犍 -犒 -犢 -犧 -犬 -犯 -状 -犷 -犸 -犹 -狀 -狂 -狄 -狈 -狎 -狐 -狒 -狗 -狙 -狞 -狠 -狡 -狩 -独 -狭 -狮 -狰 -狱 -狸 -狹 -狼 -狽 -猎 -猕 -猖 -猗 -猙 -猛 -猜 -猝 -猥 -猩 -猪 -猫 -猬 -献 -猴 -猶 -猷 -猾 -猿 -獄 -獅 -獎 -獐 -獒 -獗 -獠 -獣 -獨 -獭 -獰 -獲 -獵 -獷 -獸 -獺 -獻 -獼 -獾 -玄 -率 -玉 -王 -玑 -玖 -玛 -玟 -玠 -玥 -玩 -玫 -玮 -环 -现 -玲 -玳 -玷 -玺 -玻 -珀 -珂 -珅 -珈 -珉 -珊 -珍 -珏 -珐 -珑 -珙 -珞 -珠 -珣 -珥 -珩 -珪 -班 -珮 -珲 -珺 -現 -球 -琅 -理 -琇 -琉 -琊 -琍 -琏 -琐 -琛 -琢 -琥 -琦 -琨 -琪 -琬 -琮 -琰 -琲 -琳 -琴 -琵 -琶 -琺 -琼 -瑀 -瑁 -瑄 -瑋 -瑕 -瑗 -瑙 -瑚 -瑛 -瑜 -瑞 -瑟 -瑠 -瑣 -瑤 -瑩 -瑪 -瑯 -瑰 -瑶 -瑾 -璀 -璁 -璃 -璇 -璉 -璋 -璎 -璐 -璜 -璞 -璟 -璧 -璨 -環 -璽 -璿 -瓊 -瓏 -瓒 -瓜 -瓢 -瓣 -瓤 -瓦 -瓮 -瓯 -瓴 -瓶 -瓷 -甄 -甌 -甕 -甘 -甙 -甚 -甜 -生 -產 -産 -甥 -甦 -用 -甩 -甫 -甬 -甭 -甯 -田 -由 -甲 -申 -电 -男 -甸 -町 -画 -甾 -畀 -畅 -界 -畏 -畑 -畔 -留 -畜 -畝 -畢 -略 -畦 -番 -畫 -異 -畲 -畳 -畴 -當 -畸 -畹 -畿 -疆 -疇 -疊 -疏 -疑 -疔 -疖 -疗 -疙 -疚 -疝 -疟 -疡 -疣 -疤 -疥 -疫 -疮 -疯 -疱 -疲 -疳 -疵 -疸 -疹 -疼 -疽 -疾 -痂 -病 -症 -痈 -痉 -痊 -痍 -痒 -痔 -痕 -痘 -痙 -痛 -痞 -痠 -痢 -痣 -痤 -痧 -痨 -痪 -痫 -痰 -痱 -痴 -痹 -痺 -痼 -痿 -瘀 -瘁 -瘋 -瘍 -瘓 -瘘 -瘙 -瘟 -瘠 -瘡 -瘢 -瘤 -瘦 -瘧 -瘩 -瘪 -瘫 -瘴 -瘸 -瘾 -療 -癇 -癌 -癒 -癖 -癜 -癞 -癡 -癢 -癣 -癥 -癫 -癬 -癮 -癱 -癲 -癸 -発 -登 -發 -白 -百 -皂 -的 -皆 -皇 -皈 -皋 -皎 -皑 -皓 -皖 -皙 -皚 -皮 -皰 -皱 -皴 -皺 -皿 -盂 -盃 -盅 -盆 -盈 -益 -盎 -盏 -盐 -监 -盒 -盔 -盖 -盗 -盘 -盛 -盜 -盞 -盟 -盡 -監 -盤 -盥 -盧 -盪 -目 -盯 -盱 -盲 -直 -相 -盹 -盼 -盾 -省 -眈 -眉 -看 -県 -眙 -眞 -真 -眠 -眦 -眨 -眩 -眯 -眶 -眷 -眸 -眺 -眼 -眾 -着 -睁 -睇 -睏 -睐 -睑 -睛 -睜 -睞 -睡 -睢 -督 -睥 -睦 -睨 -睪 -睫 -睬 -睹 -睽 -睾 -睿 -瞄 -瞅 -瞇 -瞋 -瞌 -瞎 -瞑 -瞒 -瞓 -瞞 -瞟 -瞠 -瞥 -瞧 -瞩 -瞪 -瞬 -瞭 -瞰 -瞳 -瞻 -瞼 -瞿 -矇 -矍 -矗 -矚 -矛 -矜 -矢 -矣 -知 -矩 -矫 -短 -矮 -矯 -石 -矶 -矽 -矾 -矿 -码 -砂 -砌 -砍 -砒 -研 -砖 -砗 -砚 -砝 -砣 -砥 -砧 -砭 -砰 -砲 -破 -砷 -砸 -砺 -砼 -砾 -础 -硅 -硐 -硒 -硕 -硝 -硫 -硬 -确 -硯 -硼 -碁 -碇 -碉 -碌 -碍 -碎 -碑 -碓 -碗 -碘 -碚 -碛 -碟 -碣 -碧 -碩 -碰 -碱 -碳 -碴 -確 -碼 -碾 -磁 -磅 -磊 -磋 -磐 -磕 -磚 -磡 -磨 -磬 -磯 -磲 -磷 -磺 -礁 -礎 -礙 -礡 -礦 -礪 -礫 -礴 -示 -礼 -社 -祀 -祁 -祂 -祇 -祈 -祉 -祎 -祐 -祕 -祖 -祗 -祚 -祛 -祜 -祝 -神 -祟 -祠 -祢 -祥 -票 -祭 -祯 -祷 -祸 -祺 -祿 -禀 -禁 -禄 -禅 -禍 -禎 -福 -禛 -禦 -禧 -禪 -禮 -禱 -禹 -禺 -离 -禽 -禾 -禿 -秀 -私 -秃 -秆 -秉 -秋 -种 -科 -秒 -秘 -租 -秣 -秤 -秦 -秧 -秩 -秭 -积 -称 -秸 -移 -秽 -稀 -稅 -程 -稍 -税 -稔 -稗 -稚 -稜 -稞 -稟 -稠 -稣 -種 -稱 -稲 -稳 -稷 -稹 -稻 -稼 -稽 -稿 -穀 -穂 -穆 -穌 -積 -穎 -穗 -穢 -穩 -穫 -穴 -究 -穷 -穹 -空 -穿 -突 -窃 -窄 -窈 -窍 -窑 -窒 -窓 -窕 -窖 -窗 -窘 -窜 -窝 -窟 -窠 -窥 -窦 -窨 -窩 -窪 -窮 -窯 -窺 -窿 -竄 -竅 -竇 -竊 -立 -竖 -站 -竜 -竞 -竟 -章 -竣 -童 -竭 -端 -競 -竹 -竺 -竽 -竿 -笃 -笆 -笈 -笋 -笏 -笑 -笔 -笙 -笛 -笞 -笠 -符 -笨 -第 -笹 -笺 -笼 -筆 -等 -筊 -筋 -筍 -筏 -筐 -筑 -筒 -答 -策 -筛 -筝 -筠 -筱 -筲 -筵 -筷 -筹 -签 -简 -箇 -箋 -箍 -箏 -箐 -箔 -箕 -算 -箝 -管 -箩 -箫 -箭 -箱 -箴 -箸 -節 -篁 -範 -篆 -篇 -築 -篑 -篓 -篙 -篝 -篠 -篡 -篤 -篩 -篪 -篮 -篱 -篷 -簇 -簌 -簍 -簡 -簦 -簧 -簪 -簫 -簷 -簸 -簽 -簾 -簿 -籁 -籃 -籌 -籍 -籐 -籟 -籠 -籤 -籬 -籮 -籲 -米 -类 -籼 -籽 -粄 -粉 -粑 -粒 -粕 -粗 -粘 -粟 -粤 -粥 -粧 -粪 -粮 -粱 -粲 -粳 -粵 -粹 -粼 -粽 -精 -粿 -糅 -糊 -糍 -糕 -糖 -糗 -糙 -糜 -糞 -糟 -糠 -糧 -糬 -糯 -糰 -糸 -系 -糾 -紀 -紂 -約 -紅 -紉 -紊 -紋 -納 -紐 -紓 -純 -紗 -紘 -紙 -級 -紛 -紜 -素 -紡 -索 -紧 -紫 -紮 -累 -細 -紳 -紹 -紺 -終 -絃 -組 -絆 -経 -結 -絕 -絞 -絡 -絢 -給 -絨 -絮 -統 -絲 -絳 -絵 -絶 -絹 -綁 -綏 -綑 -經 -継 -続 -綜 -綠 -綢 -綦 -綫 -綬 -維 -綱 -網 -綴 -綵 -綸 -綺 -綻 -綽 -綾 -綿 -緊 -緋 -総 -緑 -緒 -緘 -線 -緝 -緞 -締 -緣 -編 -緩 -緬 -緯 -練 -緹 -緻 -縁 -縄 -縈 -縛 -縝 -縣 -縫 -縮 -縱 -縴 -縷 -總 -績 -繁 -繃 -繆 -繇 -繋 -織 -繕 -繚 -繞 -繡 -繩 -繪 -繫 -繭 -繳 -繹 -繼 -繽 -纂 -續 -纍 -纏 -纓 -纔 -纖 -纜 -纠 -红 -纣 -纤 -约 -级 -纨 -纪 -纫 -纬 -纭 -纯 -纰 -纱 -纲 -纳 -纵 -纶 -纷 -纸 -纹 -纺 -纽 -纾 -线 -绀 -练 -组 -绅 -细 -织 -终 -绊 -绍 -绎 -经 -绑 -绒 -结 -绔 -绕 -绘 -给 -绚 -绛 -络 -绝 -绞 -统 -绡 -绢 -绣 -绥 -绦 -继 -绩 -绪 -绫 -续 -绮 -绯 -绰 -绳 -维 -绵 -绶 -绷 -绸 -绻 -综 -绽 -绾 -绿 -缀 -缄 -缅 -缆 -缇 -缈 -缉 -缎 -缓 -缔 -缕 -编 -缘 -缙 -缚 -缜 -缝 -缠 -缢 -缤 -缥 -缨 -缩 -缪 -缭 -缮 -缰 -缱 -缴 -缸 -缺 -缽 -罂 -罄 -罌 -罐 -网 -罔 -罕 -罗 -罚 -罡 -罢 -罩 -罪 -置 -罰 -署 -罵 -罷 -罹 -羁 -羅 -羈 -羊 -羌 -美 -羔 -羚 -羞 -羟 -羡 -羣 -群 -羥 -羧 -羨 -義 -羯 -羲 -羸 -羹 -羽 -羿 -翁 -翅 -翊 -翌 -翎 -習 -翔 -翘 -翟 -翠 -翡 -翦 -翩 -翰 -翱 -翳 -翹 -翻 -翼 -耀 -老 -考 -耄 -者 -耆 -耋 -而 -耍 -耐 -耒 -耕 -耗 -耘 -耙 -耦 -耨 -耳 -耶 -耷 -耸 -耻 -耽 -耿 -聂 -聆 -聊 -聋 -职 -聒 -联 -聖 -聘 -聚 -聞 -聪 -聯 -聰 -聲 -聳 -聴 -聶 -職 -聽 -聾 -聿 -肃 -肄 -肅 -肆 -肇 -肉 -肋 -肌 -肏 -肓 -肖 -肘 -肚 -肛 -肝 -肠 -股 -肢 -肤 -肥 -肩 -肪 -肮 -肯 -肱 -育 -肴 -肺 -肽 -肾 -肿 -胀 -胁 -胃 -胄 -胆 -背 -胍 -胎 -胖 -胚 -胛 -胜 -胝 -胞 -胡 -胤 -胥 -胧 -胫 -胭 -胯 -胰 -胱 -胳 -胴 -胶 -胸 -胺 -能 -脂 -脅 -脆 -脇 -脈 -脉 -脊 -脍 -脏 -脐 -脑 -脓 -脖 -脘 -脚 -脛 -脣 -脩 -脫 -脯 -脱 -脲 -脳 -脸 -脹 -脾 -腆 -腈 -腊 -腋 -腌 -腎 -腐 -腑 -腓 -腔 -腕 -腥 -腦 -腩 -腫 -腭 -腮 -腰 -腱 -腳 -腴 -腸 -腹 -腺 -腻 -腼 -腾 -腿 -膀 -膈 -膊 -膏 -膑 -膘 -膚 -膛 -膜 -膝 -膠 -膦 -膨 -膩 -膳 -膺 -膻 -膽 -膾 -膿 -臀 -臂 -臃 -臆 -臉 -臊 -臍 -臓 -臘 -臟 -臣 -臥 -臧 -臨 -自 -臬 -臭 -至 -致 -臺 -臻 -臼 -臾 -舀 -舂 -舅 -舆 -與 -興 -舉 -舊 -舌 -舍 -舎 -舐 -舒 -舔 -舖 -舗 -舛 -舜 -舞 -舟 -航 -舫 -般 -舰 -舱 -舵 -舶 -舷 -舸 -船 -舺 -舾 -艇 -艋 -艘 -艙 -艦 -艮 -良 -艰 -艱 -色 -艳 -艷 -艹 -艺 -艾 -节 -芃 -芈 -芊 -芋 -芍 -芎 -芒 -芙 -芜 -芝 -芡 -芥 -芦 -芩 -芪 -芫 -芬 -芭 -芮 -芯 -花 -芳 -芷 -芸 -芹 -芻 -芽 -芾 -苁 -苄 -苇 -苋 -苍 -苏 -苑 -苒 -苓 -苔 -苕 -苗 -苛 -苜 -苞 -苟 -苡 -苣 -若 -苦 -苫 -苯 -英 -苷 -苹 -苻 -茁 -茂 -范 -茄 -茅 -茉 -茎 -茏 -茗 -茜 -茧 -茨 -茫 -茬 -茭 -茯 -茱 -茲 -茴 -茵 -茶 -茸 -茹 -茼 -荀 -荃 -荆 -草 -荊 -荏 -荐 -荒 -荔 -荖 -荘 -荚 -荞 -荟 -荠 -荡 -荣 -荤 -荥 -荧 -荨 -荪 -荫 -药 -荳 -荷 -荸 -荻 -荼 -荽 -莅 -莆 -莉 -莊 -莎 -莒 -莓 -莖 -莘 -莞 -莠 -莢 -莧 -莪 -莫 -莱 -莲 -莴 -获 -莹 -莺 -莽 -莿 -菀 -菁 -菅 -菇 -菈 -菊 -菌 -菏 -菓 -菖 -菘 -菜 -菟 -菠 -菡 -菩 -華 -菱 -菲 -菸 -菽 -萁 -萃 -萄 -萊 -萋 -萌 -萍 -萎 -萘 -萝 -萤 -营 -萦 -萧 -萨 -萩 -萬 -萱 -萵 -萸 -萼 -落 -葆 -葉 -著 -葚 -葛 -葡 -董 -葦 -葩 -葫 -葬 -葭 -葯 -葱 -葳 -葵 -葷 -葺 -蒂 -蒋 -蒐 -蒔 -蒙 -蒜 -蒞 -蒟 -蒡 -蒨 -蒲 -蒸 -蒹 -蒻 -蒼 -蒿 -蓁 -蓄 -蓆 -蓉 -蓋 -蓑 -蓓 -蓖 -蓝 -蓟 -蓦 -蓬 -蓮 -蓼 -蓿 -蔑 -蔓 -蔔 -蔗 -蔘 -蔚 -蔡 -蔣 -蔥 -蔫 -蔬 -蔭 -蔵 -蔷 -蔺 -蔻 -蔼 -蔽 -蕁 -蕃 -蕈 -蕉 -蕊 -蕎 -蕙 -蕤 -蕨 -蕩 -蕪 -蕭 -蕲 -蕴 -蕻 -蕾 -薄 -薅 -薇 -薈 -薊 -薏 -薑 -薔 -薙 -薛 -薦 -薨 -薩 -薪 -薬 -薯 -薰 -薹 -藉 -藍 -藏 -藐 -藓 -藕 -藜 -藝 -藤 -藥 -藩 -藹 -藻 -藿 -蘆 -蘇 -蘊 -蘋 -蘑 -蘚 -蘭 -蘸 -蘼 -蘿 -虎 -虏 -虐 -虑 -虔 -處 -虚 -虛 -虜 -虞 -號 -虢 -虧 -虫 -虬 -虱 -虹 -虻 -虽 -虾 -蚀 -蚁 -蚂 -蚊 -蚌 -蚓 -蚕 -蚜 -蚝 -蚣 -蚤 -蚩 -蚪 -蚯 -蚱 -蚵 -蛀 -蛆 -蛇 -蛊 -蛋 -蛎 -蛐 -蛔 -蛙 -蛛 -蛟 -蛤 -蛭 -蛮 -蛰 -蛳 -蛹 -蛻 -蛾 -蜀 -蜂 -蜃 -蜆 -蜇 -蜈 -蜊 -蜍 -蜒 -蜓 -蜕 -蜗 -蜘 -蜚 -蜜 -蜡 -蜢 -蜥 -蜱 -蜴 -蜷 -蜻 -蜿 -蝇 -蝈 -蝉 -蝌 -蝎 -蝕 -蝗 -蝙 -蝟 -蝠 -蝦 -蝨 -蝴 -蝶 -蝸 -蝼 -螂 -螃 -融 -螞 -螢 -螨 -螯 -螳 -螺 -蟀 -蟄 -蟆 -蟋 -蟎 -蟑 -蟒 -蟠 -蟬 -蟲 -蟹 -蟻 -蟾 -蠅 -蠍 -蠔 -蠕 -蠛 -蠟 -蠡 -蠢 -蠣 -蠱 -蠶 -蠹 -蠻 -血 -衄 -衅 -衆 -行 -衍 -術 -衔 -街 -衙 -衛 -衝 -衞 -衡 -衢 -衣 -补 -表 -衩 -衫 -衬 -衮 -衰 -衲 -衷 -衹 -衾 -衿 -袁 -袂 -袄 -袅 -袈 -袋 -袍 -袒 -袖 -袜 -袞 -袤 -袪 -被 -袭 -袱 -裁 -裂 -装 -裆 -裊 -裏 -裔 -裕 -裘 -裙 -補 -裝 -裟 -裡 -裤 -裨 -裱 -裳 -裴 -裸 -裹 -製 -裾 -褂 -複 -褐 -褒 -褓 -褔 -褚 -褥 -褪 -褫 -褲 -褶 -褻 -襁 -襄 -襟 -襠 -襪 -襬 -襯 -襲 -西 -要 -覃 -覆 -覇 -見 -規 -覓 -視 -覚 -覦 -覧 -親 -覬 -観 -覷 -覺 -覽 -觀 -见 -观 -规 -觅 -视 -览 -觉 -觊 -觎 -觐 -觑 -角 -觞 -解 -觥 -触 -觸 -言 -訂 -計 -訊 -討 -訓 -訕 -訖 -託 -記 -訛 -訝 -訟 -訣 -訥 -訪 -設 -許 -訳 -訴 -訶 -診 -註 -証 -詆 -詐 -詔 -評 -詛 -詞 -詠 -詡 -詢 -詣 -試 -詩 -詫 -詬 -詭 -詮 -詰 -話 -該 -詳 -詹 -詼 -誅 -誇 -誉 -誌 -認 -誓 -誕 -誘 -語 -誠 -誡 -誣 -誤 -誥 -誦 -誨 -說 -説 -読 -誰 -課 -誹 -誼 -調 -諄 -談 -請 -諏 -諒 -論 -諗 -諜 -諡 -諦 -諧 -諫 -諭 -諮 -諱 -諳 -諷 -諸 -諺 -諾 -謀 -謁 -謂 -謄 -謊 -謎 -謐 -謔 -謗 -謙 -講 -謝 -謠 -謨 -謬 -謹 -謾 -譁 -證 -譎 -譏 -識 -譙 -譚 -譜 -警 -譬 -譯 -議 -譲 -譴 -護 -譽 -讀 -變 -讓 -讚 -讞 -计 -订 -认 -讥 -讧 -讨 -让 -讪 -讫 -训 -议 -讯 -记 -讲 -讳 -讴 -讶 -讷 -许 -讹 -论 -讼 -讽 -设 -访 -诀 -证 -诃 -评 -诅 -识 -诈 -诉 -诊 -诋 -词 -诏 -译 -试 -诗 -诘 -诙 -诚 -诛 -话 -诞 -诟 -诠 -诡 -询 -诣 -诤 -该 -详 -诧 -诩 -诫 -诬 -语 -误 -诰 -诱 -诲 -说 -诵 -诶 -请 -诸 -诺 -读 -诽 -课 -诿 -谀 -谁 -调 -谄 -谅 -谆 -谈 -谊 -谋 -谌 -谍 -谎 -谏 -谐 -谑 -谒 -谓 -谔 -谕 -谗 -谘 -谙 -谚 -谛 -谜 -谟 -谢 -谣 -谤 -谥 -谦 -谧 -谨 -谩 -谪 -谬 -谭 -谯 -谱 -谲 -谴 -谶 -谷 -豁 -豆 -豇 -豈 -豉 -豊 -豌 -豎 -豐 -豔 -豚 -象 -豢 -豪 -豫 -豬 -豹 -豺 -貂 -貅 -貌 -貓 -貔 -貘 -貝 -貞 -負 -財 -貢 -貧 -貨 -販 -貪 -貫 -責 -貯 -貰 -貳 -貴 -貶 -買 -貸 -費 -貼 -貽 -貿 -賀 -賁 -賂 -賃 -賄 -資 -賈 -賊 -賑 -賓 -賜 -賞 -賠 -賡 -賢 -賣 -賤 -賦 -質 -賬 -賭 -賴 -賺 -購 -賽 -贅 -贈 -贊 -贍 -贏 -贓 -贖 -贛 -贝 -贞 -负 -贡 -财 -责 -贤 -败 -账 -货 -质 -贩 -贪 -贫 -贬 -购 -贮 -贯 -贰 -贱 -贲 -贴 -贵 -贷 -贸 -费 -贺 -贻 -贼 -贾 -贿 -赁 -赂 -赃 -资 -赅 -赈 -赊 -赋 -赌 -赎 -赏 -赐 -赓 -赔 -赖 -赘 -赚 -赛 -赝 -赞 -赠 -赡 -赢 -赣 -赤 -赦 -赧 -赫 -赭 -走 -赳 -赴 -赵 -赶 -起 -趁 -超 -越 -趋 -趕 -趙 -趟 -趣 -趨 -足 -趴 -趵 -趸 -趺 -趾 -跃 -跄 -跆 -跋 -跌 -跎 -跑 -跖 -跚 -跛 -距 -跟 -跡 -跤 -跨 -跩 -跪 -路 -跳 -践 -跷 -跹 -跺 -跻 -踉 -踊 -踌 -踏 -踐 -踝 -踞 -踟 -踢 -踩 -踪 -踮 -踱 -踴 -踵 -踹 -蹂 -蹄 -蹇 -蹈 -蹉 -蹊 -蹋 -蹑 -蹒 -蹙 -蹟 -蹣 -蹤 -蹦 -蹩 -蹬 -蹭 -蹲 -蹴 -蹶 -蹺 -蹼 -蹿 -躁 -躇 -躉 -躊 -躋 -躍 -躏 -躪 -身 -躬 -躯 -躲 -躺 -軀 -車 -軋 -軌 -軍 -軒 -軟 -転 -軸 -軼 -軽 -軾 -較 -載 -輒 -輓 -輔 -輕 -輛 -輝 -輟 -輩 -輪 -輯 -輸 -輻 -輾 -輿 -轄 -轅 -轆 -轉 -轍 -轎 -轟 -车 -轧 -轨 -轩 -转 -轭 -轮 -软 -轰 -轲 -轴 -轶 -轻 -轼 -载 -轿 -较 -辄 -辅 -辆 -辇 -辈 -辉 -辊 -辍 -辐 -辑 -输 -辕 -辖 -辗 -辘 -辙 -辛 -辜 -辞 -辟 -辣 -辦 -辨 -辩 -辫 -辭 -辮 -辯 -辰 -辱 -農 -边 -辺 -辻 -込 -辽 -达 -迁 -迂 -迄 -迅 -过 -迈 -迎 -运 -近 -返 -还 -这 -进 -远 -违 -连 -迟 -迢 -迤 -迥 -迦 -迩 -迪 -迫 -迭 -述 -迴 -迷 -迸 -迹 -迺 -追 -退 -送 -适 -逃 -逅 -逆 -选 -逊 -逍 -透 -逐 -递 -途 -逕 -逗 -這 -通 -逛 -逝 -逞 -速 -造 -逢 -連 -逮 -週 -進 -逵 -逶 -逸 -逻 -逼 -逾 -遁 -遂 -遅 -遇 -遊 -運 -遍 -過 -遏 -遐 -遑 -遒 -道 -達 -違 -遗 -遙 -遛 -遜 -遞 -遠 -遢 -遣 -遥 -遨 -適 -遭 -遮 -遲 -遴 -遵 -遶 -遷 -選 -遺 -遼 -遽 -避 -邀 -邁 -邂 -邃 -還 -邇 -邈 -邊 -邋 -邏 -邑 -邓 -邕 -邛 -邝 -邢 -那 -邦 -邨 -邪 -邬 -邮 -邯 -邰 -邱 -邳 -邵 -邸 -邹 -邺 -邻 -郁 -郅 -郊 -郎 -郑 -郜 -郝 -郡 -郢 -郤 -郦 -郧 -部 -郫 -郭 -郴 -郵 -郷 -郸 -都 -鄂 -鄉 -鄒 -鄔 -鄙 -鄞 -鄢 -鄧 -鄭 -鄰 -鄱 -鄲 -鄺 -酉 -酊 -酋 -酌 -配 -酐 -酒 -酗 -酚 -酝 -酢 -酣 -酥 -酩 -酪 -酬 -酮 -酯 -酰 -酱 -酵 -酶 -酷 -酸 -酿 -醃 -醇 -醉 -醋 -醍 -醐 -醒 -醚 -醛 -醜 -醞 -醣 -醪 -醫 -醬 -醮 -醯 -醴 -醺 -釀 -釁 -采 -釉 -释 -釋 -里 -重 -野 -量 -釐 -金 -釗 -釘 -釜 -針 -釣 -釦 -釧 -釵 -鈀 -鈉 -鈍 -鈎 -鈔 -鈕 -鈞 -鈣 -鈦 -鈪 -鈴 -鈺 -鈾 -鉀 -鉄 -鉅 -鉉 -鉑 -鉗 -鉚 -鉛 -鉤 -鉴 -鉻 -銀 -銃 -銅 -銑 -銓 -銖 -銘 -銜 -銬 -銭 -銮 -銳 -銷 -銹 -鋁 -鋅 -鋒 -鋤 -鋪 -鋰 -鋸 -鋼 -錄 -錐 -錘 -錚 -錠 -錢 -錦 -錨 -錫 -錮 -錯 -録 -錳 -錶 -鍊 -鍋 -鍍 -鍛 -鍥 -鍰 -鍵 -鍺 -鍾 -鎂 -鎊 -鎌 -鎏 -鎔 -鎖 -鎗 -鎚 -鎧 -鎬 -鎮 -鎳 -鏈 -鏖 -鏗 -鏘 -鏞 -鏟 -鏡 -鏢 -鏤 -鏽 -鐘 -鐮 -鐲 -鐳 -鐵 -鐸 -鐺 -鑄 -鑊 -鑑 -鑒 -鑣 -鑫 -鑰 -鑲 -鑼 -鑽 -鑾 -鑿 -针 -钉 -钊 -钎 -钏 -钒 -钓 -钗 -钙 -钛 -钜 -钝 -钞 -钟 -钠 -钡 -钢 -钣 -钤 -钥 -钦 -钧 -钨 -钩 -钮 -钯 -钰 -钱 -钳 -钴 -钵 -钺 -钻 -钼 -钾 -钿 -铀 -铁 -铂 -铃 -铄 -铅 -铆 -铉 -铎 -铐 -铛 -铜 -铝 -铠 -铡 -铢 -铣 -铤 -铨 -铩 -铬 -铭 -铮 -铰 -铲 -铵 -银 -铸 -铺 -链 -铿 -销 -锁 -锂 -锄 -锅 -锆 -锈 -锉 -锋 -锌 -锏 -锐 -锑 -错 -锚 -锟 -锡 -锢 -锣 -锤 -锥 -锦 -锭 -键 -锯 -锰 -锲 -锵 -锹 -锺 -锻 -镀 -镁 -镂 -镇 -镉 -镌 -镍 -镐 -镑 -镕 -镖 -镗 -镛 -镜 -镣 -镭 -镯 -镰 -镳 -镶 -長 -长 -門 -閃 -閉 -開 -閎 -閏 -閑 -閒 -間 -閔 -閘 -閡 -関 -閣 -閥 -閨 -閩 -閱 -閲 -閹 -閻 -閾 -闆 -闇 -闊 -闌 -闍 -闔 -闕 -闖 -闘 -關 -闡 -闢 -门 -闪 -闫 -闭 -问 -闯 -闰 -闲 -间 -闵 -闷 -闸 -闹 -闺 -闻 -闽 -闾 -阀 -阁 -阂 -阅 -阆 -阇 -阈 -阉 -阎 -阐 -阑 -阔 -阕 -阖 -阙 -阚 -阜 -队 -阡 -阪 -阮 -阱 -防 -阳 -阴 -阵 -阶 -阻 -阿 -陀 -陂 -附 -际 -陆 -陇 -陈 -陋 -陌 -降 -限 -陕 -陛 -陝 -陞 -陟 -陡 -院 -陣 -除 -陨 -险 -陪 -陰 -陲 -陳 -陵 -陶 -陷 -陸 -険 -陽 -隅 -隆 -隈 -隊 -隋 -隍 -階 -随 -隐 -隔 -隕 -隘 -隙 -際 -障 -隠 -隣 -隧 -隨 -險 -隱 -隴 -隶 -隸 -隻 -隼 -隽 -难 -雀 -雁 -雄 -雅 -集 -雇 -雉 -雋 -雌 -雍 -雎 -雏 -雑 -雒 -雕 -雖 -雙 -雛 -雜 -雞 -離 -難 -雨 -雪 -雯 -雰 -雲 -雳 -零 -雷 -雹 -電 -雾 -需 -霁 -霄 -霆 -震 -霈 -霉 -霊 -霍 -霎 -霏 -霑 -霓 -霖 -霜 -霞 -霧 -霭 -霰 -露 -霸 -霹 -霽 -霾 -靂 -靄 -靈 -青 -靓 -靖 -静 -靚 -靛 -靜 -非 -靠 -靡 -面 -靥 -靦 -革 -靳 -靴 -靶 -靼 -鞅 -鞋 -鞍 -鞏 -鞑 -鞘 -鞠 -鞣 -鞦 -鞭 -韆 -韋 -韌 -韓 -韜 -韦 -韧 -韩 -韬 -韭 -音 -韵 -韶 -韻 -響 -頁 -頂 -頃 -項 -順 -須 -頌 -預 -頑 -頒 -頓 -頗 -領 -頜 -頡 -頤 -頫 -頭 -頰 -頷 -頸 -頹 -頻 -頼 -顆 -題 -額 -顎 -顏 -顔 -願 -顛 -類 -顧 -顫 -顯 -顱 -顴 -页 -顶 -顷 -项 -顺 -须 -顼 -顽 -顾 -顿 -颁 -颂 -预 -颅 -领 -颇 -颈 -颉 -颊 -颌 -颍 -颐 -频 -颓 -颔 -颖 -颗 -题 -颚 -颛 -颜 -额 -颞 -颠 -颡 -颢 -颤 -颦 -颧 -風 -颯 -颱 -颳 -颶 -颼 -飄 -飆 -风 -飒 -飓 -飕 -飘 -飙 -飚 -飛 -飞 -食 -飢 -飨 -飩 -飪 -飯 -飲 -飼 -飽 -飾 -餃 -餅 -餉 -養 -餌 -餐 -餒 -餓 -餘 -餚 -餛 -餞 -餡 -館 -餮 -餵 -餾 -饅 -饈 -饋 -饌 -饍 -饑 -饒 -饕 -饗 -饞 -饥 -饨 -饪 -饬 -饭 -饮 -饯 -饰 -饱 -饲 -饴 -饵 -饶 -饷 -饺 -饼 -饽 -饿 -馀 -馁 -馄 -馅 -馆 -馈 -馋 -馍 -馏 -馒 -馔 -首 -馗 -香 -馥 -馨 -馬 -馭 -馮 -馳 -馴 -駁 -駄 -駅 -駆 -駐 -駒 -駕 -駛 -駝 -駭 -駱 -駿 -騁 -騎 -騏 -験 -騙 -騨 -騰 -騷 -驀 -驅 -驊 -驍 -驒 -驕 -驗 -驚 -驛 -驟 -驢 -驥 -马 -驭 -驮 -驯 -驰 -驱 -驳 -驴 -驶 -驷 -驸 -驹 -驻 -驼 -驾 -驿 -骁 -骂 -骄 -骅 -骆 -骇 -骈 -骊 -骋 -验 -骏 -骐 -骑 -骗 -骚 -骛 -骜 -骞 -骠 -骡 -骤 -骥 -骧 -骨 -骯 -骰 -骶 -骷 -骸 -骼 -髂 -髅 -髋 -髏 -髒 -髓 -體 -髖 -高 -髦 -髪 -髮 -髯 -髻 -鬃 -鬆 -鬍 -鬓 -鬚 -鬟 -鬢 -鬣 -鬥 -鬧 -鬱 -鬼 -魁 -魂 -魄 -魅 -魇 -魍 -魏 -魔 -魘 -魚 -魯 -魷 -鮑 -鮨 -鮪 -鮭 -鮮 -鯉 -鯊 -鯖 -鯛 -鯨 -鯰 -鯽 -鰍 -鰓 -鰭 -鰲 -鰻 -鰾 -鱈 -鱉 -鱔 -鱗 -鱷 -鱸 -鱼 -鱿 -鲁 -鲈 -鲍 -鲑 -鲛 -鲜 -鲟 -鲢 -鲤 -鲨 -鲫 -鲱 -鲲 -鲶 -鲷 -鲸 -鳃 -鳄 -鳅 -鳌 -鳍 -鳕 -鳖 -鳗 -鳝 -鳞 -鳥 -鳩 -鳳 -鳴 -鳶 -鴉 -鴕 -鴛 -鴦 -鴨 -鴻 -鴿 -鵑 -鵜 -鵝 -鵡 -鵬 -鵰 -鵲 -鶘 -鶩 -鶯 -鶴 -鷗 -鷲 -鷹 -鷺 -鸚 -鸞 -鸟 -鸠 -鸡 -鸢 -鸣 -鸥 -鸦 -鸨 -鸪 -鸭 -鸯 -鸳 -鸵 -鸽 -鸾 -鸿 -鹂 -鹃 -鹄 -鹅 -鹈 -鹉 -鹊 -鹌 -鹏 -鹑 -鹕 -鹘 -鹜 -鹞 -鹤 -鹦 -鹧 -鹫 -鹭 -鹰 -鹳 -鹵 -鹹 -鹼 -鹽 -鹿 -麂 -麋 -麒 -麓 -麗 -麝 -麟 -麥 -麦 -麩 -麴 -麵 -麸 -麺 -麻 -麼 -麽 -麾 -黃 -黄 -黍 -黎 -黏 -黑 -黒 -黔 -默 -黛 -黜 -黝 -點 -黠 -黨 -黯 -黴 -鼋 -鼎 -鼐 -鼓 -鼠 -鼬 -鼹 -鼻 -鼾 -齁 -齊 -齋 -齐 -齒 -齡 -齢 -齣 -齦 -齿 -龄 -龅 -龈 -龊 -龋 -龌 -龍 -龐 -龔 -龕 -龙 -龚 -龛 -龜 -龟 -︰ -︱ -︶ -︿ -﹁ -﹂ -﹍ -﹏ -﹐ -﹑ -﹒ -﹔ -﹕ -﹖ -﹗ -﹙ -﹚ -﹝ -﹞ -﹡ -﹣ -！ -＂ -＃ -＄ -％ -＆ -＇ -（ -） -＊ -＋ -， -－ -． -／ -０ -１ -２ -３ -４ -５ -６ -７ -８ -９ -： -； -＜ -＝ -＞ -？ -＠ -［ -＼ -］ -＾ -＿ -｀ -ａ -ｂ -ｃ -ｄ -ｅ -ｆ -ｇ -ｈ -ｉ -ｊ -ｋ -ｌ -ｍ -ｎ -ｏ -ｐ -ｑ -ｒ -ｓ -ｔ -ｕ -ｖ -ｗ -ｘ -ｙ -ｚ -｛ -｜ -｝ -～ -｡ -｢ -｣ -､ -･ -ｯ -ｰ -ｲ -ｸ -ｼ -ｽ -ﾄ -ﾉ -ﾌ -ﾗ -ﾙ -ﾝ -ﾞ -ﾟ -￣ -￥ -👍 -🔥 -😂 -😎 -... -yam -10 -2017 -12 -11 -2016 -20 -30 -15 -06 -lofter -##s -2015 -by -16 -14 -18 -13 -24 -17 -2014 -21 -##0 -22 -19 -25 -23 -com -100 -00 -05 -2013 -##a -03 -09 -08 -28 -##2 -50 -01 -04 -##1 -27 -02 -2012 -##3 -26 -##e -07 -##8 -##5 -##6 -##4 -##9 -##7 -29 -2011 -40 -##t -2010 -##o -##d -##i -2009 -##n -app -www -the -##m -31 -##c -##l -##y -##r -##g -2008 -60 -http -200 -qq -##p -80 -##f -google -pixnet -90 -cookies -tripadvisor -500 -##er -##k -35 -##h -facebook -2007 -2000 -70 -##b -of -##x -##u -45 -300 -iphone -32 -1000 -2006 -48 -ip -36 -in -38 -3d -##w -##ing -55 -ctrip -##on -##v -33 -##の -to -34 -400 -id -2005 -it -37 -windows -llc -top -99 -42 -39 -000 -led -at -##an -41 -51 -52 -46 -49 -43 -53 -44 -##z -android -58 -and -59 -2004 -56 -vr -##か -5000 -2003 -47 -blogthis -twitter -54 -##le -150 -ok -2018 -57 -75 -cn -no -ios -##in -##mm -##00 -800 -on -te -3000 -65 -2001 -360 -95 -ig -lv -120 -##ng -##を -##us -##に -pc -てす -── -600 -##te -85 -2002 -88 -##ed -html -ncc -wifi -email -64 -blog -is -##10 -##て -mail -online -##al -dvd -##ic -studio -##は -##℃ -##ia -##と -line -vip -72 -##q -98 -##ce -##en -for -##is -##ra -##es -##j -usb -net -cp -1999 -asia -4g -##cm -diy -new -3c -##お -ta -66 -language -vs -apple -tw -86 -web -##ne -ipad -62 -you -##re -101 -68 -##tion -ps -de -bt -pony -atm -##2017 -1998 -67 -##ch -ceo -##or -go -##na -av -pro -cafe -96 -pinterest -97 -63 -pixstyleme3c -##ta -more -said -##2016 -1997 -mp3 -700 -##ll -nba -jun -##20 -92 -tv -1995 -pm -61 -76 -nbsp -250 -##ie -linux -##ma -cd -110 -hd -##17 -78 -##ion -77 -6000 -am -##th -##st -94 -##se -##et -69 -180 -gdp -my -105 -81 -abc -89 -flash -79 -one -93 -1990 -1996 -##ck -gps -##も -##ly -web885 -106 -2020 -91 -##ge -4000 -1500 -xd -boss -isbn -1994 -org -##ry -me -love -##11 -0fork -73 -##12 -3g -##ter -##ar -71 -82 -##la -hotel -130 -1970 -pk -83 -87 -140 -ie -##os -##30 -##el -74 -##50 -seo -cpu -##ml -p2p -84 -may -##る -sun -tue -internet -cc -posted -youtube -##at -##ン -##man -ii -##ル -##15 -abs -nt -pdf -yahoo -ago -1980 -##it -news -mac -104 -##てす -##me -##り -java -1992 -spa -##de -##nt -hk -all -plus -la -1993 -##mb -##16 -##ve -west -##da -160 -air -##い -##ps -から -##to -1989 -logo -htc -php -https -fi -momo -##son -sat -##ke -##80 -ebd -suv -wi -day -apk -##88 -##um -mv -galaxy -wiki -or -brake -##ス -1200 -する -this -1991 -mon -##こ -❤2017 -po -##ない -javascript -life -home -june -##ss -system -900 -##ー -##０ -pp -1988 -world -fb -4k -br -##as -ic -ai -leonardo -safari -##60 -live -free -xx -wed -win7 -kiehl -##co -lg -o2o -##go -us -235 -1949 -mm -しい -vfm -kanye -##90 -##2015 -##id -jr -##ey -123 -rss -##sa -##ro -##am -##no -thu -fri -350 -##sh -##ki -103 -comments -name -##のて -##pe -##ine -max -1987 -8000 -uber -##mi -##ton -wordpress -office -1986 -1985 -##ment -107 -bd -win10 -##ld -##li -gmail -bb -dior -##rs -##ri -##rd -##ます -up -cad -##® -dr -して -read -##21 -をお -##io -##99 -url -1984 -pvc -paypal -show -policy -##40 -##ty -##18 -with -##★ -##01 -txt -102 -##ba -dna -from -post -mini -ar -taiwan -john -##ga -privacy -agoda -##13 -##ny -word -##24 -##22 -##by -##ur -##hz -1982 -##ang -265 -cookie -netscape -108 -##ka -##～ -##ad -house -share -note -ibm -code -hello -nike -sim -survey -##016 -1979 -1950 -wikia -##32 -##017 -5g -cbc -##tor -##kg -1983 -##rt -##14 -campaign -store -2500 -os -##ct -##ts -##° -170 -api -##ns -365 -excel -##な -##ao -##ら -##し -～～ -##nd -university -163 -には -518 -##70 -##ya -##il -##25 -pierre -ipo -0020 -897 -##23 -hotels -##ian -のお -125 -years -6606 -##ers -##26 -high -##day -time -##ay -bug -##line -##く -##す -##be -xp -talk2yam -yamservice -10000 -coco -##dy -sony -##ies -1978 -microsoft -david -people -##ha -1960 -instagram -intel -その -##ot -iso -1981 -##va -115 -##mo -##land -xxx -man -co -ltxsw -##ation -baby -220 -##pa -##ol -1945 -7000 -tag -450 -##ue -msn -##31 -oppo -##ト -##ca -control -##om -st -chrome -##ure -##ん -be -##き -lol -##19 -した -##bo -240 -lady -##100 -##way -##から -4600 -##ko -##do -##un -4s -corporation -168 -##ni -herme -##28 -ｃｐ -978 -##up -##06 -ui -##ds -ppt -admin -three -します -bbc -re -128 -##48 -ca -##015 -##35 -hp -##ee -tpp -##た -##ive -×× -root -##cc -##ました -##ble -##ity -adobe -park -114 -et -oled -city -##ex -##ler -##ap -china -##book -20000 -view -##ice -global -##km -your -hong -##mg -out -##ms -ng -ebay -##29 -menu -ubuntu -##cy -rom -##view -open -ktv -do -server -##lo -if -english -##ね -##５ -##oo -1600 -##02 -step1 -kong -club -135 -july -inc -1976 -mr -hi -##net -touch -##ls -##ii -michael -lcd -##05 -##33 -phone -james -step2 -1300 -ios9 -##box -dc -##２ -##ley -samsung -111 -280 -pokemon -css -##ent -##les -いいえ -##１ -s8 -atom -play -bmw -##said -sa -etf -ctrl -♥yoyo♥ -##55 -2025 -##2014 -##66 -adidas -amazon -1958 -##ber -##ner -visa -##77 -##der -1800 -connectivity -##hi -firefox -109 -118 -hr -so -style -mark -pop -ol -skip -1975 -as -##27 -##ir -##61 -190 -mba -##う -##ai -le -##ver -1900 -cafe2017 -lte -super -113 -129 -##ron -amd -like -##☆ -are -##ster -we -##sk -paul -data -international -##ft -longchamp -ssd -good -##ート -##ti -reply -##my -↓↓↓ -apr -star -##ker -source -136 -js -112 -get -force -photo -##one -126 -##2013 -##ow -link -bbs -1972 -goods -##lin -python -119 -##ip -game -##ics -##ません -blue -##● -520 -##45 -page -itunes -##03 -1955 -260 -1968 -gt -gif -618 -##ff -##47 -group -くたさい -about -bar -ganji -##nce -music -lee -not -1977 -1971 -1973 -##per -an -faq -comment -##って -days -##ock -116 -##bs -1974 -1969 -v1 -player -1956 -xbox -sql -fm -f1 -139 -##ah -210 -##lv -##mp -##000 -melody -1957 -##３ -550 -17life -199 -1966 -xml -market -##au -##71 -999 -##04 -what -gl -##95 -##age -tips -##68 -book -##ting -mysql -can -1959 -230 -##ung -wonderland -watch -10℃ -##ction -9000 -mar -mobile -1946 -1962 -article -##db -part -▲top -party -って -1967 -1964 -1948 -##07 -##ore -##op -この -dj -##78 -##38 -010 -main -225 -1965 -##ong -art -320 -ad -134 -020 -##73 -117 -pm2 -japan -228 -##08 -ts -1963 -##ica -der -sm -##36 -2019 -##wa -ct -##７ -##や -##64 -1937 -homemesh -search -##85 -##れは -##tv -##di -macbook -##９ -##くたさい -service -##♥ -type -った -750 -##ier -##si -##75 -##います -##ok -best -##ット -goris -lock -##った -cf -3m -big -##ut -ftp -carol -##vi -１０ -1961 -happy -sd -##ac -122 -anti -pe -cnn -iii -1920 -138 -##ラ -1940 -esp -jan -tags -##98 -##51 -august -vol -##86 -154 -##™ -##fs -##れ -##sion -design -ac -##ム -press -jordan -ppp -that -key -check -##６ -##tt -##㎡ -1080p -##lt -power -##42 -1952 -##bc -vivi -##ック -he -133 -121 -jpg -##rry -201 -175 -3500 -1947 -nb -##ted -##rn -しています -1954 -usd -##t00 -master -##ンク -001 -model -##58 -al -##09 -1953 -##34 -ram -goo -ても -##ui -127 -1930 -red -##ary -rpg -item -##pm -##41 -270 -##za -project -##2012 -hot -td -blogabstract -##ger -##62 -650 -##44 -gr2 -##します -##ｍ -black -electronic -nfc -year -asus -また -html5 -cindy -##hd -m3 -132 -esc -##od -booking -##53 -fed -tvb -##81 -##ina -mit -165 -##いる -chan -192 -distribution -next -になる -peter -bios -steam -cm -1941 -にも -pk10 -##ix -##65 -##91 -dec -nasa -##ana -icecat -00z -b1 -will -##46 -li -se -##ji -##み -##ard -oct -##ain -jp -##ze -##bi -cio -##56 -smart -h5 -##39 -##port -curve -vpn -##nm -##dia -utc -##あり -12345678910 -##52 -rmvb -chanel -a4 -miss -##and -##im -media -who -##63 -she -girl -5s -124 -vera -##して -class -vivo -king -##フ -##ei -national -ab -1951 -5cm -888 -145 -ipod -ap -1100 -5mm -211 -ms -2756 -##69 -mp4 -msci -##po -##89 -131 -mg -index -380 -##bit -##out -##zz -##97 -##67 -158 -apec -##８ -photoshop -opec -￥799 -ては -##96 -##tes -##ast -2g -○○ -##ール -￥2899 -##ling -##よ -##ory -1938 -##ical -kitty -content -##43 -step3 -##cn -win8 -155 -vc -1400 -iphone7 -robert -##した -tcl -137 -beauty -##87 -en -dollars -##ys -##oc -step -pay -yy -a1 -##2011 -##lly -##ks -##♪ -1939 -188 -download -1944 -sep -exe -ph -います -school -gb -center -pr -street -##board -uv -##37 -##lan -winrar -##que -##ua -##com -1942 -1936 -480 -gpu -##４ -ettoday -fu -tom -##54 -##ren -##via -149 -##72 -b2b -144 -##79 -##tch -rose -arm -mb -##49 -##ial -##nn -nvidia -step4 -mvp -00㎡ -york -156 -##イ -how -cpi -591 -2765 -gov -kg -joe -##xx -mandy -pa -##ser -copyright -fashion -1935 -don -##け -ecu -##ist -##art -erp -wap -have -##lm -talk -##ek -##ning -##if -ch -##ite -video -1943 -cs -san -iot -look -##84 -##2010 -##ku -october -##ux -trump -##hs -##ide -box -141 -first -##ins -april -##ight -##83 -185 -angel -protected -aa -151 -162 -x1 -m2 -##fe -##× -##ho -size -143 -min -ofo -fun -gomaji -ex -hdmi -food -dns -march -chris -kevin -##のか -##lla -##pp -##ec -ag -ems -6s -720p -##rm -##ham -off -##92 -asp -team -fandom -ed -299 -▌♥ -##ell -info -されています -##82 -sina -4066 -161 -##able -##ctor -330 -399 -315 -dll -rights -ltd -idc -jul -3kg -1927 -142 -ma -surface -##76 -##ク -～～～ -304 -mall -eps -146 -green -##59 -map -space -donald -v2 -sodu -##light -1931 -148 -1700 -まて -310 -reserved -htm -##han -##57 -2d -178 -mod -##ise -##tions -152 -ti -##shi -doc -1933 -icp -055 -wang -##ram -shopping -aug -##pi -##well -now -wam -b2 -からお -##hu -236 -1928 -##gb -266 -f2 -##93 -153 -mix -##ef -##uan -bwl -##plus -##res -core -##ess -tea -5℃ -hktvmall -nhk -##ate -list -##ese -301 -feb -4m -inn -ての -nov -159 -12345 -daniel -##ci -pass -##bet -##nk -coffee -202 -ssl -airbnb -##ute -fbi -woshipm -skype -ea -cg -sp -##fc -##www -yes -edge -alt -007 -##94 -fpga -##ght -##gs -iso9001 -さい -##ile -##wood -##uo -image -lin -icon -american -##em -1932 -set -says -##king -##tive -blogger -##74 -なと -256 -147 -##ox -##zy -##red -##ium -##lf -nokia -claire -##リ -##ding -november -lohas -##500 -##tic -##マ -##cs -##ある -##che -##ire -##gy -##ult -db -january -win -##カ -166 -road -ptt -##ま -##つ -198 -##fa -##mer -anna -pchome -はい -udn -ef -420 -##time -##tte -2030 -##ア -g20 -white -かかります -1929 -308 -garden -eleven -di -##おります -chen -309b -777 -172 -young -cosplay -ちてない -4500 -bat -##123 -##tra -##ては -kindle -npc -steve -etc -##ern -##｜ -call -xperia -ces -travel -sk -s7 -##ous -1934 -##int -みいたたけます -183 -edu -file -cho -qr -##car -##our -186 -##ant -##ｄ -eric -1914 -rends -##jo -##する -mastercard -##2000 -kb -##min -290 -##ino -vista -##ris -##ud -jack -2400 -##set -169 -pos -1912 -##her -##ou -taipei -しく -205 -beta -##ませんか -232 -##fi -express -255 -body -##ill -aphojoy -user -december -meiki -##ick -tweet -richard -##av -##ᆫ -iphone6 -##dd -ちてすか -views -##mark -321 -pd -##００ -times -##▲ -level -##ash -10g -point -5l -##ome -208 -koreanmall -##ak -george -q2 -206 -wma -tcp -##200 -スタッフ -full -mlb -##lle -##watch -tm -run -179 -911 -smith -business -##und -1919 -color -##tal -222 -171 -##less -moon -4399 -##rl -update -pcb -shop -499 -157 -little -なし -end -##mhz -van -dsp -easy -660 -##house -##key -history -##ｏ -oh -##001 -##hy -##web -oem -let -was -##2009 -##gg -review -##wan -182 -##°c -203 -uc -title -##val -united -233 -2021 -##ons -doi -trivago -overdope -sbs -##ance -##ち -grand -special -573032185 -imf -216 -wx17house -##so -##ーム -audi -##he -london -william -##rp -##ake -science -beach -cfa -amp -ps4 -880 -##800 -##link -##hp -crm -ferragamo -bell -make -##eng -195 -under -zh -photos -2300 -##style -##ント -via -176 -da -##gi -company -i7 -##ray -thomas -370 -ufo -i5 -##max -plc -ben -back -research -8g -173 -mike -##pc -##ッフ -september -189 -##ace -vps -february -167 -pantos -wp -lisa -1921 -★★ -jquery -night -long -offer -##berg -##news -1911 -##いて -ray -fks -wto -せます -over -164 -340 -##all -##rus -1924 -##888 -##works -blogtitle -loftpermalink -##→ -187 -martin -test -ling -km -##め -15000 -fda -v3 -##ja -##ロ -ｗedding -かある -outlet -family -##ea -をこ -##top -story -##ness -salvatore -##lu -204 -swift -215 -room -している -oracle -##ul -1925 -sam -b2c -week -pi -rock -##のは -##ａ -##けと -##ean -##300 -##gle -cctv -after -chinese -##back -powered -x2 -##tan -1918 -##nes -##イン -canon -only -181 -##zi -##las -say -##oe -184 -##sd -221 -##bot -##world -##zo -sky -made -top100 -just -1926 -pmi -802 -234 -gap -##vr -177 -les -174 -▲topoct -ball -vogue -vi -ing -ofweek -cos -##list -##ort -▲topmay -##なら -##lon -として -last -##tc -##of -##bus -##gen -real -eva -##コ -a3 -nas -##lie -##ria -##coin -##bt -▲topapr -his -212 -cat -nata -vive -health -⋯⋯ -drive -sir -▲topmar -du -cup -##カー -##ook -##よう -##sy -alex -msg -tour -しました -3ce -##word -193 -ebooks -r8 -block -318 -##より -2200 -nice -pvp -207 -months -1905 -rewards -##ther -1917 -0800 -##xi -##チ -##sc -micro -850 -gg -blogfp -op -1922 -daily -m1 -264 -true -##bb -ml -##tar -##のお -##ky -anthony -196 -253 -##yo -state -218 -##ara -##aa -##rc -##tz -##ston -より -gear -##eo -##ade -ge -see -1923 -##win -##ura -ss -heart -##den -##ita -down -##sm -el -png -2100 -610 -rakuten -whatsapp -bay -dream -add -##use -680 -311 -pad -gucci -mpv -##ode -##fo -island -▲topjun -##▼ -223 -jason -214 -chicago -##❤ -しの -##hone -io -##れる -##ことか -sogo -be2 -##ology -990 -cloud -vcd -##con -2～3 -##ford -##joy -##kb -##こさいます -##rade -but -##ach -docker -##ful -rfid -ul -##ase -hit -ford -##star -580 -##○ -１１ -a2 -sdk -reading -edited -##are -cmos -##mc -238 -siri -light -##ella -##ため -bloomberg -##read -pizza -##ison -jimmy -##vm -college -node -journal -ba -18k -##play -245 -##cer -２０ -magic -##yu -191 -jump -288 -tt -##ings -asr -##lia -3200 -step5 -network -##cd -mc -いします -1234 -pixstyleme -273 -##600 -2800 -money -★★★★★ -1280 -１２ -430 -bl -みの -act -##tus -tokyo -##rial -##life -emba -##ae -saas -tcs -##rk -##wang -summer -##sp -ko -##ving -390 -premium -##その -netflix -##ヒ -uk -mt -##lton -right -frank -two -209 -える -##ple -##cal -021 -##んな -##sen -##ville -hold -nexus -dd -##ius -てお -##mah -##なく -tila -zero -820 -ce -##tin -resort -##ws -charles -old -p10 -5d -report -##360 -##ru -##には -bus -vans -lt -##est -pv -##レ -links -rebecca -##ツ -##dm -azure -##365 -きな -limited -bit -4gb -##mon -1910 -moto -##eam -213 -1913 -var -eos -なとの -226 -blogspot -された -699 -e3 -dos -dm -fc -##ments -##ik -##kw -boy -##bin -##ata -960 -er -##せ -219 -##vin -##tu -##ula -194 -##∥ -station -##ろ -##ature -835 -files -zara -hdr -top10 -nature -950 -magazine -s6 -marriott -##シ -avira -case -##っと -tab -##ran -tony -##home -oculus -im -##ral -jean -saint -cry -307 -rosie -##force -##ini -ice -##bert -のある -##nder -##mber -pet -2600 -##◆ -plurk -▲topdec -##sis -00kg -▲topnov -720 -##ence -tim -##ω -##nc -##ても -##name -log -ips -great -ikea -malaysia -unix -##イト -3600 -##ncy -##nie -12000 -akb48 -##ye -##oid -404 -##chi -##いた -oa -xuehai -##1000 -##orm -##rf -275 -さん -##ware -##リー -980 -ho -##pro -text -##era -560 -bob -227 -##ub -##2008 -8891 -scp -avi -##zen -2022 -mi -wu -museum -qvod -apache -lake -jcb -▲topaug -★★★ -ni -##hr -hill -302 -ne -weibo -490 -ruby -##ーシ -##ヶ -##row -4d -▲topjul -iv -##ish -github -306 -mate -312 -##スト -##lot -##ane -andrew -のハイト -##tina -t1 -rf -ed2k -##vel -##900 -way -final -りの -ns -5a -705 -197 -##メ -sweet -bytes -##ene -▲topjan -231 -##cker -##2007 -##px -100g -topapp -229 -helpapp -rs -low -14k -g4g -care -630 -ldquo -あり -##fork -leave -rm -edition -##gan -##zon -##qq -▲topsep -##google -##ism -gold -224 -explorer -##zer -toyota -category -select -visual -##labels -restaurant -##md -posts -s1 -##ico -もっと -angelababy -123456 -217 -sports -s3 -mbc -1915 -してくたさい -shell -x86 -candy -##new -kbs -face -xl -470 -##here -4a -swissinfo -v8 -▲topfeb -dram -##ual -##vice -3a -##wer -sport -q1 -ios10 -public -int -card -##ｃ -ep -au -rt -##れた -1080 -bill -##mll -kim -３０ -460 -wan -##uk -##ミ -x3 -298 -0t -scott -##ming -239 -e5 -##3d -h7n9 -worldcat -brown -##あります -##vo -##led -##580 -##ax -249 -410 -##ert -paris -##～6 -polo -925 -##lr -599 -##ナ -capital -##hing -bank -cv -1g -##chat -##ｓ -##たい -adc -##ule -2m -##ｅ -digital -hotmail -268 -##pad -870 -bbq -quot -##ring -before -wali -##まて -mcu -2k -2b -という -costco -316 -north -333 -switch -##city -##ｐ -philips -##mann -management -panasonic -##cl -##vd -##ping -##rge -alice -##lk -##ましょう -css3 -##ney -vision -alpha -##ular -##400 -##tter -lz -にお -##ありません -mode -gre -1916 -pci -##tm -237 -1～2 -##yan -##そ -について -##let -##キ -work -war -coach -ah -mary -##ᅵ -huang -##pt -a8 -pt -follow -##berry -1895 -##ew -a5 -ghost -##ション -##wn -##og -south -##code -girls -##rid -action -villa -git -r11 -table -games -##cket -error -##anonymoussaid -##ag -here -##ame -##gc -qa -##■ -##lis -gmp -##gin -vmalife -##cher -yu -wedding -##tis -demo -dragon -530 -soho -social -bye -##rant -river -orz -acer -325 -##↑ -##ース -##ats -261 -del -##ven -440 -ups -##ように -##ター -305 -value -macd -yougou -##dn -661 -##ano -ll -##urt -##rent -continue -script -##wen -##ect -paper -263 -319 -shift -##chel -##フト -##cat -258 -x5 -fox -243 -##さん -car -aaa -##blog -loading -##yn -##tp -kuso -799 -si -sns -イカせるテンマ -ヒンクテンマ3 -rmb -vdc -forest -central -prime -help -ultra -##rmb -##ような -241 -square -688 -##しい -のないフロクに -##field -##reen -##ors -##ju -c1 -start -510 -##air -##map -cdn -##wo -cba -stephen -m8 -100km -##get -opera -##base -##ood -vsa -com™ -##aw -##ail -251 -なのて -count -t2 -##ᅡ -##een -2700 -hop -##gp -vsc -tree -##eg -##ose -816 -285 -##ories -##shop -alphago -v4 -1909 -simon -##ᆼ -fluke62max -zip -スホンサー -##sta -louis -cr -bas -##～10 -bc -##yer -hadoop -##ube -##wi -1906 -0755 -hola -##low -place -centre -5v -d3 -##fer -252 -##750 -##media -281 -540 -0l -exchange -262 -series -##ハー -##san -eb -##bank -##ｋ -q3 -##nge -##mail -take -##lp -259 -1888 -client -east -cache -event -vincent -##ールを -きを -##nse -sui -855 -adchoice -##и -##stry -##なたの -246 -##zone -ga -apps -sea -##ab -248 -cisco -##タ -##rner -kymco -##care -dha -##pu -##yi -minkoff -royal -p1 -への -annie -269 -collection -kpi -playstation -257 -になります -866 -bh -##bar -queen -505 -radio -1904 -andy -armani -##xy -manager -iherb -##ery -##share -spring -raid -johnson -1908 -##ob -volvo -hall -##ball -v6 -our -taylor -##hk -bi -242 -##cp -kate -bo -water -technology -##rie -サイトは -277 -##ona -##sl -hpv -303 -gtx -hip -rdquo -jayz -stone -##lex -##rum -namespace -##やり -620 -##ale -##atic -des -##erson -##ql -##ves -##type -enter -##この -##てきます -d2 -##168 -##mix -##bian -との -a9 -jj -ky -##lc -access -movie -##hc -リストに -tower -##ration -##mit -ます -##nch -ua -tel -prefix -##o2 -1907 -##point -1901 -ott -～10 -##http -##ury -baidu -##ink -member -##logy -bigbang -nownews -##js -##shot -##tb -##こと -247 -eba -##tics -##lus -ける -v5 -spark -##ama -there -##ions -god -##lls -##down -hiv -##ress -burberry -day2 -##kv -◆◆ -jeff -related -film -edit -joseph -283 -##ark -cx -32gb -order -g9 -30000 -##ans -##tty -s5 -##bee -かあります -thread -xr -buy -sh -005 -land -spotify -mx -##ari -276 -##verse -×email -sf -why -##ことて -244 -7headlines -nego -sunny -dom -exo -401 -666 -positioning -fit -rgb -##tton -278 -kiss -alexa -adam -lp -みリストを -##ｇ -mp -##ties -##llow -amy -##du -np -002 -institute -271 -##rth -##lar -2345 -590 -##des -sidebar -１５ -imax -site -##cky -##kit -##ime -##009 -season -323 -##fun -##ンター -##ひ -gogoro -a7 -pu -lily -fire -twd600 -##ッセーシを -いて -##vis -30ml -##cture -##をお -information -##オ -close -friday -##くれる -yi -nick -てすか -##tta -##tel -6500 -##lock -cbd -economy -254 -かお -267 -tinker -double -375 -8gb -voice -##app -oops -channel -today -985 -##right -raw -xyz -##＋ -jim -edm -##cent -7500 -supreme -814 -ds -##its -##asia -dropbox -##てすか -##tti -books -272 -100ml -##tle -##ller -##ken -##more -##boy -sex -309 -##dom -t3 -##ider -##なります -##unch -1903 -810 -feel -5500 -##かった -##put -により -s2 -mo -##gh -men -ka -amoled -div -##tr -##n1 -port -howard -##tags -ken -dnf -##nus -adsense -##а -ide -##へ -buff -thunder -##town -##ique -has -##body -auto -pin -##erry -tee -てした -295 -number -##the -##013 -object -psp -cool -udnbkk -16gb -##mic -miui -##tro -most -r2 -##alk -##nity -1880 -±0 -##いました -428 -s4 -law -version -##oa -n1 -sgs -docomo -##tf -##ack -henry -fc2 -##ded -##sco -##014 -##rite -286 -0mm -linkedin -##ada -##now -wii -##ndy -ucbug -##◎ -sputniknews -legalminer -##ika -##xp -2gb -##bu -q10 -oo -b6 -come -##rman -cheese -ming -maker -##gm -nikon -##fig -ppi -kelly -##ります -jchere -てきます -ted -md -003 -fgo -tech -##tto -dan -soc -##gl -##len -hair -earth -640 -521 -img -##pper -##a1 -##てきる -##ロク -acca -##ition -##ference -suite -##ig -outlook -##mond -##cation -398 -##pr -279 -101vip -358 -##999 -282 -64gb -3800 -345 -airport -##over -284 -##おり -jones -##ith -lab -##su -##いるのて -co2 -town -piece -##llo -no1 -vmware -24h -##qi -focus -reader -##admin -##ora -tb -false -##log -1898 -know -lan -838 -##ces -f4 -##ume -motel -stop -##oper -na -flickr -netcomponents -##af -##─ -pose -williams -local -##ound -##cg -##site -##iko -いお -274 -5m -gsm -con -##ath -1902 -friends -##hip -cell -317 -##rey -780 -cream -##cks -012 -##dp -facebooktwitterpinterestgoogle -sso -324 -shtml -song -swiss -##mw -##キンク -lumia -xdd -string -tiffany -522 -marc -られた -insee -russell -sc -dell -##ations -ｏｋ -camera -289 -##vs -##flow -##late -classic -287 -##nter -stay -g1 -mtv -512 -##ever -##lab -##nger -qe -sata -ryan -d1 -50ml -cms -##cing -su -292 -3300 -editor -296 -##nap -security -sunday -association -##ens -##700 -##bra -acg -##かり -sofascore -とは -mkv -##ign -jonathan -gary -build -labels -##oto -tesla -moba -qi -gohappy -general -ajax -1024 -##かる -サイト -society -##test -##urs -wps -fedora -##ich -mozilla -328 -##480 -##dr -usa -urn -##lina -##ｒ -grace -##die -##try -##ader -1250 -##なり -elle -570 -##chen -##ᆯ -price -##ten -uhz -##ough -eq -##hen -states -push -session -balance -wow -506 -##cus -##py -when -##ward -##ep -34e -wong -library -prada -##サイト -##cle -running -##ree -313 -ck -date -q4 -##ctive -##ool -##＞ -mk -##ira -##163 -388 -die -secret -rq -dota -buffet -は１ヶ -e6 -##ez -pan -368 -ha -##card -##cha -2a -##さ -alan -day3 -eye -f3 -##end -france -keep -adi -rna -tvbs -##ala -solo -nova -##え -##tail -##ょう -support -##ries -##なる -##ved -base -copy -iis -fps -##ways -hero -hgih -profile -fish -mu -ssh -entertainment -chang -##wd -click -cake -##ond -pre -##tom -kic -pixel -##ov -##fl -product -6a -##pd -dear -##gate -es -yumi -audio -##² -##sky -echo -bin -where -##ture -329 -##ape -find -sap -isis -##なと -nand -##101 -##load -##ream -band -a6 -525 -never -##post -festival -50cm -##we -555 -guide -314 -zenfone -##ike -335 -gd -forum -jessica -strong -alexander -##ould -software -allen -##ious -program -360° -else -lohasthree -##gar -することかてきます -please -##れます -rc -##ggle -##ric -bim -50000 -##own -eclipse -355 -brian -3ds -##side -061 -361 -##other -##ける -##tech -##ator -485 -engine -##ged -##ｔ -plaza -##fit -cia -ngo -westbrook -shi -tbs -50mm -##みませんか -sci -291 -reuters -##ily -contextlink -##hn -af -##cil -bridge -very -##cel -1890 -cambridge -##ize -15g -##aid -##data -790 -frm -##head -award -butler -##sun -meta -##mar -america -ps3 -puma -pmid -##すか -lc -670 -kitchen -##lic -オーフン5 -きなしソフトサーヒス -そして -day1 -future -★★★★ -##text -##page -##rris -pm1 -##ket -fans -##っています -1001 -christian -bot -kids -trackback -##hai -c3 -display -##hl -n2 -1896 -idea -さんも -##sent -airmail -##ug -##men -pwm -けます -028 -##lution -369 -852 -awards -schemas -354 -asics -wikipedia -font -##tional -##vy -c2 -293 -##れている -##dget -##ein -っている -contact -pepper -スキル -339 -##～5 -294 -##uel -##ument -730 -##hang -みてす -q5 -##sue -rain -##ndi -wei -swatch -##cept -わせ -331 -popular -##ste -##tag -p2 -501 -trc -1899 -##west -##live -justin -honda -ping -messenger -##rap -v9 -543 -##とは -unity -appqq -はすへて -025 -leo -##tone -##テ -##ass -uniqlo -##010 -502 -her -jane -memory -moneydj -##tical -human -12306 -していると -##m2 -coc -miacare -##mn -tmt -##core -vim -kk -##may -fan -target -use -too -338 -435 -2050 -867 -737 -fast -##2c -services -##ope -omega -energy -##わ -pinkoi -1a -##なから -##rain -jackson -##ement -##シャンルの -374 -366 -そんな -p9 -rd -##ᆨ -1111 -##tier -##vic -zone -##│ -385 -690 -dl -isofix -cpa -m4 -322 -kimi -めて -davis -##lay -lulu -##uck -050 -weeks -qs -##hop -920 -##ｎ -ae -##ear -～5 -eia -405 -##fly -korea -jpeg -boost -##ship -small -##リア -1860 -eur -297 -425 -valley -##iel -simple -##ude -rn -k2 -##ena -されます -non -patrick -しているから -##ナー -feed -5757 -30g -process -well -qqmei -##thing -they -aws -lu -pink -##ters -##kin -または -board -##vertisement -wine -##ien -unicode -##dge -r1 -359 -##tant -いを -##twitter -##3c -cool1 -される -##れて -##ｌ -isp -##012 -standard -45㎡2 -402 -##150 -matt -##fu -326 -##iner -googlemsn -pixnetfacebookyahoo -##ラン -x7 -886 -##uce -メーカー -sao -##ev -##きました -##file -9678 -403 -xddd -shirt -6l -##rio -##hat -3mm -givenchy -ya -bang -##lio -monday -crystal -ロクイン -##abc -336 -head -890 -ubuntuforumwikilinuxpastechat -##vc -##～20 -##rity -cnc -7866 -ipv6 -null -1897 -##ost -yang -imsean -tiger -##fet -##ンス -352 -##＝ -dji -327 -ji -maria -##come -##んて -foundation -3100 -##beth -##なった -1m -601 -active -##aft -##don -3p -sr -349 -emma -##khz -living -415 -353 -1889 -341 -709 -457 -sas -x6 -##face -pptv -x4 -##mate -han -sophie -##jing -337 -fifa -##mand -other -sale -inwedding -##gn -てきちゃいます -##mmy -##pmlast -bad -nana -nbc -してみてくたさいね -なとはお -##wu -##かあります -##あ -note7 -single -##340 -せからこ -してくたさい♪この -しにはとんとんワークケートを -するとあなたにもっとマッチした -ならワークケートへ -もみつかっちゃうかも -ワークケートの -##bel -window -##dio -##ht -union -age -382 -１４ -##ivity -##ｙ -コメント -domain -neo -##isa -##lter -5k -f5 -steven -##cts -powerpoint -tft -self -g2 -ft -##テル -zol -##act -mwc -381 -343 -もう -nbapop -408 -てある -eds -ace -##room -previous -author -tomtom -il -##ets -hu -financial -☆☆☆ -っています -bp -5t -chi -1gb -##hg -fairmont -cross -008 -gay -h2 -function -##けて -356 -also -1b -625 -##ータ -##raph -1894 -3～5 -##ils -i3 -334 -avenue -##host -による -##bon -##tsu -message -navigation -50g -fintech -h6 -##ことを -8cm -##ject -##vas -##firm -credit -##wf -xxxx -form -##nor -##space -huawei -plan -json -sbl -##dc -machine -921 -392 -wish -##120 -##sol -windows7 -edward -##ために -development -washington -##nsis -lo -818 -##sio -##ym -##bor -planet -##～8 -##wt -ieee -gpa -##めて -camp -ann -gm -##tw -##oka -connect -##rss -##work -##atus -wall -chicken -soul -2mm -##times -fa -##ather -##cord -009 -##eep -hitachi -gui -harry -##pan -e1 -disney -##press -##ーション -wind -386 -frigidaire -##tl -liu -hsu -332 -basic -von -ev -いた -てきる -スホンサーサイト -learning -##ull -expedia -archives -change -##wei -santa -cut -ins -6gb -turbo -brand -cf1 -508 -004 -return -747 -##rip -h1 -##nis -##をこ -128gb -##にお -3t -application -しており -emc -rx -##oon -384 -quick -412 -15058 -wilson -wing -chapter -##bug -beyond -##cms -##dar -##oh -zoom -e2 -trip -sb -##nba -rcep -342 -aspx -ci -080 -gc -gnu -める -##count -advanced -dance -dv -##url -##ging -367 -8591 -am09 -shadow -battle -346 -##ｉ -##cia -##という -emily -##のてす -##tation -host -ff -techorz -sars -##mini -##mporary -##ering -nc -4200 -798 -##next -cma -##mbps -##gas -##ift -##dot -##ィ -455 -##～17 -amana -##りの -426 -##ros -ir -00㎡1 -##eet -##ible -##↓ -710 -ˋ▽ˊ -##aka -dcs -iq -##ｖ -l1 -##lor -maggie -##011 -##iu -588 -##～1 -830 -##gt -1tb -articles -create -##burg -##iki -database -fantasy -##rex -##cam -dlc -dean -##you -hard -path -gaming -victoria -maps -cb -##lee -##itor -overchicstoretvhome -systems -##xt -416 -p3 -sarah -760 -##nan -407 -486 -x9 -install -second -626 -##ann -##ph -##rcle -##nic -860 -##nar -ec -##とう -768 -metro -chocolate -##rian -～4 -##table -##しています -skin -##sn -395 -mountain -##0mm -inparadise -6m -7x24 -ib -4800 -##jia -eeworld -creative -g5 -g3 -357 -parker -ecfa -village -からの -18000 -sylvia -サーヒス -hbl -##ques -##onsored -##x2 -##きます -##v4 -##tein -ie6 -383 -##stack -389 -ver -##ads -##baby -sound -bbe -##110 -##lone -##uid -ads -022 -gundam -351 -thinkpad -006 -scrum -match -##ave -mems -##470 -##oy -##なりました -##talk -glass -lamigo -span -##eme -job -##a5 -jay -wade -kde -498 -##lace -ocean -tvg -##covery -##r3 -##ners -##rea -junior -think -##aine -cover -##ision -##sia -↓↓ -##bow -msi -413 -458 -406 -##love -711 -801 -soft -z2 -##pl -456 -1840 -mobil -mind -##uy -427 -nginx -##oi -めた -##rr -6221 -##mple -##sson -##ーシてす -371 -##nts -91tv -comhd -crv3000 -##uard -1868 -397 -deep -lost -field -gallery -##bia -rate -spf -redis -traction -930 -icloud -011 -なら -fe -jose -372 -##tory -into -sohu -fx -899 -379 -kicstart2 -##hia -すく -##～3 -##sit -ra -２４ -##walk -##xure -500g -##pact -pacific -xa -natural -carlo -##250 -##walker -1850 -##can -cto -gigi -516 -##サー -pen -##hoo -ob -matlab -##ｂ -##yy -13913459 -##iti -mango -##bbs -sense -c5 -oxford -##ニア -walker -jennifer -##ola -course -##bre -701 -##pus -##rder -lucky -075 -##ぁ -ivy -なお -##nia -sotheby -side -##ugh -joy -##orage -##ush -##bat -##dt -364 -r9 -##2d -##gio -511 -country -wear -##lax -##～7 -##moon -393 -seven -study -411 -348 -lonzo -8k -##ェ -evolution -##イフ -##kk -gs -kd -##レス -arduino -344 -b12 -##lux -arpg -##rdon -cook -##x5 -dark -five -##als -##ida -とても -sign -362 -##ちの -something -20mm -##nda -387 -##posted -fresh -tf -1870 -422 -cam -##mine -##skip -##form -##ssion -education -394 -##tee -dyson -stage -##jie -want -##night -epson -pack -あります -##ppy -テリヘル -##█ -wd -##eh -##rence -left -##lvin -golden -mhz -discovery -##trix -##n2 -loft -##uch -##dra -##sse -speed -～1 -1mdb -sorry -welcome -##urn -wave -gaga -##lmer -teddy -##160 -トラックハック -せよ -611 -##f2016 -378 -rp -##sha -rar -##あなたに -##きた -840 -holiday -##ュー -373 -074 -##vg -##nos -##rail -gartner -gi -6p -##dium -kit -488 -b3 -eco -##ろう -20g -sean -##stone -autocad -nu -##np -f16 -write -029 -m5 -##ias -images -atp -##dk -fsm -504 -1350 -ve -52kb -##xxx -##のに -##cake -414 -unit -lim -ru -1v -##ification -published -angela -16g -analytics -ak -##ｑ -##nel -gmt -##icon -again -##₂ -##bby -ios11 -445 -かこさいます -waze -いてす -##ハ -9985 -##ust -##ティー -framework -##007 -iptv -delete -52sykb -cl -wwdc -027 -30cm -##fw -##ての -1389 -##xon -brandt -##ses -##dragon -tc -vetements -anne -monte -modern -official -##へて -##ere -##nne -##oud -もちろん -５０ -etnews -##a2 -##graphy -421 -863 -##ちゃん -444 -##rtex -##てお -l2 -##gma -mount -ccd -たと -archive -morning -tan -ddos -e7 -##ホ -day4 -##ウ -gis -453 -its -495 -factory -bruce -pg -##ito -ってくたさい -guest -cdma -##lling -536 -n3 -しかし -3～4 -mega -eyes -ro -１３ -women -dac -church -##jun -singapore -##facebook -6991 -starbucks -##tos -##stin -##shine -zen -##mu -tina -20℃ -1893 -##たけて -503 -465 -request -##gence -qt -##っ -1886 -347 -363 -q7 -##zzi -diary -##tore -409 -##ead -468 -cst -##osa -canada -agent -va -##jiang -##ちは -##ーク -##lam -sg -##nix -##sday -##よって -g6 -##master -bing -##zl -charlie -１６ -8mm -nb40 -##ーン -thai -##ルフ -ln284ct -##itz -##2f -bonnie -##food -##lent -originals -##stro -##lts -418 -∟∣ -##bscribe -children -ntd -yesstyle -##かも -hmv -##tment -d5 -2cm -arts -sms -##pn -##я -##いい -topios9 -539 -lifestyle -virtual -##ague -xz -##deo -muji -024 -unt -##nnis -##ᅩ -faq1 -1884 -396 -##ette -fly -64㎡ -はしめまして -441 -curry -##pop -のこ -release -##← -##◆◆ -##cast -073 -ありな -500ml -##ews -5c -##stle -ios7 -##ima -787 -dog -lenovo -##r4 -roger -013 -cbs -vornado -100m -417 -##desk -##クok -##ald -1867 -9595 -2900 -##van -oil -##ｘ -some -break -common -##jy -##lines -g7 -twice -419 -ella -nano -belle -にこ -##mes -##self -##note -jb -##ことかてきます -benz -##との -##ova -451 -save -##wing -##ますのて -kai -りは -##hua -##rect -rainer -##unge -448 -##0m -adsl -##かな -guestname -##uma -##kins -##zu -tokichoi -##price -county -##med -##mus -rmk -391 -address -vm -えて -openload -##group -##hin -##iginal -amg -urban -##oz -jobs -emi -##public -beautiful -##sch -album -##dden -##bell -jerry -works -hostel -miller -##drive -##rmin -##１０ -376 -boot -828 -##370 -##fx -##cm～ -1885 -##nome -##ctionary -##oman -##lish -##cr -##hm -433 -##how -432 -francis -xi -c919 -b5 -evernote -##uc -vga -##3000 -coupe -##urg -##cca -##uality -019 -6g -れる -multi -##また -##ett -em -hey -##ani -##tax -##rma -inside -than -740 -leonnhurt -##jin -ict -れた -bird -notes -200mm -くの -##dical -##lli -result -442 -iu -ee -438 -smap -gopro -##last -yin -pure -998 -32g -けた -5kg -##dan -##rame -mama -##oot -bean -marketing -##hur -2l -bella -sync -xuite -##ground -515 -discuz -##getrelax -##ince -##bay -##5s -cj -##イス -gmat -apt -##pass -jing -##rix -c4 -rich -##とても -niusnews -##ello -bag -770 -##eting -##mobile -１８ -culture -015 -##のてすか -377 -1020 -area -##ience -616 -details -gp -universal -silver -dit -はお -private -ddd -u11 -kanshu -##ified -fung -##nny -dx -##520 -tai -475 -023 -##fr -##lean -3s -##pin -429 -##rin -25000 -ly -rick -##bility -usb3 -banner -##baru -##gion -metal -dt -vdf -1871 -karl -qualcomm -bear -1010 -oldid -ian -jo -##tors -population -##ernel -1882 -mmorpg -##mv -##bike -603 -##© -ww -friend -##ager -exhibition -##del -##pods -fpx -structure -##free -##tings -kl -##rley -##copyright -##mma -california -3400 -orange -yoga -4l -canmake -honey -##anda -##コメント -595 -nikkie -##ルハイト -dhl -publishing -##mall -##gnet -20cm -513 -##クセス -##┅ -e88 -970 -##dog -fishbase -##! -##" -### -##$ -##% -##& -##' -##( -##) -##* -##+ -##, -##- -##. -##/ -##: -##; -##< -##= -##> -##? -##@ -##[ -##\ -##] -##^ -##_ -##{ -##| -##} -##~ -##£ -##¤ -##¥ -##§ -##« -##± -##³ -##µ -##· -##¹ -##º -##» -##¼ -##ß -##æ -##÷ -##ø -##đ -##ŋ -##ɔ -##ə -##ɡ -##ʰ -##ˇ -##ˈ -##ˊ -##ˋ -##ˍ -##ː -##˙ -##˚ -##ˢ -##α -##β -##γ -##δ -##ε -##η -##θ -##ι -##κ -##λ -##μ -##ν -##ο -##π -##ρ -##ς -##σ -##τ -##υ -##φ -##χ -##ψ -##б -##в -##г -##д -##е -##ж -##з -##к -##л -##м -##н -##о -##п -##р -##с -##т -##у -##ф -##х -##ц -##ч -##ш -##ы -##ь -##і -##ا -##ب -##ة -##ت -##د -##ر -##س -##ع -##ل -##م -##ن -##ه -##و -##ي -##۩ -##ก -##ง -##น -##ม -##ย -##ร -##อ -##า -##เ -##๑ -##་ -##ღ -##ᄀ -##ᄁ -##ᄂ -##ᄃ -##ᄅ -##ᄆ -##ᄇ -##ᄈ -##ᄉ -##ᄋ -##ᄌ -##ᄎ -##ᄏ -##ᄐ -##ᄑ -##ᄒ -##ᅢ -##ᅣ -##ᅥ -##ᅦ -##ᅧ -##ᅨ -##ᅪ -##ᅬ -##ᅭ -##ᅮ -##ᅯ -##ᅲ -##ᅳ -##ᅴ -##ᆷ -##ᆸ -##ᆺ -##ᆻ -##ᗜ -##ᵃ -##ᵉ -##ᵍ -##ᵏ -##ᵐ -##ᵒ -##ᵘ -##‖ -##„ -##† -##• -##‥ -##‧ -##
 -##‰ -##′ -##″ -##‹ -##› -##※ -##‿ -##⁄ -##ⁱ -##⁺ -##ⁿ -##₁ -##₃ -##₄ -##€ -##№ -##ⅰ -##ⅱ -##ⅲ -##ⅳ -##ⅴ -##↔ -##↗ -##↘ -##⇒ -##∀ -##− -##∕ -##∙ -##√ -##∞ -##∟ -##∠ -##∣ -##∩ -##∮ -##∶ -##∼ -##∽ -##≈ -##≒ -##≡ -##≤ -##≥ -##≦ -##≧ -##≪ -##≫ -##⊙ -##⋅ -##⋈ -##⋯ -##⌒ -##① -##② -##③ -##④ -##⑤ -##⑥ -##⑦ -##⑧ -##⑨ -##⑩ -##⑴ -##⑵ -##⑶ -##⑷ -##⑸ -##⒈ -##⒉ -##⒊ -##⒋ -##ⓒ -##ⓔ -##ⓘ -##━ -##┃ -##┆ -##┊ -##┌ -##└ -##├ -##┣ -##═ -##║ -##╚ -##╞ -##╠ -##╭ -##╮ -##╯ -##╰ -##╱ -##╳ -##▂ -##▃ -##▅ -##▇ -##▉ -##▋ -##▌ -##▍ -##▎ -##□ -##▪ -##▫ -##▬ -##△ -##▶ -##► -##▽ -##◇ -##◕ -##◠ -##◢ -##◤ -##☀ -##☕ -##☞ -##☺ -##☼ -##♀ -##♂ -##♠ -##♡ -##♣ -##♦ -##♫ -##♬ -##✈ -##✔ -##✕ -##✖ -##✦ -##✨ -##✪ -##✰ -##✿ -##❀ -##➜ -##➤ -##⦿ -##、 -##。 -##〃 -##々 -##〇 -##〈 -##〉 -##《 -##》 -##「 -##」 -##『 -##』 -##【 -##】 -##〓 -##〔 -##〕 -##〖 -##〗 -##〜 -##〝 -##〞 -##ぃ -##ぇ -##ぬ -##ふ -##ほ -##む -##ゃ -##ゅ -##ゆ -##ょ -##゜ -##ゝ -##ァ -##ゥ -##エ -##ォ -##ケ -##サ -##セ -##ソ -##ッ -##ニ -##ヌ -##ネ -##ノ -##ヘ -##モ -##ャ -##ヤ -##ュ -##ユ -##ョ -##ヨ -##ワ -##ヲ -##・ -##ヽ -##ㄅ -##ㄆ -##ㄇ -##ㄉ -##ㄋ -##ㄌ -##ㄍ -##ㄎ -##ㄏ -##ㄒ -##ㄚ -##ㄛ -##ㄞ -##ㄟ -##ㄢ -##ㄤ -##ㄥ -##ㄧ -##ㄨ -##ㆍ -##㈦ -##㊣ -##㗎 -##一 -##丁 -##七 -##万 -##丈 -##三 -##上 -##下 -##不 -##与 -##丐 -##丑 -##专 -##且 -##丕 -##世 -##丘 -##丙 -##业 -##丛 -##东 -##丝 -##丞 -##丟 -##両 -##丢 -##两 -##严 -##並 -##丧 -##丨 -##个 -##丫 -##中 -##丰 -##串 -##临 -##丶 -##丸 -##丹 -##为 -##主 -##丼 -##丽 -##举 -##丿 -##乂 -##乃 -##久 -##么 -##义 -##之 -##乌 -##乍 -##乎 -##乏 -##乐 -##乒 -##乓 -##乔 -##乖 -##乗 -##乘 -##乙 -##乜 -##九 -##乞 -##也 -##习 -##乡 -##书 -##乩 -##买 -##乱 -##乳 -##乾 -##亀 -##亂 -##了 -##予 -##争 -##事 -##二 -##于 -##亏 -##云 -##互 -##五 -##井 -##亘 -##亙 -##亚 -##些 -##亜 -##亞 -##亟 -##亡 -##亢 -##交 -##亥 -##亦 -##产 -##亨 -##亩 -##享 -##京 -##亭 -##亮 -##亲 -##亳 -##亵 -##人 -##亿 -##什 -##仁 -##仃 -##仄 -##仅 -##仆 -##仇 -##今 -##介 -##仍 -##从 -##仏 -##仑 -##仓 -##仔 -##仕 -##他 -##仗 -##付 -##仙 -##仝 -##仞 -##仟 -##代 -##令 -##以 -##仨 -##仪 -##们 -##仮 -##仰 -##仲 -##件 -##价 -##任 -##份 -##仿 -##企 -##伉 -##伊 -##伍 -##伎 -##伏 -##伐 -##休 -##伕 -##众 -##优 -##伙 -##会 -##伝 -##伞 -##伟 -##传 -##伢 -##伤 -##伦 -##伪 -##伫 -##伯 -##估 -##伴 -##伶 -##伸 -##伺 -##似 -##伽 -##佃 -##但 -##佇 -##佈 -##位 -##低 -##住 -##佐 -##佑 -##体 -##佔 -##何 -##佗 -##佘 -##余 -##佚 -##佛 -##作 -##佝 -##佞 -##佟 -##你 -##佢 -##佣 -##佤 -##佥 -##佩 -##佬 -##佯 -##佰 -##佳 -##併 -##佶 -##佻 -##佼 -##使 -##侃 -##侄 -##來 -##侈 -##例 -##侍 -##侏 -##侑 -##侖 -##侗 -##供 -##依 -##侠 -##価 -##侣 -##侥 -##侦 -##侧 -##侨 -##侬 -##侮 -##侯 -##侵 -##侶 -##侷 -##便 -##係 -##促 -##俄 -##俊 -##俎 -##俏 -##俐 -##俑 -##俗 -##俘 -##俚 -##保 -##俞 -##俟 -##俠 -##信 -##俨 -##俩 -##俪 -##俬 -##俭 -##修 -##俯 -##俱 -##俳 -##俸 -##俺 -##俾 -##倆 -##倉 -##個 -##倌 -##倍 -##倏 -##們 -##倒 -##倔 -##倖 -##倘 -##候 -##倚 -##倜 -##借 -##倡 -##値 -##倦 -##倩 -##倪 -##倫 -##倬 -##倭 -##倶 -##债 -##值 -##倾 -##偃 -##假 -##偈 -##偉 -##偌 -##偎 -##偏 -##偕 -##做 -##停 -##健 -##側 -##偵 -##偶 -##偷 -##偻 -##偽 -##偿 -##傀 -##傅 -##傍 -##傑 -##傘 -##備 -##傚 -##傢 -##傣 -##傥 -##储 -##傩 -##催 -##傭 -##傲 -##傳 -##債 -##傷 -##傻 -##傾 -##僅 -##働 -##像 -##僑 -##僕 -##僖 -##僚 -##僥 -##僧 -##僭 -##僮 -##僱 -##僵 -##價 -##僻 -##儀 -##儂 -##億 -##儆 -##儉 -##儋 -##儒 -##儕 -##儘 -##償 -##儡 -##優 -##儲 -##儷 -##儼 -##儿 -##兀 -##允 -##元 -##兄 -##充 -##兆 -##兇 -##先 -##光 -##克 -##兌 -##免 -##児 -##兑 -##兒 -##兔 -##兖 -##党 -##兜 -##兢 -##入 -##內 -##全 -##兩 -##八 -##公 -##六 -##兮 -##兰 -##共 -##兲 -##关 -##兴 -##兵 -##其 -##具 -##典 -##兹 -##养 -##兼 -##兽 -##冀 -##内 -##円 -##冇 -##冈 -##冉 -##冊 -##册 -##再 -##冏 -##冒 -##冕 -##冗 -##写 -##军 -##农 -##冠 -##冢 -##冤 -##冥 -##冨 -##冪 -##冬 -##冯 -##冰 -##冲 -##决 -##况 -##冶 -##冷 -##冻 -##冼 -##冽 -##冾 -##净 -##凄 -##准 -##凇 -##凈 -##凉 -##凋 -##凌 -##凍 -##减 -##凑 -##凛 -##凜 -##凝 -##几 -##凡 -##凤 -##処 -##凪 -##凭 -##凯 -##凰 -##凱 -##凳 -##凶 -##凸 -##凹 -##出 -##击 -##函 -##凿 -##刀 -##刁 -##刃 -##分 -##切 -##刈 -##刊 -##刍 -##刎 -##刑 -##划 -##列 -##刘 -##则 -##刚 -##创 -##初 -##删 -##判 -##別 -##刨 -##利 -##刪 -##别 -##刮 -##到 -##制 -##刷 -##券 -##刹 -##刺 -##刻 -##刽 -##剁 -##剂 -##剃 -##則 -##剉 -##削 -##剋 -##剌 -##前 -##剎 -##剐 -##剑 -##剔 -##剖 -##剛 -##剜 -##剝 -##剣 -##剤 -##剥 -##剧 -##剩 -##剪 -##副 -##割 -##創 -##剷 -##剽 -##剿 -##劃 -##劇 -##劈 -##劉 -##劊 -##劍 -##劏 -##劑 -##力 -##劝 -##办 -##功 -##加 -##务 -##劣 -##动 -##助 -##努 -##劫 -##劭 -##励 -##劲 -##劳 -##労 -##劵 -##効 -##劾 -##势 -##勁 -##勃 -##勇 -##勉 -##勋 -##勐 -##勒 -##動 -##勖 -##勘 -##務 -##勛 -##勝 -##勞 -##募 -##勢 -##勤 -##勧 -##勳 -##勵 -##勸 -##勺 -##勻 -##勾 -##勿 -##匀 -##包 -##匆 -##匈 -##匍 -##匐 -##匕 -##化 -##北 -##匙 -##匝 -##匠 -##匡 -##匣 -##匪 -##匮 -##匯 -##匱 -##匹 -##区 -##医 -##匾 -##匿 -##區 -##十 -##千 -##卅 -##升 -##午 -##卉 -##半 -##卍 -##华 -##协 -##卑 -##卒 -##卓 -##協 -##单 -##卖 -##南 -##単 -##博 -##卜 -##卞 -##卟 -##占 -##卡 -##卢 -##卤 -##卦 -##卧 -##卫 -##卮 -##卯 -##印 -##危 -##即 -##却 -##卵 -##卷 -##卸 -##卻 -##卿 -##厂 -##厄 -##厅 -##历 -##厉 -##压 -##厌 -##厕 -##厘 -##厚 -##厝 -##原 -##厢 -##厥 -##厦 -##厨 -##厩 -##厭 -##厮 -##厲 -##厳 -##去 -##县 -##叁 -##参 -##參 -##又 -##叉 -##及 -##友 -##双 -##反 -##収 -##发 -##叔 -##取 -##受 -##变 -##叙 -##叛 -##叟 -##叠 -##叡 -##叢 -##口 -##古 -##句 -##另 -##叨 -##叩 -##只 -##叫 -##召 -##叭 -##叮 -##可 -##台 -##叱 -##史 -##右 -##叵 -##叶 -##号 -##司 -##叹 -##叻 -##叼 -##叽 -##吁 -##吃 -##各 -##吆 -##合 -##吉 -##吊 -##吋 -##同 -##名 -##后 -##吏 -##吐 -##向 -##吒 -##吓 -##吕 -##吖 -##吗 -##君 -##吝 -##吞 -##吟 -##吠 -##吡 -##否 -##吧 -##吨 -##吩 -##含 -##听 -##吭 -##吮 -##启 -##吱 -##吳 -##吴 -##吵 -##吶 -##吸 -##吹 -##吻 -##吼 -##吽 -##吾 -##呀 -##呂 -##呃 -##呆 -##呈 -##告 -##呋 -##呎 -##呐 -##呓 -##呕 -##呗 -##员 -##呛 -##呜 -##呢 -##呤 -##呦 -##周 -##呱 -##呲 -##味 -##呵 -##呷 -##呸 -##呻 -##呼 -##命 -##咀 -##咁 -##咂 -##咄 -##咆 -##咋 -##和 -##咎 -##咏 -##咐 -##咒 -##咔 -##咕 -##咖 -##咗 -##咘 -##咙 -##咚 -##咛 -##咣 -##咤 -##咦 -##咧 -##咨 -##咩 -##咪 -##咫 -##咬 -##咭 -##咯 -##咱 -##咲 -##咳 -##咸 -##咻 -##咽 -##咿 -##哀 -##品 -##哂 -##哄 -##哆 -##哇 -##哈 -##哉 -##哋 -##哌 -##响 -##哎 -##哏 -##哐 -##哑 -##哒 -##哔 -##哗 -##哟 -##員 -##哥 -##哦 -##哧 -##哨 -##哩 -##哪 -##哭 -##哮 -##哲 -##哺 -##哼 -##哽 -##唁 -##唄 -##唆 -##唇 -##唉 -##唏 -##唐 -##唑 -##唔 -##唠 -##唤 -##唧 -##唬 -##售 -##唯 -##唰 -##唱 -##唳 -##唷 -##唸 -##唾 -##啃 -##啄 -##商 -##啉 -##啊 -##問 -##啓 -##啕 -##啖 -##啜 -##啞 -##啟 -##啡 -##啤 -##啥 -##啦 -##啧 -##啪 -##啫 -##啬 -##啮 -##啰 -##啱 -##啲 -##啵 -##啶 -##啷 -##啸 -##啻 -##啼 -##啾 -##喀 -##喂 -##喃 -##善 -##喆 -##喇 -##喉 -##喊 -##喋 -##喎 -##喏 -##喔 -##喘 -##喙 -##喚 -##喜 -##喝 -##喟 -##喧 -##喪 -##喫 -##喬 -##單 -##喰 -##喱 -##喲 -##喳 -##喵 -##営 -##喷 -##喹 -##喺 -##喻 -##喽 -##嗅 -##嗆 -##嗇 -##嗎 -##嗑 -##嗒 -##嗓 -##嗔 -##嗖 -##嗚 -##嗜 -##嗝 -##嗟 -##嗡 -##嗣 -##嗤 -##嗦 -##嗨 -##嗪 -##嗬 -##嗯 -##嗰 -##嗲 -##嗳 -##嗶 -##嗷 -##嗽 -##嘀 -##嘅 -##嘆 -##嘈 -##嘉 -##嘌 -##嘍 -##嘎 -##嘔 -##嘖 -##嘗 -##嘘 -##嘚 -##嘛 -##嘜 -##嘞 -##嘟 -##嘢 -##嘣 -##嘤 -##嘧 -##嘩 -##嘭 -##嘮 -##嘯 -##嘰 -##嘱 -##嘲 -##嘴 -##嘶 -##嘸 -##嘹 -##嘻 -##嘿 -##噁 -##噌 -##噎 -##噓 -##噔 -##噗 -##噙 -##噜 -##噠 -##噢 -##噤 -##器 -##噩 -##噪 -##噬 -##噱 -##噴 -##噶 -##噸 -##噹 -##噻 -##噼 -##嚀 -##嚇 -##嚎 -##嚏 -##嚐 -##嚓 -##嚕 -##嚟 -##嚣 -##嚥 -##嚨 -##嚮 -##嚴 -##嚷 -##嚼 -##囂 -##囉 -##囊 -##囍 -##囑 -##囔 -##囗 -##囚 -##四 -##囝 -##回 -##囟 -##因 -##囡 -##团 -##団 -##囤 -##囧 -##囪 -##囫 -##园 -##困 -##囱 -##囲 -##図 -##围 -##囹 -##固 -##国 -##图 -##囿 -##圃 -##圄 -##圆 -##圈 -##國 -##圍 -##圏 -##園 -##圓 -##圖 -##團 -##圜 -##土 -##圣 -##圧 -##在 -##圩 -##圭 -##地 -##圳 -##场 -##圻 -##圾 -##址 -##坂 -##均 -##坊 -##坍 -##坎 -##坏 -##坐 -##坑 -##块 -##坚 -##坛 -##坝 -##坞 -##坟 -##坠 -##坡 -##坤 -##坦 -##坨 -##坪 -##坯 -##坳 -##坵 -##坷 -##垂 -##垃 -##垄 -##型 -##垒 -##垚 -##垛 -##垠 -##垢 -##垣 -##垦 -##垩 -##垫 -##垭 -##垮 -##垵 -##埂 -##埃 -##埋 -##城 -##埔 -##埕 -##埗 -##域 -##埠 -##埤 -##埵 -##執 -##埸 -##培 -##基 -##埼 -##堀 -##堂 -##堃 -##堅 -##堆 -##堇 -##堑 -##堕 -##堙 -##堡 -##堤 -##堪 -##堯 -##堰 -##報 -##場 -##堵 -##堺 -##堿 -##塊 -##塌 -##塑 -##塔 -##塗 -##塘 -##塚 -##塞 -##塢 -##塩 -##填 -##塬 -##塭 -##塵 -##塾 -##墀 -##境 -##墅 -##墉 -##墊 -##墒 -##墓 -##増 -##墘 -##墙 -##墜 -##增 -##墟 -##墨 -##墩 -##墮 -##墳 -##墻 -##墾 -##壁 -##壅 -##壆 -##壇 -##壊 -##壑 -##壓 -##壕 -##壘 -##壞 -##壟 -##壢 -##壤 -##壩 -##士 -##壬 -##壮 -##壯 -##声 -##売 -##壳 -##壶 -##壹 -##壺 -##壽 -##处 -##备 -##変 -##复 -##夏 -##夔 -##夕 -##外 -##夙 -##多 -##夜 -##够 -##夠 -##夢 -##夥 -##大 -##天 -##太 -##夫 -##夭 -##央 -##夯 -##失 -##头 -##夷 -##夸 -##夹 -##夺 -##夾 -##奂 -##奄 -##奇 -##奈 -##奉 -##奋 -##奎 -##奏 -##奐 -##契 -##奔 -##奕 -##奖 -##套 -##奘 -##奚 -##奠 -##奢 -##奥 -##奧 -##奪 -##奬 -##奮 -##女 -##奴 -##奶 -##奸 -##她 -##好 -##如 -##妃 -##妄 -##妆 -##妇 -##妈 -##妊 -##妍 -##妒 -##妓 -##妖 -##妘 -##妙 -##妝 -##妞 -##妣 -##妤 -##妥 -##妨 -##妩 -##妪 -##妮 -##妲 -##妳 -##妹 -##妻 -##妾 -##姆 -##姉 -##姊 -##始 -##姍 -##姐 -##姑 -##姒 -##姓 -##委 -##姗 -##姚 -##姜 -##姝 -##姣 -##姥 -##姦 -##姨 -##姪 -##姫 -##姬 -##姹 -##姻 -##姿 -##威 -##娃 -##娄 -##娅 -##娆 -##娇 -##娉 -##娑 -##娓 -##娘 -##娛 -##娜 -##娟 -##娠 -##娣 -##娥 -##娩 -##娱 -##娲 -##娴 -##娶 -##娼 -##婀 -##婁 -##婆 -##婉 -##婊 -##婕 -##婚 -##婢 -##婦 -##婧 -##婪 -##婭 -##婴 -##婵 -##婶 -##婷 -##婺 -##婿 -##媒 -##媚 -##媛 -##媞 -##媧 -##媲 -##媳 -##媽 -##媾 -##嫁 -##嫂 -##嫉 -##嫌 -##嫑 -##嫔 -##嫖 -##嫘 -##嫚 -##嫡 -##嫣 -##嫦 -##嫩 -##嫲 -##嫵 -##嫻 -##嬅 -##嬉 -##嬌 -##嬗 -##嬛 -##嬢 -##嬤 -##嬪 -##嬰 -##嬴 -##嬷 -##嬸 -##嬿 -##孀 -##孃 -##子 -##孑 -##孔 -##孕 -##孖 -##字 -##存 -##孙 -##孚 -##孛 -##孜 -##孝 -##孟 -##孢 -##季 -##孤 -##学 -##孩 -##孪 -##孫 -##孬 -##孰 -##孱 -##孳 -##孵 -##學 -##孺 -##孽 -##孿 -##宁 -##它 -##宅 -##宇 -##守 -##安 -##宋 -##完 -##宏 -##宓 -##宕 -##宗 -##官 -##宙 -##定 -##宛 -##宜 -##宝 -##实 -##実 -##宠 -##审 -##客 -##宣 -##室 -##宥 -##宦 -##宪 -##宫 -##宮 -##宰 -##害 -##宴 -##宵 -##家 -##宸 -##容 -##宽 -##宾 -##宿 -##寂 -##寄 -##寅 -##密 -##寇 -##富 -##寐 -##寒 -##寓 -##寛 -##寝 -##寞 -##察 -##寡 -##寢 -##寥 -##實 -##寧 -##寨 -##審 -##寫 -##寬 -##寮 -##寰 -##寵 -##寶 -##寸 -##对 -##寺 -##寻 -##导 -##対 -##寿 -##封 -##専 -##射 -##将 -##將 -##專 -##尉 -##尊 -##尋 -##對 -##導 -##小 -##少 -##尔 -##尕 -##尖 -##尘 -##尚 -##尝 -##尤 -##尧 -##尬 -##就 -##尴 -##尷 -##尸 -##尹 -##尺 -##尻 -##尼 -##尽 -##尾 -##尿 -##局 -##屁 -##层 -##屄 -##居 -##屆 -##屈 -##屉 -##届 -##屋 -##屌 -##屍 -##屎 -##屏 -##屐 -##屑 -##展 -##屜 -##属 -##屠 -##屡 -##屢 -##層 -##履 -##屬 -##屯 -##山 -##屹 -##屿 -##岀 -##岁 -##岂 -##岌 -##岐 -##岑 -##岔 -##岖 -##岗 -##岘 -##岙 -##岚 -##岛 -##岡 -##岩 -##岫 -##岬 -##岭 -##岱 -##岳 -##岷 -##岸 -##峇 -##峋 -##峒 -##峙 -##峡 -##峤 -##峥 -##峦 -##峨 -##峪 -##峭 -##峯 -##峰 -##峴 -##島 -##峻 -##峽 -##崁 -##崂 -##崆 -##崇 -##崎 -##崑 -##崔 -##崖 -##崗 -##崙 -##崛 -##崧 -##崩 -##崭 -##崴 -##崽 -##嵇 -##嵊 -##嵋 -##嵌 -##嵐 -##嵘 -##嵩 -##嵬 -##嵯 -##嶂 -##嶄 -##嶇 -##嶋 -##嶙 -##嶺 -##嶼 -##嶽 -##巅 -##巍 -##巒 -##巔 -##巖 -##川 -##州 -##巡 -##巢 -##工 -##左 -##巧 -##巨 -##巩 -##巫 -##差 -##己 -##已 -##巳 -##巴 -##巷 -##巻 -##巽 -##巾 -##巿 -##币 -##市 -##布 -##帅 -##帆 -##师 -##希 -##帐 -##帑 -##帕 -##帖 -##帘 -##帚 -##帛 -##帜 -##帝 -##帥 -##带 -##帧 -##師 -##席 -##帮 -##帯 -##帰 -##帳 -##帶 -##帷 -##常 -##帼 -##帽 -##幀 -##幂 -##幄 -##幅 -##幌 -##幔 -##幕 -##幟 -##幡 -##幢 -##幣 -##幫 -##干 -##平 -##年 -##并 -##幸 -##幹 -##幺 -##幻 -##幼 -##幽 -##幾 -##广 -##庁 -##広 -##庄 -##庆 -##庇 -##床 -##序 -##庐 -##库 -##应 -##底 -##庖 -##店 -##庙 -##庚 -##府 -##庞 -##废 -##庠 -##度 -##座 -##庫 -##庭 -##庵 -##庶 -##康 -##庸 -##庹 -##庾 -##廁 -##廂 -##廃 -##廈 -##廉 -##廊 -##廓 -##廖 -##廚 -##廝 -##廟 -##廠 -##廢 -##廣 -##廬 -##廳 -##延 -##廷 -##建 -##廿 -##开 -##弁 -##异 -##弃 -##弄 -##弈 -##弊 -##弋 -##式 -##弑 -##弒 -##弓 -##弔 -##引 -##弗 -##弘 -##弛 -##弟 -##张 -##弥 -##弦 -##弧 -##弩 -##弭 -##弯 -##弱 -##張 -##強 -##弹 -##强 -##弼 -##弾 -##彅 -##彆 -##彈 -##彌 -##彎 -##归 -##当 -##录 -##彗 -##彙 -##彝 -##形 -##彤 -##彥 -##彦 -##彧 -##彩 -##彪 -##彫 -##彬 -##彭 -##彰 -##影 -##彷 -##役 -##彻 -##彼 -##彿 -##往 -##征 -##径 -##待 -##徇 -##很 -##徉 -##徊 -##律 -##後 -##徐 -##徑 -##徒 -##従 -##徕 -##得 -##徘 -##徙 -##徜 -##從 -##徠 -##御 -##徨 -##復 -##循 -##徬 -##微 -##徳 -##徴 -##徵 -##德 -##徹 -##徼 -##徽 -##心 -##必 -##忆 -##忌 -##忍 -##忏 -##忐 -##忑 -##忒 -##忖 -##志 -##忘 -##忙 -##応 -##忠 -##忡 -##忤 -##忧 -##忪 -##快 -##忱 -##念 -##忻 -##忽 -##忿 -##怀 -##态 -##怂 -##怅 -##怆 -##怎 -##怏 -##怒 -##怔 -##怕 -##怖 -##怙 -##怜 -##思 -##怠 -##怡 -##急 -##怦 -##性 -##怨 -##怪 -##怯 -##怵 -##总 -##怼 -##恁 -##恃 -##恆 -##恋 -##恍 -##恐 -##恒 -##恕 -##恙 -##恚 -##恢 -##恣 -##恤 -##恥 -##恨 -##恩 -##恪 -##恫 -##恬 -##恭 -##息 -##恰 -##恳 -##恵 -##恶 -##恸 -##恺 -##恻 -##恼 -##恿 -##悄 -##悅 -##悉 -##悌 -##悍 -##悔 -##悖 -##悚 -##悟 -##悠 -##患 -##悦 -##您 -##悩 -##悪 -##悬 -##悯 -##悱 -##悲 -##悴 -##悵 -##悶 -##悸 -##悻 -##悼 -##悽 -##情 -##惆 -##惇 -##惊 -##惋 -##惑 -##惕 -##惘 -##惚 -##惜 -##惟 -##惠 -##惡 -##惦 -##惧 -##惨 -##惩 -##惫 -##惬 -##惭 -##惮 -##惯 -##惰 -##惱 -##想 -##惴 -##惶 -##惹 -##惺 -##愁 -##愆 -##愈 -##愉 -##愍 -##意 -##愕 -##愚 -##愛 -##愜 -##感 -##愣 -##愤 -##愧 -##愫 -##愷 -##愿 -##慄 -##慈 -##態 -##慌 -##慎 -##慑 -##慕 -##慘 -##慚 -##慟 -##慢 -##慣 -##慧 -##慨 -##慫 -##慮 -##慰 -##慳 -##慵 -##慶 -##慷 -##慾 -##憂 -##憊 -##憋 -##憎 -##憐 -##憑 -##憔 -##憚 -##憤 -##憧 -##憨 -##憩 -##憫 -##憬 -##憲 -##憶 -##憾 -##懂 -##懇 -##懈 -##應 -##懊 -##懋 -##懑 -##懒 -##懦 -##懲 -##懵 -##懶 -##懷 -##懸 -##懺 -##懼 -##懾 -##懿 -##戀 -##戈 -##戊 -##戌 -##戍 -##戎 -##戏 -##成 -##我 -##戒 -##戕 -##或 -##战 -##戚 -##戛 -##戟 -##戡 -##戦 -##截 -##戬 -##戮 -##戰 -##戲 -##戳 -##戴 -##戶 -##户 -##戸 -##戻 -##戾 -##房 -##所 -##扁 -##扇 -##扈 -##扉 -##手 -##才 -##扎 -##扑 -##扒 -##打 -##扔 -##払 -##托 -##扛 -##扣 -##扦 -##执 -##扩 -##扪 -##扫 -##扬 -##扭 -##扮 -##扯 -##扰 -##扱 -##扳 -##扶 -##批 -##扼 -##找 -##承 -##技 -##抄 -##抉 -##把 -##抑 -##抒 -##抓 -##投 -##抖 -##抗 -##折 -##抚 -##抛 -##抜 -##択 -##抟 -##抠 -##抡 -##抢 -##护 -##报 -##抨 -##披 -##抬 -##抱 -##抵 -##抹 -##押 -##抽 -##抿 -##拂 -##拄 -##担 -##拆 -##拇 -##拈 -##拉 -##拋 -##拌 -##拍 -##拎 -##拐 -##拒 -##拓 -##拔 -##拖 -##拗 -##拘 -##拙 -##拚 -##招 -##拜 -##拟 -##拡 -##拢 -##拣 -##拥 -##拦 -##拧 -##拨 -##择 -##括 -##拭 -##拮 -##拯 -##拱 -##拳 -##拴 -##拷 -##拼 -##拽 -##拾 -##拿 -##持 -##挂 -##指 -##挈 -##按 -##挎 -##挑 -##挖 -##挙 -##挚 -##挛 -##挝 -##挞 -##挟 -##挠 -##挡 -##挣 -##挤 -##挥 -##挨 -##挪 -##挫 -##振 -##挲 -##挹 -##挺 -##挽 -##挾 -##捂 -##捅 -##捆 -##捉 -##捋 -##捌 -##捍 -##捎 -##捏 -##捐 -##捕 -##捞 -##损 -##捡 -##换 -##捣 -##捧 -##捨 -##捩 -##据 -##捱 -##捲 -##捶 -##捷 -##捺 -##捻 -##掀 -##掂 -##掃 -##掇 -##授 -##掉 -##掌 -##掏 -##掐 -##排 -##掖 -##掘 -##掙 -##掛 -##掠 -##採 -##探 -##掣 -##接 -##控 -##推 -##掩 -##措 -##掬 -##掰 -##掲 -##掳 -##掴 -##掷 -##掸 -##掺 -##揀 -##揃 -##揄 -##揆 -##揉 -##揍 -##描 -##提 -##插 -##揖 -##揚 -##換 -##握 -##揣 -##揩 -##揪 -##揭 -##揮 -##援 -##揶 -##揸 -##揹 -##揽 -##搀 -##搁 -##搂 -##搅 -##損 -##搏 -##搐 -##搓 -##搔 -##搖 -##搗 -##搜 -##搞 -##搡 -##搪 -##搬 -##搭 -##搵 -##搶 -##携 -##搽 -##摀 -##摁 -##摄 -##摆 -##摇 -##摈 -##摊 -##摒 -##摔 -##摘 -##摞 -##摟 -##摧 -##摩 -##摯 -##摳 -##摸 -##摹 -##摺 -##摻 -##撂 -##撃 -##撅 -##撇 -##撈 -##撐 -##撑 -##撒 -##撓 -##撕 -##撚 -##撞 -##撤 -##撥 -##撩 -##撫 -##撬 -##播 -##撮 -##撰 -##撲 -##撵 -##撷 -##撸 -##撻 -##撼 -##撿 -##擀 -##擁 -##擂 -##擄 -##擅 -##擇 -##擊 -##擋 -##操 -##擎 -##擒 -##擔 -##擘 -##據 -##擞 -##擠 -##擡 -##擢 -##擦 -##擬 -##擰 -##擱 -##擲 -##擴 -##擷 -##擺 -##擼 -##擾 -##攀 -##攏 -##攒 -##攔 -##攘 -##攙 -##攜 -##攝 -##攞 -##攢 -##攣 -##攤 -##攥 -##攪 -##攫 -##攬 -##支 -##收 -##攸 -##改 -##攻 -##放 -##政 -##故 -##效 -##敌 -##敍 -##敎 -##敏 -##救 -##敕 -##敖 -##敗 -##敘 -##教 -##敛 -##敝 -##敞 -##敢 -##散 -##敦 -##敬 -##数 -##敲 -##整 -##敵 -##敷 -##數 -##斂 -##斃 -##文 -##斋 -##斌 -##斎 -##斐 -##斑 -##斓 -##斗 -##料 -##斛 -##斜 -##斟 -##斡 -##斤 -##斥 -##斧 -##斩 -##斫 -##斬 -##断 -##斯 -##新 -##斷 -##方 -##於 -##施 -##旁 -##旃 -##旅 -##旋 -##旌 -##旎 -##族 -##旖 -##旗 -##无 -##既 -##日 -##旦 -##旧 -##旨 -##早 -##旬 -##旭 -##旮 -##旱 -##时 -##旷 -##旺 -##旻 -##昀 -##昂 -##昆 -##昇 -##昉 -##昊 -##昌 -##明 -##昏 -##易 -##昔 -##昕 -##昙 -##星 -##映 -##春 -##昧 -##昨 -##昭 -##是 -##昱 -##昴 -##昵 -##昶 -##昼 -##显 -##晁 -##時 -##晃 -##晉 -##晋 -##晌 -##晏 -##晒 -##晓 -##晔 -##晕 -##晖 -##晗 -##晚 -##晝 -##晞 -##晟 -##晤 -##晦 -##晨 -##晩 -##普 -##景 -##晰 -##晴 -##晶 -##晷 -##智 -##晾 -##暂 -##暄 -##暇 -##暈 -##暉 -##暌 -##暐 -##暑 -##暖 -##暗 -##暝 -##暢 -##暧 -##暨 -##暫 -##暮 -##暱 -##暴 -##暸 -##暹 -##曄 -##曆 -##曇 -##曉 -##曖 -##曙 -##曜 -##曝 -##曠 -##曦 -##曬 -##曰 -##曲 -##曳 -##更 -##書 -##曹 -##曼 -##曾 -##替 -##最 -##會 -##月 -##有 -##朋 -##服 -##朐 -##朔 -##朕 -##朗 -##望 -##朝 -##期 -##朦 -##朧 -##木 -##未 -##末 -##本 -##札 -##朮 -##术 -##朱 -##朴 -##朵 -##机 -##朽 -##杀 -##杂 -##权 -##杆 -##杈 -##杉 -##李 -##杏 -##材 -##村 -##杓 -##杖 -##杜 -##杞 -##束 -##杠 -##条 -##来 -##杨 -##杭 -##杯 -##杰 -##東 -##杳 -##杵 -##杷 -##杼 -##松 -##板 -##极 -##构 -##枇 -##枉 -##枋 -##析 -##枕 -##林 -##枚 -##果 -##枝 -##枢 -##枣 -##枪 -##枫 -##枭 -##枯 -##枰 -##枱 -##枳 -##架 -##枷 -##枸 -##柄 -##柏 -##某 -##柑 -##柒 -##染 -##柔 -##柘 -##柚 -##柜 -##柞 -##柠 -##柢 -##查 -##柩 -##柬 -##柯 -##柱 -##柳 -##柴 -##柵 -##査 -##柿 -##栀 -##栃 -##栄 -##栅 -##标 -##栈 -##栉 -##栋 -##栎 -##栏 -##树 -##栓 -##栖 -##栗 -##校 -##栩 -##株 -##样 -##核 -##根 -##格 -##栽 -##栾 -##桀 -##桁 -##桂 -##桃 -##桅 -##框 -##案 -##桉 -##桌 -##桎 -##桐 -##桑 -##桓 -##桔 -##桜 -##桠 -##桡 -##桢 -##档 -##桥 -##桦 -##桧 -##桨 -##桩 -##桶 -##桿 -##梁 -##梅 -##梆 -##梏 -##梓 -##梗 -##條 -##梟 -##梢 -##梦 -##梧 -##梨 -##梭 -##梯 -##械 -##梳 -##梵 -##梶 -##检 -##棂 -##棄 -##棉 -##棋 -##棍 -##棒 -##棕 -##棗 -##棘 -##棚 -##棟 -##棠 -##棣 -##棧 -##森 -##棱 -##棲 -##棵 -##棹 -##棺 -##椁 -##椅 -##椋 -##植 -##椎 -##椒 -##検 -##椪 -##椭 -##椰 -##椹 -##椽 -##椿 -##楂 -##楊 -##楓 -##楔 -##楚 -##楝 -##楞 -##楠 -##楣 -##楨 -##楫 -##業 -##楮 -##極 -##楷 -##楸 -##楹 -##楼 -##楽 -##概 -##榄 -##榆 -##榈 -##榉 -##榔 -##榕 -##榖 -##榛 -##榜 -##榨 -##榫 -##榭 -##榮 -##榱 -##榴 -##榷 -##榻 -##槁 -##槃 -##構 -##槌 -##槍 -##槎 -##槐 -##槓 -##様 -##槛 -##槟 -##槤 -##槭 -##槲 -##槳 -##槻 -##槽 -##槿 -##樁 -##樂 -##樊 -##樑 -##樓 -##標 -##樞 -##樟 -##模 -##樣 -##権 -##横 -##樫 -##樯 -##樱 -##樵 -##樸 -##樹 -##樺 -##樽 -##樾 -##橄 -##橇 -##橋 -##橐 -##橘 -##橙 -##機 -##橡 -##橢 -##橫 -##橱 -##橹 -##橼 -##檀 -##檄 -##檎 -##檐 -##檔 -##檗 -##檜 -##檢 -##檬 -##檯 -##檳 -##檸 -##檻 -##櫃 -##櫚 -##櫛 -##櫥 -##櫸 -##櫻 -##欄 -##權 -##欒 -##欖 -##欠 -##次 -##欢 -##欣 -##欧 -##欲 -##欸 -##欺 -##欽 -##款 -##歆 -##歇 -##歉 -##歌 -##歎 -##歐 -##歓 -##歙 -##歛 -##歡 -##止 -##正 -##此 -##步 -##武 -##歧 -##歩 -##歪 -##歯 -##歲 -##歳 -##歴 -##歷 -##歸 -##歹 -##死 -##歼 -##殁 -##殃 -##殆 -##殇 -##殉 -##殊 -##残 -##殒 -##殓 -##殖 -##殘 -##殞 -##殡 -##殤 -##殭 -##殯 -##殲 -##殴 -##段 -##殷 -##殺 -##殼 -##殿 -##毀 -##毁 -##毂 -##毅 -##毆 -##毋 -##母 -##毎 -##每 -##毒 -##毓 -##比 -##毕 -##毗 -##毘 -##毙 -##毛 -##毡 -##毫 -##毯 -##毽 -##氈 -##氏 -##氐 -##民 -##氓 -##气 -##氖 -##気 -##氙 -##氛 -##氟 -##氡 -##氢 -##氣 -##氤 -##氦 -##氧 -##氨 -##氪 -##氫 -##氮 -##氯 -##氰 -##氲 -##水 -##氷 -##永 -##氹 -##氾 -##汀 -##汁 -##求 -##汆 -##汇 -##汉 -##汎 -##汐 -##汕 -##汗 -##汙 -##汛 -##汝 -##汞 -##江 -##池 -##污 -##汤 -##汨 -##汩 -##汪 -##汰 -##汲 -##汴 -##汶 -##汹 -##決 -##汽 -##汾 -##沁 -##沂 -##沃 -##沅 -##沈 -##沉 -##沌 -##沏 -##沐 -##沒 -##沓 -##沖 -##沙 -##沛 -##沟 -##没 -##沢 -##沣 -##沥 -##沦 -##沧 -##沪 -##沫 -##沭 -##沮 -##沱 -##河 -##沸 -##油 -##治 -##沼 -##沽 -##沾 -##沿 -##況 -##泄 -##泉 -##泊 -##泌 -##泓 -##法 -##泗 -##泛 -##泞 -##泠 -##泡 -##波 -##泣 -##泥 -##注 -##泪 -##泫 -##泮 -##泯 -##泰 -##泱 -##泳 -##泵 -##泷 -##泸 -##泻 -##泼 -##泽 -##泾 -##洁 -##洄 -##洋 -##洒 -##洗 -##洙 -##洛 -##洞 -##津 -##洩 -##洪 -##洮 -##洱 -##洲 -##洵 -##洶 -##洸 -##洹 -##活 -##洼 -##洽 -##派 -##流 -##浃 -##浄 -##浅 -##浆 -##浇 -##浊 -##测 -##济 -##浏 -##浑 -##浒 -##浓 -##浔 -##浙 -##浚 -##浜 -##浣 -##浦 -##浩 -##浪 -##浬 -##浮 -##浯 -##浴 -##海 -##浸 -##涂 -##涅 -##涇 -##消 -##涉 -##涌 -##涎 -##涓 -##涔 -##涕 -##涙 -##涛 -##涝 -##涞 -##涟 -##涠 -##涡 -##涣 -##涤 -##润 -##涧 -##涨 -##涩 -##涪 -##涮 -##涯 -##液 -##涵 -##涸 -##涼 -##涿 -##淀 -##淄 -##淅 -##淆 -##淇 -##淋 -##淌 -##淑 -##淒 -##淖 -##淘 -##淙 -##淚 -##淞 -##淡 -##淤 -##淦 -##淨 -##淩 -##淪 -##淫 -##淬 -##淮 -##深 -##淳 -##淵 -##混 -##淹 -##淺 -##添 -##淼 -##清 -##済 -##渉 -##渊 -##渋 -##渍 -##渎 -##渐 -##渔 -##渗 -##渙 -##渚 -##減 -##渝 -##渠 -##渡 -##渣 -##渤 -##渥 -##渦 -##温 -##測 -##渭 -##港 -##渲 -##渴 -##游 -##渺 -##渾 -##湃 -##湄 -##湊 -##湍 -##湖 -##湘 -##湛 -##湟 -##湧 -##湫 -##湮 -##湯 -##湳 -##湾 -##湿 -##満 -##溃 -##溅 -##溉 -##溏 -##源 -##準 -##溜 -##溝 -##溟 -##溢 -##溥 -##溧 -##溪 -##溫 -##溯 -##溱 -##溴 -##溶 -##溺 -##溼 -##滁 -##滂 -##滄 -##滅 -##滇 -##滋 -##滌 -##滑 -##滓 -##滔 -##滕 -##滙 -##滚 -##滝 -##滞 -##滟 -##满 -##滢 -##滤 -##滥 -##滦 -##滨 -##滩 -##滬 -##滯 -##滲 -##滴 -##滷 -##滸 -##滾 -##滿 -##漁 -##漂 -##漆 -##漉 -##漏 -##漓 -##演 -##漕 -##漠 -##漢 -##漣 -##漩 -##漪 -##漫 -##漬 -##漯 -##漱 -##漲 -##漳 -##漸 -##漾 -##漿 -##潆 -##潇 -##潋 -##潍 -##潑 -##潔 -##潘 -##潛 -##潜 -##潞 -##潟 -##潢 -##潤 -##潦 -##潧 -##潭 -##潮 -##潰 -##潴 -##潸 -##潺 -##潼 -##澀 -##澄 -##澆 -##澈 -##澍 -##澎 -##澗 -##澜 -##澡 -##澤 -##澧 -##澱 -##澳 -##澹 -##激 -##濁 -##濂 -##濃 -##濑 -##濒 -##濕 -##濘 -##濛 -##濟 -##濠 -##濡 -##濤 -##濫 -##濬 -##濮 -##濯 -##濱 -##濺 -##濾 -##瀅 -##瀆 -##瀉 -##瀋 -##瀏 -##瀑 -##瀕 -##瀘 -##瀚 -##瀛 -##瀝 -##瀞 -##瀟 -##瀧 -##瀨 -##瀬 -##瀰 -##瀾 -##灌 -##灏 -##灑 -##灘 -##灝 -##灞 -##灣 -##火 -##灬 -##灭 -##灯 -##灰 -##灵 -##灶 -##灸 -##灼 -##災 -##灾 -##灿 -##炀 -##炁 -##炅 -##炉 -##炊 -##炎 -##炒 -##炔 -##炕 -##炖 -##炙 -##炜 -##炫 -##炬 -##炭 -##炮 -##炯 -##炳 -##炷 -##炸 -##点 -##為 -##炼 -##炽 -##烁 -##烂 -##烃 -##烈 -##烊 -##烏 -##烘 -##烙 -##烛 -##烟 -##烤 -##烦 -##烧 -##烨 -##烩 -##烫 -##烬 -##热 -##烯 -##烷 -##烹 -##烽 -##焉 -##焊 -##焕 -##焖 -##焗 -##焘 -##焙 -##焚 -##焜 -##無 -##焦 -##焯 -##焰 -##焱 -##然 -##焼 -##煅 -##煉 -##煊 -##煌 -##煎 -##煒 -##煖 -##煙 -##煜 -##煞 -##煤 -##煥 -##煦 -##照 -##煨 -##煩 -##煮 -##煲 -##煸 -##煽 -##熄 -##熊 -##熏 -##熒 -##熔 -##熙 -##熟 -##熠 -##熨 -##熬 -##熱 -##熵 -##熹 -##熾 -##燁 -##燃 -##燄 -##燈 -##燉 -##燊 -##燎 -##燒 -##燔 -##燕 -##燙 -##燜 -##營 -##燥 -##燦 -##燧 -##燭 -##燮 -##燴 -##燻 -##燼 -##燿 -##爆 -##爍 -##爐 -##爛 -##爪 -##爬 -##爭 -##爰 -##爱 -##爲 -##爵 -##父 -##爷 -##爸 -##爹 -##爺 -##爻 -##爽 -##爾 -##牆 -##片 -##版 -##牌 -##牍 -##牒 -##牙 -##牛 -##牝 -##牟 -##牠 -##牡 -##牢 -##牦 -##牧 -##物 -##牯 -##牲 -##牴 -##牵 -##特 -##牺 -##牽 -##犀 -##犁 -##犄 -##犊 -##犍 -##犒 -##犢 -##犧 -##犬 -##犯 -##状 -##犷 -##犸 -##犹 -##狀 -##狂 -##狄 -##狈 -##狎 -##狐 -##狒 -##狗 -##狙 -##狞 -##狠 -##狡 -##狩 -##独 -##狭 -##狮 -##狰 -##狱 -##狸 -##狹 -##狼 -##狽 -##猎 -##猕 -##猖 -##猗 -##猙 -##猛 -##猜 -##猝 -##猥 -##猩 -##猪 -##猫 -##猬 -##献 -##猴 -##猶 -##猷 -##猾 -##猿 -##獄 -##獅 -##獎 -##獐 -##獒 -##獗 -##獠 -##獣 -##獨 -##獭 -##獰 -##獲 -##獵 -##獷 -##獸 -##獺 -##獻 -##獼 -##獾 -##玄 -##率 -##玉 -##王 -##玑 -##玖 -##玛 -##玟 -##玠 -##玥 -##玩 -##玫 -##玮 -##环 -##现 -##玲 -##玳 -##玷 -##玺 -##玻 -##珀 -##珂 -##珅 -##珈 -##珉 -##珊 -##珍 -##珏 -##珐 -##珑 -##珙 -##珞 -##珠 -##珣 -##珥 -##珩 -##珪 -##班 -##珮 -##珲 -##珺 -##現 -##球 -##琅 -##理 -##琇 -##琉 -##琊 -##琍 -##琏 -##琐 -##琛 -##琢 -##琥 -##琦 -##琨 -##琪 -##琬 -##琮 -##琰 -##琲 -##琳 -##琴 -##琵 -##琶 -##琺 -##琼 -##瑀 -##瑁 -##瑄 -##瑋 -##瑕 -##瑗 -##瑙 -##瑚 -##瑛 -##瑜 -##瑞 -##瑟 -##瑠 -##瑣 -##瑤 -##瑩 -##瑪 -##瑯 -##瑰 -##瑶 -##瑾 -##璀 -##璁 -##璃 -##璇 -##璉 -##璋 -##璎 -##璐 -##璜 -##璞 -##璟 -##璧 -##璨 -##環 -##璽 -##璿 -##瓊 -##瓏 -##瓒 -##瓜 -##瓢 -##瓣 -##瓤 -##瓦 -##瓮 -##瓯 -##瓴 -##瓶 -##瓷 -##甄 -##甌 -##甕 -##甘 -##甙 -##甚 -##甜 -##生 -##產 -##産 -##甥 -##甦 -##用 -##甩 -##甫 -##甬 -##甭 -##甯 -##田 -##由 -##甲 -##申 -##电 -##男 -##甸 -##町 -##画 -##甾 -##畀 -##畅 -##界 -##畏 -##畑 -##畔 -##留 -##畜 -##畝 -##畢 -##略 -##畦 -##番 -##畫 -##異 -##畲 -##畳 -##畴 -##當 -##畸 -##畹 -##畿 -##疆 -##疇 -##疊 -##疏 -##疑 -##疔 -##疖 -##疗 -##疙 -##疚 -##疝 -##疟 -##疡 -##疣 -##疤 -##疥 -##疫 -##疮 -##疯 -##疱 -##疲 -##疳 -##疵 -##疸 -##疹 -##疼 -##疽 -##疾 -##痂 -##病 -##症 -##痈 -##痉 -##痊 -##痍 -##痒 -##痔 -##痕 -##痘 -##痙 -##痛 -##痞 -##痠 -##痢 -##痣 -##痤 -##痧 -##痨 -##痪 -##痫 -##痰 -##痱 -##痴 -##痹 -##痺 -##痼 -##痿 -##瘀 -##瘁 -##瘋 -##瘍 -##瘓 -##瘘 -##瘙 -##瘟 -##瘠 -##瘡 -##瘢 -##瘤 -##瘦 -##瘧 -##瘩 -##瘪 -##瘫 -##瘴 -##瘸 -##瘾 -##療 -##癇 -##癌 -##癒 -##癖 -##癜 -##癞 -##癡 -##癢 -##癣 -##癥 -##癫 -##癬 -##癮 -##癱 -##癲 -##癸 -##発 -##登 -##發 -##白 -##百 -##皂 -##的 -##皆 -##皇 -##皈 -##皋 -##皎 -##皑 -##皓 -##皖 -##皙 -##皚 -##皮 -##皰 -##皱 -##皴 -##皺 -##皿 -##盂 -##盃 -##盅 -##盆 -##盈 -##益 -##盎 -##盏 -##盐 -##监 -##盒 -##盔 -##盖 -##盗 -##盘 -##盛 -##盜 -##盞 -##盟 -##盡 -##監 -##盤 -##盥 -##盧 -##盪 -##目 -##盯 -##盱 -##盲 -##直 -##相 -##盹 -##盼 -##盾 -##省 -##眈 -##眉 -##看 -##県 -##眙 -##眞 -##真 -##眠 -##眦 -##眨 -##眩 -##眯 -##眶 -##眷 -##眸 -##眺 -##眼 -##眾 -##着 -##睁 -##睇 -##睏 -##睐 -##睑 -##睛 -##睜 -##睞 -##睡 -##睢 -##督 -##睥 -##睦 -##睨 -##睪 -##睫 -##睬 -##睹 -##睽 -##睾 -##睿 -##瞄 -##瞅 -##瞇 -##瞋 -##瞌 -##瞎 -##瞑 -##瞒 -##瞓 -##瞞 -##瞟 -##瞠 -##瞥 -##瞧 -##瞩 -##瞪 -##瞬 -##瞭 -##瞰 -##瞳 -##瞻 -##瞼 -##瞿 -##矇 -##矍 -##矗 -##矚 -##矛 -##矜 -##矢 -##矣 -##知 -##矩 -##矫 -##短 -##矮 -##矯 -##石 -##矶 -##矽 -##矾 -##矿 -##码 -##砂 -##砌 -##砍 -##砒 -##研 -##砖 -##砗 -##砚 -##砝 -##砣 -##砥 -##砧 -##砭 -##砰 -##砲 -##破 -##砷 -##砸 -##砺 -##砼 -##砾 -##础 -##硅 -##硐 -##硒 -##硕 -##硝 -##硫 -##硬 -##确 -##硯 -##硼 -##碁 -##碇 -##碉 -##碌 -##碍 -##碎 -##碑 -##碓 -##碗 -##碘 -##碚 -##碛 -##碟 -##碣 -##碧 -##碩 -##碰 -##碱 -##碳 -##碴 -##確 -##碼 -##碾 -##磁 -##磅 -##磊 -##磋 -##磐 -##磕 -##磚 -##磡 -##磨 -##磬 -##磯 -##磲 -##磷 -##磺 -##礁 -##礎 -##礙 -##礡 -##礦 -##礪 -##礫 -##礴 -##示 -##礼 -##社 -##祀 -##祁 -##祂 -##祇 -##祈 -##祉 -##祎 -##祐 -##祕 -##祖 -##祗 -##祚 -##祛 -##祜 -##祝 -##神 -##祟 -##祠 -##祢 -##祥 -##票 -##祭 -##祯 -##祷 -##祸 -##祺 -##祿 -##禀 -##禁 -##禄 -##禅 -##禍 -##禎 -##福 -##禛 -##禦 -##禧 -##禪 -##禮 -##禱 -##禹 -##禺 -##离 -##禽 -##禾 -##禿 -##秀 -##私 -##秃 -##秆 -##秉 -##秋 -##种 -##科 -##秒 -##秘 -##租 -##秣 -##秤 -##秦 -##秧 -##秩 -##秭 -##积 -##称 -##秸 -##移 -##秽 -##稀 -##稅 -##程 -##稍 -##税 -##稔 -##稗 -##稚 -##稜 -##稞 -##稟 -##稠 -##稣 -##種 -##稱 -##稲 -##稳 -##稷 -##稹 -##稻 -##稼 -##稽 -##稿 -##穀 -##穂 -##穆 -##穌 -##積 -##穎 -##穗 -##穢 -##穩 -##穫 -##穴 -##究 -##穷 -##穹 -##空 -##穿 -##突 -##窃 -##窄 -##窈 -##窍 -##窑 -##窒 -##窓 -##窕 -##窖 -##窗 -##窘 -##窜 -##窝 -##窟 -##窠 -##窥 -##窦 -##窨 -##窩 -##窪 -##窮 -##窯 -##窺 -##窿 -##竄 -##竅 -##竇 -##竊 -##立 -##竖 -##站 -##竜 -##竞 -##竟 -##章 -##竣 -##童 -##竭 -##端 -##競 -##竹 -##竺 -##竽 -##竿 -##笃 -##笆 -##笈 -##笋 -##笏 -##笑 -##笔 -##笙 -##笛 -##笞 -##笠 -##符 -##笨 -##第 -##笹 -##笺 -##笼 -##筆 -##等 -##筊 -##筋 -##筍 -##筏 -##筐 -##筑 -##筒 -##答 -##策 -##筛 -##筝 -##筠 -##筱 -##筲 -##筵 -##筷 -##筹 -##签 -##简 -##箇 -##箋 -##箍 -##箏 -##箐 -##箔 -##箕 -##算 -##箝 -##管 -##箩 -##箫 -##箭 -##箱 -##箴 -##箸 -##節 -##篁 -##範 -##篆 -##篇 -##築 -##篑 -##篓 -##篙 -##篝 -##篠 -##篡 -##篤 -##篩 -##篪 -##篮 -##篱 -##篷 -##簇 -##簌 -##簍 -##簡 -##簦 -##簧 -##簪 -##簫 -##簷 -##簸 -##簽 -##簾 -##簿 -##籁 -##籃 -##籌 -##籍 -##籐 -##籟 -##籠 -##籤 -##籬 -##籮 -##籲 -##米 -##类 -##籼 -##籽 -##粄 -##粉 -##粑 -##粒 -##粕 -##粗 -##粘 -##粟 -##粤 -##粥 -##粧 -##粪 -##粮 -##粱 -##粲 -##粳 -##粵 -##粹 -##粼 -##粽 -##精 -##粿 -##糅 -##糊 -##糍 -##糕 -##糖 -##糗 -##糙 -##糜 -##糞 -##糟 -##糠 -##糧 -##糬 -##糯 -##糰 -##糸 -##系 -##糾 -##紀 -##紂 -##約 -##紅 -##紉 -##紊 -##紋 -##納 -##紐 -##紓 -##純 -##紗 -##紘 -##紙 -##級 -##紛 -##紜 -##素 -##紡 -##索 -##紧 -##紫 -##紮 -##累 -##細 -##紳 -##紹 -##紺 -##終 -##絃 -##組 -##絆 -##経 -##結 -##絕 -##絞 -##絡 -##絢 -##給 -##絨 -##絮 -##統 -##絲 -##絳 -##絵 -##絶 -##絹 -##綁 -##綏 -##綑 -##經 -##継 -##続 -##綜 -##綠 -##綢 -##綦 -##綫 -##綬 -##維 -##綱 -##網 -##綴 -##綵 -##綸 -##綺 -##綻 -##綽 -##綾 -##綿 -##緊 -##緋 -##総 -##緑 -##緒 -##緘 -##線 -##緝 -##緞 -##締 -##緣 -##編 -##緩 -##緬 -##緯 -##練 -##緹 -##緻 -##縁 -##縄 -##縈 -##縛 -##縝 -##縣 -##縫 -##縮 -##縱 -##縴 -##縷 -##總 -##績 -##繁 -##繃 -##繆 -##繇 -##繋 -##織 -##繕 -##繚 -##繞 -##繡 -##繩 -##繪 -##繫 -##繭 -##繳 -##繹 -##繼 -##繽 -##纂 -##續 -##纍 -##纏 -##纓 -##纔 -##纖 -##纜 -##纠 -##红 -##纣 -##纤 -##约 -##级 -##纨 -##纪 -##纫 -##纬 -##纭 -##纯 -##纰 -##纱 -##纲 -##纳 -##纵 -##纶 -##纷 -##纸 -##纹 -##纺 -##纽 -##纾 -##线 -##绀 -##练 -##组 -##绅 -##细 -##织 -##终 -##绊 -##绍 -##绎 -##经 -##绑 -##绒 -##结 -##绔 -##绕 -##绘 -##给 -##绚 -##绛 -##络 -##绝 -##绞 -##统 -##绡 -##绢 -##绣 -##绥 -##绦 -##继 -##绩 -##绪 -##绫 -##续 -##绮 -##绯 -##绰 -##绳 -##维 -##绵 -##绶 -##绷 -##绸 -##绻 -##综 -##绽 -##绾 -##绿 -##缀 -##缄 -##缅 -##缆 -##缇 -##缈 -##缉 -##缎 -##缓 -##缔 -##缕 -##编 -##缘 -##缙 -##缚 -##缜 -##缝 -##缠 -##缢 -##缤 -##缥 -##缨 -##缩 -##缪 -##缭 -##缮 -##缰 -##缱 -##缴 -##缸 -##缺 -##缽 -##罂 -##罄 -##罌 -##罐 -##网 -##罔 -##罕 -##罗 -##罚 -##罡 -##罢 -##罩 -##罪 -##置 -##罰 -##署 -##罵 -##罷 -##罹 -##羁 -##羅 -##羈 -##羊 -##羌 -##美 -##羔 -##羚 -##羞 -##羟 -##羡 -##羣 -##群 -##羥 -##羧 -##羨 -##義 -##羯 -##羲 -##羸 -##羹 -##羽 -##羿 -##翁 -##翅 -##翊 -##翌 -##翎 -##習 -##翔 -##翘 -##翟 -##翠 -##翡 -##翦 -##翩 -##翰 -##翱 -##翳 -##翹 -##翻 -##翼 -##耀 -##老 -##考 -##耄 -##者 -##耆 -##耋 -##而 -##耍 -##耐 -##耒 -##耕 -##耗 -##耘 -##耙 -##耦 -##耨 -##耳 -##耶 -##耷 -##耸 -##耻 -##耽 -##耿 -##聂 -##聆 -##聊 -##聋 -##职 -##聒 -##联 -##聖 -##聘 -##聚 -##聞 -##聪 -##聯 -##聰 -##聲 -##聳 -##聴 -##聶 -##職 -##聽 -##聾 -##聿 -##肃 -##肄 -##肅 -##肆 -##肇 -##肉 -##肋 -##肌 -##肏 -##肓 -##肖 -##肘 -##肚 -##肛 -##肝 -##肠 -##股 -##肢 -##肤 -##肥 -##肩 -##肪 -##肮 -##肯 -##肱 -##育 -##肴 -##肺 -##肽 -##肾 -##肿 -##胀 -##胁 -##胃 -##胄 -##胆 -##背 -##胍 -##胎 -##胖 -##胚 -##胛 -##胜 -##胝 -##胞 -##胡 -##胤 -##胥 -##胧 -##胫 -##胭 -##胯 -##胰 -##胱 -##胳 -##胴 -##胶 -##胸 -##胺 -##能 -##脂 -##脅 -##脆 -##脇 -##脈 -##脉 -##脊 -##脍 -##脏 -##脐 -##脑 -##脓 -##脖 -##脘 -##脚 -##脛 -##脣 -##脩 -##脫 -##脯 -##脱 -##脲 -##脳 -##脸 -##脹 -##脾 -##腆 -##腈 -##腊 -##腋 -##腌 -##腎 -##腐 -##腑 -##腓 -##腔 -##腕 -##腥 -##腦 -##腩 -##腫 -##腭 -##腮 -##腰 -##腱 -##腳 -##腴 -##腸 -##腹 -##腺 -##腻 -##腼 -##腾 -##腿 -##膀 -##膈 -##膊 -##膏 -##膑 -##膘 -##膚 -##膛 -##膜 -##膝 -##膠 -##膦 -##膨 -##膩 -##膳 -##膺 -##膻 -##膽 -##膾 -##膿 -##臀 -##臂 -##臃 -##臆 -##臉 -##臊 -##臍 -##臓 -##臘 -##臟 -##臣 -##臥 -##臧 -##臨 -##自 -##臬 -##臭 -##至 -##致 -##臺 -##臻 -##臼 -##臾 -##舀 -##舂 -##舅 -##舆 -##與 -##興 -##舉 -##舊 -##舌 -##舍 -##舎 -##舐 -##舒 -##舔 -##舖 -##舗 -##舛 -##舜 -##舞 -##舟 -##航 -##舫 -##般 -##舰 -##舱 -##舵 -##舶 -##舷 -##舸 -##船 -##舺 -##舾 -##艇 -##艋 -##艘 -##艙 -##艦 -##艮 -##良 -##艰 -##艱 -##色 -##艳 -##艷 -##艹 -##艺 -##艾 -##节 -##芃 -##芈 -##芊 -##芋 -##芍 -##芎 -##芒 -##芙 -##芜 -##芝 -##芡 -##芥 -##芦 -##芩 -##芪 -##芫 -##芬 -##芭 -##芮 -##芯 -##花 -##芳 -##芷 -##芸 -##芹 -##芻 -##芽 -##芾 -##苁 -##苄 -##苇 -##苋 -##苍 -##苏 -##苑 -##苒 -##苓 -##苔 -##苕 -##苗 -##苛 -##苜 -##苞 -##苟 -##苡 -##苣 -##若 -##苦 -##苫 -##苯 -##英 -##苷 -##苹 -##苻 -##茁 -##茂 -##范 -##茄 -##茅 -##茉 -##茎 -##茏 -##茗 -##茜 -##茧 -##茨 -##茫 -##茬 -##茭 -##茯 -##茱 -##茲 -##茴 -##茵 -##茶 -##茸 -##茹 -##茼 -##荀 -##荃 -##荆 -##草 -##荊 -##荏 -##荐 -##荒 -##荔 -##荖 -##荘 -##荚 -##荞 -##荟 -##荠 -##荡 -##荣 -##荤 -##荥 -##荧 -##荨 -##荪 -##荫 -##药 -##荳 -##荷 -##荸 -##荻 -##荼 -##荽 -##莅 -##莆 -##莉 -##莊 -##莎 -##莒 -##莓 -##莖 -##莘 -##莞 -##莠 -##莢 -##莧 -##莪 -##莫 -##莱 -##莲 -##莴 -##获 -##莹 -##莺 -##莽 -##莿 -##菀 -##菁 -##菅 -##菇 -##菈 -##菊 -##菌 -##菏 -##菓 -##菖 -##菘 -##菜 -##菟 -##菠 -##菡 -##菩 -##華 -##菱 -##菲 -##菸 -##菽 -##萁 -##萃 -##萄 -##萊 -##萋 -##萌 -##萍 -##萎 -##萘 -##萝 -##萤 -##营 -##萦 -##萧 -##萨 -##萩 -##萬 -##萱 -##萵 -##萸 -##萼 -##落 -##葆 -##葉 -##著 -##葚 -##葛 -##葡 -##董 -##葦 -##葩 -##葫 -##葬 -##葭 -##葯 -##葱 -##葳 -##葵 -##葷 -##葺 -##蒂 -##蒋 -##蒐 -##蒔 -##蒙 -##蒜 -##蒞 -##蒟 -##蒡 -##蒨 -##蒲 -##蒸 -##蒹 -##蒻 -##蒼 -##蒿 -##蓁 -##蓄 -##蓆 -##蓉 -##蓋 -##蓑 -##蓓 -##蓖 -##蓝 -##蓟 -##蓦 -##蓬 -##蓮 -##蓼 -##蓿 -##蔑 -##蔓 -##蔔 -##蔗 -##蔘 -##蔚 -##蔡 -##蔣 -##蔥 -##蔫 -##蔬 -##蔭 -##蔵 -##蔷 -##蔺 -##蔻 -##蔼 -##蔽 -##蕁 -##蕃 -##蕈 -##蕉 -##蕊 -##蕎 -##蕙 -##蕤 -##蕨 -##蕩 -##蕪 -##蕭 -##蕲 -##蕴 -##蕻 -##蕾 -##薄 -##薅 -##薇 -##薈 -##薊 -##薏 -##薑 -##薔 -##薙 -##薛 -##薦 -##薨 -##薩 -##薪 -##薬 -##薯 -##薰 -##薹 -##藉 -##藍 -##藏 -##藐 -##藓 -##藕 -##藜 -##藝 -##藤 -##藥 -##藩 -##藹 -##藻 -##藿 -##蘆 -##蘇 -##蘊 -##蘋 -##蘑 -##蘚 -##蘭 -##蘸 -##蘼 -##蘿 -##虎 -##虏 -##虐 -##虑 -##虔 -##處 -##虚 -##虛 -##虜 -##虞 -##號 -##虢 -##虧 -##虫 -##虬 -##虱 -##虹 -##虻 -##虽 -##虾 -##蚀 -##蚁 -##蚂 -##蚊 -##蚌 -##蚓 -##蚕 -##蚜 -##蚝 -##蚣 -##蚤 -##蚩 -##蚪 -##蚯 -##蚱 -##蚵 -##蛀 -##蛆 -##蛇 -##蛊 -##蛋 -##蛎 -##蛐 -##蛔 -##蛙 -##蛛 -##蛟 -##蛤 -##蛭 -##蛮 -##蛰 -##蛳 -##蛹 -##蛻 -##蛾 -##蜀 -##蜂 -##蜃 -##蜆 -##蜇 -##蜈 -##蜊 -##蜍 -##蜒 -##蜓 -##蜕 -##蜗 -##蜘 -##蜚 -##蜜 -##蜡 -##蜢 -##蜥 -##蜱 -##蜴 -##蜷 -##蜻 -##蜿 -##蝇 -##蝈 -##蝉 -##蝌 -##蝎 -##蝕 -##蝗 -##蝙 -##蝟 -##蝠 -##蝦 -##蝨 -##蝴 -##蝶 -##蝸 -##蝼 -##螂 -##螃 -##融 -##螞 -##螢 -##螨 -##螯 -##螳 -##螺 -##蟀 -##蟄 -##蟆 -##蟋 -##蟎 -##蟑 -##蟒 -##蟠 -##蟬 -##蟲 -##蟹 -##蟻 -##蟾 -##蠅 -##蠍 -##蠔 -##蠕 -##蠛 -##蠟 -##蠡 -##蠢 -##蠣 -##蠱 -##蠶 -##蠹 -##蠻 -##血 -##衄 -##衅 -##衆 -##行 -##衍 -##術 -##衔 -##街 -##衙 -##衛 -##衝 -##衞 -##衡 -##衢 -##衣 -##补 -##表 -##衩 -##衫 -##衬 -##衮 -##衰 -##衲 -##衷 -##衹 -##衾 -##衿 -##袁 -##袂 -##袄 -##袅 -##袈 -##袋 -##袍 -##袒 -##袖 -##袜 -##袞 -##袤 -##袪 -##被 -##袭 -##袱 -##裁 -##裂 -##装 -##裆 -##裊 -##裏 -##裔 -##裕 -##裘 -##裙 -##補 -##裝 -##裟 -##裡 -##裤 -##裨 -##裱 -##裳 -##裴 -##裸 -##裹 -##製 -##裾 -##褂 -##複 -##褐 -##褒 -##褓 -##褔 -##褚 -##褥 -##褪 -##褫 -##褲 -##褶 -##褻 -##襁 -##襄 -##襟 -##襠 -##襪 -##襬 -##襯 -##襲 -##西 -##要 -##覃 -##覆 -##覇 -##見 -##規 -##覓 -##視 -##覚 -##覦 -##覧 -##親 -##覬 -##観 -##覷 -##覺 -##覽 -##觀 -##见 -##观 -##规 -##觅 -##视 -##览 -##觉 -##觊 -##觎 -##觐 -##觑 -##角 -##觞 -##解 -##觥 -##触 -##觸 -##言 -##訂 -##計 -##訊 -##討 -##訓 -##訕 -##訖 -##託 -##記 -##訛 -##訝 -##訟 -##訣 -##訥 -##訪 -##設 -##許 -##訳 -##訴 -##訶 -##診 -##註 -##証 -##詆 -##詐 -##詔 -##評 -##詛 -##詞 -##詠 -##詡 -##詢 -##詣 -##試 -##詩 -##詫 -##詬 -##詭 -##詮 -##詰 -##話 -##該 -##詳 -##詹 -##詼 -##誅 -##誇 -##誉 -##誌 -##認 -##誓 -##誕 -##誘 -##語 -##誠 -##誡 -##誣 -##誤 -##誥 -##誦 -##誨 -##說 -##説 -##読 -##誰 -##課 -##誹 -##誼 -##調 -##諄 -##談 -##請 -##諏 -##諒 -##論 -##諗 -##諜 -##諡 -##諦 -##諧 -##諫 -##諭 -##諮 -##諱 -##諳 -##諷 -##諸 -##諺 -##諾 -##謀 -##謁 -##謂 -##謄 -##謊 -##謎 -##謐 -##謔 -##謗 -##謙 -##講 -##謝 -##謠 -##謨 -##謬 -##謹 -##謾 -##譁 -##證 -##譎 -##譏 -##識 -##譙 -##譚 -##譜 -##警 -##譬 -##譯 -##議 -##譲 -##譴 -##護 -##譽 -##讀 -##變 -##讓 -##讚 -##讞 -##计 -##订 -##认 -##讥 -##讧 -##讨 -##让 -##讪 -##讫 -##训 -##议 -##讯 -##记 -##讲 -##讳 -##讴 -##讶 -##讷 -##许 -##讹 -##论 -##讼 -##讽 -##设 -##访 -##诀 -##证 -##诃 -##评 -##诅 -##识 -##诈 -##诉 -##诊 -##诋 -##词 -##诏 -##译 -##试 -##诗 -##诘 -##诙 -##诚 -##诛 -##话 -##诞 -##诟 -##诠 -##诡 -##询 -##诣 -##诤 -##该 -##详 -##诧 -##诩 -##诫 -##诬 -##语 -##误 -##诰 -##诱 -##诲 -##说 -##诵 -##诶 -##请 -##诸 -##诺 -##读 -##诽 -##课 -##诿 -##谀 -##谁 -##调 -##谄 -##谅 -##谆 -##谈 -##谊 -##谋 -##谌 -##谍 -##谎 -##谏 -##谐 -##谑 -##谒 -##谓 -##谔 -##谕 -##谗 -##谘 -##谙 -##谚 -##谛 -##谜 -##谟 -##谢 -##谣 -##谤 -##谥 -##谦 -##谧 -##谨 -##谩 -##谪 -##谬 -##谭 -##谯 -##谱 -##谲 -##谴 -##谶 -##谷 -##豁 -##豆 -##豇 -##豈 -##豉 -##豊 -##豌 -##豎 -##豐 -##豔 -##豚 -##象 -##豢 -##豪 -##豫 -##豬 -##豹 -##豺 -##貂 -##貅 -##貌 -##貓 -##貔 -##貘 -##貝 -##貞 -##負 -##財 -##貢 -##貧 -##貨 -##販 -##貪 -##貫 -##責 -##貯 -##貰 -##貳 -##貴 -##貶 -##買 -##貸 -##費 -##貼 -##貽 -##貿 -##賀 -##賁 -##賂 -##賃 -##賄 -##資 -##賈 -##賊 -##賑 -##賓 -##賜 -##賞 -##賠 -##賡 -##賢 -##賣 -##賤 -##賦 -##質 -##賬 -##賭 -##賴 -##賺 -##購 -##賽 -##贅 -##贈 -##贊 -##贍 -##贏 -##贓 -##贖 -##贛 -##贝 -##贞 -##负 -##贡 -##财 -##责 -##贤 -##败 -##账 -##货 -##质 -##贩 -##贪 -##贫 -##贬 -##购 -##贮 -##贯 -##贰 -##贱 -##贲 -##贴 -##贵 -##贷 -##贸 -##费 -##贺 -##贻 -##贼 -##贾 -##贿 -##赁 -##赂 -##赃 -##资 -##赅 -##赈 -##赊 -##赋 -##赌 -##赎 -##赏 -##赐 -##赓 -##赔 -##赖 -##赘 -##赚 -##赛 -##赝 -##赞 -##赠 -##赡 -##赢 -##赣 -##赤 -##赦 -##赧 -##赫 -##赭 -##走 -##赳 -##赴 -##赵 -##赶 -##起 -##趁 -##超 -##越 -##趋 -##趕 -##趙 -##趟 -##趣 -##趨 -##足 -##趴 -##趵 -##趸 -##趺 -##趾 -##跃 -##跄 -##跆 -##跋 -##跌 -##跎 -##跑 -##跖 -##跚 -##跛 -##距 -##跟 -##跡 -##跤 -##跨 -##跩 -##跪 -##路 -##跳 -##践 -##跷 -##跹 -##跺 -##跻 -##踉 -##踊 -##踌 -##踏 -##踐 -##踝 -##踞 -##踟 -##踢 -##踩 -##踪 -##踮 -##踱 -##踴 -##踵 -##踹 -##蹂 -##蹄 -##蹇 -##蹈 -##蹉 -##蹊 -##蹋 -##蹑 -##蹒 -##蹙 -##蹟 -##蹣 -##蹤 -##蹦 -##蹩 -##蹬 -##蹭 -##蹲 -##蹴 -##蹶 -##蹺 -##蹼 -##蹿 -##躁 -##躇 -##躉 -##躊 -##躋 -##躍 -##躏 -##躪 -##身 -##躬 -##躯 -##躲 -##躺 -##軀 -##車 -##軋 -##軌 -##軍 -##軒 -##軟 -##転 -##軸 -##軼 -##軽 -##軾 -##較 -##載 -##輒 -##輓 -##輔 -##輕 -##輛 -##輝 -##輟 -##輩 -##輪 -##輯 -##輸 -##輻 -##輾 -##輿 -##轄 -##轅 -##轆 -##轉 -##轍 -##轎 -##轟 -##车 -##轧 -##轨 -##轩 -##转 -##轭 -##轮 -##软 -##轰 -##轲 -##轴 -##轶 -##轻 -##轼 -##载 -##轿 -##较 -##辄 -##辅 -##辆 -##辇 -##辈 -##辉 -##辊 -##辍 -##辐 -##辑 -##输 -##辕 -##辖 -##辗 -##辘 -##辙 -##辛 -##辜 -##辞 -##辟 -##辣 -##辦 -##辨 -##辩 -##辫 -##辭 -##辮 -##辯 -##辰 -##辱 -##農 -##边 -##辺 -##辻 -##込 -##辽 -##达 -##迁 -##迂 -##迄 -##迅 -##过 -##迈 -##迎 -##运 -##近 -##返 -##还 -##这 -##进 -##远 -##违 -##连 -##迟 -##迢 -##迤 -##迥 -##迦 -##迩 -##迪 -##迫 -##迭 -##述 -##迴 -##迷 -##迸 -##迹 -##迺 -##追 -##退 -##送 -##适 -##逃 -##逅 -##逆 -##选 -##逊 -##逍 -##透 -##逐 -##递 -##途 -##逕 -##逗 -##這 -##通 -##逛 -##逝 -##逞 -##速 -##造 -##逢 -##連 -##逮 -##週 -##進 -##逵 -##逶 -##逸 -##逻 -##逼 -##逾 -##遁 -##遂 -##遅 -##遇 -##遊 -##運 -##遍 -##過 -##遏 -##遐 -##遑 -##遒 -##道 -##達 -##違 -##遗 -##遙 -##遛 -##遜 -##遞 -##遠 -##遢 -##遣 -##遥 -##遨 -##適 -##遭 -##遮 -##遲 -##遴 -##遵 -##遶 -##遷 -##選 -##遺 -##遼 -##遽 -##避 -##邀 -##邁 -##邂 -##邃 -##還 -##邇 -##邈 -##邊 -##邋 -##邏 -##邑 -##邓 -##邕 -##邛 -##邝 -##邢 -##那 -##邦 -##邨 -##邪 -##邬 -##邮 -##邯 -##邰 -##邱 -##邳 -##邵 -##邸 -##邹 -##邺 -##邻 -##郁 -##郅 -##郊 -##郎 -##郑 -##郜 -##郝 -##郡 -##郢 -##郤 -##郦 -##郧 -##部 -##郫 -##郭 -##郴 -##郵 -##郷 -##郸 -##都 -##鄂 -##鄉 -##鄒 -##鄔 -##鄙 -##鄞 -##鄢 -##鄧 -##鄭 -##鄰 -##鄱 -##鄲 -##鄺 -##酉 -##酊 -##酋 -##酌 -##配 -##酐 -##酒 -##酗 -##酚 -##酝 -##酢 -##酣 -##酥 -##酩 -##酪 -##酬 -##酮 -##酯 -##酰 -##酱 -##酵 -##酶 -##酷 -##酸 -##酿 -##醃 -##醇 -##醉 -##醋 -##醍 -##醐 -##醒 -##醚 -##醛 -##醜 -##醞 -##醣 -##醪 -##醫 -##醬 -##醮 -##醯 -##醴 -##醺 -##釀 -##釁 -##采 -##釉 -##释 -##釋 -##里 -##重 -##野 -##量 -##釐 -##金 -##釗 -##釘 -##釜 -##針 -##釣 -##釦 -##釧 -##釵 -##鈀 -##鈉 -##鈍 -##鈎 -##鈔 -##鈕 -##鈞 -##鈣 -##鈦 -##鈪 -##鈴 -##鈺 -##鈾 -##鉀 -##鉄 -##鉅 -##鉉 -##鉑 -##鉗 -##鉚 -##鉛 -##鉤 -##鉴 -##鉻 -##銀 -##銃 -##銅 -##銑 -##銓 -##銖 -##銘 -##銜 -##銬 -##銭 -##銮 -##銳 -##銷 -##銹 -##鋁 -##鋅 -##鋒 -##鋤 -##鋪 -##鋰 -##鋸 -##鋼 -##錄 -##錐 -##錘 -##錚 -##錠 -##錢 -##錦 -##錨 -##錫 -##錮 -##錯 -##録 -##錳 -##錶 -##鍊 -##鍋 -##鍍 -##鍛 -##鍥 -##鍰 -##鍵 -##鍺 -##鍾 -##鎂 -##鎊 -##鎌 -##鎏 -##鎔 -##鎖 -##鎗 -##鎚 -##鎧 -##鎬 -##鎮 -##鎳 -##鏈 -##鏖 -##鏗 -##鏘 -##鏞 -##鏟 -##鏡 -##鏢 -##鏤 -##鏽 -##鐘 -##鐮 -##鐲 -##鐳 -##鐵 -##鐸 -##鐺 -##鑄 -##鑊 -##鑑 -##鑒 -##鑣 -##鑫 -##鑰 -##鑲 -##鑼 -##鑽 -##鑾 -##鑿 -##针 -##钉 -##钊 -##钎 -##钏 -##钒 -##钓 -##钗 -##钙 -##钛 -##钜 -##钝 -##钞 -##钟 -##钠 -##钡 -##钢 -##钣 -##钤 -##钥 -##钦 -##钧 -##钨 -##钩 -##钮 -##钯 -##钰 -##钱 -##钳 -##钴 -##钵 -##钺 -##钻 -##钼 -##钾 -##钿 -##铀 -##铁 -##铂 -##铃 -##铄 -##铅 -##铆 -##铉 -##铎 -##铐 -##铛 -##铜 -##铝 -##铠 -##铡 -##铢 -##铣 -##铤 -##铨 -##铩 -##铬 -##铭 -##铮 -##铰 -##铲 -##铵 -##银 -##铸 -##铺 -##链 -##铿 -##销 -##锁 -##锂 -##锄 -##锅 -##锆 -##锈 -##锉 -##锋 -##锌 -##锏 -##锐 -##锑 -##错 -##锚 -##锟 -##锡 -##锢 -##锣 -##锤 -##锥 -##锦 -##锭 -##键 -##锯 -##锰 -##锲 -##锵 -##锹 -##锺 -##锻 -##镀 -##镁 -##镂 -##镇 -##镉 -##镌 -##镍 -##镐 -##镑 -##镕 -##镖 -##镗 -##镛 -##镜 -##镣 -##镭 -##镯 -##镰 -##镳 -##镶 -##長 -##长 -##門 -##閃 -##閉 -##開 -##閎 -##閏 -##閑 -##閒 -##間 -##閔 -##閘 -##閡 -##関 -##閣 -##閥 -##閨 -##閩 -##閱 -##閲 -##閹 -##閻 -##閾 -##闆 -##闇 -##闊 -##闌 -##闍 -##闔 -##闕 -##闖 -##闘 -##關 -##闡 -##闢 -##门 -##闪 -##闫 -##闭 -##问 -##闯 -##闰 -##闲 -##间 -##闵 -##闷 -##闸 -##闹 -##闺 -##闻 -##闽 -##闾 -##阀 -##阁 -##阂 -##阅 -##阆 -##阇 -##阈 -##阉 -##阎 -##阐 -##阑 -##阔 -##阕 -##阖 -##阙 -##阚 -##阜 -##队 -##阡 -##阪 -##阮 -##阱 -##防 -##阳 -##阴 -##阵 -##阶 -##阻 -##阿 -##陀 -##陂 -##附 -##际 -##陆 -##陇 -##陈 -##陋 -##陌 -##降 -##限 -##陕 -##陛 -##陝 -##陞 -##陟 -##陡 -##院 -##陣 -##除 -##陨 -##险 -##陪 -##陰 -##陲 -##陳 -##陵 -##陶 -##陷 -##陸 -##険 -##陽 -##隅 -##隆 -##隈 -##隊 -##隋 -##隍 -##階 -##随 -##隐 -##隔 -##隕 -##隘 -##隙 -##際 -##障 -##隠 -##隣 -##隧 -##隨 -##險 -##隱 -##隴 -##隶 -##隸 -##隻 -##隼 -##隽 -##难 -##雀 -##雁 -##雄 -##雅 -##集 -##雇 -##雉 -##雋 -##雌 -##雍 -##雎 -##雏 -##雑 -##雒 -##雕 -##雖 -##雙 -##雛 -##雜 -##雞 -##離 -##難 -##雨 -##雪 -##雯 -##雰 -##雲 -##雳 -##零 -##雷 -##雹 -##電 -##雾 -##需 -##霁 -##霄 -##霆 -##震 -##霈 -##霉 -##霊 -##霍 -##霎 -##霏 -##霑 -##霓 -##霖 -##霜 -##霞 -##霧 -##霭 -##霰 -##露 -##霸 -##霹 -##霽 -##霾 -##靂 -##靄 -##靈 -##青 -##靓 -##靖 -##静 -##靚 -##靛 -##靜 -##非 -##靠 -##靡 -##面 -##靥 -##靦 -##革 -##靳 -##靴 -##靶 -##靼 -##鞅 -##鞋 -##鞍 -##鞏 -##鞑 -##鞘 -##鞠 -##鞣 -##鞦 -##鞭 -##韆 -##韋 -##韌 -##韓 -##韜 -##韦 -##韧 -##韩 -##韬 -##韭 -##音 -##韵 -##韶 -##韻 -##響 -##頁 -##頂 -##頃 -##項 -##順 -##須 -##頌 -##預 -##頑 -##頒 -##頓 -##頗 -##領 -##頜 -##頡 -##頤 -##頫 -##頭 -##頰 -##頷 -##頸 -##頹 -##頻 -##頼 -##顆 -##題 -##額 -##顎 -##顏 -##顔 -##願 -##顛 -##類 -##顧 -##顫 -##顯 -##顱 -##顴 -##页 -##顶 -##顷 -##项 -##顺 -##须 -##顼 -##顽 -##顾 -##顿 -##颁 -##颂 -##预 -##颅 -##领 -##颇 -##颈 -##颉 -##颊 -##颌 -##颍 -##颐 -##频 -##颓 -##颔 -##颖 -##颗 -##题 -##颚 -##颛 -##颜 -##额 -##颞 -##颠 -##颡 -##颢 -##颤 -##颦 -##颧 -##風 -##颯 -##颱 -##颳 -##颶 -##颼 -##飄 -##飆 -##风 -##飒 -##飓 -##飕 -##飘 -##飙 -##飚 -##飛 -##飞 -##食 -##飢 -##飨 -##飩 -##飪 -##飯 -##飲 -##飼 -##飽 -##飾 -##餃 -##餅 -##餉 -##養 -##餌 -##餐 -##餒 -##餓 -##餘 -##餚 -##餛 -##餞 -##餡 -##館 -##餮 -##餵 -##餾 -##饅 -##饈 -##饋 -##饌 -##饍 -##饑 -##饒 -##饕 -##饗 -##饞 -##饥 -##饨 -##饪 -##饬 -##饭 -##饮 -##饯 -##饰 -##饱 -##饲 -##饴 -##饵 -##饶 -##饷 -##饺 -##饼 -##饽 -##饿 -##馀 -##馁 -##馄 -##馅 -##馆 -##馈 -##馋 -##馍 -##馏 -##馒 -##馔 -##首 -##馗 -##香 -##馥 -##馨 -##馬 -##馭 -##馮 -##馳 -##馴 -##駁 -##駄 -##駅 -##駆 -##駐 -##駒 -##駕 -##駛 -##駝 -##駭 -##駱 -##駿 -##騁 -##騎 -##騏 -##験 -##騙 -##騨 -##騰 -##騷 -##驀 -##驅 -##驊 -##驍 -##驒 -##驕 -##驗 -##驚 -##驛 -##驟 -##驢 -##驥 -##马 -##驭 -##驮 -##驯 -##驰 -##驱 -##驳 -##驴 -##驶 -##驷 -##驸 -##驹 -##驻 -##驼 -##驾 -##驿 -##骁 -##骂 -##骄 -##骅 -##骆 -##骇 -##骈 -##骊 -##骋 -##验 -##骏 -##骐 -##骑 -##骗 -##骚 -##骛 -##骜 -##骞 -##骠 -##骡 -##骤 -##骥 -##骧 -##骨 -##骯 -##骰 -##骶 -##骷 -##骸 -##骼 -##髂 -##髅 -##髋 -##髏 -##髒 -##髓 -##體 -##髖 -##高 -##髦 -##髪 -##髮 -##髯 -##髻 -##鬃 -##鬆 -##鬍 -##鬓 -##鬚 -##鬟 -##鬢 -##鬣 -##鬥 -##鬧 -##鬱 -##鬼 -##魁 -##魂 -##魄 -##魅 -##魇 -##魍 -##魏 -##魔 -##魘 -##魚 -##魯 -##魷 -##鮑 -##鮨 -##鮪 -##鮭 -##鮮 -##鯉 -##鯊 -##鯖 -##鯛 -##鯨 -##鯰 -##鯽 -##鰍 -##鰓 -##鰭 -##鰲 -##鰻 -##鰾 -##鱈 -##鱉 -##鱔 -##鱗 -##鱷 -##鱸 -##鱼 -##鱿 -##鲁 -##鲈 -##鲍 -##鲑 -##鲛 -##鲜 -##鲟 -##鲢 -##鲤 -##鲨 -##鲫 -##鲱 -##鲲 -##鲶 -##鲷 -##鲸 -##鳃 -##鳄 -##鳅 -##鳌 -##鳍 -##鳕 -##鳖 -##鳗 -##鳝 -##鳞 -##鳥 -##鳩 -##鳳 -##鳴 -##鳶 -##鴉 -##鴕 -##鴛 -##鴦 -##鴨 -##鴻 -##鴿 -##鵑 -##鵜 -##鵝 -##鵡 -##鵬 -##鵰 -##鵲 -##鶘 -##鶩 -##鶯 -##鶴 -##鷗 -##鷲 -##鷹 -##鷺 -##鸚 -##鸞 -##鸟 -##鸠 -##鸡 -##鸢 -##鸣 -##鸥 -##鸦 -##鸨 -##鸪 -##鸭 -##鸯 -##鸳 -##鸵 -##鸽 -##鸾 -##鸿 -##鹂 -##鹃 -##鹄 -##鹅 -##鹈 -##鹉 -##鹊 -##鹌 -##鹏 -##鹑 -##鹕 -##鹘 -##鹜 -##鹞 -##鹤 -##鹦 -##鹧 -##鹫 -##鹭 -##鹰 -##鹳 -##鹵 -##鹹 -##鹼 -##鹽 -##鹿 -##麂 -##麋 -##麒 -##麓 -##麗 -##麝 -##麟 -##麥 -##麦 -##麩 -##麴 -##麵 -##麸 -##麺 -##麻 -##麼 -##麽 -##麾 -##黃 -##黄 -##黍 -##黎 -##黏 -##黑 -##黒 -##黔 -##默 -##黛 -##黜 -##黝 -##點 -##黠 -##黨 -##黯 -##黴 -##鼋 -##鼎 -##鼐 -##鼓 -##鼠 -##鼬 -##鼹 -##鼻 -##鼾 -##齁 -##齊 -##齋 -##齐 -##齒 -##齡 -##齢 -##齣 -##齦 -##齿 -##龄 -##龅 -##龈 -##龊 -##龋 -##龌 -##龍 -##龐 -##龔 -##龕 -##龙 -##龚 -##龛 -##龜 -##龟 -##︰ -##︱ -##︶ -##︿ -##﹁ -##﹂ -##﹍ -##﹏ -##﹐ -##﹑ -##﹒ -##﹔ -##﹕ -##﹖ -##﹗ -##﹙ -##﹚ -##﹝ -##﹞ -##﹡ -##﹣ -##！ -##＂ -##＃ -##＄ -##％ -##＆ -##＇ -##（ -##） -##＊ -##， -##－ -##． -##／ -##： -##； -##＜ -##？ -##＠ -##［ -##＼ -##］ -##＾ -##＿ -##｀ -##ｆ -##ｈ -##ｊ -##ｕ -##ｗ -##ｚ -##｛ -##｝ -##｡ -##｢ -##｣ -##､ -##･ -##ｯ -##ｰ -##ｲ -##ｸ -##ｼ -##ｽ -##ﾄ -##ﾉ -##ﾌ -##ﾗ -##ﾙ -##ﾝ -##ﾞ -##ﾟ -##￣ -##￥ -##👍 -##🔥 -##😂 -##😎 From c530e15e091c394cf1707c86b0d552a03f498224 Mon Sep 17 00:00:00 2001 From: chenzomi Date: Tue, 30 Jun 2020 14:04:15 +0800 Subject: [PATCH 209/254] add mobilenet v2 quant and resnet50 quant to model_zoo --- model_zoo/lenet_quant/README.md | 6 +- model_zoo/lenet_quant/eval.py | 2 +- model_zoo/lenet_quant/eval_quant.py | 4 +- model_zoo/lenet_quant/train.py | 5 +- model_zoo/lenet_quant/train_quant.py | 11 +- model_zoo/mobilenetv2/scripts/run_train.sh | 2 +- model_zoo/mobilenetv2_quant/Readme.md | 142 ++++++++++ model_zoo/mobilenetv2_quant/eval.py | 76 ++++++ .../mobilenetv2_quant/scripts/run_infer.sh | 53 ++++ .../scripts/run_infer_quant.sh | 54 ++++ .../mobilenetv2_quant/scripts/run_train.sh | 62 +++++ .../scripts/run_train_quant.sh | 63 +++++ model_zoo/mobilenetv2_quant/src/config.py | 60 +++++ model_zoo/mobilenetv2_quant/src/dataset.py | 156 +++++++++++ model_zoo/mobilenetv2_quant/src/launch.py | 166 ++++++++++++ .../mobilenetv2_quant/src/lr_generator.py | 54 ++++ .../mobilenetv2_quant/src/mobilenetV2.py | 231 ++++++++++++++++ model_zoo/mobilenetv2_quant/src/utils.py | 113 ++++++++ model_zoo/mobilenetv2_quant/train.py | 131 +++++++++ model_zoo/mobilenetv3/scripts/run_train.sh | 2 +- model_zoo/resnet50_quant/Readme.md | 122 +++++++++ model_zoo/resnet50_quant/eval.py | 78 ++++++ .../resnet50_quant/models/resnet_quant.py | 251 ++++++++++++++++++ model_zoo/resnet50_quant/scripts/run_infer.sh | 54 ++++ model_zoo/resnet50_quant/scripts/run_train.sh | 62 +++++ model_zoo/resnet50_quant/src/config.py | 68 +++++ model_zoo/resnet50_quant/src/crossentropy.py | 39 +++ model_zoo/resnet50_quant/src/dataset.py | 157 +++++++++++ model_zoo/resnet50_quant/src/launch.py | 165 ++++++++++++ model_zoo/resnet50_quant/src/lr_generator.py | 87 ++++++ model_zoo/resnet50_quant/src/utils.py | 46 ++++ model_zoo/resnet50_quant/train.py | 153 +++++++++++ 32 files changed, 2659 insertions(+), 16 deletions(-) create mode 100644 model_zoo/mobilenetv2_quant/Readme.md create mode 100644 model_zoo/mobilenetv2_quant/eval.py create mode 100644 model_zoo/mobilenetv2_quant/scripts/run_infer.sh create mode 100644 model_zoo/mobilenetv2_quant/scripts/run_infer_quant.sh create mode 100644 model_zoo/mobilenetv2_quant/scripts/run_train.sh create mode 100644 model_zoo/mobilenetv2_quant/scripts/run_train_quant.sh create mode 100644 model_zoo/mobilenetv2_quant/src/config.py create mode 100644 model_zoo/mobilenetv2_quant/src/dataset.py create mode 100644 model_zoo/mobilenetv2_quant/src/launch.py create mode 100644 model_zoo/mobilenetv2_quant/src/lr_generator.py create mode 100644 model_zoo/mobilenetv2_quant/src/mobilenetV2.py create mode 100644 model_zoo/mobilenetv2_quant/src/utils.py create mode 100644 model_zoo/mobilenetv2_quant/train.py create mode 100644 model_zoo/resnet50_quant/Readme.md create mode 100755 model_zoo/resnet50_quant/eval.py create mode 100755 model_zoo/resnet50_quant/models/resnet_quant.py create mode 100644 model_zoo/resnet50_quant/scripts/run_infer.sh create mode 100644 model_zoo/resnet50_quant/scripts/run_train.sh create mode 100755 model_zoo/resnet50_quant/src/config.py create mode 100644 model_zoo/resnet50_quant/src/crossentropy.py create mode 100755 model_zoo/resnet50_quant/src/dataset.py create mode 100644 model_zoo/resnet50_quant/src/launch.py create mode 100755 model_zoo/resnet50_quant/src/lr_generator.py create mode 100644 model_zoo/resnet50_quant/src/utils.py create mode 100755 model_zoo/resnet50_quant/train.py diff --git a/model_zoo/lenet_quant/README.md b/model_zoo/lenet_quant/README.md index 2f949f6d76..2fd3e129a2 100644 --- a/model_zoo/lenet_quant/README.md +++ b/model_zoo/lenet_quant/README.md @@ -33,7 +33,7 @@ Then you will get the following display ```bash >>> Found existing installation: mindspore-ascend >>> Uninstalling mindspore-ascend: ->>> Successfully uninstalled mindspore-ascend. +>>> Successfully uninstalled mindspore-ascend. ``` ### Prepare Dataset @@ -186,7 +186,7 @@ model = Model(network, net_loss, net_opt, metrics={"Accuracy": Accuracy()}) ### train quantization aware model -Also, you can just run this command instread. +Also, you can just run this command instead. ```python python train_quant.py --data_path MNIST_Data --device_target Ascend --ckpt_path checkpoint_lenet.ckpt @@ -235,7 +235,7 @@ The top1 accuracy would display on shell. Here are some optional parameters: ```bash ---device_target {Ascend,GPU,CPU} +--device_target {Ascend,GPU} device where the code will be implemented (default: Ascend) --data_path DATA_PATH path where the dataset is saved diff --git a/model_zoo/lenet_quant/eval.py b/model_zoo/lenet_quant/eval.py index d94e77279f..c0293ae1f7 100644 --- a/model_zoo/lenet_quant/eval.py +++ b/model_zoo/lenet_quant/eval.py @@ -31,7 +31,7 @@ from src.lenet_fusion import LeNet5 as LeNet5Fusion parser = argparse.ArgumentParser(description='MindSpore MNIST Example') parser.add_argument('--device_target', type=str, default="Ascend", - choices=['Ascend', 'GPU', 'CPU'], + choices=['Ascend', 'GPU'], help='device where the code will be implemented (default: Ascend)') parser.add_argument('--data_path', type=str, default="./MNIST_Data", help='path where the dataset is saved') diff --git a/model_zoo/lenet_quant/eval_quant.py b/model_zoo/lenet_quant/eval_quant.py index 2c2477123f..bc9b62121d 100644 --- a/model_zoo/lenet_quant/eval_quant.py +++ b/model_zoo/lenet_quant/eval_quant.py @@ -32,7 +32,7 @@ from src.lenet_fusion import LeNet5 as LeNet5Fusion parser = argparse.ArgumentParser(description='MindSpore MNIST Example') parser.add_argument('--device_target', type=str, default="Ascend", - choices=['Ascend', 'GPU', 'CPU'], + choices=['Ascend', 'GPU'], help='device where the code will be implemented (default: Ascend)') parser.add_argument('--data_path', type=str, default="./MNIST_Data", help='path where the dataset is saved') @@ -61,7 +61,7 @@ if __name__ == "__main__": model = Model(network, net_loss, net_opt, metrics={"Accuracy": Accuracy()}) # load quantization aware network checkpoint - param_dict = load_checkpoint(args.ckpt_path, model_type="quant") + param_dict = load_checkpoint(args.ckpt_path) load_param_into_net(network, param_dict) print("============== Starting Testing ==============") diff --git a/model_zoo/lenet_quant/train.py b/model_zoo/lenet_quant/train.py index b6040776ef..a34b6d5ed6 100644 --- a/model_zoo/lenet_quant/train.py +++ b/model_zoo/lenet_quant/train.py @@ -31,7 +31,7 @@ from src.lenet_fusion import LeNet5 as LeNet5Fusion parser = argparse.ArgumentParser(description='MindSpore MNIST Example') parser.add_argument('--device_target', type=str, default="Ascend", - choices=['Ascend', 'GPU', 'CPU'], + choices=['Ascend', 'GPU'], help='device where the code will be implemented (default: Ascend)') parser.add_argument('--data_path', type=str, default="./MNIST_Data", help='path where the dataset is saved') @@ -56,8 +56,7 @@ if __name__ == "__main__": # call back and monitor time_cb = TimeMonitor(data_size=ds_train.get_dataset_size()) config_ckpt = CheckpointConfig(save_checkpoint_steps=cfg.epoch_size * step_size, - keep_checkpoint_max=cfg.keep_checkpoint_max, - model_type=network.type) + keep_checkpoint_max=cfg.keep_checkpoint_max) ckpt_callback = ModelCheckpoint(prefix="checkpoint_lenet", config=config_ckpt) # define model diff --git a/model_zoo/lenet_quant/train_quant.py b/model_zoo/lenet_quant/train_quant.py index eb1f783a7c..ba54e63d80 100644 --- a/model_zoo/lenet_quant/train_quant.py +++ b/model_zoo/lenet_quant/train_quant.py @@ -33,7 +33,7 @@ from src.lenet_fusion import LeNet5 as LeNet5Fusion parser = argparse.ArgumentParser(description='MindSpore MNIST Example') parser.add_argument('--device_target', type=str, default="Ascend", - choices=['Ascend', 'GPU', 'CPU'], + choices=['Ascend', 'GPU'], help='device where the code will be implemented (default: Ascend)') parser.add_argument('--data_path', type=str, default="./MNIST_Data", help='path where the dataset is saved') @@ -50,11 +50,13 @@ if __name__ == "__main__": # define fusion network network = LeNet5Fusion(cfg.num_classes) + + # convert fusion network to quantization aware network + network = quant.convert_quant_network(network, quant_delay=0, bn_fold=False, freeze_bn=10000) + # load quantization aware network checkpoint param_dict = load_checkpoint(args.ckpt_path, network.type) load_param_into_net(network, param_dict) - # convert fusion network to quantization aware network - network = quant.convert_quant_network(network, quant_delay=0, bn_fold=False, freeze_bn=10000) # define network loss net_loss = nn.SoftmaxCrossEntropyWithLogits(is_grad=False, sparse=True, reduction="mean") @@ -64,8 +66,7 @@ if __name__ == "__main__": # call back and monitor time_cb = TimeMonitor(data_size=ds_train.get_dataset_size()) config_ckpt = CheckpointConfig(save_checkpoint_steps=cfg.epoch_size * step_size, - keep_checkpoint_max=cfg.keep_checkpoint_max, - model_type="quant") + keep_checkpoint_max=cfg.keep_checkpoint_max) ckpt_callback = ModelCheckpoint(prefix="checkpoint_lenet", config=config_ckpt) # define model diff --git a/model_zoo/mobilenetv2/scripts/run_train.sh b/model_zoo/mobilenetv2/scripts/run_train.sh index fc013d474c..9b9c13a006 100644 --- a/model_zoo/mobilenetv2/scripts/run_train.sh +++ b/model_zoo/mobilenetv2/scripts/run_train.sh @@ -30,7 +30,7 @@ run_ascend() BASEPATH=$(cd "`dirname $0`" || exit; pwd) export PYTHONPATH=${BASEPATH}:$PYTHONPATH - if [ -d "train" ]; + if [ -d "../train" ]; then rm -rf ../train fi diff --git a/model_zoo/mobilenetv2_quant/Readme.md b/model_zoo/mobilenetv2_quant/Readme.md new file mode 100644 index 0000000000..81be5d519c --- /dev/null +++ b/model_zoo/mobilenetv2_quant/Readme.md @@ -0,0 +1,142 @@ +# MobileNetV2 Quantization Aware Training + +MobileNetV2 is a significant improvement over MobileNetV1 and pushes the state of the art for mobile visual recognition including classification, object detection and semantic segmentation. + +MobileNetV2 builds upon the ideas from MobileNetV1, using depthwise separable convolution as efficient building blocks. However, V2 introduces two new features to the architecture: 1) linear bottlenecks between the layers, and 2) shortcut connections between the bottlenecks1. + +Training MobileNetV2 with ImageNet dataset in MindSpore with quantization aware training. + +This is the simple and basic tutorial for constructing a network in MindSpore with quantization aware. + +In this readme tutorial, you will: + +1. Train a MindSpore fusion MobileNetV2 model for ImageNet from scratch using `nn.Conv2dBnAct` and `nn.DenseBnAct`. +2. Fine tune the fusion model by applying the quantization aware training auto network converter API `convert_quant_network`, after the network convergence then export a quantization aware model checkpoint file. + +[Paper](https://arxiv.org/pdf/1801.04381) Sandler, Mark, et al. "Mobilenetv2: Inverted residuals and linear bottlenecks." Proceedings of the IEEE conference on computer vision and pattern recognition. 2018. + +# Dataset + +Dataset use: ImageNet + +- Dataset size: about 125G + - Train: 120G, 1281167 images: 1000 directories + - Test: 5G, 50000 images: images should be classified into 1000 directories firstly, just like train images +- Data format: RGB images. + - Note: Data will be processed in src/dataset.py + +# Environment Requirements + +- Hardware（Ascend) + - Prepare hardware environment with Ascend processor. If you want to try Ascend, please send the [application form](https://obs-9be7.obs.cn-east-2.myhuaweicloud.com/file/other/Ascend%20Model%20Zoo%E4%BD%93%E9%AA%8C%E8%B5%84%E6%BA%90%E7%94%B3%E8%AF%B7%E8%A1%A8.docx) to ascend@huawei.com. Once approved, you can get the resources. +- Framework + - [MindSpore](http://10.90.67.50/mindspore/archive/20200506/OpenSource/me_vm_x86/) +- For more information, please check the resources below： + - [MindSpore tutorials](https://www.mindspore.cn/tutorial/zh-CN/master/index.html) + - [MindSpore API](https://www.mindspore.cn/api/zh-CN/master/index.html) + + +# Script description + +## Script and sample code + +```python +├── mobilenetv2_quant + ├── Readme.md + ├── scripts + │ ├──run_train.sh + │ ├──run_infer.sh + │ ├──run_train_quant.sh + │ ├──run_infer_quant.sh + ├── src + │ ├──config.py + │ ├──dataset.py + │ ├──luanch.py + │ ├──lr_generator.py + │ ├──mobilenetV2.py + ├── train.py + ├── eval.py +``` + +## Training process + +### Train MobileNetV2 model + +Train a MindSpore fusion MobileNetV2 model for ImageNet, like: + +- sh run_train.sh Ascend [DEVICE_NUM] [SERVER_IP(x.x.x.x)] [VISIABLE_DEVICES(0,1,2,3,4,5,6,7)] [DATASET_PATH] [CKPT_PATH] + +You can just run this command instead. + +``` bash +>>> sh run_train.sh Ascend 4 192.168.0.1 0,1,2,3 ~/imagenet/train/ ~/mobilenet.ckpt +``` + +Training result will be stored in the example path. Checkpoints will be stored at `. /checkpoint` by default, and training log will be redirected to `./train/train.log` like followings. + +``` +>>> epoch: [ 0/200], step:[ 624/ 625], loss:[5.258/5.258], time:[140412.236], lr:[0.100] +>>> epoch time: 140522.500, per step time: 224.836, avg loss: 5.258 +>>> epoch: [ 1/200], step:[ 624/ 625], loss:[3.917/3.917], time:[138221.250], lr:[0.200] +>>> epoch time: 138331.250, per step time: 221.330, avg loss: 3.917 +``` + +### Evaluate MobileNetV2 model + +Evaluate a MindSpore fusion MobileNetV2 model for ImageNet, like: + +- sh run_infer.sh Ascend [DATASET_PATH] [CHECKPOINT_PATH] + +You can just run this command instead. + +``` bash +>>> sh run_infer.sh Ascend ~/imagenet/val/ ~/train/mobilenet-200_625.ckpt +``` + +Inference result will be stored in the example path, you can find result like the followings in `val.log`. + +``` +>>> result: {'acc': 0.71976314102564111} ckpt=/path/to/checkpoint/mobilenet-200_625.ckpt +``` + +### Fine-tune for quantization aware training + +Fine tune the fusion model by applying the quantization aware training auto network converter API `convert_quant_network`, after the network convergence then export a quantization aware model checkpoint file. + +- sh run_train_quant.sh Ascend [DEVICE_NUM] [SERVER_IP(x.x.x.x)] [VISIABLE_DEVICES(0,1,2,3,4,5,6,7)] [DATASET_PATH] [CKPT_PATH] + +You can just run this command instead. + +``` bash +>>> sh run_train_quant.sh Ascend 4 192.168.0.1 0,1,2,3 ~/imagenet/train/ ~/mobilenet.ckpt +``` + +Training result will be stored in the example path. Checkpoints will be stored at `. /checkpoint` by default, and training log will be redirected to `./train/train.log` like followings. + +``` +>>> epoch: [ 0/60], step:[ 624/ 625], loss:[5.258/5.258], time:[140412.236], lr:[0.100] +>>> epoch time: 140522.500, per step time: 224.836, avg loss: 5.258 +>>> epoch: [ 1/60], step:[ 624/ 625], loss:[3.917/3.917], time:[138221.250], lr:[0.200] +>>> epoch time: 138331.250, per step time: 221.330, avg loss: 3.917 +``` + +### Evaluate quantization aware training model + +Evaluate a MindSpore fusion MobileNetV2 model for ImageNet by applying the quantization aware training, like: + +- sh run_infer_quant.sh Ascend [DATASET_PATH] [CHECKPOINT_PATH] + +You can just run this command instead. + +``` bash +>>> sh run_infer_quant.sh Ascend ~/imagenet/val/ ~/train/mobilenet-60_625.ckpt +``` + +Inference result will be stored in the example path, you can find result like the followings in `val.log`. + +``` +>>> result: {'acc': 0.71976314102564111} ckpt=/path/to/checkpoint/mobilenet-60_625.ckpt +``` + +# ModelZoo Homepage + [Link](https://gitee.com/mindspore/mindspore/tree/master/mindspore/model_zoo) diff --git a/model_zoo/mobilenetv2_quant/eval.py b/model_zoo/mobilenetv2_quant/eval.py new file mode 100644 index 0000000000..0976abbe99 --- /dev/null +++ b/model_zoo/mobilenetv2_quant/eval.py @@ -0,0 +1,76 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Evaluate MobilenetV2 on ImageNet""" + +import os +import argparse + +from mindspore import context +from mindspore import nn +from mindspore.train.model import Model +from mindspore.train.serialization import load_checkpoint, load_param_into_net +from mindspore.train.quant import quant + +from src.mobilenetV2 import mobilenetV2 +from src.dataset import create_dataset +from src.config import config_ascend + +parser = argparse.ArgumentParser(description='Image classification') +parser.add_argument('--checkpoint_path', type=str, default=None, help='Checkpoint file path') +parser.add_argument('--dataset_path', type=str, default=None, help='Dataset path') +parser.add_argument('--device_target', type=str, default=None, help='Run device target') +parser.add_argument('--quantization_aware', type=bool, default=False, help='Use quantization aware training') +args_opt = parser.parse_args() + +if __name__ == '__main__': + config_device_target = None + if args_opt.device_target == "Ascend": + config_device_target = config_ascend + device_id = int(os.getenv('DEVICE_ID')) + context.set_context(mode=context.GRAPH_MODE, device_target="Ascend", + device_id=device_id, save_graphs=False) + + else: + raise ValueError("Unsupported device target: {}.".format(args_opt.device_target)) + + # define fusion network + network = mobilenetV2(num_classes=config_device_target.num_classes) + if args_opt.quantization_aware: + # convert fusion network to quantization aware network + network = quant.convert_quant_network(network, bn_fold=True, per_channel=[True, False], symmetric=[True, False]) + # define network loss + loss = nn.SoftmaxCrossEntropyWithLogits(is_grad=False, sparse=True, reduction='mean') + + # define dataset + dataset = create_dataset(dataset_path=args_opt.dataset_path, + do_train=False, + config=config_device_target, + device_target=args_opt.device_target, + batch_size=config_device_target.batch_size) + step_size = dataset.get_dataset_size() + + # load checkpoint + if args_opt.checkpoint_path: + param_dict = load_checkpoint(args_opt.checkpoint_path) + load_param_into_net(network, param_dict) + network.set_train(False) + + # define model + model = Model(network, loss_fn=loss, metrics={'acc'}) + + print("============== Starting Validation ==============") + res = model.eval(dataset) + print("result:", res, "ckpt=", args_opt.checkpoint_path) + print("============== End Validation ==============") diff --git a/model_zoo/mobilenetv2_quant/scripts/run_infer.sh b/model_zoo/mobilenetv2_quant/scripts/run_infer.sh new file mode 100644 index 0000000000..308723af2a --- /dev/null +++ b/model_zoo/mobilenetv2_quant/scripts/run_infer.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +if [ $# != 3 ] +then + echo "Ascend: sh run_infer.sh [PLATFORM] [DATASET_PATH] [CHECKPOINT_PATH]" +exit 1 +fi + +# check dataset path +if [ ! -d $2 ] && [ ! -f $2 ] +then + echo "error: DATASET_PATH=$2 is not a directory or file" +exit 1 +fi + +# check checkpoint file +if [ ! -f $3 ] +then + echo "error: CHECKPOINT_PATH=$3 is not a file" +exit 1 +fi + +# set environment +BASEPATH=$(cd "`dirname $0`" || exit; pwd) +export DEVICE_ID=0 +export RANK_ID=0 +export RANK_SIZE=1 +if [ -d "../eval" ]; +then + rm -rf ../eval +fi +mkdir ../eval +cd ../eval || exit + +# launch +python ${BASEPATH}/../eval.py \ + --device_target=$1 \ + --dataset_path=$2 \ + --checkpoint_path=$3 \ + &> infer.log & # dataset val folder path diff --git a/model_zoo/mobilenetv2_quant/scripts/run_infer_quant.sh b/model_zoo/mobilenetv2_quant/scripts/run_infer_quant.sh new file mode 100644 index 0000000000..f8f3c10619 --- /dev/null +++ b/model_zoo/mobilenetv2_quant/scripts/run_infer_quant.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +if [ $# != 3 ] +then + echo "Ascend: sh run_infer.sh [PLATFORM] [DATASET_PATH] [CHECKPOINT_PATH]" +exit 1 +fi + +# check dataset path +if [ ! -d $2 ] && [ ! -f $2 ] +then + echo "error: DATASET_PATH=$2 is not a directory or file" +exit 1 +fi + +# check checkpoint file +if [ ! -f $3 ] +then + echo "error: CHECKPOINT_PATH=$3 is not a file" +exit 1 +fi + +# set environment +BASEPATH=$(cd "`dirname $0`" || exit; pwd) +export DEVICE_ID=0 +export RANK_ID=0 +export RANK_SIZE=1 +if [ -d "../eval" ]; +then + rm -rf ../eval +fi +mkdir ../eval +cd ../eval || exit + +# launch +python ${BASEPATH}/../eval.py \ + --device_target=$1 \ + --dataset_path=$2 \ + --checkpoint_path=$3 \ + --quantization_aware=True \ + &> infer.log & # dataset val folder path diff --git a/model_zoo/mobilenetv2_quant/scripts/run_train.sh b/model_zoo/mobilenetv2_quant/scripts/run_train.sh new file mode 100644 index 0000000000..59b105f92e --- /dev/null +++ b/model_zoo/mobilenetv2_quant/scripts/run_train.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +run_ascend() +{ + if [ $2 -lt 1 ] && [ $2 -gt 8 ] + then + echo "error: DEVICE_NUM=$2 is not in (1-9)" + exit 1 + fi + + if [ ! -d $5 ] && [ ! -f $5 ] + then + echo "error: DATASET_PATH=$5 is not a directory or file" + exit 1 + fi + + BASEPATH=$(cd "`dirname $0`" || exit; pwd) + export PYTHONPATH=${BASEPATH}:$PYTHONPATH + if [ -d "../train" ]; + then + rm -rf ../train + fi + mkdir ../train + cd ../train || exit + python ${BASEPATH}/../src/launch.py \ + --nproc_per_node=$2 \ + --visible_devices=$4 \ + --server_id=$3 \ + --training_script=${BASEPATH}/../train.py \ + --dataset_path=$5 \ + --pre_trained=$6 \ + --device_target=$1 &> train.log & # dataset train folder +} + +if [ $# -gt 6 ] || [ $# -lt 4 ] +then + echo "Usage:\n \ + Ascend: sh run_train.sh Ascend [DEVICE_NUM] [SERVER_IP(x.x.x.x)] [VISIABLE_DEVICES(0,1,2,3,4,5,6,7)] [DATASET_PATH] [CKPT_PATH]\n \ + " +exit 1 +fi + +if [ $1 = "Ascend" ] ; then + run_ascend "$@" +else + echo "Unsupported device target." +fi; + diff --git a/model_zoo/mobilenetv2_quant/scripts/run_train_quant.sh b/model_zoo/mobilenetv2_quant/scripts/run_train_quant.sh new file mode 100644 index 0000000000..c82d1b0da5 --- /dev/null +++ b/model_zoo/mobilenetv2_quant/scripts/run_train_quant.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +run_ascend() +{ + if [ $2 -lt 1 ] && [ $2 -gt 8 ] + then + echo "error: DEVICE_NUM=$2 is not in (1-9)" + exit 1 + fi + + if [ ! -d $5 ] && [ ! -f $5 ] + then + echo "error: DATASET_PATH=$5 is not a directory or file" + exit 1 + fi + + BASEPATH=$(cd "`dirname $0`" || exit; pwd) + export PYTHONPATH=${BASEPATH}:$PYTHONPATH + if [ -d "../train" ]; + then + rm -rf ../train + fi + mkdir ../train + cd ../train || exit + python ${BASEPATH}/../src/launch.py \ + --nproc_per_node=$2 \ + --visible_devices=$4 \ + --server_id=$3 \ + --training_script=${BASEPATH}/../train.py \ + --dataset_path=$5 \ + --pre_trained=$6 \ + --quantization_aware=True \ + --device_target=$1 &> train.log & # dataset train folder +} + +if [ $# -gt 6 ] || [ $# -lt 4 ] +then + echo "Usage:\n \ + Ascend: sh run_train.sh Ascend [DEVICE_NUM] [SERVER_IP(x.x.x.x)] [VISIABLE_DEVICES(0,1,2,3,4,5,6,7)] [DATASET_PATH] [CKPT_PATH]\n \ + " +exit 1 +fi + +if [ $1 = "Ascend" ] ; then + run_ascend "$@" +else + echo "Unsupported device target." +fi; + diff --git a/model_zoo/mobilenetv2_quant/src/config.py b/model_zoo/mobilenetv2_quant/src/config.py new file mode 100644 index 0000000000..97fbc52e12 --- /dev/null +++ b/model_zoo/mobilenetv2_quant/src/config.py @@ -0,0 +1,60 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +network config setting, will be used in train.py and eval.py +""" +from easydict import EasyDict as ed + +config_ascend = ed({ + "num_classes": 1000, + "image_height": 224, + "image_width": 224, + "batch_size": 256, + "data_load_mode": "mindrecord", + "epoch_size": 200, + "start_epoch": 0, + "warmup_epochs": 4, + "lr": 0.4, + "momentum": 0.9, + "weight_decay": 4e-5, + "label_smooth": 0.1, + "loss_scale": 1024, + "save_checkpoint": True, + "save_checkpoint_epochs": 1, + "keep_checkpoint_max": 200, + "save_checkpoint_path": "./checkpoint", + "quantization_aware": False, +}) + +config_ascend_quant = ed({ + "num_classes": 1000, + "image_height": 224, + "image_width": 224, + "batch_size": 192, + "data_load_mode": "mindrecord", + "epoch_size": 60, + "start_epoch": 200, + "warmup_epochs": 1, + "lr": 0.3, + "momentum": 0.9, + "weight_decay": 4e-5, + "label_smooth": 0.1, + "loss_scale": 1024, + "save_checkpoint": True, + "save_checkpoint_epochs": 1, + "keep_checkpoint_max": 200, + "save_checkpoint_path": "./checkpoint", + "quantization_aware": True, +}) diff --git a/model_zoo/mobilenetv2_quant/src/dataset.py b/model_zoo/mobilenetv2_quant/src/dataset.py new file mode 100644 index 0000000000..105a5e1396 --- /dev/null +++ b/model_zoo/mobilenetv2_quant/src/dataset.py @@ -0,0 +1,156 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +create train or eval dataset. +""" +import os +from functools import partial +import mindspore.common.dtype as mstype +import mindspore.dataset.engine as de +import mindspore.dataset.transforms.vision.c_transforms as C +import mindspore.dataset.transforms.c_transforms as C2 +import mindspore.dataset.transforms.vision.py_transforms as P +from src.config import config_ascend + + +def create_dataset(dataset_path, do_train, config, device_target, repeat_num=1, batch_size=32): + """ + create a train or eval dataset + + Args: + dataset_path(string): the path of dataset. + do_train(bool): whether dataset is used for train or eval. + repeat_num(int): the repeat times of dataset. Default: 1. + batch_size(int): the batch size of dataset. Default: 32. + + Returns: + dataset + """ + if device_target == "Ascend": + rank_size = int(os.getenv("RANK_SIZE")) + rank_id = int(os.getenv("RANK_ID")) + columns_list = ['image', 'label'] + if config_ascend.data_load_mode == "mindrecord": + load_func = partial(de.MindDataset, dataset_path, columns_list) + else: + load_func = partial(de.ImageFolderDatasetV2, dataset_path) + if do_train: + if rank_size == 1: + ds = load_func(num_parallel_workers=8, shuffle=True) + else: + ds = load_func(num_parallel_workers=8, shuffle=True, + num_shards=rank_size, shard_id=rank_id) + else: + ds = load_func(num_parallel_workers=8, shuffle=False) + else: + raise ValueError("Unsupport device_target.") + + resize_height = config.image_height + + if do_train: + buffer_size = 20480 + # apply shuffle operations + ds = ds.shuffle(buffer_size=buffer_size) + + # define map operations + decode_op = C.Decode() + resize_crop_decode_op = C.RandomCropDecodeResize(resize_height, scale=(0.08, 1.0), ratio=(0.75, 1.333)) + horizontal_flip_op = C.RandomHorizontalFlip(prob=0.5) + + resize_op = C.Resize(256) + center_crop = C.CenterCrop(resize_height) + normalize_op = C.Normalize(mean=[0.485 * 255, 0.456 * 255, 0.406 * 255], + std=[0.229 * 255, 0.224 * 255, 0.225 * 255]) + change_swap_op = C.HWC2CHW() + + if do_train: + trans = [resize_crop_decode_op, horizontal_flip_op, normalize_op, change_swap_op] + else: + trans = [decode_op, resize_op, center_crop, normalize_op, change_swap_op] + + type_cast_op = C2.TypeCast(mstype.int32) + + ds = ds.map(input_columns="image", operations=trans, num_parallel_workers=16) + ds = ds.map(input_columns="label", operations=type_cast_op, num_parallel_workers=8) + + # apply batch operations + ds = ds.batch(batch_size, drop_remainder=True) + + # apply dataset repeat operation + ds = ds.repeat(repeat_num) + + return ds + + +def create_dataset_py(dataset_path, do_train, config, device_target, repeat_num=1, batch_size=32): + """ + create a train or eval dataset + + Args: + dataset_path(string): the path of dataset. + do_train(bool): whether dataset is used for train or eval. + repeat_num(int): the repeat times of dataset. Default: 1. + batch_size(int): the batch size of dataset. Default: 32. + + Returns: + dataset + """ + if device_target == "Ascend": + rank_size = int(os.getenv("RANK_SIZE")) + rank_id = int(os.getenv("RANK_ID")) + if do_train: + if rank_size == 1: + ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True) + else: + ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True, + num_shards=rank_size, shard_id=rank_id) + else: + ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=False) + else: + raise ValueError("Unsupported device target.") + + resize_height = config.image_height + + if do_train: + buffer_size = 20480 + # apply shuffle operations + ds = ds.shuffle(buffer_size=buffer_size) + + # define map operations + decode_op = P.Decode() + resize_crop_op = P.RandomResizedCrop(resize_height, scale=(0.08, 1.0), ratio=(0.75, 1.333)) + horizontal_flip_op = P.RandomHorizontalFlip(prob=0.5) + + resize_op = P.Resize(256) + center_crop = P.CenterCrop(resize_height) + to_tensor = P.ToTensor() + normalize_op = P.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) + + if do_train: + trans = [decode_op, resize_crop_op, horizontal_flip_op, to_tensor, normalize_op] + else: + trans = [decode_op, resize_op, center_crop, to_tensor, normalize_op] + + compose = P.ComposeOp(trans) + + ds = ds.map(input_columns="image", operations=compose(), num_parallel_workers=8, python_multiprocessing=True) + + # apply batch operations + ds = ds.batch(batch_size, drop_remainder=True) + + # apply dataset repeat operation + ds = ds.repeat(repeat_num) + + return ds diff --git a/model_zoo/mobilenetv2_quant/src/launch.py b/model_zoo/mobilenetv2_quant/src/launch.py new file mode 100644 index 0000000000..08477a363a --- /dev/null +++ b/model_zoo/mobilenetv2_quant/src/launch.py @@ -0,0 +1,166 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""launch train script""" +import os +import sys +import json +import subprocess +import shutil +import platform +from argparse import ArgumentParser + + +def parse_args(): + """ + parse args . + + Args: + + Returns: + args. + + Examples: + >>> parse_args() + """ + parser = ArgumentParser(description="mindspore distributed training launch " + "helper utilty that will spawn up " + "multiple distributed processes") + parser.add_argument("--nproc_per_node", type=int, default=1, + help="The number of processes to launch on each node, " + "for D training, this is recommended to be set " + "to the number of D in your system so that " + "each process can be bound to a single D.") + parser.add_argument("--visible_devices", type=str, default="0,1,2,3,4,5,6,7", + help="will use the visible devices sequentially") + parser.add_argument("--server_id", type=str, default="", + help="server ip") + parser.add_argument("--training_script", type=str, + help="The full path to the single D training " + "program/script to be launched in parallel, " + "followed by all the arguments for the " + "training script") + # rest from the training program + args, unknown = parser.parse_known_args() + args.training_script_args = unknown + return args + + +def main(): + print("start", __file__) + args = parse_args() + print(args) + visible_devices = args.visible_devices.split(',') + assert os.path.isfile(args.training_script) + assert len(visible_devices) >= args.nproc_per_node + print('visible_devices:{}'.format(visible_devices)) + if not args.server_id: + print('pleaser input server ip!!!') + exit(0) + print('server_id:{}'.format(args.server_id)) + + # construct hccn_table + hccn_configs = open('/etc/hccn.conf', 'r').readlines() + device_ips = {} + for hccn_item in hccn_configs: + hccn_item = hccn_item.strip() + if hccn_item.startswith('address_'): + device_id, device_ip = hccn_item.split('=') + device_id = device_id.split('_')[1] + device_ips[device_id] = device_ip + print('device_id:{}, device_ip:{}'.format(device_id, device_ip)) + hccn_table = {} + arch = platform.processor() + hccn_table['board_id'] = {'aarch64': '0x002f', 'x86_64': '0x0000'}[arch] + hccn_table['chip_info'] = '910' + hccn_table['deploy_mode'] = 'lab' + hccn_table['group_count'] = '1' + hccn_table['group_list'] = [] + instance_list = [] + usable_dev = '' + for instance_id in range(args.nproc_per_node): + instance = {} + instance['devices'] = [] + device_id = visible_devices[instance_id] + device_ip = device_ips[device_id] + usable_dev += str(device_id) + instance['devices'].append({ + 'device_id': device_id, + 'device_ip': device_ip, + }) + instance['rank_id'] = str(instance_id) + instance['server_id'] = args.server_id + instance_list.append(instance) + hccn_table['group_list'].append({ + 'device_num': str(args.nproc_per_node), + 'server_num': '1', + 'group_name': '', + 'instance_count': str(args.nproc_per_node), + 'instance_list': instance_list, + }) + hccn_table['para_plane_nic_location'] = 'device' + hccn_table['para_plane_nic_name'] = [] + for instance_id in range(args.nproc_per_node): + eth_id = visible_devices[instance_id] + hccn_table['para_plane_nic_name'].append('eth{}'.format(eth_id)) + hccn_table['para_plane_nic_num'] = str(args.nproc_per_node) + hccn_table['status'] = 'completed' + + # save hccn_table to file + table_path = os.getcwd() + if not os.path.exists(table_path): + os.mkdir(table_path) + table_fn = os.path.join(table_path, + 'rank_table_{}p_{}_{}.json'.format(args.nproc_per_node, usable_dev, args.server_id)) + with open(table_fn, 'w') as table_fp: + json.dump(hccn_table, table_fp, indent=4) + sys.stdout.flush() + + # spawn the processes + processes = [] + cmds = [] + log_files = [] + env = os.environ.copy() + env['RANK_SIZE'] = str(args.nproc_per_node) + cur_path = os.getcwd() + for rank_id in range(0, args.nproc_per_node): + os.chdir(cur_path) + device_id = visible_devices[rank_id] + device_dir = os.path.join(cur_path, 'device{}'.format(rank_id)) + env['RANK_ID'] = str(rank_id) + env['DEVICE_ID'] = str(device_id) + if args.nproc_per_node > 1: + env['MINDSPORE_HCCL_CONFIG_PATH'] = table_fn + env['RANK_TABLE_FILE'] = table_fn + if os.path.exists(device_dir): + shutil.rmtree(device_dir) + os.mkdir(device_dir) + os.chdir(device_dir) + cmd = [sys.executable, '-u'] + cmd.append(args.training_script) + cmd.extend(args.training_script_args) + log_file = open('{dir}/log{id}.log'.format(dir=device_dir, id=rank_id), 'w') + process = subprocess.Popen(cmd, stdout=log_file, stderr=log_file, env=env) + processes.append(process) + cmds.append(cmd) + log_files.append(log_file) + for process, cmd, log_file in zip(processes, cmds, log_files): + process.wait() + if process.returncode != 0: + raise subprocess.CalledProcessError(returncode=process, cmd=cmd) + log_file.close() + + +if __name__ == "__main__": + main() diff --git a/model_zoo/mobilenetv2_quant/src/lr_generator.py b/model_zoo/mobilenetv2_quant/src/lr_generator.py new file mode 100644 index 0000000000..68bbfe3158 --- /dev/null +++ b/model_zoo/mobilenetv2_quant/src/lr_generator.py @@ -0,0 +1,54 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""learning rate generator""" +import math +import numpy as np + + +def get_lr(global_step, lr_init, lr_end, lr_max, warmup_epochs, total_epochs, steps_per_epoch): + """ + generate learning rate array + + Args: + global_step(int): total steps of the training + lr_init(float): init learning rate + lr_end(float): end learning rate + lr_max(float): max learning rate + warmup_epochs(int): number of warmup epochs + total_epochs(int): total epoch of training + steps_per_epoch(int): steps of one epoch + + Returns: + np.array, learning rate array + """ + lr_each_step = [] + total_steps = steps_per_epoch * total_epochs + warmup_steps = steps_per_epoch * warmup_epochs + for i in range(total_steps): + if i < warmup_steps: + lr = lr_init + (lr_max - lr_init) * i / warmup_steps + else: + lr = lr_end + \ + (lr_max - lr_end) * \ + (1. + math.cos(math.pi * (i - warmup_steps) / (total_steps - warmup_steps))) / 2. + if lr < 0.0: + lr = 0.0 + lr_each_step.append(lr) + + current_step = global_step + lr_each_step = np.array(lr_each_step).astype(np.float32) + learning_rate = lr_each_step[current_step:] + + return learning_rate diff --git a/model_zoo/mobilenetv2_quant/src/mobilenetV2.py b/model_zoo/mobilenetv2_quant/src/mobilenetV2.py new file mode 100644 index 0000000000..25dccfed10 --- /dev/null +++ b/model_zoo/mobilenetv2_quant/src/mobilenetV2.py @@ -0,0 +1,231 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""MobileNetV2 Quant model define""" + +import numpy as np + +import mindspore.nn as nn +from mindspore.ops import operations as P +from mindspore import Tensor + +__all__ = ['mobilenetV2'] + + +def _make_divisible(v, divisor, min_value=None): + if min_value is None: + min_value = divisor + new_v = max(min_value, int(v + divisor / 2) // divisor * divisor) + # Make sure that round down does not go down by more than 10%. + if new_v < 0.9 * v: + new_v += divisor + return new_v + + +class GlobalAvgPooling(nn.Cell): + """ + Global avg pooling definition. + + Args: + + Returns: + Tensor, output tensor. + + Examples: + >>> GlobalAvgPooling() + """ + + def __init__(self): + super(GlobalAvgPooling, self).__init__() + self.mean = P.ReduceMean(keep_dims=False) + + def construct(self, x): + x = self.mean(x, (2, 3)) + return x + + +class ConvBNReLU(nn.Cell): + """ + Convolution/Depthwise fused with Batchnorm and ReLU block definition. + + Args: + in_planes (int): Input channel. + out_planes (int): Output channel. + kernel_size (int): Input kernel size. + stride (int): Stride size for the first convolutional layer. Default: 1. + groups (int): channel group. Convolution is 1 while Depthiwse is input channel. Default: 1. + + Returns: + Tensor, output tensor. + + Examples: + >>> ConvBNReLU(16, 256, kernel_size=1, stride=1, groups=1) + """ + + def __init__(self, in_planes, out_planes, kernel_size=3, stride=1, groups=1): + super(ConvBNReLU, self).__init__() + padding = (kernel_size - 1) // 2 + self.conv = nn.Conv2dBnAct(in_planes, out_planes, kernel_size, + stride=stride, + pad_mode='pad', + padding=padding, + group=groups, + has_bn=True, + activation='relu') + + def construct(self, x): + x = self.conv(x) + return x + + +class InvertedResidual(nn.Cell): + """ + Mobilenetv2 residual block definition. + + Args: + inp (int): Input channel. + oup (int): Output channel. + stride (int): Stride size for the first convolutional layer. Default: 1. + expand_ratio (int): expand ration of input channel + + Returns: + Tensor, output tensor. + + Examples: + >>> ResidualBlock(3, 256, 1, 1) + """ + + def __init__(self, inp, oup, stride, expand_ratio): + super(InvertedResidual, self).__init__() + assert stride in [1, 2] + + hidden_dim = int(round(inp * expand_ratio)) + self.use_res_connect = stride == 1 and inp == oup + + layers = [] + if expand_ratio != 1: + layers.append(ConvBNReLU(inp, hidden_dim, kernel_size=1)) + layers.extend([ + # dw + ConvBNReLU(hidden_dim, hidden_dim, stride=stride, groups=hidden_dim), + # pw-linear + nn.Conv2dBnAct(hidden_dim, oup, kernel_size=1, stride=1, pad_mode='pad', padding=0, group=1, has_bn=True) + ]) + self.conv = nn.SequentialCell(layers) + self.add = P.TensorAdd() + + def construct(self, x): + out = self.conv(x) + if self.use_res_connect: + out = self.add(out, x) + return out + + +class mobilenetV2(nn.Cell): + """ + mobilenetV2 fusion architecture. + + Args: + class_num (Cell): number of classes. + width_mult (int): Channels multiplier for round to 8/16 and others. Default is 1. + has_dropout (bool): Is dropout used. Default is false + inverted_residual_setting (list): Inverted residual settings. Default is None + round_nearest (list): Channel round to . Default is 8 + Returns: + Tensor, output tensor. + + Examples: + >>> mobilenetV2(num_classes=1000) + """ + + def __init__(self, num_classes=1000, width_mult=1., + has_dropout=False, inverted_residual_setting=None, round_nearest=8): + super(mobilenetV2, self).__init__() + block = InvertedResidual + input_channel = 32 + last_channel = 1280 + # setting of inverted residual blocks + self.cfgs = inverted_residual_setting + if inverted_residual_setting is None: + self.cfgs = [ + # t, c, n, s + [1, 16, 1, 1], + [6, 24, 2, 2], + [6, 32, 3, 2], + [6, 64, 4, 2], + [6, 96, 3, 1], + [6, 160, 3, 2], + [6, 320, 1, 1], + ] + + # building first layer + input_channel = _make_divisible(input_channel * width_mult, round_nearest) + self.out_channels = _make_divisible(last_channel * max(1.0, width_mult), round_nearest) + + features = [ConvBNReLU(3, input_channel, stride=2)] + # building inverted residual blocks + for t, c, n, s in self.cfgs: + output_channel = _make_divisible(c * width_mult, round_nearest) + for i in range(n): + stride = s if i == 0 else 1 + features.append(block(input_channel, output_channel, stride, expand_ratio=t)) + input_channel = output_channel + # building last several layers + features.append(ConvBNReLU(input_channel, self.out_channels, kernel_size=1)) + # make it nn.CellList + self.features = nn.SequentialCell(features) + # mobilenet head + head = ([GlobalAvgPooling(), + nn.DenseBnAct(self.out_channels, num_classes, has_bias=True, has_bn=False) + ] if not has_dropout else + [GlobalAvgPooling(), + nn.Dropout(0.2), + nn.DenseBnAct(self.out_channels, num_classes, has_bias=True, has_bn=False) + ]) + self.head = nn.SequentialCell(head) + + # init weights + self._initialize_weights() + + def construct(self, x): + x = self.features(x) + x = self.head(x) + return x + + def _initialize_weights(self): + """ + Initialize weights. + + Args: + + Returns: + None. + + Examples: + >>> _initialize_weights() + """ + for _, m in self.cells_and_names(): + if isinstance(m, nn.Conv2d): + n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels + w = Tensor(np.random.normal(0, np.sqrt(2. / n), m.weight.data.shape).astype("float32")) + m.weight.set_parameter_data(w) + if m.bias is not None: + m.bias.set_parameter_data(Tensor(np.zeros(m.bias.data.shape, dtype="float32"))) + elif isinstance(m, nn.BatchNorm2d): + m.gamma.set_parameter_data(Tensor(np.ones(m.gamma.data.shape, dtype="float32"))) + m.beta.set_parameter_data(Tensor(np.zeros(m.beta.data.shape, dtype="float32"))) + elif isinstance(m, nn.Dense): + m.weight.set_parameter_data(Tensor(np.random.normal(0, 0.01, m.weight.data.shape).astype("float32"))) + if m.bias is not None: + m.bias.set_parameter_data(Tensor(np.zeros(m.bias.data.shape, dtype="float32"))) diff --git a/model_zoo/mobilenetv2_quant/src/utils.py b/model_zoo/mobilenetv2_quant/src/utils.py new file mode 100644 index 0000000000..8690d9c380 --- /dev/null +++ b/model_zoo/mobilenetv2_quant/src/utils.py @@ -0,0 +1,113 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""MobileNetV2 utils""" + +import time +import numpy as np + +from mindspore.train.callback import Callback +from mindspore import Tensor +from mindspore import nn +from mindspore.nn.loss.loss import _Loss +from mindspore.ops import operations as P +from mindspore.ops import functional as F +from mindspore.common import dtype as mstype + + +class Monitor(Callback): + """ + Monitor loss and time. + + Args: + lr_init (numpy array): train lr + + Returns: + None + + Examples: + >>> Monitor(100,lr_init=Tensor([0.05]*100).asnumpy()) + """ + + def __init__(self, lr_init=None): + super(Monitor, self).__init__() + self.lr_init = lr_init + self.lr_init_len = len(lr_init) + + def epoch_begin(self, run_context): + self.losses = [] + self.epoch_time = time.time() + + def epoch_end(self, run_context): + cb_params = run_context.original_args() + + epoch_mseconds = (time.time() - self.epoch_time) * 1000 + per_step_mseconds = epoch_mseconds / cb_params.batch_num + print("epoch time: {:5.3f}, per step time: {:5.3f}, avg loss: {:5.3f}".format(epoch_mseconds, + per_step_mseconds, + np.mean(self.losses))) + + def step_begin(self, run_context): + self.step_time = time.time() + + def step_end(self, run_context): + cb_params = run_context.original_args() + step_mseconds = (time.time() - self.step_time) * 1000 + step_loss = cb_params.net_outputs + + if isinstance(step_loss, (tuple, list)) and isinstance(step_loss[0], Tensor): + step_loss = step_loss[0] + if isinstance(step_loss, Tensor): + step_loss = np.mean(step_loss.asnumpy()) + + self.losses.append(step_loss) + cur_step_in_epoch = (cb_params.cur_step_num - 1) % cb_params.batch_num + + print("epoch: [{:3d}/{:3d}], step:[{:5d}/{:5d}], loss:[{:5.3f}/{:5.3f}], time:[{:5.3f}], lr:[{:5.5f}]".format( + cb_params.cur_epoch_num - + 1, cb_params.epoch_num, cur_step_in_epoch, cb_params.batch_num, step_loss, + np.mean(self.losses), step_mseconds, self.lr_init[cb_params.cur_step_num - 1])) + + +class CrossEntropyWithLabelSmooth(_Loss): + """ + CrossEntropyWith LabelSmooth. + + Args: + smooth_factor (float): smooth factor, default=0. + num_classes (int): num classes + + Returns: + None. + + Examples: + >>> CrossEntropyWithLabelSmooth(smooth_factor=0., num_classes=1000) + """ + + def __init__(self, smooth_factor=0., num_classes=1000): + super(CrossEntropyWithLabelSmooth, self).__init__() + self.onehot = P.OneHot() + self.on_value = Tensor(1.0 - smooth_factor, mstype.float32) + self.off_value = Tensor(1.0 * smooth_factor / + (num_classes - 1), mstype.float32) + self.ce = nn.SoftmaxCrossEntropyWithLogits() + self.mean = P.ReduceMean(False) + self.cast = P.Cast() + + def construct(self, logit, label): + one_hot_label = self.onehot(self.cast(label, mstype.int32), F.shape(logit)[1], + self.on_value, self.off_value) + out_loss = self.ce(logit, one_hot_label) + out_loss = self.mean(out_loss, 0) + return out_loss diff --git a/model_zoo/mobilenetv2_quant/train.py b/model_zoo/mobilenetv2_quant/train.py new file mode 100644 index 0000000000..1302c3cf27 --- /dev/null +++ b/model_zoo/mobilenetv2_quant/train.py @@ -0,0 +1,131 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Train mobilenetV2 on ImageNet""" + +import os +import argparse +import random +import numpy as np + +from mindspore import context +from mindspore import Tensor +from mindspore import nn +from mindspore.train.model import Model, ParallelMode +from mindspore.train.callback import ModelCheckpoint, CheckpointConfig +from mindspore.train.serialization import load_checkpoint, load_param_into_net +from mindspore.communication.management import init +from mindspore.train.quant import quant +import mindspore.dataset.engine as de + +from src.dataset import create_dataset +from src.lr_generator import get_lr +from src.utils import Monitor, CrossEntropyWithLabelSmooth +from src.config import config_ascend, config_ascend_quant +from src.mobilenetV2 import mobilenetV2 + +random.seed(1) +np.random.seed(1) +de.config.set_seed(1) + +parser = argparse.ArgumentParser(description='Image classification') +parser.add_argument('--dataset_path', type=str, default=None, help='Dataset path') +parser.add_argument('--pre_trained', type=str, default=None, help='Pertained checkpoint path') +parser.add_argument('--device_target', type=str, default=None, help='Run device target') +parser.add_argument('--quantization_aware', type=bool, default=False, help='Use quantization aware training') +args_opt = parser.parse_args() + +if args_opt.device_target == "Ascend": + device_id = int(os.getenv('DEVICE_ID')) + rank_id = int(os.getenv('RANK_ID')) + rank_size = int(os.getenv('RANK_SIZE')) + run_distribute = rank_size > 1 + device_id = int(os.getenv('DEVICE_ID')) + context.set_context(mode=context.GRAPH_MODE, + device_target="Ascend", + device_id=device_id, save_graphs=False) +else: + raise ValueError("Unsupported device target.") + +if __name__ == '__main__': + # train on ascend + config = config_ascend_quant if args_opt.quantization_aware else config_ascend + print("training args: {}".format(args_opt)) + print("training configure: {}".format(config)) + print("parallel args: rank_id {}, device_id {}, rank_size {}".format(rank_id, device_id, rank_size)) + epoch_size = config.epoch_size + + # distribute init + if run_distribute: + context.set_auto_parallel_context(device_num=rank_size, + parallel_mode=ParallelMode.DATA_PARALLEL, + parameter_broadcast=True, + mirror_mean=True) + init() + + # define network + network = mobilenetV2(num_classes=config.num_classes) + # define loss + if config.label_smooth > 0: + loss = CrossEntropyWithLabelSmooth(smooth_factor=config.label_smooth, num_classes=config.num_classes) + else: + loss = nn.SoftmaxCrossEntropyWithLogits(is_grad=False, sparse=True, reduction='mean') + # define dataset + dataset = create_dataset(dataset_path=args_opt.dataset_path, + do_train=True, + config=config, + device_target=args_opt.device_target, + repeat_num=epoch_size, + batch_size=config.batch_size) + step_size = dataset.get_dataset_size() + # load pre trained ckpt + if args_opt.pre_trained: + param_dict = load_checkpoint(args_opt.pre_trained) + load_param_into_net(network, param_dict) + + # convert fusion network to quantization aware network + if config.quantization_aware: + network = quant.convert_quant_network(network, + bn_fold=True, + per_channel=[True, False], + symmetric=[True, False]) + + # get learning rate + lr = Tensor(get_lr(global_step=config.start_epoch * step_size, + lr_init=0, + lr_end=0, + lr_max=config.lr, + warmup_epochs=config.warmup_epochs, + total_epochs=epoch_size + config.start_epoch, + steps_per_epoch=step_size)) + + # define optimization + opt = nn.Momentum(filter(lambda x: x.requires_grad, network.get_parameters()), lr, config.momentum, + config.weight_decay) + # define model + model = Model(network, loss_fn=loss, optimizer=opt) + + print("============== Starting Training ==============") + callback = None + if rank_id == 0: + callback = [Monitor(lr_init=lr.asnumpy())] + if config.save_checkpoint: + config_ck = CheckpointConfig(save_checkpoint_steps=config.save_checkpoint_epochs * step_size, + keep_checkpoint_max=config.keep_checkpoint_max) + ckpt_cb = ModelCheckpoint(prefix="mobilenetV2", + directory=config.save_checkpoint_path, + config=config_ck) + callback += [ckpt_cb] + model.train(epoch_size, dataset, callbacks=callback) + print("============== End Training ==============") diff --git a/model_zoo/mobilenetv3/scripts/run_train.sh b/model_zoo/mobilenetv3/scripts/run_train.sh index 78b79b235f..4912c895de 100644 --- a/model_zoo/mobilenetv3/scripts/run_train.sh +++ b/model_zoo/mobilenetv3/scripts/run_train.sh @@ -29,7 +29,7 @@ run_ascend() BASEPATH=$(cd "`dirname $0`" || exit; pwd) export PYTHONPATH=${BASEPATH}:$PYTHONPATH - if [ -d "train" ]; + if [ -d "../train" ]; then rm -rf ../train fi diff --git a/model_zoo/resnet50_quant/Readme.md b/model_zoo/resnet50_quant/Readme.md new file mode 100644 index 0000000000..9e843b2223 --- /dev/null +++ b/model_zoo/resnet50_quant/Readme.md @@ -0,0 +1,122 @@ +# ResNet-50_quant Example + +## Description + +This is an example of training ResNet-50_quant with ImageNet2012 dataset in MindSpore. + +## Requirements + +- Install [MindSpore](https://www.mindspore.cn/install/en). + +- Download the dataset ImageNet2012 + +> Unzip the ImageNet2012 dataset to any path you want and the folder structure should include train and eval dataset as follows: +> ``` +> . +> ├── ilsvrc # train dataset +> └── ilsvrc_eval # infer dataset: images should be classified into 1000 directories firstly, just like train images +> ``` + + +## Example structure + +```shell +. +├── Resnet50_quant + ├── Readme.md + ├── scripts + │ ├──run_train.sh + │ ├──run_eval.sh + ├── src + │ ├──config.py + │ ├──crossentropy.py + │ ├──dataset.py + │ ├──luanch.py + │ ├──lr_generator.py + │ ├──utils.py + ├── models + │ ├──resnet_quant.py + ├── train.py + ├── eval.py +``` + + +## Parameter configuration + +Parameters for both training and inference can be set in config.py. + +``` +"class_num": 1001, # dataset class number +"batch_size": 32, # batch size of input tensor +"loss_scale": 1024, # loss scale +"momentum": 0.9, # momentum optimizer +"weight_decay": 1e-4, # weight decay +"epoch_size": 120, # only valid for taining, which is always 1 for inference +"pretrained_epoch_size": 90, # epoch size that model has been trained before load pretrained checkpoint +"buffer_size": 1000, # number of queue size in data preprocessing +"image_height": 224, # image height +"image_width": 224, # image width +"save_checkpoint": True, # whether save checkpoint or not +"save_checkpoint_epochs": 1, # the epoch interval between two checkpoints. By default, the last checkpoint will be saved after the last epoch +"keep_checkpoint_max": 50, # only keep the last keep_checkpoint_max checkpoint +"save_checkpoint_path": "./", # path to save checkpoint relative to the executed path +"warmup_epochs": 0, # number of warmup epoch +"lr_decay_mode": "cosine", # decay mode for generating learning rate +"label_smooth": True, # label smooth +"label_smooth_factor": 0.1, # label smooth factor +"lr_init": 0, # initial learning rate +"lr_max": 0.005, # maximum learning rate +``` + +## Running the example + +### Train + +### Usage + +- Ascend: sh run_train.sh Ascend [DEVICE_NUM] [SERVER_IP(x.x.x.x)] [VISIABLE_DEVICES(0,1,2,3,4,5,6,7)] [DATASET_PATH] [CKPT_PATH] + + +### Launch + +``` +# training example + Ascend: sh run_train.sh Ascend 8 192.168.0.1 0,1,2,3,4,5,6,7 ~/imagenet/train/ +``` + +### Result + +Training result will be stored in the example path. Checkpoints will be stored at `. /checkpoint` by default, and training log will be redirected to `./train/train.log` like followings. + +``` +epoch: 1 step: 5004, loss is 4.8995576 +epoch: 2 step: 5004, loss is 3.9235563 +epoch: 3 step: 5004, loss is 3.833077 +epoch: 4 step: 5004, loss is 3.2795618 +epoch: 5 step: 5004, loss is 3.1978393 +``` + +## Eval process + +### Usage + +- Ascend: sh run_infer.sh Ascend [DATASET_PATH] [CHECKPOINT_PATH] + +### Launch + +``` +# infer example + Ascend: sh run_infer.sh Ascend ~/imagenet/val/ ~/checkpoint/resnet50-110_5004.ckpt +``` + + +> checkpoint can be produced in training process. + +#### Result + +Inference result will be stored in the example path, whose folder name is "infer". Under this, you can find result like the followings in log. + +``` +result: {'acc': 0.75.252054737516005} ckpt=train_parallel0/resnet-110_5004.ckpt +``` + diff --git a/model_zoo/resnet50_quant/eval.py b/model_zoo/resnet50_quant/eval.py new file mode 100755 index 0000000000..481e4bb853 --- /dev/null +++ b/model_zoo/resnet50_quant/eval.py @@ -0,0 +1,78 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Evaluate Resnet50 on ImageNet""" + +import os +import argparse + +from src.config import quant_set, config_quant, config_noquant +from src.dataset import create_dataset +from src.crossentropy import CrossEntropy +from src.utils import _load_param_into_net +from models.resnet_quant import resnet50_quant + +from mindspore import context +from mindspore.train.model import Model +from mindspore.train.serialization import load_checkpoint +from mindspore.train.quant import quant + +parser = argparse.ArgumentParser(description='Image classification') +parser.add_argument('--checkpoint_path', type=str, default=None, help='Checkpoint file path') +parser.add_argument('--dataset_path', type=str, default=None, help='Dataset path') +parser.add_argument('--device_target', type=str, default='Ascend', help='Device target') +args_opt = parser.parse_args() + +context.set_context(mode=context.GRAPH_MODE, device_target=args_opt.device_target, save_graphs=False) +config = config_quant if quant_set.quantization_aware else config_noquant + +if args_opt.device_target == "Ascend": + device_id = int(os.getenv('DEVICE_ID')) + context.set_context(device_id=device_id) + +if __name__ == '__main__': + # define fusion network + net = resnet50_quant(class_num=config.class_num) + if quant_set.quantization_aware: + # convert fusion network to quantization aware network + net = quant.convert_quant_network(net, + bn_fold=True, + per_channel=[True, False], + symmetric=[True, False]) + # define network loss + if not config.use_label_smooth: + config.label_smooth_factor = 0.0 + loss = CrossEntropy(smooth_factor=config.label_smooth_factor, + num_classes=config.class_num) + + # define dataset + dataset = create_dataset(dataset_path=args_opt.dataset_path, + do_train=False, + batch_size=config.batch_size, + target=args_opt.device_target) + step_size = dataset.get_dataset_size() + + # load checkpoint + if args_opt.checkpoint_path: + param_dict = load_checkpoint(args_opt.checkpoint_path) + _load_param_into_net(net, param_dict) + net.set_train(False) + + # define model + model = Model(net, loss_fn=loss, metrics={'acc'}) + + print("============== Starting Validation ==============") + res = model.eval(dataset) + print("result:", res, "ckpt=", args_opt.checkpoint_path) + print("============== End Validation ==============") diff --git a/model_zoo/resnet50_quant/models/resnet_quant.py b/model_zoo/resnet50_quant/models/resnet_quant.py new file mode 100755 index 0000000000..63fa32222d --- /dev/null +++ b/model_zoo/resnet50_quant/models/resnet_quant.py @@ -0,0 +1,251 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""ResNet.""" +import mindspore.nn as nn +from mindspore.ops import operations as P + + +class ConvBNReLU(nn.Cell): + """ + Convolution/Depthwise fused with Batchnorm and ReLU block definition. + + Args: + in_planes (int): Input channel. + out_planes (int): Output channel. + kernel_size (int): Input kernel size. + stride (int): Stride size for the first convolutional layer. Default: 1. + groups (int): channel group. Convolution is 1 while Depthiwse is input channel. Default: 1. + + Returns: + Tensor, output tensor. + + Examples: + >>> ConvBNReLU(16, 256, kernel_size=1, stride=1, groups=1) + """ + + def __init__(self, in_planes, out_planes, kernel_size=3, stride=1, groups=1): + super(ConvBNReLU, self).__init__() + padding = (kernel_size - 1) // 2 + conv = nn.Conv2dBnAct(in_planes, out_planes, kernel_size, stride, pad_mode='pad', padding=padding, + group=groups, has_bn=True, activation='relu') + self.features = conv + + def construct(self, x): + output = self.features(x) + return output + + +class ResidualBlock(nn.Cell): + """ + ResNet V1 residual block definition. + + Args: + in_channel (int): Input channel. + out_channel (int): Output channel. + stride (int): Stride size for the first convolutional layer. Default: 1. + + Returns: + Tensor, output tensor. + + Examples: + >>> ResidualBlock(3, 256, stride=2) + """ + expansion = 4 + + def __init__(self, + in_channel, + out_channel, + stride=1): + super(ResidualBlock, self).__init__() + + channel = out_channel // self.expansion + self.conv1 = ConvBNReLU(in_channel, channel, kernel_size=1, stride=1) + self.conv2 = ConvBNReLU(channel, channel, kernel_size=3, stride=stride) + self.conv3 = nn.Conv2dBnAct(channel, out_channel, kernel_size=1, stride=1, pad_mode='same', padding=0, + has_bn=True, activation='relu') + + self.down_sample = False + if stride != 1 or in_channel != out_channel: + self.down_sample = True + self.down_sample_layer = None + + if self.down_sample: + self.down_sample_layer = nn.Conv2dBnAct(in_channel, out_channel, + kernel_size=1, stride=stride, + pad_mode='same', padding=0, has_bn=True, activation='relu') + self.add = P.TensorAdd() + self.relu = P.ReLU() + + def construct(self, x): + identity = x + out = self.conv1(x) + out = self.conv2(out) + out = self.conv3(out) + + if self.down_sample: + identity = self.down_sample_layer(identity) + + out = self.add(out, identity) + out = self.relu(out) + + return out + + +class ResNet(nn.Cell): + """ + ResNet architecture. + + Args: + block (Cell): Block for network. + layer_nums (list): Numbers of block in different layers. + in_channels (list): Input channel in each layer. + out_channels (list): Output channel in each layer. + strides (list): Stride size in each layer. + num_classes (int): The number of classes that the training images are belonging to. + Returns: + Tensor, output tensor. + + Examples: + >>> ResNet(ResidualBlock, + >>> [3, 4, 6, 3], + >>> [64, 256, 512, 1024], + >>> [256, 512, 1024, 2048], + >>> [1, 2, 2, 2], + >>> 10) + """ + + def __init__(self, + block, + layer_nums, + in_channels, + out_channels, + strides, + num_classes): + super(ResNet, self).__init__() + + if not len(layer_nums) == len(in_channels) == len(out_channels) == 4: + raise ValueError("the length of layer_num, in_channels, out_channels list must be 4!") + + self.conv1 = ConvBNReLU(3, 64, kernel_size=7, stride=2) + self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, pad_mode="same") + + self.layer1 = self._make_layer(block, + layer_nums[0], + in_channel=in_channels[0], + out_channel=out_channels[0], + stride=strides[0]) + self.layer2 = self._make_layer(block, + layer_nums[1], + in_channel=in_channels[1], + out_channel=out_channels[1], + stride=strides[1]) + self.layer3 = self._make_layer(block, + layer_nums[2], + in_channel=in_channels[2], + out_channel=out_channels[2], + stride=strides[2]) + self.layer4 = self._make_layer(block, + layer_nums[3], + in_channel=in_channels[3], + out_channel=out_channels[3], + stride=strides[3]) + + self.mean = P.ReduceMean(keep_dims=True) + self.flatten = nn.Flatten() + self.end_point = nn.DenseBnAct(out_channels[3], num_classes, has_bias=True, has_bn=False) + + def _make_layer(self, block, layer_num, in_channel, out_channel, stride): + """ + Make stage network of ResNet. + + Args: + block (Cell): Resnet block. + layer_num (int): Layer number. + in_channel (int): Input channel. + out_channel (int): Output channel. + stride (int): Stride size for the first convolutional layer. + + Returns: + SequentialCell, the output layer. + + Examples: + >>> _make_layer(ResidualBlock, 3, 128, 256, 2) + """ + layers = [] + + resnet_block = block(in_channel, out_channel, stride=stride) + layers.append(resnet_block) + + for _ in range(1, layer_num): + resnet_block = block(out_channel, out_channel, stride=1) + layers.append(resnet_block) + + return nn.SequentialCell(layers) + + def construct(self, x): + x = self.conv1(x) + c1 = self.maxpool(x) + + c2 = self.layer1(c1) + c3 = self.layer2(c2) + c4 = self.layer3(c3) + c5 = self.layer4(c4) + + out = self.mean(c5, (2, 3)) + out = self.flatten(out) + out = self.end_point(out) + return out + + +def resnet50_quant(class_num=10001): + """ + Get ResNet50 neural network. + + Args: + class_num (int): Class number. + + Returns: + Cell, cell instance of ResNet50 neural network. + + Examples: + >>> net = resnet50_quant(10) + """ + return ResNet(ResidualBlock, + [3, 4, 6, 3], + [64, 256, 512, 1024], + [256, 512, 1024, 2048], + [1, 2, 2, 2], + class_num) + + +def resnet101_quant(class_num=1001): + """ + Get ResNet101 neural network. + + Args: + class_num (int): Class number. + + Returns: + Cell, cell instance of ResNet101 neural network. + + Examples: + >>> net = resnet101(1001) + """ + return ResNet(ResidualBlock, + [3, 4, 23, 3], + [64, 256, 512, 1024], + [256, 512, 1024, 2048], + [1, 2, 2, 2], + class_num) diff --git a/model_zoo/resnet50_quant/scripts/run_infer.sh b/model_zoo/resnet50_quant/scripts/run_infer.sh new file mode 100644 index 0000000000..1d74f63737 --- /dev/null +++ b/model_zoo/resnet50_quant/scripts/run_infer.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +if [ $# != 3 ] +then + echo "Ascend: sh run_infer.sh [PLATFORM] [DATASET_PATH] [CHECKPOINT_PATH]" +exit 1 +fi + +# check dataset path +if [ ! -d $2 ] && [ ! -f $2 ] +then + echo "error: DATASET_PATH=$2 is not a directory or file" +exit 1 +fi + +# check checkpoint file +if [ ! -f $3 ] +then + echo "error: CHECKPOINT_PATH=$3 is not a file" +exit 1 +fi + +# set environment +BASEPATH=$(cd "`dirname $0`" || exit; pwd) +export PYTHONPATH=${BASEPATH}:$PYTHONPATH +export DEVICE_ID=0 +export RANK_ID=0 +export RANK_SIZE=1 +if [ -d "../eval" ]; +then + rm -rf ../eval +fi +mkdir ../eval +cd ../eval || exit + +# luanch +python ${BASEPATH}/../eval.py \ + --device_target=$1 \ + --dataset_path=$2 \ + --checkpoint_path=$3 \ + &> infer.log & # dataset val folder path diff --git a/model_zoo/resnet50_quant/scripts/run_train.sh b/model_zoo/resnet50_quant/scripts/run_train.sh new file mode 100644 index 0000000000..a427201587 --- /dev/null +++ b/model_zoo/resnet50_quant/scripts/run_train.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +run_ascend() +{ + if [ $2 -lt 1 ] && [ $2 -gt 8 ] + then + echo "error: DEVICE_NUM=$2 is not in (1-8)" + exit 1 + fi + + if [ ! -d $5 ] && [ ! -f $5 ] + then + echo "error: DATASET_PATH=$5 is not a directory or file" + exit 1 + fi + + BASEPATH=$(cd "`dirname $0`" || exit; pwd) + export PYTHONPATH=${BASEPATH}:$PYTHONPATH + if [ -d "../train" ]; + then + rm -rf ../train + fi + mkdir ../train + cd ../train || exit + python ${BASEPATH}/../src/launch.py \ + --nproc_per_node=$2 \ + --visible_devices=$4 \ + --server_id=$3 \ + --training_script=${BASEPATH}/../train.py \ + --dataset_path=$5 \ + --pre_trained=$6 \ + --device_target=$1 &> train.log & # dataset train folder +} + +if [ $# -gt 6 ] || [ $# -lt 4 ] +then + echo "Usage:\n \ + Ascend: sh run_train.sh Ascend [DEVICE_NUM] [SERVER_IP(x.x.x.x)] [VISIABLE_DEVICES(0,1,2,3,4,5,6,7)] [DATASET_PATH] [CKPT_PATH]\n \ + " +exit 1 +fi + +if [ $1 = "Ascend" ] ; then + run_ascend "$@" +else + echo "not support platform" +fi; + diff --git a/model_zoo/resnet50_quant/src/config.py b/model_zoo/resnet50_quant/src/config.py new file mode 100755 index 0000000000..523702dc43 --- /dev/null +++ b/model_zoo/resnet50_quant/src/config.py @@ -0,0 +1,68 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +network config setting, will be used in train.py and eval.py +""" +from easydict import EasyDict as ed + +quant_set = ed({ + "quantization_aware": True, +}) +config_noquant = ed({ + "class_num": 1001, + "batch_size": 32, + "loss_scale": 1024, + "momentum": 0.9, + "weight_decay": 1e-4, + "epoch_size": 90, + "pretrained_epoch_size": 1, + "buffer_size": 1000, + "image_height": 224, + "image_width": 224, + "data_load_mode": "mindrecord", + "save_checkpoint": True, + "save_checkpoint_epochs": 1, + "keep_checkpoint_max": 50, + "save_checkpoint_path": "./", + "warmup_epochs": 0, + "lr_decay_mode": "cosine", + "use_label_smooth": True, + "label_smooth_factor": 0.1, + "lr_init": 0, + "lr_max": 0.1, +}) +config_quant = ed({ + "class_num": 1001, + "batch_size": 32, + "loss_scale": 1024, + "momentum": 0.9, + "weight_decay": 1e-4, + "epoch_size": 120, + "pretrained_epoch_size": 90, + "buffer_size": 1000, + "image_height": 224, + "image_width": 224, + "data_load_mode": "mindrecord", + "save_checkpoint": True, + "save_checkpoint_epochs": 1, + "keep_checkpoint_max": 50, + "save_checkpoint_path": "./", + "warmup_epochs": 0, + "lr_decay_mode": "cosine", + "use_label_smooth": True, + "label_smooth_factor": 0.1, + "lr_init": 0, + "lr_max": 0.005, +}) diff --git a/model_zoo/resnet50_quant/src/crossentropy.py b/model_zoo/resnet50_quant/src/crossentropy.py new file mode 100644 index 0000000000..b078b29f6f --- /dev/null +++ b/model_zoo/resnet50_quant/src/crossentropy.py @@ -0,0 +1,39 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""define loss function for network""" +from mindspore.nn.loss.loss import _Loss +from mindspore.ops import operations as P +from mindspore.ops import functional as F +from mindspore import Tensor +from mindspore.common import dtype as mstype +import mindspore.nn as nn + + +class CrossEntropy(_Loss): + """the redefined loss function with SoftmaxCrossEntropyWithLogits""" + + def __init__(self, smooth_factor=0, num_classes=1001): + super(CrossEntropy, self).__init__() + self.onehot = P.OneHot() + self.on_value = Tensor(1.0 - smooth_factor, mstype.float32) + self.off_value = Tensor(1.0 * smooth_factor / (num_classes - 1), mstype.float32) + self.ce = nn.SoftmaxCrossEntropyWithLogits() + self.mean = P.ReduceMean(False) + + def construct(self, logit, label): + one_hot_label = self.onehot(label, F.shape(logit)[1], self.on_value, self.off_value) + loss = self.ce(logit, one_hot_label) + loss = self.mean(loss, 0) + return loss diff --git a/model_zoo/resnet50_quant/src/dataset.py b/model_zoo/resnet50_quant/src/dataset.py new file mode 100755 index 0000000000..73c0780090 --- /dev/null +++ b/model_zoo/resnet50_quant/src/dataset.py @@ -0,0 +1,157 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +create train or eval dataset. +""" +import os +from functools import partial +import mindspore.common.dtype as mstype +import mindspore.dataset.engine as de +import mindspore.dataset.transforms.vision.c_transforms as C +import mindspore.dataset.transforms.c_transforms as C2 +import mindspore.dataset.transforms.vision.py_transforms as P +from mindspore.communication.management import init, get_rank, get_group_size +from src.config import quant_set, config_quant, config_noquant + +config = config_quant if quant_set.quantization_aware else config_noquant + + +def create_dataset(dataset_path, do_train, repeat_num=1, batch_size=32, target="Ascend"): + """ + create a train or eval dataset + + Args: + dataset_path(string): the path of dataset. + do_train(bool): whether dataset is used for train or eval. + repeat_num(int): the repeat times of dataset. Default: 1 + batch_size(int): the batch size of dataset. Default: 32 + target(str): the device target. Default: Ascend + + Returns: + dataset + """ + if target == "Ascend": + device_num = int(os.getenv("RANK_SIZE")) + rank_id = int(os.getenv("RANK_ID")) + else: + init("nccl") + rank_id = get_rank() + device_num = get_group_size() + + columns_list = ['image', 'label'] + if config.data_load_mode == "mindrecord": + load_func = partial(de.MindDataset, dataset_path, columns_list) + else: + load_func = partial(de.ImageFolderDatasetV2, dataset_path) + if device_num == 1: + ds = load_func(num_parallel_workers=8, shuffle=True) + else: + ds = load_func(num_parallel_workers=8, shuffle=True, + num_shards=device_num, shard_id=rank_id) + + image_size = config.image_height + mean = [0.485 * 255, 0.456 * 255, 0.406 * 255] + std = [0.229 * 255, 0.224 * 255, 0.225 * 255] + + # define map operations + if do_train: + trans = [ + C.RandomCropDecodeResize(image_size, scale=(0.08, 1.0), ratio=(0.75, 1.333)), + C.RandomHorizontalFlip(prob=0.5), + C.Normalize(mean=mean, std=std), + C.HWC2CHW() + ] + else: + trans = [ + C.Decode(), + C.Resize(256), + C.CenterCrop(image_size), + C.Normalize(mean=mean, std=std), + C.HWC2CHW() + ] + + type_cast_op = C2.TypeCast(mstype.int32) + + ds = ds.map(input_columns="image", num_parallel_workers=8, operations=trans) + ds = ds.map(input_columns="label", num_parallel_workers=8, operations=type_cast_op) + + # apply batch operations + ds = ds.batch(batch_size, drop_remainder=True) + + # apply dataset repeat operation + ds = ds.repeat(repeat_num) + + return ds + + +def create_dataset_py(dataset_path, do_train, repeat_num=1, batch_size=32, target="Ascend"): + """ + create a train or eval dataset + + Args: + dataset_path(string): the path of dataset. + do_train(bool): whether dataset is used for train or eval. + repeat_num(int): the repeat times of dataset. Default: 1 + batch_size(int): the batch size of dataset. Default: 32 + target(str): the device target. Default: Ascend + + Returns: + dataset + """ + if target == "Ascend": + device_num = int(os.getenv("RANK_SIZE")) + rank_id = int(os.getenv("RANK_ID")) + else: + init("nccl") + rank_id = get_rank() + device_num = get_group_size() + + if do_train: + if device_num == 1: + ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True) + else: + ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True, + num_shards=device_num, shard_id=rank_id) + else: + ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=False) + + image_size = 224 + + # define map operations + decode_op = P.Decode() + resize_crop_op = P.RandomResizedCrop(image_size, scale=(0.08, 1.0), ratio=(0.75, 1.333)) + horizontal_flip_op = P.RandomHorizontalFlip(prob=0.5) + + resize_op = P.Resize(256) + center_crop = P.CenterCrop(image_size) + to_tensor = P.ToTensor() + normalize_op = P.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) + + # define map operations + if do_train: + trans = [decode_op, resize_crop_op, horizontal_flip_op, to_tensor, normalize_op] + else: + trans = [decode_op, resize_op, center_crop, to_tensor, normalize_op] + + compose = P.ComposeOp(trans) + ds = ds.map(input_columns="image", operations=compose(), num_parallel_workers=8, python_multiprocessing=True) + + # apply batch operations + ds = ds.batch(batch_size, drop_remainder=True) + + # apply dataset repeat operation + ds = ds.repeat(repeat_num) + + return ds diff --git a/model_zoo/resnet50_quant/src/launch.py b/model_zoo/resnet50_quant/src/launch.py new file mode 100644 index 0000000000..abba92a540 --- /dev/null +++ b/model_zoo/resnet50_quant/src/launch.py @@ -0,0 +1,165 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""launch train script""" +import os +import sys +import json +import subprocess +import shutil +import platform +from argparse import ArgumentParser + +def parse_args(): + """ + parse args . + + Args: + + Returns: + args. + + Examples: + >>> parse_args() + """ + parser = ArgumentParser(description="mindspore distributed training launch " + "helper utilty that will spawn up " + "multiple distributed processes") + parser.add_argument("--nproc_per_node", type=int, default=1, + help="The number of processes to launch on each node, " + "for D training, this is recommended to be set " + "to the number of D in your system so that " + "each process can be bound to a single D.") + parser.add_argument("--visible_devices", type=str, default="0,1,2,3,4,5,6,7", + help="will use the visible devices sequentially") + parser.add_argument("--server_id", type=str, default="", + help="server ip") + parser.add_argument("--training_script", type=str, + help="The full path to the single D training " + "program/script to be launched in parallel, " + "followed by all the arguments for the " + "training script") + # rest from the training program + args, unknown = parser.parse_known_args() + args.training_script_args = unknown + return args + + +def main(): + print("start", __file__) + args = parse_args() + print(args) + visible_devices = args.visible_devices.split(',') + assert os.path.isfile(args.training_script) + assert len(visible_devices) >= args.nproc_per_node + print('visible_devices:{}'.format(visible_devices)) + if not args.server_id: + print('pleaser input server ip!!!') + exit(0) + print('server_id:{}'.format(args.server_id)) + + # construct hccn_table + hccn_configs = open('/etc/hccn.conf', 'r').readlines() + device_ips = {} + for hccn_item in hccn_configs: + hccn_item = hccn_item.strip() + if hccn_item.startswith('address_'): + device_id, device_ip = hccn_item.split('=') + device_id = device_id.split('_')[1] + device_ips[device_id] = device_ip + print('device_id:{}, device_ip:{}'.format(device_id, device_ip)) + hccn_table = {} + arch = platform.processor() + hccn_table['board_id'] = {'aarch64': '0x002f', 'x86_64': '0x0000'}[arch] + hccn_table['chip_info'] = '910' + hccn_table['deploy_mode'] = 'lab' + hccn_table['group_count'] = '1' + hccn_table['group_list'] = [] + instance_list = [] + usable_dev = '' + for instance_id in range(args.nproc_per_node): + instance = {} + instance['devices'] = [] + device_id = visible_devices[instance_id] + device_ip = device_ips[device_id] + usable_dev += str(device_id) + instance['devices'].append({ + 'device_id': device_id, + 'device_ip': device_ip, + }) + instance['rank_id'] = str(instance_id) + instance['server_id'] = args.server_id + instance_list.append(instance) + hccn_table['group_list'].append({ + 'device_num': str(args.nproc_per_node), + 'server_num': '1', + 'group_name': '', + 'instance_count': str(args.nproc_per_node), + 'instance_list': instance_list, + }) + hccn_table['para_plane_nic_location'] = 'device' + hccn_table['para_plane_nic_name'] = [] + for instance_id in range(args.nproc_per_node): + eth_id = visible_devices[instance_id] + hccn_table['para_plane_nic_name'].append('eth{}'.format(eth_id)) + hccn_table['para_plane_nic_num'] = str(args.nproc_per_node) + hccn_table['status'] = 'completed' + + # save hccn_table to file + table_path = os.getcwd() + if not os.path.exists(table_path): + os.mkdir(table_path) + table_fn = os.path.join(table_path, + 'rank_table_{}p_{}_{}.json'.format(args.nproc_per_node, usable_dev, args.server_id)) + with open(table_fn, 'w') as table_fp: + json.dump(hccn_table, table_fp, indent=4) + sys.stdout.flush() + + # spawn the processes + processes = [] + cmds = [] + log_files = [] + env = os.environ.copy() + env['RANK_SIZE'] = str(args.nproc_per_node) + cur_path = os.getcwd() + for rank_id in range(0, args.nproc_per_node): + os.chdir(cur_path) + device_id = visible_devices[rank_id] + device_dir = os.path.join(cur_path, 'device{}'.format(rank_id)) + env['RANK_ID'] = str(rank_id) + env['DEVICE_ID'] = str(device_id) + if args.nproc_per_node > 1: + env['MINDSPORE_HCCL_CONFIG_PATH'] = table_fn + env['RANK_TABLE_FILE'] = table_fn + if os.path.exists(device_dir): + shutil.rmtree(device_dir) + os.mkdir(device_dir) + os.chdir(device_dir) + cmd = [sys.executable, '-u'] + cmd.append(args.training_script) + cmd.extend(args.training_script_args) + log_file = open('{dir}/log{id}.log'.format(dir=device_dir, id=rank_id), 'w') + process = subprocess.Popen(cmd, stdout=log_file, stderr=log_file, env=env) + processes.append(process) + cmds.append(cmd) + log_files.append(log_file) + for process, cmd, log_file in zip(processes, cmds, log_files): + process.wait() + if process.returncode != 0: + raise subprocess.CalledProcessError(returncode=process, cmd=cmd) + log_file.close() + + +if __name__ == "__main__": + main() diff --git a/model_zoo/resnet50_quant/src/lr_generator.py b/model_zoo/resnet50_quant/src/lr_generator.py new file mode 100755 index 0000000000..4a57be2f01 --- /dev/null +++ b/model_zoo/resnet50_quant/src/lr_generator.py @@ -0,0 +1,87 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""learning rate generator""" +import math +import numpy as np + + +def get_lr(lr_init, lr_end, lr_max, warmup_epochs, total_epochs, steps_per_epoch, lr_decay_mode): + """ + generate learning rate array + + Args: + lr_init(float): init learning rate + lr_end(float): end learning rate + lr_max(float): max learning rate + warmup_epochs(int): number of warmup epochs + total_epochs(int): total epoch of training + steps_per_epoch(int): steps of one epoch + lr_decay_mode(string): learning rate decay mode, including steps, poly, cosine or default + + Returns: + np.array, learning rate array + """ + lr_each_step = [] + total_steps = steps_per_epoch * total_epochs + warmup_steps = steps_per_epoch * warmup_epochs + if lr_decay_mode == 'steps': + decay_epoch_index = [0.3 * total_steps, 0.6 * total_steps, 0.8 * total_steps] + for i in range(total_steps): + if i < decay_epoch_index[0]: + lr = lr_max + elif i < decay_epoch_index[1]: + lr = lr_max * 0.1 + elif i < decay_epoch_index[2]: + lr = lr_max * 0.01 + else: + lr = lr_max * 0.001 + lr_each_step.append(lr) + elif lr_decay_mode == 'poly': + if warmup_steps != 0: + inc_each_step = (float(lr_max) - float(lr_init)) / float(warmup_steps) + else: + inc_each_step = 0 + for i in range(total_steps): + if i < warmup_steps: + lr = float(lr_init) + inc_each_step * float(i) + else: + base = (1.0 - (float(i) - float(warmup_steps)) / (float(total_steps) - float(warmup_steps))) + lr = float(lr_max) * base * base + if lr < 0.0: + lr = 0.0 + lr_each_step.append(lr) + elif lr_decay_mode == 'cosine': + decay_steps = total_steps - warmup_steps + for i in range(total_steps): + if i < warmup_steps: + lr_inc = (float(lr_max) - float(lr_init)) / float(warmup_steps) + lr = float(lr_init) + lr_inc * (i + 1) + else: + linear_decay = (total_steps - i) / decay_steps + cosine_decay = 0.5 * (1 + math.cos(math.pi * 2 * 0.47 * i / decay_steps)) + decayed = linear_decay * cosine_decay + 0.00001 + lr = lr_max * decayed + lr_each_step.append(lr) + else: + for i in range(total_steps): + if i < warmup_steps: + lr = lr_init + (lr_max - lr_init) * i / warmup_steps + else: + lr = lr_max - (lr_max - lr_end) * (i - warmup_steps) / (total_steps - warmup_steps) + lr_each_step.append(lr) + + learning_rate = np.array(lr_each_step).astype(np.float32) + + return learning_rate diff --git a/model_zoo/resnet50_quant/src/utils.py b/model_zoo/resnet50_quant/src/utils.py new file mode 100644 index 0000000000..846fd7b894 --- /dev/null +++ b/model_zoo/resnet50_quant/src/utils.py @@ -0,0 +1,46 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""utils script""" + +def _load_param_into_net(model, params_dict): + """ + load fp32 model parameters to quantization model. + + Args: + model: quantization model + params_dict: f32 param + + Returns: + None + """ + iterable_dict = { + 'weight': iter([item for item in params_dict.items() if item[0].endswith('weight')]), + 'bias': iter([item for item in params_dict.items() if item[0].endswith('bias')]), + 'gamma': iter([item for item in params_dict.items() if item[0].endswith('gamma')]), + 'beta': iter([item for item in params_dict.items() if item[0].endswith('beta')]), + 'moving_mean': iter([item for item in params_dict.items() if item[0].endswith('moving_mean')]), + 'moving_variance': iter( + [item for item in params_dict.items() if item[0].endswith('moving_variance')]), + 'minq': iter([item for item in params_dict.items() if item[0].endswith('minq')]), + 'maxq': iter([item for item in params_dict.items() if item[0].endswith('maxq')]) + } + for name, param in model.parameters_and_names(): + key_name = name.split(".")[-1] + if key_name not in iterable_dict.keys(): + continue + value_param = next(iterable_dict[key_name], None) + if value_param is not None: + param.set_parameter_data(value_param[1].data) + print(f'init model param {name} with checkpoint param {value_param[0]}') diff --git a/model_zoo/resnet50_quant/train.py b/model_zoo/resnet50_quant/train.py new file mode 100755 index 0000000000..b026f97278 --- /dev/null +++ b/model_zoo/resnet50_quant/train.py @@ -0,0 +1,153 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Train Resnet50 on ImageNet""" + +import os +import argparse + +from mindspore import context +from mindspore import Tensor +from mindspore.parallel._auto_parallel_context import auto_parallel_context +from mindspore.nn.optim.momentum import Momentum +from mindspore.train.model import Model, ParallelMode +from mindspore.train.callback import ModelCheckpoint, CheckpointConfig, LossMonitor, TimeMonitor +from mindspore.train.loss_scale_manager import FixedLossScaleManager +from mindspore.train.serialization import load_checkpoint +from mindspore.train.quant import quant +from mindspore.communication.management import init +import mindspore.nn as nn +import mindspore.common.initializer as weight_init + +from models.resnet_quant import resnet50_quant +from src.dataset import create_dataset +from src.lr_generator import get_lr +from src.config import quant_set, config_quant, config_noquant +from src.crossentropy import CrossEntropy +from src.utils import _load_param_into_net + +parser = argparse.ArgumentParser(description='Image classification') +parser.add_argument('--run_distribute', type=bool, default=False, help='Run distribute') +parser.add_argument('--device_num', type=int, default=1, help='Device num.') +parser.add_argument('--dataset_path', type=str, default=None, help='Dataset path') +parser.add_argument('--device_target', type=str, default='Ascend', help='Device target') +parser.add_argument('--pre_trained', type=str, default=None, help='Pertained checkpoint path') +args_opt = parser.parse_args() +config = config_quant if quant_set.quantization_aware else config_noquant + +if args_opt.device_target == "Ascend": + device_id = int(os.getenv('DEVICE_ID')) + rank_id = int(os.getenv('RANK_ID')) + rank_size = int(os.getenv('RANK_SIZE')) + run_distribute = rank_size > 1 + context.set_context(mode=context.GRAPH_MODE, + device_target="Ascend", + save_graphs=False, + device_id=device_id, + enable_auto_mixed_precision=True) +else: + raise ValueError("Unsupported device target.") + +if __name__ == '__main__': + # train on ascend + print("training args: {}".format(args_opt)) + print("training configure: {}".format(config)) + print("parallel args: rank_id {}, device_id {}, rank_size {}".format(rank_id, device_id, rank_size)) + epoch_size = config.epoch_size + + # distribute init + if run_distribute: + context.set_auto_parallel_context(device_num=rank_size, + parallel_mode=ParallelMode.DATA_PARALLEL, + parameter_broadcast=True, + mirror_mean=True) + init() + context.set_auto_parallel_context(device_num=args_opt.device_num, + parallel_mode=ParallelMode.DATA_PARALLEL, + mirror_mean=True) + auto_parallel_context().set_all_reduce_fusion_split_indices([107, 160]) + + # define network + net = resnet50_quant(class_num=config.class_num) + net.set_train(True) + + # weight init and load checkpoint file + if args_opt.pre_trained: + param_dict = load_checkpoint(args_opt.pre_trained) + _load_param_into_net(net, param_dict) + epoch_size = config.epoch_size - config.pretrained_epoch_size + else: + for _, cell in net.cells_and_names(): + if isinstance(cell, nn.Conv2d): + cell.weight.default_input = weight_init.initializer(weight_init.XavierUniform(), + cell.weight.default_input.shape, + cell.weight.default_input.dtype).to_tensor() + if isinstance(cell, nn.Dense): + cell.weight.default_input = weight_init.initializer(weight_init.TruncatedNormal(), + cell.weight.default_input.shape, + cell.weight.default_input.dtype).to_tensor() + if not config.use_label_smooth: + config.label_smooth_factor = 0.0 + loss = CrossEntropy(smooth_factor=config.label_smooth_factor, num_classes=config.class_num) + loss_scale = FixedLossScaleManager(config.loss_scale, drop_overflow_update=False) + + # define dataset + dataset = create_dataset(dataset_path=args_opt.dataset_path, + do_train=True, + repeat_num=epoch_size, + batch_size=config.batch_size, + target=args_opt.device_target) + step_size = dataset.get_dataset_size() + + if quant_set.quantization_aware: + # convert fusion network to quantization aware network + net = quant.convert_quant_network(net, bn_fold=True, per_channel=[True, False], symmetric=[True, False]) + + # get learning rate + lr = get_lr(lr_init=config.lr_init, + lr_end=0.0, + lr_max=config.lr_max, + warmup_epochs=config.warmup_epochs, + total_epochs=config.epoch_size, + steps_per_epoch=step_size, + lr_decay_mode='cosine') + if args_opt.pre_trained: + lr = lr[config.pretrained_epoch_size * step_size:] + lr = Tensor(lr) + + # define optimization + opt = Momentum(filter(lambda x: x.requires_grad, net.get_parameters()), lr, config.momentum, + config.weight_decay, config.loss_scale) + + # define model + if quant_set.quantization_aware: + model = Model(net, loss_fn=loss, optimizer=opt, loss_scale_manager=loss_scale, metrics={'acc'}) + else: + model = Model(net, loss_fn=loss, optimizer=opt, loss_scale_manager=loss_scale, metrics={'acc'}, + amp_level="O2") + + print("============== Starting Training ==============") + time_callback = TimeMonitor(data_size=step_size) + loss_callback = LossMonitor() + callbacks = [time_callback, loss_callback] + if rank_id == 0: + if config.save_checkpoint: + config_ckpt = CheckpointConfig(save_checkpoint_steps=config.save_checkpoint_epochs * step_size, + keep_checkpoint_max=config.keep_checkpoint_max) + ckpt_callback = ModelCheckpoint(prefix="ResNet50", + directory=config.save_checkpoint_path, + config=config_ckpt) + callbacks += [ckpt_callback] + model.train(epoch_size, dataset, callbacks=callbacks) + print("============== End Training ==============") From 44efabe27946891c4d1b83f823c1c54095f1a16c Mon Sep 17 00:00:00 2001 From: shenwei Date: Thu, 2 Jul 2020 09:58:19 +0800 Subject: [PATCH 210/254] fix api document --- mindspore/dataset/engine/datasets.py | 76 ++++++++++++++-------------- 1 file changed, 38 insertions(+), 38 deletions(-) diff --git a/mindspore/dataset/engine/datasets.py b/mindspore/dataset/engine/datasets.py index f2c1642df5..ae0dc6789e 100644 --- a/mindspore/dataset/engine/datasets.py +++ b/mindspore/dataset/engine/datasets.py @@ -4421,23 +4421,7 @@ class CelebADataset(MappableDataset): The generated dataset has two columns ['image', 'attr']. The type of the image tensor is uint8. The attr tensor is uint32 and one hot type. - Args: - dataset_dir (str): Path to the root directory that contains the dataset. - num_parallel_workers (int, optional): Number of workers to read the data (default=value set in the config). - shuffle (bool, optional): Whether to perform shuffle on the dataset (default=None). - dataset_type (str): one of 'all', 'train', 'valid' or 'test'. - sampler (Sampler, optional): Object used to choose samples from the dataset (default=None). - decode (bool, optional): decode the images after reading (default=False). - extensions (list[str], optional): List of file extensions to be - included in the dataset (default=None). - num_samples (int, optional): The number of images to be included in the dataset. - (default=None, all images). - num_shards (int, optional): Number of shards that the dataset should be divided - into (default=None). - shard_id (int, optional): The shard ID within num_shards (default=None). This - argument should be specified only when num_shards is also specified. - - Citation of CelebA dataset. + Citation of CelebA dataset. .. code-block:: @@ -4455,9 +4439,9 @@ class CelebADataset(MappableDataset): bibsource = {dblp computer science bibliography, https://dblp.org}, howpublished = {http://mmlab.ie.cuhk.edu.hk/projects/CelebA.html}, description = {CelebFaces Attributes Dataset (CelebA) is a large-scale face attributes dataset - with more than 200K celebrity images, each with 40 attribute annotations. The - images in this dataset cover large pose variations and background clutter. CelebA - has large diversities, large quantities, and rich annotations, including + with more than 200K celebrity images, each with 40 attribute annotations. + The images in this dataset cover large pose variations and background clutter. + CelebA has large diversities, large quantities, and rich annotations, including * 10,177 number of identities, * 202,599 number of face images, and * 5 landmark locations, 40 binary attributes annotations per image. @@ -4465,6 +4449,22 @@ class CelebADataset(MappableDataset): vision tasks: face attribute recognition, face detection, landmark (or facial part) localization, and face editing & synthesis.} } + + Args: + dataset_dir (str): Path to the root directory that contains the dataset. + num_parallel_workers (int, optional): Number of workers to read the data (default=value set in the config). + shuffle (bool, optional): Whether to perform shuffle on the dataset (default=None). + dataset_type (str): one of 'all', 'train', 'valid' or 'test'. + sampler (Sampler, optional): Object used to choose samples from the dataset (default=None). + decode (bool, optional): decode the images after reading (default=False). + extensions (list[str], optional): List of file extensions to be + included in the dataset (default=None). + num_samples (int, optional): The number of images to be included in the dataset. + (default=None, all images). + num_shards (int, optional): Number of shards that the dataset should be divided + into (default=None). + shard_id (int, optional): The shard ID within num_shards (default=None). This + argument should be specified only when num_shards is also specified. """ @check_celebadataset @@ -4542,6 +4542,24 @@ class CLUEDataset(SourceDataset): models, corpus and leaderboard. Here we bring in classification task of CLUE, which are AFQMC, TNEWS, IFLYTEK, CMNLI, WSC and CSL. + Citation of CLUE dataset. + + .. code-block:: + + @article{CLUEbenchmark, + title = {CLUE: A Chinese Language Understanding Evaluation Benchmark}, + author = {Liang Xu, Xuanwei Zhang, Lu Li, Hai Hu, Chenjie Cao, Weitang Liu, Junyi Li, Yudong Li, + Kai Sun, Yechen Xu, Yiming Cui, Cong Yu, Qianqian Dong, Yin Tian, Dian Yu, Bo Shi, Jun Zeng, + Rongzhao Wang, Weijian Xie, Yanting Li, Yina Patterson, Zuoyu Tian, Yiwen Zhang, He Zhou, + Shaoweihua Liu, Qipeng Zhao, Cong Yue, Xinrui Zhang, Zhengliang Yang, Zhenzhong Lan}, + journal = {arXiv preprint arXiv:2004.05986}, + year = {2020}, + howpublished = {https://github.com/CLUEbenchmark/CLUE}, + description = {CLUE, a Chinese Language Understanding Evaluation benchmark. It contains eight different + tasks, including single-sentence classification, sentence pair classification, and machine + reading comprehension.} + } + Args: dataset_files (str or list[str]): String or list of files to be read or glob strings to search for a pattern of files. The list will be sorted in a lexicographical order. @@ -4564,24 +4582,6 @@ class CLUEDataset(SourceDataset): shard_id (int, optional): The shard ID within num_shards (default=None). This argument should be specified only when num_shards is also specified. - Citation of CLUE dataset. - - .. code-block:: - - @article{CLUEbenchmark, - title = {CLUE: A Chinese Language Understanding Evaluation Benchmark}, - author = {Liang Xu, Xuanwei Zhang, Lu Li, Hai Hu, Chenjie Cao, Weitang Liu, Junyi Li, Yudong Li, - Kai Sun, Yechen Xu, Yiming Cui, Cong Yu, Qianqian Dong, Yin Tian, Dian Yu, Bo Shi, Jun Zeng, - Rongzhao Wang, Weijian Xie, Yanting Li, Yina Patterson, Zuoyu Tian, Yiwen Zhang, He Zhou, - Shaoweihua Liu, Qipeng Zhao, Cong Yue, Xinrui Zhang, Zhengliang Yang, Zhenzhong Lan}, - journal = {arXiv preprint arXiv:2004.05986}, - year = {2020}, - howpublished = {https://github.com/CLUEbenchmark/CLUE}, - description = {CLUE, a Chinese Language Understanding Evaluation benchmark. It contains eight different - tasks, including single-sentence classification, sentence pair classification, and machine - reading comprehension.} - } - Examples: >>> import mindspore.dataset as ds >>> dataset_files = ["/path/to/1", "/path/to/2"] # contains 1 or multiple text files From 128dac7ccfab3df63e4843b79a2cecdce23e8742 Mon Sep 17 00:00:00 2001 From: liuwenhao4 Date: Thu, 11 Jun 2020 17:25:59 +0800 Subject: [PATCH 211/254] Support dynamic broadcast reduce format with operator --- mindspore/ops/_op_impl/tbe/acos.py | 6 ++--- mindspore/ops/_op_impl/tbe/acosh.py | 6 ++--- mindspore/ops/_op_impl/tbe/add.py | 9 ++------ mindspore/ops/_op_impl/tbe/asin.py | 4 ++-- mindspore/ops/_op_impl/tbe/asinh.py | 4 ++-- mindspore/ops/_op_impl/tbe/atan.py | 6 ++--- mindspore/ops/_op_impl/tbe/atan2.py | 6 ++--- mindspore/ops/_op_impl/tbe/atanh.py | 6 ++--- mindspore/ops/_op_impl/tbe/batch_matmul.py | 7 +++--- mindspore/ops/_op_impl/tbe/bessel_i0e.py | 4 ++-- mindspore/ops/_op_impl/tbe/bessel_i1e.py | 4 ++-- mindspore/ops/_op_impl/tbe/bias_add.py | 6 ++--- mindspore/ops/_op_impl/tbe/bitwise_and.py | 5 ++-- mindspore/ops/_op_impl/tbe/bitwise_or.py | 5 ++-- mindspore/ops/_op_impl/tbe/bitwise_xor.py | 5 ++-- .../ops/_op_impl/tbe/bn_training_reduce.py | 4 ++-- .../_op_impl/tbe/bn_training_reduce_grad.py | 8 +++---- .../_op_impl/tbe/bn_training_update_grad.py | 8 +++---- .../ops/_op_impl/tbe/bn_training_update_v2.py | 12 +++++----- mindspore/ops/_op_impl/tbe/ceil.py | 5 ++-- mindspore/ops/_op_impl/tbe/check_valid.py | 5 ++-- mindspore/ops/_op_impl/tbe/clip_by_value.py | 10 ++++---- mindspore/ops/_op_impl/tbe/concat.py | 23 +------------------ mindspore/ops/_op_impl/tbe/conv2d.py | 2 ++ mindspore/ops/_op_impl/tbe/cos.py | 6 ++--- mindspore/ops/_op_impl/tbe/cosh.py | 4 ++-- mindspore/ops/_op_impl/tbe/div.py | 16 +++++-------- mindspore/ops/_op_impl/tbe/div_no_nan.py | 16 +++++-------- mindspore/ops/_op_impl/tbe/dropout_do_mask.py | 3 +-- mindspore/ops/_op_impl/tbe/elu.py | 4 ++-- mindspore/ops/_op_impl/tbe/equal.py | 16 +++++-------- mindspore/ops/_op_impl/tbe/erf.py | 4 ++-- mindspore/ops/_op_impl/tbe/erfc.py | 4 ++-- mindspore/ops/_op_impl/tbe/exp.py | 7 +++--- mindspore/ops/_op_impl/tbe/expm1.py | 4 ++-- mindspore/ops/_op_impl/tbe/floor.py | 5 ++-- mindspore/ops/_op_impl/tbe/floor_div.py | 11 +++++---- mindspore/ops/_op_impl/tbe/floor_mod.py | 7 +++--- mindspore/ops/_op_impl/tbe/fused_mul_add.py | 16 +------------ mindspore/ops/_op_impl/tbe/gelu.py | 11 +++------ mindspore/ops/_op_impl/tbe/greater.py | 16 +++++-------- mindspore/ops/_op_impl/tbe/greater_equal.py | 16 +++++-------- mindspore/ops/_op_impl/tbe/inv.py | 7 +++--- mindspore/ops/_op_impl/tbe/inv_grad.py | 9 ++++---- mindspore/ops/_op_impl/tbe/invert.py | 5 ++-- mindspore/ops/_op_impl/tbe/layer_norm.py | 16 ++++--------- .../tbe/layer_norm_beta_gamma_backprop.py | 16 ++++--------- .../ops/_op_impl/tbe/layer_norm_x_backprop.py | 16 ++++--------- mindspore/ops/_op_impl/tbe/less.py | 16 +++++-------- mindspore/ops/_op_impl/tbe/less_equal.py | 16 +++++-------- mindspore/ops/_op_impl/tbe/log.py | 7 +++--- mindspore/ops/_op_impl/tbe/log1p.py | 7 +++--- mindspore/ops/_op_impl/tbe/logical_and.py | 6 ++--- mindspore/ops/_op_impl/tbe/logical_not.py | 6 ++--- mindspore/ops/_op_impl/tbe/logical_or.py | 6 ++--- mindspore/ops/_op_impl/tbe/maximum.py | 10 ++++---- mindspore/ops/_op_impl/tbe/maximum_grad.py | 19 ++++++--------- mindspore/ops/_op_impl/tbe/minimum.py | 10 ++++---- mindspore/ops/_op_impl/tbe/minimum_grad.py | 19 ++++++--------- mindspore/ops/_op_impl/tbe/neg.py | 12 ++++------ mindspore/ops/_op_impl/tbe/not_equal.py | 16 +++++-------- mindspore/ops/_op_impl/tbe/ones_like.py | 11 +++++---- mindspore/ops/_op_impl/tbe/pow.py | 11 +++++---- mindspore/ops/_op_impl/tbe/reciprocal.py | 7 +----- mindspore/ops/_op_impl/tbe/reduce_all.py | 4 ++-- mindspore/ops/_op_impl/tbe/reduce_max.py | 13 ++++++----- mindspore/ops/_op_impl/tbe/reduce_min.py | 13 ++++------- mindspore/ops/_op_impl/tbe/reduce_prod.py | 13 ++++------- mindspore/ops/_op_impl/tbe/reduce_sum.py | 5 ++-- mindspore/ops/_op_impl/tbe/relu.py | 14 ++++------- mindspore/ops/_op_impl/tbe/relu6.py | 10 ++++---- mindspore/ops/_op_impl/tbe/relu_grad.py | 16 +++++-------- mindspore/ops/_op_impl/tbe/round.py | 10 ++++---- mindspore/ops/_op_impl/tbe/rsqrt.py | 11 +++------ mindspore/ops/_op_impl/tbe/select.py | 11 +-------- mindspore/ops/_op_impl/tbe/sigmoid.py | 13 +++-------- mindspore/ops/_op_impl/tbe/sign.py | 6 ++--- mindspore/ops/_op_impl/tbe/sin.py | 6 ++--- mindspore/ops/_op_impl/tbe/sinh.py | 4 ++-- .../ops/_op_impl/tbe/softmax_grad_ext.py | 10 ++------ mindspore/ops/_op_impl/tbe/softplus.py | 4 ++-- mindspore/ops/_op_impl/tbe/softplus_grad.py | 4 ++-- mindspore/ops/_op_impl/tbe/split_d.py | 23 +------------------ mindspore/ops/_op_impl/tbe/sqrt.py | 9 +++----- mindspore/ops/_op_impl/tbe/square.py | 13 ++++------- mindspore/ops/_op_impl/tbe/sub.py | 10 ++++---- mindspore/ops/_op_impl/tbe/tanh.py | 9 +++----- .../ops/_op_impl/tbe/unsorted_segment_sum.py | 11 +-------- 88 files changed, 297 insertions(+), 519 deletions(-) diff --git a/mindspore/ops/_op_impl/tbe/acos.py b/mindspore/ops/_op_impl/tbe/acos.py index 98516f4496..974276ca20 100644 --- a/mindspore/ops/_op_impl/tbe/acos.py +++ b/mindspore/ops/_op_impl/tbe/acos.py @@ -26,10 +26,8 @@ acos_op_info = TBERegOp("ACos") \ .op_pattern("formatAgnostic") \ .input(0, "x", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD) \ + .dtype_format(DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/acosh.py b/mindspore/ops/_op_impl/tbe/acosh.py index 0bf8755bc0..bbc35f2533 100644 --- a/mindspore/ops/_op_impl/tbe/acosh.py +++ b/mindspore/ops/_op_impl/tbe/acosh.py @@ -26,10 +26,8 @@ acosh_op_info = TBERegOp("Acosh") \ .op_pattern("formatAgnostic") \ .input(0, "x", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD) \ + .dtype_format(DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/add.py b/mindspore/ops/_op_impl/tbe/add.py index d3db3de0ad..01eb6e881e 100644 --- a/mindspore/ops/_op_impl/tbe/add.py +++ b/mindspore/ops/_op_impl/tbe/add.py @@ -23,16 +23,11 @@ add_op_info = TBERegOp("Add") \ .compute_cost(10) \ .kernel_name("add") \ .partial_flag(True) \ + .op_pattern("dynamicFormat") \ .input(0, "x1", False, "required", "all") \ .input(1, "x2", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .op_pattern("dynamicFormat") \ - .dtype_format(DataType.I32_Default, DataType.I32_Default, DataType.I32_Default) \ - .dtype_format(DataType.I32_5HD, DataType.I32_5HD, DataType.I32_5HD) \ - .dtype_format(DataType.F16_Default, DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD, DataType.F16_5HD) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default, DataType.F32_Default) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD, DataType.F32_5HD) \ + .dtype_format(DataType.None_None, DataType.None_None, DataType.None_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/asin.py b/mindspore/ops/_op_impl/tbe/asin.py index b4e3dbcca3..7a6bdfa087 100644 --- a/mindspore/ops/_op_impl/tbe/asin.py +++ b/mindspore/ops/_op_impl/tbe/asin.py @@ -26,8 +26,8 @@ asin_op_info = TBERegOp("Asin") \ .op_pattern("formatAgnostic") \ .input(0, "x", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD) \ + .dtype_format(DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/asinh.py b/mindspore/ops/_op_impl/tbe/asinh.py index 1720f48638..5702b865ad 100644 --- a/mindspore/ops/_op_impl/tbe/asinh.py +++ b/mindspore/ops/_op_impl/tbe/asinh.py @@ -26,8 +26,8 @@ asinh_op_info = TBERegOp("Asinh") \ .op_pattern("formatAgnostic") \ .input(0, "x", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD) \ + .dtype_format(DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/atan.py b/mindspore/ops/_op_impl/tbe/atan.py index 293839eaf0..6dcbce0d70 100644 --- a/mindspore/ops/_op_impl/tbe/atan.py +++ b/mindspore/ops/_op_impl/tbe/atan.py @@ -26,10 +26,8 @@ atan_op_info = TBERegOp("Atan") \ .op_pattern("formatAgnostic") \ .input(0, "x", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD) \ + .dtype_format(DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/atan2.py b/mindspore/ops/_op_impl/tbe/atan2.py index 26ffdcb59a..f6f11af10c 100644 --- a/mindspore/ops/_op_impl/tbe/atan2.py +++ b/mindspore/ops/_op_impl/tbe/atan2.py @@ -27,10 +27,8 @@ atan2_op_info = TBERegOp("Atan2") \ .input(0, "x1", False, "required", "all") \ .input(1, "x2", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.F16_Default, DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD, DataType.F16_5HD) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default, DataType.F32_Default) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD, DataType.F32_5HD) \ + .dtype_format(DataType.F16_None, DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/atanh.py b/mindspore/ops/_op_impl/tbe/atanh.py index f60b01967c..d5de93a7bd 100644 --- a/mindspore/ops/_op_impl/tbe/atanh.py +++ b/mindspore/ops/_op_impl/tbe/atanh.py @@ -26,10 +26,8 @@ atanh_op_info = TBERegOp("Atanh") \ .op_pattern("formatAgnostic") \ .input(0, "x", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD) \ + .dtype_format(DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/batch_matmul.py b/mindspore/ops/_op_impl/tbe/batch_matmul.py index 02f2dd5880..fd1935d369 100644 --- a/mindspore/ops/_op_impl/tbe/batch_matmul.py +++ b/mindspore/ops/_op_impl/tbe/batch_matmul.py @@ -30,10 +30,9 @@ batch_matmul_op_info = TBERegOp("BatchMatMul") \ .input(2, "bias", False, "optional", "all") \ .output(0, "y", False, "required", "all") \ .op_pattern("dynamicFormat") \ - .dtype_format(DataType.I32_Default, DataType.I32_Default, DataType.I32_Default, DataType.I32_Default) \ - .dtype_format(DataType.F16_Default, DataType.F16_Default, DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F16_FracNZ, DataType.F16_FracNZ, DataType.F16_Default, DataType.F16_FracNZ) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default, DataType.F32_Default, DataType.F32_Default) \ + .dtype_format(DataType.I32_None, DataType.I32_None, DataType.I32_None, DataType.I32_None) \ + .dtype_format(DataType.F16_None, DataType.F16_None, DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None, DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/bessel_i0e.py b/mindspore/ops/_op_impl/tbe/bessel_i0e.py index ad0030d93a..d1d05d43cb 100644 --- a/mindspore/ops/_op_impl/tbe/bessel_i0e.py +++ b/mindspore/ops/_op_impl/tbe/bessel_i0e.py @@ -26,8 +26,8 @@ bessel_i0e_op_info = TBERegOp("BesselI0e") \ .op_pattern("formatAgnostic") \ .input(0, "x", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD) \ + .dtype_format(DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/bessel_i1e.py b/mindspore/ops/_op_impl/tbe/bessel_i1e.py index 39abb5dad4..a9bc270d0f 100644 --- a/mindspore/ops/_op_impl/tbe/bessel_i1e.py +++ b/mindspore/ops/_op_impl/tbe/bessel_i1e.py @@ -26,8 +26,8 @@ bessel_i1e_op_info = TBERegOp("BesselI1e") \ .op_pattern("formatAgnostic") \ .input(0, "x", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD) \ + .dtype_format(DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/bias_add.py b/mindspore/ops/_op_impl/tbe/bias_add.py index 5ab1916299..8fe792d2ff 100644 --- a/mindspore/ops/_op_impl/tbe/bias_add.py +++ b/mindspore/ops/_op_impl/tbe/bias_add.py @@ -28,9 +28,9 @@ bias_add_grad_op_info = TBERegOp("BiasAdd") \ .input(1, "bias", False, "required", "all") \ .output(0, "y", False, "required", "all") \ .op_pattern("dynamicFormat") \ - .dtype_format(DataType.I32_Default, DataType.I32_Default, DataType.I32_Default) \ - .dtype_format(DataType.F16_Default, DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default, DataType.F32_Default) \ + .dtype_format(DataType.I32_None, DataType.I32_None, DataType.I32_None) \ + .dtype_format(DataType.F16_None, DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/bitwise_and.py b/mindspore/ops/_op_impl/tbe/bitwise_and.py index 30a79e5243..332cf32ee1 100644 --- a/mindspore/ops/_op_impl/tbe/bitwise_and.py +++ b/mindspore/ops/_op_impl/tbe/bitwise_and.py @@ -26,8 +26,9 @@ bitwise_and_op_info = TBERegOp("BitwiseAnd") \ .input(0, "x1", False, "required", "all") \ .input(1, "x2", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.I16_Default, DataType.I16_Default, DataType.I16_Default) \ - .dtype_format(DataType.U16_Default, DataType.U16_Default, DataType.U16_Default) \ + .op_pattern("broadcast") \ + .dtype_format(DataType.I16_None, DataType.I16_None, DataType.I16_None) \ + .dtype_format(DataType.U16_None, DataType.U16_None, DataType.U16_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/bitwise_or.py b/mindspore/ops/_op_impl/tbe/bitwise_or.py index a80bc908b0..6af6a03eb1 100644 --- a/mindspore/ops/_op_impl/tbe/bitwise_or.py +++ b/mindspore/ops/_op_impl/tbe/bitwise_or.py @@ -26,8 +26,9 @@ bitwise_or_op_info = TBERegOp("BitwiseOr") \ .input(0, "x1", False, "required", "all") \ .input(1, "x2", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.I16_Default, DataType.I16_Default, DataType.I16_Default) \ - .dtype_format(DataType.U16_Default, DataType.U16_Default, DataType.U16_Default) \ + .op_pattern("broadcast") \ + .dtype_format(DataType.I16_None, DataType.I16_None, DataType.I16_None) \ + .dtype_format(DataType.U16_None, DataType.U16_None, DataType.U16_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/bitwise_xor.py b/mindspore/ops/_op_impl/tbe/bitwise_xor.py index c606877ff6..8073d13e62 100644 --- a/mindspore/ops/_op_impl/tbe/bitwise_xor.py +++ b/mindspore/ops/_op_impl/tbe/bitwise_xor.py @@ -26,8 +26,9 @@ bitwise_xor_op_info = TBERegOp("BitwiseXor") \ .input(0, "x1", False, "required", "all") \ .input(1, "x2", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.I16_Default, DataType.I16_Default, DataType.I16_Default) \ - .dtype_format(DataType.U16_Default, DataType.U16_Default, DataType.U16_Default) \ + .op_pattern("broadcast") \ + .dtype_format(DataType.I16_None, DataType.I16_None, DataType.I16_None) \ + .dtype_format(DataType.U16_None, DataType.U16_None, DataType.U16_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/bn_training_reduce.py b/mindspore/ops/_op_impl/tbe/bn_training_reduce.py index f33cba2110..0982202d2b 100644 --- a/mindspore/ops/_op_impl/tbe/bn_training_reduce.py +++ b/mindspore/ops/_op_impl/tbe/bn_training_reduce.py @@ -27,8 +27,8 @@ bn_training_reduce_op_info = TBERegOp("BNTrainingReduce") \ .output(0, "sum", False, "required", "all") \ .output(1, "square_sum", False, "required", "all") \ .op_pattern("dynamicFormat") \ - .dtype_format(DataType.F16_5HD, DataType.F32_5HD, DataType.F32_5HD) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD, DataType.F32_5HD) \ + .dtype_format(DataType.F16_None, DataType.F32_None, DataType.F32_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/bn_training_reduce_grad.py b/mindspore/ops/_op_impl/tbe/bn_training_reduce_grad.py index 89736a0097..9e39a0325d 100644 --- a/mindspore/ops/_op_impl/tbe/bn_training_reduce_grad.py +++ b/mindspore/ops/_op_impl/tbe/bn_training_reduce_grad.py @@ -33,10 +33,10 @@ bn_training_reduce_grad_op_info = TBERegOp("BNTrainingReduceGrad") \ .input(6, "batch_variance", False, "required", "all") \ .output(0, "y", False, "required", "all", reshape_type="NC") \ .op_pattern("dynamicFormat") \ - .dtype_format(DataType.F16_5HD, DataType.F32_5HD, DataType.F32_5HD, DataType.F32_5HD, - DataType.F32_5HD, DataType.F32_5HD, DataType.F32_5HD, DataType.F16_5HD) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD, DataType.F32_5HD, DataType.F32_5HD, - DataType.F32_5HD, DataType.F32_5HD, DataType.F32_5HD, DataType.F32_5HD) \ + .dtype_format(DataType.F16_None, DataType.F32_None, DataType.F32_None, DataType.F32_None, + DataType.F32_None, DataType.F32_None, DataType.F32_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None, DataType.F32_None, DataType.F32_None, + DataType.F32_None, DataType.F32_None, DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/bn_training_update_grad.py b/mindspore/ops/_op_impl/tbe/bn_training_update_grad.py index 1aa822a3c1..3739e9cfaf 100644 --- a/mindspore/ops/_op_impl/tbe/bn_training_update_grad.py +++ b/mindspore/ops/_op_impl/tbe/bn_training_update_grad.py @@ -31,10 +31,10 @@ bn_training_update_grad_op_info = TBERegOp("BNTrainingUpdateGrad") \ .output(0, "diff_scale", False, "required", "all") \ .output(1, "diff_offset", False, "required", "all") \ .op_pattern("dynamicFormat") \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD, DataType.F32_5HD, DataType.F32_5HD, - DataType.F32_5HD, DataType.F32_5HD) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD, DataType.F32_5HD, DataType.F32_5HD, - DataType.F32_5HD, DataType.F32_5HD) \ + .dtype_format(DataType.F16_None, DataType.F16_None, DataType.F32_None, DataType.F32_None, + DataType.F32_None, DataType.F32_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None, DataType.F32_None, DataType.F32_None, + DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/bn_training_update_v2.py b/mindspore/ops/_op_impl/tbe/bn_training_update_v2.py index a54d91a483..1a0f65c6e5 100644 --- a/mindspore/ops/_op_impl/tbe/bn_training_update_v2.py +++ b/mindspore/ops/_op_impl/tbe/bn_training_update_v2.py @@ -33,12 +33,12 @@ bn_training_update_v2_op_info = TBERegOp("BNTrainingUpdateV2") \ .output(1, "batch_mean", False, "required", "all") \ .output(2, "batch_variance", False, "required", "all") \ .op_pattern("dynamicFormat") \ - .dtype_format(DataType.F16_5HD, DataType.F32_5HD, DataType.F32_5HD, - DataType.F32_5HD, DataType.F32_5HD, DataType.F16_5HD, - DataType.F32_5HD, DataType.F32_5HD) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD, DataType.F32_5HD, - DataType.F32_5HD, DataType.F32_5HD, DataType.F32_5HD, - DataType.F32_5HD, DataType.F32_5HD) \ + .dtype_format(DataType.F16_None, DataType.F32_None, DataType.F32_None, + DataType.F32_None, DataType.F32_None, DataType.F16_None, + DataType.F32_None, DataType.F32_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None, DataType.F32_None, + DataType.F32_None, DataType.F32_None, DataType.F32_None, + DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/ceil.py b/mindspore/ops/_op_impl/tbe/ceil.py index d9a127603f..769d1738da 100644 --- a/mindspore/ops/_op_impl/tbe/ceil.py +++ b/mindspore/ops/_op_impl/tbe/ceil.py @@ -25,8 +25,9 @@ ceil_op_info = TBERegOp("Ceil") \ .partial_flag(True) \ .input(0, "x", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default) \ + .op_pattern("formatAgnostic") \ + .dtype_format(DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/check_valid.py b/mindspore/ops/_op_impl/tbe/check_valid.py index 9c489b64c5..c5e218b049 100644 --- a/mindspore/ops/_op_impl/tbe/check_valid.py +++ b/mindspore/ops/_op_impl/tbe/check_valid.py @@ -26,8 +26,9 @@ check_valid_op_info = TBERegOp("CheckValid") \ .input(0, "bbox_tensor", False, "required", "all") \ .input(1, "img_tas", False, "required", "all") \ .output(0, "valid_tensor", False, "required", "all") \ - .dtype_format(DataType.F16_Default, DataType.F16_Default, DataType.I8_Default) \ - .dtype_format(DataType.F16_Default, DataType.F16_Default, DataType.BOOL_Default) \ + .op_pattern("broadcast") \ + .dtype_format(DataType.F16_None, DataType.F16_None, DataType.I8_None) \ + .dtype_format(DataType.F16_None, DataType.F16_None, DataType.BOOL_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/clip_by_value.py b/mindspore/ops/_op_impl/tbe/clip_by_value.py index 4ddc6c3c0f..3b51c14160 100644 --- a/mindspore/ops/_op_impl/tbe/clip_by_value.py +++ b/mindspore/ops/_op_impl/tbe/clip_by_value.py @@ -28,12 +28,10 @@ clip_by_value_op_info = TBERegOp("ClipByValue") \ .input(1, "clip_value_min", False, "required", "all") \ .input(2, "clip_value_max", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.I32_Default, DataType.I32_Default, DataType.I32_Default, DataType.I32_Default) \ - .dtype_format(DataType.I32_5HD, DataType.I32_5HD, DataType.I32_5HD, DataType.I32_5HD) \ - .dtype_format(DataType.F16_Default, DataType.F16_Default, DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD, DataType.F16_5HD, DataType.F16_5HD) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default, DataType.F32_Default, DataType.F32_Default) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD, DataType.F32_5HD, DataType.F32_5HD) \ + .op_pattern("broadcast") \ + .dtype_format(DataType.I32_None, DataType.I32_None, DataType.I32_None, DataType.I32_None) \ + .dtype_format(DataType.F16_None, DataType.F16_None, DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None, DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/concat.py b/mindspore/ops/_op_impl/tbe/concat.py index 0bf636016f..af8a6e24d8 100644 --- a/mindspore/ops/_op_impl/tbe/concat.py +++ b/mindspore/ops/_op_impl/tbe/concat.py @@ -27,28 +27,7 @@ concat_op_info = TBERegOp("Concat") \ .input(0, "input_values", False, "dynamic", "all") \ .output(0, "output_data", False, "required", "all") \ .op_pattern("dynamicFormat") \ - .dtype_format(DataType.BOOL_Default, DataType.BOOL_Default) \ - .dtype_format(DataType.BOOL_5HD, DataType.BOOL_5HD) \ - .dtype_format(DataType.I8_Default, DataType.I8_Default) \ - .dtype_format(DataType.I8_5HD, DataType.I8_5HD) \ - .dtype_format(DataType.U8_Default, DataType.U8_Default) \ - .dtype_format(DataType.U8_5HD, DataType.U8_5HD) \ - .dtype_format(DataType.I16_Default, DataType.I16_Default) \ - .dtype_format(DataType.I16_5HD, DataType.I16_5HD) \ - .dtype_format(DataType.U16_Default, DataType.U16_Default) \ - .dtype_format(DataType.U16_5HD, DataType.U16_5HD) \ - .dtype_format(DataType.I32_Default, DataType.I32_Default) \ - .dtype_format(DataType.I32_5HD, DataType.I32_5HD) \ - .dtype_format(DataType.U32_Default, DataType.U32_Default) \ - .dtype_format(DataType.U32_5HD, DataType.U32_5HD) \ - .dtype_format(DataType.I64_Default, DataType.I64_Default) \ - .dtype_format(DataType.I64_5HD, DataType.I64_5HD) \ - .dtype_format(DataType.U64_Default, DataType.U64_Default) \ - .dtype_format(DataType.U64_5HD, DataType.U64_5HD) \ - .dtype_format(DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD) \ + .dtype_format(DataType.None_None, DataType.None_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/conv2d.py b/mindspore/ops/_op_impl/tbe/conv2d.py index a2879d521a..5531b573c3 100644 --- a/mindspore/ops/_op_impl/tbe/conv2d.py +++ b/mindspore/ops/_op_impl/tbe/conv2d.py @@ -33,7 +33,9 @@ conv2d_op_info = TBERegOp("Conv2D") \ .input(2, "bias", False, "optional", "all") \ .input(3, "offset_w", False, "optional", "all") \ .output(0, "y", True, "required", "all") \ + .op_pattern("dynamicFormat") \ .dtype_format(DataType.F16_None, DataType.F16_None, DataType.F16_None, DataType.I8_None, DataType.F16_None) \ + .dtype_format(DataType.I8_None, DataType.I8_None, DataType.I32_None, DataType.I8_None, DataType.I32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/cos.py b/mindspore/ops/_op_impl/tbe/cos.py index 3acb0c2a7e..4d8d8047aa 100644 --- a/mindspore/ops/_op_impl/tbe/cos.py +++ b/mindspore/ops/_op_impl/tbe/cos.py @@ -26,10 +26,8 @@ cos_op_info = TBERegOp("Cos") \ .op_pattern("formatAgnostic") \ .input(0, "x", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD) \ + .dtype_format(DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/cosh.py b/mindspore/ops/_op_impl/tbe/cosh.py index 75d48293e9..b6710669d9 100644 --- a/mindspore/ops/_op_impl/tbe/cosh.py +++ b/mindspore/ops/_op_impl/tbe/cosh.py @@ -26,8 +26,8 @@ cosh_op_info = TBERegOp("Cosh") \ .op_pattern("formatAgnostic") \ .input(0, "x", False, "required", "all") \ .output(0, "y", True, "required", "all") \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD) \ + .dtype_format(DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/div.py b/mindspore/ops/_op_impl/tbe/div.py index 2a83745399..de36df902f 100644 --- a/mindspore/ops/_op_impl/tbe/div.py +++ b/mindspore/ops/_op_impl/tbe/div.py @@ -26,16 +26,12 @@ div_op_info = TBERegOp("Div") \ .input(0, "x1", False, "required", "all") \ .input(1, "x2", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.I8_Default, DataType.I8_Default, DataType.I8_Default) \ - .dtype_format(DataType.I8_5HD, DataType.I8_5HD, DataType.I8_5HD) \ - .dtype_format(DataType.U8_Default, DataType.U8_Default, DataType.U8_Default) \ - .dtype_format(DataType.U8_5HD, DataType.U8_5HD, DataType.U8_5HD) \ - .dtype_format(DataType.I32_Default, DataType.I32_Default, DataType.I32_Default) \ - .dtype_format(DataType.I32_5HD, DataType.I32_5HD, DataType.I32_5HD) \ - .dtype_format(DataType.F16_Default, DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD, DataType.F16_5HD) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default, DataType.F32_Default) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD, DataType.F32_5HD) \ + .op_pattern("broadcast") \ + .dtype_format(DataType.I8_None, DataType.I8_None, DataType.I8_None) \ + .dtype_format(DataType.U8_None, DataType.U8_None, DataType.U8_None) \ + .dtype_format(DataType.I32_None, DataType.I32_None, DataType.I32_None) \ + .dtype_format(DataType.F16_None, DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/div_no_nan.py b/mindspore/ops/_op_impl/tbe/div_no_nan.py index 893b38042e..431c1303b0 100644 --- a/mindspore/ops/_op_impl/tbe/div_no_nan.py +++ b/mindspore/ops/_op_impl/tbe/div_no_nan.py @@ -26,16 +26,12 @@ div_no_nan_op_info = TBERegOp("DivNoNan") \ .input(0, "x1", False, "required", "all") \ .input(1, "x2", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.I8_Default, DataType.I8_Default, DataType.I8_Default) \ - .dtype_format(DataType.I8_5HD, DataType.I8_5HD, DataType.I8_5HD) \ - .dtype_format(DataType.U8_Default, DataType.U8_Default, DataType.U8_Default) \ - .dtype_format(DataType.U8_5HD, DataType.U8_5HD, DataType.U8_5HD) \ - .dtype_format(DataType.I32_Default, DataType.I32_Default, DataType.I32_Default) \ - .dtype_format(DataType.I32_5HD, DataType.I32_5HD, DataType.I32_5HD) \ - .dtype_format(DataType.F16_Default, DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD, DataType.F16_5HD) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default, DataType.F32_Default) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD, DataType.F32_5HD) \ + .op_pattern("broadcast") \ + .dtype_format(DataType.I8_None, DataType.I8_None, DataType.I8_None) \ + .dtype_format(DataType.U8_None, DataType.U8_None, DataType.U8_None) \ + .dtype_format(DataType.I32_None, DataType.I32_None, DataType.I32_None) \ + .dtype_format(DataType.F16_None, DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/dropout_do_mask.py b/mindspore/ops/_op_impl/tbe/dropout_do_mask.py index a24e02f964..0b5a5e87f4 100644 --- a/mindspore/ops/_op_impl/tbe/dropout_do_mask.py +++ b/mindspore/ops/_op_impl/tbe/dropout_do_mask.py @@ -28,8 +28,7 @@ drop_out_do_mask_op_info = TBERegOp("DropoutDoMask") \ .input(2, "keep_prob", False, "required", "all") \ .output(0, "y", False, "required", "all") \ .op_pattern("dynamicFormat") \ - .dtype_format(DataType.F16_Default, DataType.U8_Default, DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F32_Default, DataType.U8_Default, DataType.F32_Default, DataType.F32_Default) \ + .dtype_format(DataType.None_None, DataType.None_None, DataType.None_None, DataType.None_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/elu.py b/mindspore/ops/_op_impl/tbe/elu.py index e61e2851af..75e9d3e889 100644 --- a/mindspore/ops/_op_impl/tbe/elu.py +++ b/mindspore/ops/_op_impl/tbe/elu.py @@ -27,8 +27,8 @@ elu_op_info = TBERegOp("Elu") \ .attr("alpha", "optional", "float", "all", "1.0") \ .input(0, "x", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default) \ + .dtype_format(DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/equal.py b/mindspore/ops/_op_impl/tbe/equal.py index 594fb51cb5..aabb51edad 100644 --- a/mindspore/ops/_op_impl/tbe/equal.py +++ b/mindspore/ops/_op_impl/tbe/equal.py @@ -26,16 +26,12 @@ equal_op_info = TBERegOp("Equal") \ .input(0, "x1", False, "required", "all") \ .input(1, "x2", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.I8_Default, DataType.I8_Default, DataType.BOOL_Default) \ - .dtype_format(DataType.I8_5HD, DataType.I8_5HD, DataType.BOOL_5HD) \ - .dtype_format(DataType.U8_Default, DataType.U8_Default, DataType.BOOL_Default) \ - .dtype_format(DataType.U8_5HD, DataType.U8_5HD, DataType.BOOL_5HD) \ - .dtype_format(DataType.I32_Default, DataType.I32_Default, DataType.BOOL_Default) \ - .dtype_format(DataType.I32_5HD, DataType.I32_5HD, DataType.BOOL_5HD) \ - .dtype_format(DataType.F16_Default, DataType.F16_Default, DataType.BOOL_Default) \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD, DataType.BOOL_5HD) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default, DataType.BOOL_Default) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD, DataType.BOOL_5HD) \ + .op_pattern("broadcast") \ + .dtype_format(DataType.I8_None, DataType.I8_None, DataType.BOOL_None) \ + .dtype_format(DataType.U8_None, DataType.U8_None, DataType.BOOL_None) \ + .dtype_format(DataType.I32_None, DataType.I32_None, DataType.BOOL_None) \ + .dtype_format(DataType.F16_None, DataType.F16_None, DataType.BOOL_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None, DataType.BOOL_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/erf.py b/mindspore/ops/_op_impl/tbe/erf.py index 4c4893d505..ff34a4db14 100644 --- a/mindspore/ops/_op_impl/tbe/erf.py +++ b/mindspore/ops/_op_impl/tbe/erf.py @@ -26,8 +26,8 @@ erf_op_info = TBERegOp("Erf") \ .op_pattern("formatAgnostic") \ .input(0, "x", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default) \ + .dtype_format(DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/erfc.py b/mindspore/ops/_op_impl/tbe/erfc.py index 7b0eccf52e..93d9dec070 100644 --- a/mindspore/ops/_op_impl/tbe/erfc.py +++ b/mindspore/ops/_op_impl/tbe/erfc.py @@ -26,8 +26,8 @@ erfc_op_info = TBERegOp("Erfc") \ .op_pattern("formatAgnostic") \ .input(0, "x", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default) \ + .dtype_format(DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/exp.py b/mindspore/ops/_op_impl/tbe/exp.py index 545845a3b0..9448399866 100644 --- a/mindspore/ops/_op_impl/tbe/exp.py +++ b/mindspore/ops/_op_impl/tbe/exp.py @@ -25,10 +25,9 @@ exp_op_info = TBERegOp("Exp") \ .partial_flag(True) \ .input(0, "x", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD) \ + .op_pattern("formatAgnostic") \ + .dtype_format(DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/expm1.py b/mindspore/ops/_op_impl/tbe/expm1.py index a126aca36f..f8f5479b6f 100644 --- a/mindspore/ops/_op_impl/tbe/expm1.py +++ b/mindspore/ops/_op_impl/tbe/expm1.py @@ -26,8 +26,8 @@ expm1_op_info = TBERegOp("Expm1") \ .op_pattern("formatAgnostic") \ .input(0, "x", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default) \ + .dtype_format(DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/floor.py b/mindspore/ops/_op_impl/tbe/floor.py index e02bce4aa3..dbdd7927ef 100644 --- a/mindspore/ops/_op_impl/tbe/floor.py +++ b/mindspore/ops/_op_impl/tbe/floor.py @@ -25,8 +25,9 @@ floor_op_info = TBERegOp("Floor") \ .partial_flag(True) \ .input(0, "x", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default) \ + .op_pattern("formatAgnostic") \ + .dtype_format(DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/floor_div.py b/mindspore/ops/_op_impl/tbe/floor_div.py index 74fd594901..c700e8d15a 100644 --- a/mindspore/ops/_op_impl/tbe/floor_div.py +++ b/mindspore/ops/_op_impl/tbe/floor_div.py @@ -26,11 +26,12 @@ floordiv_op_info = TBERegOp("FloorDiv") \ .input(0, "x1", False, "required", "all") \ .input(1, "x2", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.I8_Default, DataType.I8_Default, DataType.I8_Default) \ - .dtype_format(DataType.U8_Default, DataType.U8_Default, DataType.U8_Default) \ - .dtype_format(DataType.I32_Default, DataType.I32_Default, DataType.I32_Default) \ - .dtype_format(DataType.F16_Default, DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default, DataType.F32_Default) \ + .op_pattern("broadcast") \ + .dtype_format(DataType.I8_None, DataType.I8_None, DataType.I8_None) \ + .dtype_format(DataType.U8_None, DataType.U8_None, DataType.U8_None) \ + .dtype_format(DataType.I32_None, DataType.I32_None, DataType.I32_None) \ + .dtype_format(DataType.F16_None, DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/floor_mod.py b/mindspore/ops/_op_impl/tbe/floor_mod.py index 031f160e0a..72cd6499d4 100644 --- a/mindspore/ops/_op_impl/tbe/floor_mod.py +++ b/mindspore/ops/_op_impl/tbe/floor_mod.py @@ -26,9 +26,10 @@ floor_mod_op_info = TBERegOp("FloorMod") \ .input(0, "x1", False, "required", "all") \ .input(1, "x2", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.F16_Default, DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default, DataType.F32_Default) \ - .dtype_format(DataType.I32_Default, DataType.I32_Default, DataType.I32_Default) \ + .op_pattern("broadcast") \ + .dtype_format(DataType.F16_None, DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None, DataType.F32_None) \ + .dtype_format(DataType.I32_None, DataType.I32_None, DataType.I32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/fused_mul_add.py b/mindspore/ops/_op_impl/tbe/fused_mul_add.py index fa104fb561..3a7efee792 100644 --- a/mindspore/ops/_op_impl/tbe/fused_mul_add.py +++ b/mindspore/ops/_op_impl/tbe/fused_mul_add.py @@ -28,21 +28,7 @@ fused_mul_add_op_info = TBERegOp("FusedMulAdd") \ .input(2, "x3", False, "required", "all") \ .output(0, "y", False, "required", "all") \ .op_pattern("dynamicFormat") \ - .dtype_format(DataType.I32_Default, DataType.I32_Default, DataType.I32_Default, DataType.I32_Default) \ - .dtype_format(DataType.I32_5HD, DataType.I32_5HD, DataType.I32_5HD, DataType.I32_5HD) \ - .dtype_format(DataType.I32_FracZ, DataType.I32_FracZ, DataType.I32_FracZ, DataType.I32_FracZ) \ - .dtype_format(DataType.I32_FracNZ, DataType.I32_FracNZ, DataType.I32_FracNZ, DataType.I32_FracNZ) \ - .dtype_format(DataType.I32_C1HWNCoC0, DataType.I32_C1HWNCoC0, DataType.I32_C1HWNCoC0, DataType.I32_C1HWNCoC0) \ - .dtype_format(DataType.F16_Default, DataType.F16_Default, DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD, DataType.F16_5HD, DataType.F16_5HD) \ - .dtype_format(DataType.F16_FracZ, DataType.F16_FracZ, DataType.F16_FracZ, DataType.F16_FracZ) \ - .dtype_format(DataType.F16_FracNZ, DataType.F16_FracNZ, DataType.F16_FracNZ, DataType.F16_FracNZ) \ - .dtype_format(DataType.F16_C1HWNCoC0, DataType.F16_C1HWNCoC0, DataType.F16_C1HWNCoC0, DataType.F16_C1HWNCoC0) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default, DataType.F32_Default, DataType.F32_Default) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD, DataType.F32_5HD, DataType.F32_5HD) \ - .dtype_format(DataType.F32_FracZ, DataType.F32_FracZ, DataType.F32_FracZ, DataType.F32_FracZ) \ - .dtype_format(DataType.F32_FracNZ, DataType.F32_FracNZ, DataType.F32_FracNZ, DataType.F32_FracNZ) \ - .dtype_format(DataType.F32_C1HWNCoC0, DataType.F32_C1HWNCoC0, DataType.F32_C1HWNCoC0, DataType.F32_C1HWNCoC0) \ + .dtype_format(DataType.None_None, DataType.None_None, DataType.None_None, DataType.None_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/gelu.py b/mindspore/ops/_op_impl/tbe/gelu.py index 9d4b2ed7f3..a539cb8ef3 100644 --- a/mindspore/ops/_op_impl/tbe/gelu.py +++ b/mindspore/ops/_op_impl/tbe/gelu.py @@ -25,14 +25,9 @@ gelu_op_info = TBERegOp("Gelu") \ .partial_flag(True) \ .input(0, "x", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD) \ - .dtype_format(DataType.F16_FracZ, DataType.F16_FracZ) \ - .dtype_format(DataType.F16_FracNZ, DataType.F16_FracNZ) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD) \ - .dtype_format(DataType.F32_FracZ, DataType.F32_FracZ) \ - .dtype_format(DataType.F32_FracNZ, DataType.F32_FracNZ) \ + .op_pattern("formatAgnostic") \ + .dtype_format(DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/greater.py b/mindspore/ops/_op_impl/tbe/greater.py index 90c680ab04..acbc7ef729 100644 --- a/mindspore/ops/_op_impl/tbe/greater.py +++ b/mindspore/ops/_op_impl/tbe/greater.py @@ -26,16 +26,12 @@ greater_op_info = TBERegOp("Greater") \ .input(0, "x1", False, "required", "all") \ .input(1, "x2", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.I8_Default, DataType.I8_Default, DataType.BOOL_Default) \ - .dtype_format(DataType.I8_5HD, DataType.I8_5HD, DataType.BOOL_5HD) \ - .dtype_format(DataType.U8_Default, DataType.U8_Default, DataType.BOOL_Default) \ - .dtype_format(DataType.U8_5HD, DataType.U8_5HD, DataType.BOOL_5HD) \ - .dtype_format(DataType.I32_Default, DataType.I32_Default, DataType.BOOL_Default) \ - .dtype_format(DataType.I32_5HD, DataType.I32_5HD, DataType.BOOL_5HD) \ - .dtype_format(DataType.F16_Default, DataType.F16_Default, DataType.BOOL_Default) \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD, DataType.BOOL_5HD) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default, DataType.BOOL_Default) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD, DataType.BOOL_5HD) \ + .op_pattern("broadcast") \ + .dtype_format(DataType.I8_None, DataType.I8_None, DataType.BOOL_None) \ + .dtype_format(DataType.U8_None, DataType.U8_None, DataType.BOOL_None) \ + .dtype_format(DataType.I32_None, DataType.I32_None, DataType.BOOL_None) \ + .dtype_format(DataType.F16_None, DataType.F16_None, DataType.BOOL_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None, DataType.BOOL_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/greater_equal.py b/mindspore/ops/_op_impl/tbe/greater_equal.py index 5609f15f18..86aba300c1 100644 --- a/mindspore/ops/_op_impl/tbe/greater_equal.py +++ b/mindspore/ops/_op_impl/tbe/greater_equal.py @@ -26,16 +26,12 @@ greater_equal_op_info = TBERegOp("GreaterEqual") \ .input(0, "x1", False, "required", "all") \ .input(1, "x2", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.I8_Default, DataType.I8_Default, DataType.BOOL_Default) \ - .dtype_format(DataType.I8_5HD, DataType.I8_5HD, DataType.BOOL_5HD) \ - .dtype_format(DataType.U8_Default, DataType.U8_Default, DataType.BOOL_Default) \ - .dtype_format(DataType.U8_5HD, DataType.U8_5HD, DataType.BOOL_5HD) \ - .dtype_format(DataType.I32_Default, DataType.I32_Default, DataType.BOOL_Default) \ - .dtype_format(DataType.I32_5HD, DataType.I32_5HD, DataType.BOOL_5HD) \ - .dtype_format(DataType.F16_Default, DataType.F16_Default, DataType.BOOL_Default) \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD, DataType.BOOL_5HD) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default, DataType.BOOL_Default) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD, DataType.BOOL_5HD) \ + .op_pattern("broadcast") \ + .dtype_format(DataType.I8_None, DataType.I8_None, DataType.BOOL_None) \ + .dtype_format(DataType.U8_None, DataType.U8_None, DataType.BOOL_None) \ + .dtype_format(DataType.I32_None, DataType.I32_None, DataType.BOOL_None) \ + .dtype_format(DataType.F16_None, DataType.F16_None, DataType.BOOL_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None, DataType.BOOL_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/inv.py b/mindspore/ops/_op_impl/tbe/inv.py index e2b749a5aa..a838a31d4c 100644 --- a/mindspore/ops/_op_impl/tbe/inv.py +++ b/mindspore/ops/_op_impl/tbe/inv.py @@ -25,9 +25,10 @@ inv_op_info = TBERegOp("Inv") \ .partial_flag(True) \ .input(0, "x", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.I32_Default, DataType.I32_Default) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default) \ - .dtype_format(DataType.F16_Default, DataType.F16_Default) \ + .op_pattern("formatAgnostic") \ + .dtype_format(DataType.I32_None, DataType.I32_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None) \ + .dtype_format(DataType.F16_None, DataType.F16_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/inv_grad.py b/mindspore/ops/_op_impl/tbe/inv_grad.py index 70626b8808..a81c1df624 100644 --- a/mindspore/ops/_op_impl/tbe/inv_grad.py +++ b/mindspore/ops/_op_impl/tbe/inv_grad.py @@ -26,10 +26,11 @@ inv_grad_op_info = TBERegOp("InvGrad") \ .input(0, "x", False, "required", "all") \ .input(1, "grad", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.F16_Default, DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default, DataType.F32_Default) \ - .dtype_format(DataType.I32_Default, DataType.I32_Default, DataType.I32_Default) \ - .dtype_format(DataType.I8_Default, DataType.I8_Default, DataType.I8_Default) \ + .op_pattern("broadcast") \ + .dtype_format(DataType.F16_None, DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None, DataType.F32_None) \ + .dtype_format(DataType.I32_None, DataType.I32_None, DataType.I32_None) \ + .dtype_format(DataType.I8_None, DataType.I8_None, DataType.I8_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/invert.py b/mindspore/ops/_op_impl/tbe/invert.py index 887eee45e7..73b33a5764 100644 --- a/mindspore/ops/_op_impl/tbe/invert.py +++ b/mindspore/ops/_op_impl/tbe/invert.py @@ -25,8 +25,9 @@ invert_op_info = TBERegOp("Invert") \ .partial_flag(True) \ .input(0, "x", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.I16_Default, DataType.I16_Default) \ - .dtype_format(DataType.U16_Default, DataType.U16_Default) \ + .op_pattern("formatAgnostic") \ + .dtype_format(DataType.I16_None, DataType.I16_None) \ + .dtype_format(DataType.U16_None, DataType.U16_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/layer_norm.py b/mindspore/ops/_op_impl/tbe/layer_norm.py index 03ddd2dc6c..2414b9bcb3 100644 --- a/mindspore/ops/_op_impl/tbe/layer_norm.py +++ b/mindspore/ops/_op_impl/tbe/layer_norm.py @@ -33,18 +33,10 @@ layer_norm_op_info = TBERegOp("LayerNorm") \ .output(1, "mean", False, "required", "all") \ .output(2, "variance", False, "required", "all") \ .op_pattern("dynamicFormat") \ - .dtype_format(DataType.F16_Default, DataType.F16_Default, DataType.F16_Default, DataType.F16_Default, - DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD, DataType.F16_5HD, DataType.F16_5HD, - DataType.F16_5HD, DataType.F16_5HD) \ - .dtype_format(DataType.F16_FracNZ, DataType.F16_Default, DataType.F16_Default, DataType.F16_FracNZ, - DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default, DataType.F32_Default, DataType.F32_Default, - DataType.F32_Default, DataType.F32_Default) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD, DataType.F32_5HD, DataType.F32_5HD, - DataType.F32_5HD, DataType.F32_5HD) \ - .dtype_format(DataType.F32_FracNZ, DataType.F32_Default, DataType.F32_Default, DataType.F32_FracNZ, - DataType.F32_Default, DataType.F32_Default) \ + .dtype_format(DataType.F16_None, DataType.F16_None, DataType.F16_None, DataType.F16_None, + DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None, DataType.F32_None, DataType.F32_None, + DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/layer_norm_beta_gamma_backprop.py b/mindspore/ops/_op_impl/tbe/layer_norm_beta_gamma_backprop.py index deca384032..8fec87d6d7 100644 --- a/mindspore/ops/_op_impl/tbe/layer_norm_beta_gamma_backprop.py +++ b/mindspore/ops/_op_impl/tbe/layer_norm_beta_gamma_backprop.py @@ -31,18 +31,10 @@ layer_norm_beta_gamma_backprop_op_info = TBERegOp("LayerNormBetaGammaBackprop") .output(0, "pd_gamma", False, "required", "all") \ .output(1, "pd_beta", False, "required", "all") \ .op_pattern("dynamicFormat") \ - .dtype_format(DataType.F16_Default, DataType.F16_Default, DataType.F16_Default, DataType.F16_Default, - DataType.F32_Default, DataType.F32_Default) \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD, DataType.F16_5HD, DataType.F16_5HD, - DataType.F32_5HD, DataType.F32_5HD) \ - .dtype_format(DataType.F16_FracNZ, DataType.F16_FracNZ, DataType.F16_Default, DataType.F16_Default, - DataType.F32_Default, DataType.F32_Default) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default, DataType.F32_Default, DataType.F32_Default, - DataType.F32_Default, DataType.F32_Default) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD, DataType.F32_5HD, DataType.F32_5HD, - DataType.F32_5HD, DataType.F32_5HD) \ - .dtype_format(DataType.F32_FracNZ, DataType.F32_FracNZ, DataType.F32_Default, DataType.F32_Default, - DataType.F32_Default, DataType.F32_Default) \ + .dtype_format(DataType.F16_None, DataType.F16_None, DataType.F16_None, DataType.F16_None, + DataType.F32_None, DataType.F32_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None, DataType.F32_None, DataType.F32_None, + DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/layer_norm_x_backprop.py b/mindspore/ops/_op_impl/tbe/layer_norm_x_backprop.py index 1d4f1ef231..ac2d28e9b8 100644 --- a/mindspore/ops/_op_impl/tbe/layer_norm_x_backprop.py +++ b/mindspore/ops/_op_impl/tbe/layer_norm_x_backprop.py @@ -30,18 +30,10 @@ layer_norm_x_backprop_op_info = TBERegOp("LayerNormXBackprop") \ .input(4, "gamma", False, "required", "all") \ .output(0, "pd_x", False, "required", "all") \ .op_pattern("dynamicFormat") \ - .dtype_format(DataType.F16_Default, DataType.F16_Default, DataType.F16_Default, DataType.F16_Default, - DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD, DataType.F16_5HD, DataType.F16_5HD, - DataType.F16_5HD, DataType.F16_5HD) \ - .dtype_format(DataType.F16_FracNZ, DataType.F16_FracNZ, DataType.F16_Default, DataType.F16_Default, - DataType.F16_Default, DataType.F16_FracNZ) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default, DataType.F32_Default, DataType.F32_Default, - DataType.F32_Default, DataType.F32_Default) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD, DataType.F32_5HD, DataType.F32_5HD, - DataType.F32_5HD, DataType.F32_5HD) \ - .dtype_format(DataType.F32_FracNZ, DataType.F32_FracNZ, DataType.F32_Default, DataType.F32_Default, - DataType.F32_Default, DataType.F32_FracNZ) \ + .dtype_format(DataType.F16_None, DataType.F16_None, DataType.F16_None, DataType.F16_None, + DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None, DataType.F32_None, DataType.F32_None, + DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/less.py b/mindspore/ops/_op_impl/tbe/less.py index 947c40b949..9197e831f1 100644 --- a/mindspore/ops/_op_impl/tbe/less.py +++ b/mindspore/ops/_op_impl/tbe/less.py @@ -26,16 +26,12 @@ less_op_info = TBERegOp("Less") \ .input(0, "x1", False, "required", "all") \ .input(1, "x2", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.I8_Default, DataType.I8_Default, DataType.BOOL_Default) \ - .dtype_format(DataType.I8_5HD, DataType.I8_5HD, DataType.BOOL_5HD) \ - .dtype_format(DataType.U8_Default, DataType.U8_Default, DataType.BOOL_Default) \ - .dtype_format(DataType.U8_5HD, DataType.U8_5HD, DataType.BOOL_5HD) \ - .dtype_format(DataType.I32_Default, DataType.I32_Default, DataType.BOOL_Default) \ - .dtype_format(DataType.I32_5HD, DataType.I32_5HD, DataType.BOOL_5HD) \ - .dtype_format(DataType.F16_Default, DataType.F16_Default, DataType.BOOL_Default) \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD, DataType.BOOL_5HD) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default, DataType.BOOL_Default) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD, DataType.BOOL_5HD) \ + .op_pattern("broadcast") \ + .dtype_format(DataType.I8_None, DataType.I8_None, DataType.BOOL_None) \ + .dtype_format(DataType.U8_None, DataType.U8_None, DataType.BOOL_None) \ + .dtype_format(DataType.I32_None, DataType.I32_None, DataType.BOOL_None) \ + .dtype_format(DataType.F16_None, DataType.F16_None, DataType.BOOL_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None, DataType.BOOL_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/less_equal.py b/mindspore/ops/_op_impl/tbe/less_equal.py index 14cf7c8906..83ee8a1c36 100644 --- a/mindspore/ops/_op_impl/tbe/less_equal.py +++ b/mindspore/ops/_op_impl/tbe/less_equal.py @@ -28,16 +28,12 @@ less_equal_op_info = TBERegOp("LessEqual") \ .input(0, "x1", False, "required", "all") \ .input(1, "x2", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.I8_Default, DataType.I8_Default, DataType.BOOL_Default) \ - .dtype_format(DataType.I8_5HD, DataType.I8_5HD, DataType.BOOL_5HD) \ - .dtype_format(DataType.U8_Default, DataType.U8_Default, DataType.BOOL_Default) \ - .dtype_format(DataType.U8_5HD, DataType.U8_5HD, DataType.BOOL_5HD) \ - .dtype_format(DataType.I32_Default, DataType.I32_Default, DataType.BOOL_Default) \ - .dtype_format(DataType.I32_5HD, DataType.I32_5HD, DataType.BOOL_5HD) \ - .dtype_format(DataType.F16_Default, DataType.F16_Default, DataType.BOOL_Default) \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD, DataType.BOOL_5HD) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default, DataType.BOOL_Default) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD, DataType.BOOL_5HD) \ + .op_pattern("broadcast") \ + .dtype_format(DataType.I8_None, DataType.I8_None, DataType.BOOL_None) \ + .dtype_format(DataType.U8_None, DataType.U8_None, DataType.BOOL_None) \ + .dtype_format(DataType.I32_None, DataType.I32_None, DataType.BOOL_None) \ + .dtype_format(DataType.F16_None, DataType.F16_None, DataType.BOOL_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None, DataType.BOOL_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/log.py b/mindspore/ops/_op_impl/tbe/log.py index b7da647248..e5895066d0 100644 --- a/mindspore/ops/_op_impl/tbe/log.py +++ b/mindspore/ops/_op_impl/tbe/log.py @@ -25,10 +25,9 @@ log_op_info = TBERegOp("Log") \ .partial_flag(True) \ .input(0, "x", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD) \ + .op_pattern("formatAgnostic") \ + .dtype_format(DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/log1p.py b/mindspore/ops/_op_impl/tbe/log1p.py index a4c6377b6a..6ae151ebf9 100644 --- a/mindspore/ops/_op_impl/tbe/log1p.py +++ b/mindspore/ops/_op_impl/tbe/log1p.py @@ -25,10 +25,9 @@ log1p_op_info = TBERegOp("Log1p") \ .partial_flag(True) \ .input(0, "x", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD) \ + .op_pattern("formatAgnostic") \ + .dtype_format(DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/logical_and.py b/mindspore/ops/_op_impl/tbe/logical_and.py index 925a4e82d8..8347d0a03f 100644 --- a/mindspore/ops/_op_impl/tbe/logical_and.py +++ b/mindspore/ops/_op_impl/tbe/logical_and.py @@ -26,10 +26,8 @@ logical_and_op_info = TBERegOp("LogicalAnd") \ .input(0, "x1", False, "required", "all") \ .input(1, "x2", False, "required", "all") \ .output(0, "y", True, "required", "all") \ - .dtype_format(DataType.BOOL_Default, DataType.BOOL_Default, DataType.BOOL_Default) \ - .dtype_format(DataType.BOOL_FracZ, DataType.BOOL_FracZ, DataType.BOOL_FracZ) \ - .dtype_format(DataType.BOOL_C1HWNCoC0, DataType.BOOL_C1HWNCoC0, DataType.BOOL_C1HWNCoC0) \ - .dtype_format(DataType.BOOL_5HD, DataType.BOOL_5HD, DataType.BOOL_5HD) \ + .op_pattern("broadcast") \ + .dtype_format(DataType.BOOL_None, DataType.BOOL_None, DataType.BOOL_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/logical_not.py b/mindspore/ops/_op_impl/tbe/logical_not.py index 3d40441156..17c26b5cfd 100644 --- a/mindspore/ops/_op_impl/tbe/logical_not.py +++ b/mindspore/ops/_op_impl/tbe/logical_not.py @@ -25,10 +25,8 @@ logical_not_op_info = TBERegOp("LogicalNot") \ .partial_flag(True) \ .input(0, "x", False, "required", "all") \ .output(0, "y", True, "required", "all") \ - .dtype_format(DataType.BOOL_Default, DataType.BOOL_Default) \ - .dtype_format(DataType.BOOL_FracZ, DataType.BOOL_FracZ) \ - .dtype_format(DataType.BOOL_C1HWNCoC0, DataType.BOOL_C1HWNCoC0) \ - .dtype_format(DataType.BOOL_5HD, DataType.BOOL_5HD) \ + .op_pattern("formatAgnostic") \ + .dtype_format(DataType.BOOL_None, DataType.BOOL_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/logical_or.py b/mindspore/ops/_op_impl/tbe/logical_or.py index bf8d82c656..c99a31f534 100644 --- a/mindspore/ops/_op_impl/tbe/logical_or.py +++ b/mindspore/ops/_op_impl/tbe/logical_or.py @@ -26,10 +26,8 @@ logical_or_op_info = TBERegOp("LogicalOr") \ .input(0, "x1", False, "required", "all") \ .input(1, "x2", False, "required", "all") \ .output(0, "y", True, "required", "all") \ - .dtype_format(DataType.BOOL_Default, DataType.BOOL_Default, DataType.BOOL_Default) \ - .dtype_format(DataType.BOOL_FracZ, DataType.BOOL_FracZ, DataType.BOOL_FracZ) \ - .dtype_format(DataType.BOOL_C1HWNCoC0, DataType.BOOL_C1HWNCoC0, DataType.BOOL_C1HWNCoC0) \ - .dtype_format(DataType.BOOL_5HD, DataType.BOOL_5HD, DataType.BOOL_5HD) \ + .op_pattern("broadcast") \ + .dtype_format(DataType.BOOL_None, DataType.BOOL_None, DataType.BOOL_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/maximum.py b/mindspore/ops/_op_impl/tbe/maximum.py index 6fb7d05e03..b4740ca55d 100644 --- a/mindspore/ops/_op_impl/tbe/maximum.py +++ b/mindspore/ops/_op_impl/tbe/maximum.py @@ -26,12 +26,10 @@ maximum_op_info = TBERegOp("Maximum") \ .input(0, "x1", False, "required", "all") \ .input(1, "x2", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.I32_Default, DataType.I32_Default, DataType.I32_Default) \ - .dtype_format(DataType.I32_5HD, DataType.I32_5HD, DataType.I32_5HD) \ - .dtype_format(DataType.F16_Default, DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD, DataType.F16_5HD) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default, DataType.F32_Default) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD, DataType.F32_5HD) \ + .op_pattern("broadcast") \ + .dtype_format(DataType.I32_None, DataType.I32_None, DataType.I32_None) \ + .dtype_format(DataType.F16_None, DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/maximum_grad.py b/mindspore/ops/_op_impl/tbe/maximum_grad.py index b9bc9c09f8..341b7dafbd 100644 --- a/mindspore/ops/_op_impl/tbe/maximum_grad.py +++ b/mindspore/ops/_op_impl/tbe/maximum_grad.py @@ -30,18 +30,13 @@ maximum_grad_op_info = TBERegOp("MaximumGrad") \ .input(2, "x2", False, "required", "all") \ .output(0, "y1", False, "required", "all") \ .output(1, "y2", False, "required", "all") \ - .dtype_format(DataType.I32_Default, DataType.I32_Default, DataType.I32_Default, DataType.I32_Default, - DataType.I32_Default) \ - .dtype_format(DataType.I32_5HD, DataType.I32_5HD, DataType.I32_5HD, DataType.I32_5HD, - DataType.I32_5HD) \ - .dtype_format(DataType.F16_Default, DataType.F16_Default, DataType.F16_Default, DataType.F16_Default, - DataType.F16_Default) \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD, DataType.F16_5HD, DataType.F16_5HD, - DataType.F16_5HD) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default, DataType.F32_Default, DataType.F32_Default, - DataType.F32_Default) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD, DataType.F32_5HD, DataType.F32_5HD, - DataType.F32_5HD) \ + .op_pattern("broadcast") \ + .dtype_format(DataType.I32_None, DataType.I32_None, DataType.I32_None, DataType.I32_None, + DataType.I32_None) \ + .dtype_format(DataType.F16_None, DataType.F16_None, DataType.F16_None, DataType.F16_None, + DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None, DataType.F32_None, DataType.F32_None, + DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/minimum.py b/mindspore/ops/_op_impl/tbe/minimum.py index 1cebfd3dad..cc1347b616 100644 --- a/mindspore/ops/_op_impl/tbe/minimum.py +++ b/mindspore/ops/_op_impl/tbe/minimum.py @@ -27,12 +27,10 @@ minimum_op_info = TBERegOp("Minimum") \ .input(0, "x1", False, "required", "all") \ .input(1, "x2", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.I32_Default, DataType.I32_Default, DataType.I32_Default) \ - .dtype_format(DataType.I32_5HD, DataType.I32_5HD, DataType.I32_5HD) \ - .dtype_format(DataType.F16_Default, DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD, DataType.F16_5HD) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default, DataType.F32_Default) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD, DataType.F32_5HD) \ + .op_pattern("broadcast") \ + .dtype_format(DataType.I32_None, DataType.I32_None, DataType.I32_None) \ + .dtype_format(DataType.F16_None, DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/minimum_grad.py b/mindspore/ops/_op_impl/tbe/minimum_grad.py index c3ea1c3a56..8ad459af17 100644 --- a/mindspore/ops/_op_impl/tbe/minimum_grad.py +++ b/mindspore/ops/_op_impl/tbe/minimum_grad.py @@ -30,18 +30,13 @@ minimum_grad_op_info = TBERegOp("MinimumGrad") \ .input(2, "x2", False, "required", "all") \ .output(0, "y1", False, "required", "all") \ .output(1, "y2", False, "required", "all") \ - .dtype_format(DataType.I32_Default, DataType.I32_Default, DataType.I32_Default, DataType.I32_Default, - DataType.I32_Default) \ - .dtype_format(DataType.I32_5HD, DataType.I32_5HD, DataType.I32_5HD, DataType.I32_5HD, - DataType.I32_5HD) \ - .dtype_format(DataType.F16_Default, DataType.F16_Default, DataType.F16_Default, DataType.F16_Default, - DataType.F16_Default) \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD, DataType.F16_5HD, DataType.F16_5HD, - DataType.F16_5HD) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default, DataType.F32_Default, DataType.F32_Default, - DataType.F32_Default) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD, DataType.F32_5HD, DataType.F32_5HD, - DataType.F32_5HD) \ + .op_pattern("broadcast") \ + .dtype_format(DataType.I32_None, DataType.I32_None, DataType.I32_None, DataType.I32_None, + DataType.I32_None) \ + .dtype_format(DataType.F16_None, DataType.F16_None, DataType.F16_None, DataType.F16_None, + DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None, DataType.F32_None, DataType.F32_None, + DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/neg.py b/mindspore/ops/_op_impl/tbe/neg.py index feb648f056..ee281a3bb2 100644 --- a/mindspore/ops/_op_impl/tbe/neg.py +++ b/mindspore/ops/_op_impl/tbe/neg.py @@ -25,14 +25,10 @@ neg_op_info = TBERegOp("Neg") \ .partial_flag(True) \ .input(0, "x", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.I8_Default, DataType.I8_Default) \ - .dtype_format(DataType.I8_5HD, DataType.I8_5HD) \ - .dtype_format(DataType.I32_Default, DataType.I32_Default) \ - .dtype_format(DataType.I32_5HD, DataType.I32_5HD) \ - .dtype_format(DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD) \ + .op_pattern("formatAgnostic") \ + .dtype_format(DataType.I32_None, DataType.I32_None) \ + .dtype_format(DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/not_equal.py b/mindspore/ops/_op_impl/tbe/not_equal.py index bd801d9a40..80993a905e 100644 --- a/mindspore/ops/_op_impl/tbe/not_equal.py +++ b/mindspore/ops/_op_impl/tbe/not_equal.py @@ -26,16 +26,12 @@ not_equal_op_info = TBERegOp("NotEqual") \ .input(0, "x1", False, "required", "all") \ .input(1, "x2", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.I8_Default, DataType.I8_Default, DataType.BOOL_Default) \ - .dtype_format(DataType.I8_5HD, DataType.I8_5HD, DataType.BOOL_5HD) \ - .dtype_format(DataType.U8_Default, DataType.U8_Default, DataType.BOOL_Default) \ - .dtype_format(DataType.U8_5HD, DataType.U8_5HD, DataType.BOOL_5HD) \ - .dtype_format(DataType.I32_Default, DataType.I32_Default, DataType.BOOL_Default) \ - .dtype_format(DataType.I32_5HD, DataType.I32_5HD, DataType.BOOL_5HD) \ - .dtype_format(DataType.F16_Default, DataType.F16_Default, DataType.BOOL_Default) \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD, DataType.BOOL_5HD) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default, DataType.BOOL_Default) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD, DataType.BOOL_5HD) \ + .op_pattern("broadcast") \ + .dtype_format(DataType.I8_None, DataType.I8_None, DataType.BOOL_None) \ + .dtype_format(DataType.U8_None, DataType.U8_None, DataType.BOOL_None) \ + .dtype_format(DataType.I32_None, DataType.I32_None, DataType.BOOL_None) \ + .dtype_format(DataType.F16_None, DataType.F16_None, DataType.BOOL_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None, DataType.BOOL_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/ones_like.py b/mindspore/ops/_op_impl/tbe/ones_like.py index ae6871cac5..9b31d6d4fa 100644 --- a/mindspore/ops/_op_impl/tbe/ones_like.py +++ b/mindspore/ops/_op_impl/tbe/ones_like.py @@ -25,11 +25,12 @@ ones_like_op_info = TBERegOp("OnesLike") \ .partial_flag(True) \ .input(0, "x", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.U8_Default, DataType.U8_Default) \ - .dtype_format(DataType.I8_Default, DataType.I8_Default) \ - .dtype_format(DataType.I32_Default, DataType.I32_Default) \ - .dtype_format(DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default) \ + .op_pattern("formatAgnostic") \ + .dtype_format(DataType.U8_None, DataType.U8_None) \ + .dtype_format(DataType.I8_None, DataType.I8_None) \ + .dtype_format(DataType.I32_None, DataType.I32_None) \ + .dtype_format(DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/pow.py b/mindspore/ops/_op_impl/tbe/pow.py index 223a139252..39e55aab23 100644 --- a/mindspore/ops/_op_impl/tbe/pow.py +++ b/mindspore/ops/_op_impl/tbe/pow.py @@ -26,11 +26,12 @@ pow_op_info = TBERegOp("Pow") \ .input(0, "x1", False, "required", "all") \ .input(1, "x2", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.I8_Default, DataType.I8_Default, DataType.I8_Default) \ - .dtype_format(DataType.U8_Default, DataType.U8_Default, DataType.U8_Default) \ - .dtype_format(DataType.I32_Default, DataType.I32_Default, DataType.I32_Default) \ - .dtype_format(DataType.F16_Default, DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default, DataType.F32_Default) \ + .op_pattern("broadcast") \ + .dtype_format(DataType.I8_None, DataType.I8_None, DataType.I8_None) \ + .dtype_format(DataType.U8_None, DataType.U8_None, DataType.U8_None) \ + .dtype_format(DataType.I32_None, DataType.I32_None, DataType.I32_None) \ + .dtype_format(DataType.F16_None, DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/reciprocal.py b/mindspore/ops/_op_impl/tbe/reciprocal.py index 77f3bfac27..c620fb17a6 100644 --- a/mindspore/ops/_op_impl/tbe/reciprocal.py +++ b/mindspore/ops/_op_impl/tbe/reciprocal.py @@ -26,12 +26,7 @@ reciprocal_op_info = TBERegOp("Reciprocal") \ .input(0, "x", False, "required", "all") \ .output(0, "y", False, "required", "all") \ .op_pattern("dynamicFormat") \ - .dtype_format(DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD) \ - .dtype_format(DataType.F16_NHWC, DataType.F16_NHWC) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD) \ - .dtype_format(DataType.F32_NHWC, DataType.F32_NHWC) \ + .dtype_format(DataType.None_None, DataType.None_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/reduce_all.py b/mindspore/ops/_op_impl/tbe/reduce_all.py index a372dcf6b7..17f5574819 100644 --- a/mindspore/ops/_op_impl/tbe/reduce_all.py +++ b/mindspore/ops/_op_impl/tbe/reduce_all.py @@ -27,8 +27,8 @@ reduce_all_op_info = TBERegOp("ReduceAll") \ .attr("keep_dims", "optional", "bool", "all") \ .input(0, "x", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.BOOL_Default, DataType.BOOL_Default) \ - .dtype_format(DataType.BOOL_FracZ, DataType.BOOL_FracZ) \ + .op_pattern("reduce") \ + .dtype_format(DataType.BOOL_None, DataType.BOOL_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/reduce_max.py b/mindspore/ops/_op_impl/tbe/reduce_max.py index ab0e766f59..a5d5ce3e8c 100644 --- a/mindspore/ops/_op_impl/tbe/reduce_max.py +++ b/mindspore/ops/_op_impl/tbe/reduce_max.py @@ -27,12 +27,13 @@ reduce_max_d_op_info = TBERegOp("ReduceMax") \ .attr("keep_dims", "optional", "bool", "all") \ .input(0, "x", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.BOOL_Default, DataType.BOOL_Default) \ - .dtype_format(DataType.I8_Default, DataType.I8_Default) \ - .dtype_format(DataType.U8_Default, DataType.U8_Default) \ - .dtype_format(DataType.I32_Default, DataType.I32_Default) \ - .dtype_format(DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default) \ + .op_pattern("reduce") \ + .dtype_format(DataType.BOOL_None, DataType.BOOL_None) \ + .dtype_format(DataType.I8_None, DataType.I8_None) \ + .dtype_format(DataType.U8_None, DataType.U8_None) \ + .dtype_format(DataType.I32_None, DataType.I32_None) \ + .dtype_format(DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/reduce_min.py b/mindspore/ops/_op_impl/tbe/reduce_min.py index f1601ebc94..d64dd7ba5c 100644 --- a/mindspore/ops/_op_impl/tbe/reduce_min.py +++ b/mindspore/ops/_op_impl/tbe/reduce_min.py @@ -27,14 +27,11 @@ reduce_min_op_info = TBERegOp("ReduceMin") \ .attr("keep_dims", "required", "bool", "all") \ .input(0, "x", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.I8_Default, DataType.I8_Default) \ - .dtype_format(DataType.I8_FracZ, DataType.I8_FracZ) \ - .dtype_format(DataType.U8_Default, DataType.U8_Default) \ - .dtype_format(DataType.U8_FracZ, DataType.U8_FracZ) \ - .dtype_format(DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F16_FracZ, DataType.F16_FracZ) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default) \ - .dtype_format(DataType.F32_FracZ, DataType.F32_FracZ) \ + .op_pattern("reduce") \ + .dtype_format(DataType.I8_None, DataType.I8_None) \ + .dtype_format(DataType.U8_None, DataType.U8_None) \ + .dtype_format(DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/reduce_prod.py b/mindspore/ops/_op_impl/tbe/reduce_prod.py index 416418e140..b6f1386ce2 100644 --- a/mindspore/ops/_op_impl/tbe/reduce_prod.py +++ b/mindspore/ops/_op_impl/tbe/reduce_prod.py @@ -27,14 +27,11 @@ reduce_prod_op_info = TBERegOp("ReduceProd") \ .attr("keep_dims", "optional", "bool", "all") \ .input(0, "x", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.I8_Default, DataType.I8_Default) \ - .dtype_format(DataType.I8_FracZ, DataType.I8_FracZ) \ - .dtype_format(DataType.U8_Default, DataType.U8_Default) \ - .dtype_format(DataType.U8_FracZ, DataType.U8_FracZ) \ - .dtype_format(DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F16_FracZ, DataType.F16_FracZ) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default) \ - .dtype_format(DataType.F32_FracZ, DataType.F32_FracZ) \ + .op_pattern("reduce") \ + .dtype_format(DataType.I8_None, DataType.I8_None) \ + .dtype_format(DataType.U8_None, DataType.U8_None) \ + .dtype_format(DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/reduce_sum.py b/mindspore/ops/_op_impl/tbe/reduce_sum.py index 2f76f74562..72b7803f94 100644 --- a/mindspore/ops/_op_impl/tbe/reduce_sum.py +++ b/mindspore/ops/_op_impl/tbe/reduce_sum.py @@ -27,8 +27,9 @@ reduce_sum_op_info = TBERegOp("ReduceSum") \ .attr("keep_dims", "optional", "bool", "all") \ .input(0, "x", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default) \ + .op_pattern("reduce") \ + .dtype_format(DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/relu.py b/mindspore/ops/_op_impl/tbe/relu.py index 8a7d023afd..1a586e02a8 100644 --- a/mindspore/ops/_op_impl/tbe/relu.py +++ b/mindspore/ops/_op_impl/tbe/relu.py @@ -25,15 +25,11 @@ relu_op_info = TBERegOp("ReLU") \ .partial_flag(True) \ .input(0, "x", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.I8_Default, DataType.I8_Default) \ - .dtype_format(DataType.I8_5HD, DataType.I8_5HD) \ - .dtype_format(DataType.I32_Default, DataType.I32_Default) \ - .dtype_format(DataType.I32_5HD, DataType.I32_5HD) \ - .dtype_format(DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD) \ - .dtype_format(DataType.F16_FracNZ, DataType.F16_FracNZ) \ + .op_pattern("formatAgnostic") \ + .dtype_format(DataType.I8_None, DataType.I8_None) \ + .dtype_format(DataType.I32_None, DataType.I32_None) \ + .dtype_format(DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/relu6.py b/mindspore/ops/_op_impl/tbe/relu6.py index d9bd7f9f8e..9a68774d63 100644 --- a/mindspore/ops/_op_impl/tbe/relu6.py +++ b/mindspore/ops/_op_impl/tbe/relu6.py @@ -25,12 +25,10 @@ relu6_op_info = TBERegOp("ReLU6") \ .partial_flag(True) \ .input(0, "x", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD) \ - .dtype_format(DataType.I32_Default, DataType.I32_Default) \ - .dtype_format(DataType.I32_5HD, DataType.I32_5HD) \ + .op_pattern("formatAgnostic") \ + .dtype_format(DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None) \ + .dtype_format(DataType.I32_None, DataType.I32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/relu_grad.py b/mindspore/ops/_op_impl/tbe/relu_grad.py index 0f9e962cfd..040294f973 100644 --- a/mindspore/ops/_op_impl/tbe/relu_grad.py +++ b/mindspore/ops/_op_impl/tbe/relu_grad.py @@ -26,16 +26,12 @@ relugrad_op_info = TBERegOp("ReluGrad") \ .input(0, "gradients", False, "required", "all") \ .input(1, "features", False, "required", "all") \ .output(0, "backprops", True, "required", "all") \ - .dtype_format(DataType.I8_Default, DataType.I8_Default, DataType.I8_Default) \ - .dtype_format(DataType.I8_5HD, DataType.I8_5HD, DataType.I8_5HD) \ - .dtype_format(DataType.U8_Default, DataType.U8_Default, DataType.U8_Default) \ - .dtype_format(DataType.U8_5HD, DataType.U8_5HD, DataType.U8_5HD) \ - .dtype_format(DataType.I32_Default, DataType.I32_Default, DataType.I32_Default) \ - .dtype_format(DataType.I32_5HD, DataType.I32_5HD, DataType.I32_5HD) \ - .dtype_format(DataType.F16_Default, DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD, DataType.F16_5HD) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default, DataType.F32_Default) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD, DataType.F32_5HD) \ + .op_pattern("broadcast") \ + .dtype_format(DataType.I8_None, DataType.I8_None, DataType.I8_None) \ + .dtype_format(DataType.U8_None, DataType.U8_None, DataType.U8_None) \ + .dtype_format(DataType.I32_None, DataType.I32_None, DataType.I32_None) \ + .dtype_format(DataType.F16_None, DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/round.py b/mindspore/ops/_op_impl/tbe/round.py index 4559a3def0..64e94e8c42 100644 --- a/mindspore/ops/_op_impl/tbe/round.py +++ b/mindspore/ops/_op_impl/tbe/round.py @@ -25,12 +25,10 @@ round_op_info = TBERegOp("Round") \ .partial_flag(True) \ .input(0, "x", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD) \ - .dtype_format(DataType.F16_FracZ, DataType.F16_FracZ) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD) \ - .dtype_format(DataType.F32_FracZ, DataType.F32_FracZ) \ + .op_pattern("formatAgnostic") \ + .dtype_format(DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None) \ + .dtype_format(DataType.I32_None, DataType.I32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/rsqrt.py b/mindspore/ops/_op_impl/tbe/rsqrt.py index b0830cf484..e080bf9772 100644 --- a/mindspore/ops/_op_impl/tbe/rsqrt.py +++ b/mindspore/ops/_op_impl/tbe/rsqrt.py @@ -25,14 +25,9 @@ rsqrt_op_info = TBERegOp("Rsqrt") \ .partial_flag(True) \ .input(0, "x", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD) \ - .dtype_format(DataType.F16_FracZ, DataType.F16_FracZ) \ - .dtype_format(DataType.F16_C1HWNCoC0, DataType.F16_C1HWNCoC0) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD) \ - .dtype_format(DataType.F32_FracZ, DataType.F32_FracZ) \ - .dtype_format(DataType.F32_C1HWNCoC0, DataType.F32_C1HWNCoC0) \ + .op_pattern("formatAgnostic") \ + .dtype_format(DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/select.py b/mindspore/ops/_op_impl/tbe/select.py index e924f05021..97e6baa58e 100644 --- a/mindspore/ops/_op_impl/tbe/select.py +++ b/mindspore/ops/_op_impl/tbe/select.py @@ -28,16 +28,7 @@ select_op_info = TBERegOp("Select") \ .input(2, "x2", False, "required", "all") \ .output(0, "y", False, "required", "all") \ .op_pattern("dynamicFormat") \ - .dtype_format(DataType.BOOL_Default, DataType.I8_Default, DataType.I8_Default, DataType.I8_Default) \ - .dtype_format(DataType.BOOL_Default, DataType.U8_Default, DataType.U8_Default, DataType.U8_Default) \ - .dtype_format(DataType.BOOL_Default, DataType.I32_Default, DataType.I32_Default, DataType.I32_Default) \ - .dtype_format(DataType.BOOL_Default, DataType.F16_Default, DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.BOOL_Default, DataType.F32_Default, DataType.F32_Default, DataType.F32_Default) \ - .dtype_format(DataType.BOOL_5HD, DataType.I8_5HD, DataType.I8_5HD, DataType.I8_5HD) \ - .dtype_format(DataType.BOOL_5HD, DataType.U8_5HD, DataType.U8_5HD, DataType.U8_5HD) \ - .dtype_format(DataType.BOOL_5HD, DataType.I32_5HD, DataType.I32_5HD, DataType.I32_5HD) \ - .dtype_format(DataType.BOOL_5HD, DataType.F16_5HD, DataType.F16_5HD, DataType.F16_5HD) \ - .dtype_format(DataType.BOOL_5HD, DataType.F32_5HD, DataType.F32_5HD, DataType.F32_5HD) \ + .dtype_format(DataType.None_None, DataType.None_None, DataType.None_None, DataType.None_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/sigmoid.py b/mindspore/ops/_op_impl/tbe/sigmoid.py index 38413c0432..312fdda8d0 100644 --- a/mindspore/ops/_op_impl/tbe/sigmoid.py +++ b/mindspore/ops/_op_impl/tbe/sigmoid.py @@ -25,16 +25,9 @@ sigmoid_op_info = TBERegOp("Sigmoid") \ .partial_flag(True) \ .input(0, "x", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD) \ - .dtype_format(DataType.F16_FracZ, DataType.F16_FracZ) \ - .dtype_format(DataType.F16_FracNZ, DataType.F16_FracNZ) \ - .dtype_format(DataType.F16_C1HWNCoC0, DataType.F16_C1HWNCoC0) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD) \ - .dtype_format(DataType.F32_FracZ, DataType.F32_FracZ) \ - .dtype_format(DataType.F32_FracNZ, DataType.F32_FracNZ) \ - .dtype_format(DataType.F32_C1HWNCoC0, DataType.F32_C1HWNCoC0) \ + .op_pattern("formatAgnostic") \ + .dtype_format(DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/sign.py b/mindspore/ops/_op_impl/tbe/sign.py index 99f7970316..7a7e290649 100644 --- a/mindspore/ops/_op_impl/tbe/sign.py +++ b/mindspore/ops/_op_impl/tbe/sign.py @@ -26,9 +26,9 @@ sign_op_info = TBERegOp("Sign") \ .op_pattern("formatAgnostic") \ .input(0, "x", None, "required", None) \ .output(0, "y", True, "required", "all") \ - .dtype_format(DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default) \ - .dtype_format(DataType.I32_Default, DataType.I32_Default) \ + .dtype_format(DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None) \ + .dtype_format(DataType.I32_None, DataType.I32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/sin.py b/mindspore/ops/_op_impl/tbe/sin.py index f01f687926..6d32e13750 100644 --- a/mindspore/ops/_op_impl/tbe/sin.py +++ b/mindspore/ops/_op_impl/tbe/sin.py @@ -26,10 +26,8 @@ sin_op_info = TBERegOp("Sin") \ .op_pattern("formatAgnostic") \ .input(0, "x", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD) \ + .dtype_format(DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/sinh.py b/mindspore/ops/_op_impl/tbe/sinh.py index 27eb66d274..6bff4c9ca2 100644 --- a/mindspore/ops/_op_impl/tbe/sinh.py +++ b/mindspore/ops/_op_impl/tbe/sinh.py @@ -26,8 +26,8 @@ sinh_op_info = TBERegOp("Sinh") \ .op_pattern("formatAgnostic") \ .input(0, "x", False, "required", "all") \ .output(0, "y", True, "required", "all") \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD) \ + .dtype_format(DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/softmax_grad_ext.py b/mindspore/ops/_op_impl/tbe/softmax_grad_ext.py index d43183dcb7..dcbe47cbff 100644 --- a/mindspore/ops/_op_impl/tbe/softmax_grad_ext.py +++ b/mindspore/ops/_op_impl/tbe/softmax_grad_ext.py @@ -31,14 +31,8 @@ softmax_grad_ext_op_info = TBERegOp("SoftmaxGradExt") \ .input(2, "x2", False, "required", "all") \ .output(0, "y", True, "required", "all") \ .op_pattern("dynamicFormat") \ - .dtype_format(DataType.F16_Default, DataType.F16_Default, - DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD, - DataType.F16_5HD, DataType.F16_5HD) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default, - DataType.F32_Default, DataType.F32_Default) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD, - DataType.F32_5HD, DataType.F32_5HD) \ + .dtype_format(DataType.None_None, DataType.None_None, + DataType.None_None, DataType.None_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/softplus.py b/mindspore/ops/_op_impl/tbe/softplus.py index 92261d91ef..1678b00474 100644 --- a/mindspore/ops/_op_impl/tbe/softplus.py +++ b/mindspore/ops/_op_impl/tbe/softplus.py @@ -26,8 +26,8 @@ softplus_op_info = TBERegOp("Softplus") \ .op_pattern("formatAgnostic") \ .input(0, "x", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default) \ + .dtype_format(DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/softplus_grad.py b/mindspore/ops/_op_impl/tbe/softplus_grad.py index 3dc0e7ee0c..3260fb7ad9 100644 --- a/mindspore/ops/_op_impl/tbe/softplus_grad.py +++ b/mindspore/ops/_op_impl/tbe/softplus_grad.py @@ -27,8 +27,8 @@ softplus_grad_op_info = TBERegOp("SoftplusGrad") \ .input(0, "gradients", False, "required", "all") \ .input(1, "features", False, "required", "all") \ .output(0, "backprops", False, "required", "all") \ - .dtype_format(DataType.F16_Default, DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default, DataType.F32_Default) \ + .dtype_format(DataType.F16_None, DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/split_d.py b/mindspore/ops/_op_impl/tbe/split_d.py index d2faf31096..80557dd701 100644 --- a/mindspore/ops/_op_impl/tbe/split_d.py +++ b/mindspore/ops/_op_impl/tbe/split_d.py @@ -28,28 +28,7 @@ split_d_op_info = TBERegOp("Split") \ .input(0, "value", False, "required", "all") \ .output(0, "output", False, "dynamic", "all") \ .op_pattern("dynamicFormat") \ - .dtype_format(DataType.BOOL_Default, DataType.BOOL_Default) \ - .dtype_format(DataType.BOOL_NHWC, DataType.BOOL_NHWC) \ - .dtype_format(DataType.I8_Default, DataType.I8_Default) \ - .dtype_format(DataType.I8_NHWC, DataType.I8_NHWC) \ - .dtype_format(DataType.U8_Default, DataType.U8_Default) \ - .dtype_format(DataType.U8_NHWC, DataType.U8_NHWC) \ - .dtype_format(DataType.I16_Default, DataType.I16_Default) \ - .dtype_format(DataType.I16_NHWC, DataType.I16_NHWC) \ - .dtype_format(DataType.U16_Default, DataType.U16_Default) \ - .dtype_format(DataType.U16_NHWC, DataType.U16_NHWC) \ - .dtype_format(DataType.I32_Default, DataType.I32_Default) \ - .dtype_format(DataType.I32_NHWC, DataType.I32_NHWC) \ - .dtype_format(DataType.U32_Default, DataType.U32_Default) \ - .dtype_format(DataType.U32_NHWC, DataType.U32_NHWC) \ - .dtype_format(DataType.I64_Default, DataType.I64_Default) \ - .dtype_format(DataType.I64_NHWC, DataType.I64_NHWC) \ - .dtype_format(DataType.U64_Default, DataType.U64_Default) \ - .dtype_format(DataType.U64_NHWC, DataType.U64_NHWC) \ - .dtype_format(DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F16_NHWC, DataType.F16_NHWC) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default) \ - .dtype_format(DataType.F32_NHWC, DataType.F32_NHWC) \ + .dtype_format(DataType.None_None, DataType.None_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/sqrt.py b/mindspore/ops/_op_impl/tbe/sqrt.py index f9e339713b..0c2610f8d1 100644 --- a/mindspore/ops/_op_impl/tbe/sqrt.py +++ b/mindspore/ops/_op_impl/tbe/sqrt.py @@ -25,12 +25,9 @@ sqrt_op_info = TBERegOp("Sqrt") \ .partial_flag(True) \ .input(0, "x", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD) \ - .dtype_format(DataType.F16_NHWC, DataType.F16_NHWC) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD) \ - .dtype_format(DataType.F32_NHWC, DataType.F32_NHWC) \ + .op_pattern("formatAgnostic") \ + .dtype_format(DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/square.py b/mindspore/ops/_op_impl/tbe/square.py index c3eeb12780..aa1d9875cb 100644 --- a/mindspore/ops/_op_impl/tbe/square.py +++ b/mindspore/ops/_op_impl/tbe/square.py @@ -25,15 +25,10 @@ square_op_info = TBERegOp("Square") \ .partial_flag(True) \ .input(0, "x", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.I32_Default, DataType.I32_Default) \ - .dtype_format(DataType.I32_5HD, DataType.I32_5HD) \ - .dtype_format(DataType.I32_NHWC, DataType.I32_NHWC) \ - .dtype_format(DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD) \ - .dtype_format(DataType.F16_NHWC, DataType.F16_NHWC) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD) \ - .dtype_format(DataType.F32_NHWC, DataType.F32_NHWC) \ + .op_pattern("formatAgnostic") \ + .dtype_format(DataType.I32_None, DataType.I32_None) \ + .dtype_format(DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/sub.py b/mindspore/ops/_op_impl/tbe/sub.py index 8e97681c6b..e2454acd35 100644 --- a/mindspore/ops/_op_impl/tbe/sub.py +++ b/mindspore/ops/_op_impl/tbe/sub.py @@ -26,12 +26,10 @@ sub_op_info = TBERegOp("Sub") \ .input(0, "x1", False, "required", "all") \ .input(1, "x2", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.I32_Default, DataType.I32_Default, DataType.I32_Default) \ - .dtype_format(DataType.F16_Default, DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default, DataType.F32_Default) \ - .dtype_format(DataType.I32_5HD, DataType.I32_5HD, DataType.I32_5HD) \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD, DataType.F16_5HD) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD, DataType.F32_5HD) \ + .op_pattern("broadcast") \ + .dtype_format(DataType.I32_None, DataType.I32_None, DataType.I32_None) \ + .dtype_format(DataType.F16_None, DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/tanh.py b/mindspore/ops/_op_impl/tbe/tanh.py index c5b1caf1dd..7dff31518f 100644 --- a/mindspore/ops/_op_impl/tbe/tanh.py +++ b/mindspore/ops/_op_impl/tbe/tanh.py @@ -25,12 +25,9 @@ tanh_op_info = TBERegOp("Tanh") \ .partial_flag(True) \ .input(0, "x", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD) \ - .dtype_format(DataType.F32_FracNZ, DataType.F32_FracNZ) \ - .dtype_format(DataType.F16_FracNZ, DataType.F16_FracNZ) \ + .op_pattern("formatAgnostic") \ + .dtype_format(DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/unsorted_segment_sum.py b/mindspore/ops/_op_impl/tbe/unsorted_segment_sum.py index b1f81b72b0..10cc3db0b0 100644 --- a/mindspore/ops/_op_impl/tbe/unsorted_segment_sum.py +++ b/mindspore/ops/_op_impl/tbe/unsorted_segment_sum.py @@ -28,16 +28,7 @@ unsorted_segment_sum_op_info = TBERegOp("UnsortedSegmentSum") \ .input(1, "segment_ids", False, "required", "all") \ .output(0, "y", False, "required", "all") \ .op_pattern("dynamicFormat") \ - .dtype_format(DataType.I8_Default, DataType.I32_Default, DataType.I8_Default) \ - .dtype_format(DataType.I8_5HD, DataType.I32_5HD, DataType.I8_5HD) \ - .dtype_format(DataType.U8_Default, DataType.I32_Default, DataType.U8_Default) \ - .dtype_format(DataType.U8_5HD, DataType.I32_5HD, DataType.U8_5HD) \ - .dtype_format(DataType.I32_Default, DataType.I32_Default, DataType.I32_Default) \ - .dtype_format(DataType.I32_5HD, DataType.I32_5HD, DataType.I32_5HD) \ - .dtype_format(DataType.F16_Default, DataType.I32_Default, DataType.F16_Default) \ - .dtype_format(DataType.F16_5HD, DataType.I32_5HD, DataType.F16_5HD) \ - .dtype_format(DataType.F32_Default, DataType.I32_Default, DataType.F32_Default) \ - .dtype_format(DataType.F32_5HD, DataType.I32_5HD, DataType.F32_5HD) \ + .dtype_format(DataType.None_None, DataType.None_None, DataType.None_None) \ .get_op_info() From 3521d4fbda46ebed78f5456d1cb3e686c5cb9208 Mon Sep 17 00:00:00 2001 From: zhoufeng Date: Thu, 2 Jul 2020 11:01:27 +0800 Subject: [PATCH 212/254] process empty graph Signed-off-by: zhoufeng --- mindspore/ccsrc/session/ascend_session.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/mindspore/ccsrc/session/ascend_session.cc b/mindspore/ccsrc/session/ascend_session.cc index 05ebcaccf8..f669f89b66 100644 --- a/mindspore/ccsrc/session/ascend_session.cc +++ b/mindspore/ccsrc/session/ascend_session.cc @@ -296,6 +296,7 @@ GraphId AscendSession::CompileGraph(NotNull func_graph) { // empty graph dont entry to backend if (root_graph->execution_order().empty()) { MS_LOG(INFO) << root_graph->ToString() << " is empty graph."; + root_graph->set_executable(false); InitRuntimeResource(); return root_graph->graph_id(); } From a04d497118781c238b369878a2bc624f232aa68c Mon Sep 17 00:00:00 2001 From: chentingting Date: Thu, 2 Jul 2020 10:17:27 +0800 Subject: [PATCH 213/254] fix gcn import error --- model_zoo/gcn/src/gcn.py | 117 ----------------------------------- model_zoo/gcn/src/metrics.py | 116 ++++++++++++++++++++++++++++++++++ model_zoo/gcn/train.py | 7 ++- tests/st/gnn/gcn/test_gcn.py | 3 +- 4 files changed, 122 insertions(+), 121 deletions(-) diff --git a/model_zoo/gcn/src/gcn.py b/model_zoo/gcn/src/gcn.py index 74199490b6..6e01c45d8b 100644 --- a/model_zoo/gcn/src/gcn.py +++ b/model_zoo/gcn/src/gcn.py @@ -15,13 +15,9 @@ """GCN.""" import numpy as np from mindspore import nn -from mindspore.common.parameter import ParameterTuple -from mindspore.ops import composite as C -from mindspore.ops import functional as F from mindspore.ops import operations as P from mindspore import Tensor from mindspore.nn.layer.activation import get_activation -from model_zoo.gcn.src.metrics import Loss, Accuracy def glorot(shape): @@ -105,116 +101,3 @@ class GCN(nn.Cell): output0 = self.layer0(self.adj, self.feature) output1 = self.layer1(self.adj, output0) return output1 - - -class LossAccuracyWrapper(nn.Cell): - """ - Wraps the GCN model with loss and accuracy cell. - - Args: - network (Cell): GCN network. - label (numpy.ndarray): Dataset labels. - mask (numpy.ndarray): Mask for training, evaluation or test. - weight_decay (float): Weight decay parameter for weight of the first convolution layer. - """ - - def __init__(self, network, label, mask, weight_decay): - super(LossAccuracyWrapper, self).__init__() - self.network = network - self.loss = Loss(label, mask, weight_decay, network.trainable_params()[0]) - self.accuracy = Accuracy(label, mask) - - def construct(self): - preds = self.network() - loss = self.loss(preds) - accuracy = self.accuracy(preds) - return loss, accuracy - - -class LossWrapper(nn.Cell): - """ - Wraps the GCN model with loss. - - Args: - network (Cell): GCN network. - label (numpy.ndarray): Dataset labels. - mask (numpy.ndarray): Mask for training. - weight_decay (float): Weight decay parameter for weight of the first convolution layer. - """ - - def __init__(self, network, label, mask, weight_decay): - super(LossWrapper, self).__init__() - self.network = network - self.loss = Loss(label, mask, weight_decay, network.trainable_params()[0]) - - def construct(self): - preds = self.network() - loss = self.loss(preds) - return loss - - -class TrainOneStepCell(nn.Cell): - r""" - Network training package class. - - Wraps the network with an optimizer. The resulting Cell be trained without inputs. - Backward graph will be created in the construct function to do parameter updating. Different - parallel modes are available to run the training. - - Args: - network (Cell): The training network. - optimizer (Cell): Optimizer for updating the weights. - sens (Number): The scaling number to be filled as the input of backpropagation. Default value is 1.0. - - Outputs: - Tensor, a scalar Tensor with shape :math:`()`. - - Examples: - >>> net = Net() - >>> loss_fn = nn.SoftmaxCrossEntropyWithLogits() - >>> optim = nn.Momentum(net.trainable_params(), learning_rate=0.1, momentum=0.9) - >>> loss_net = nn.WithLossCell(net, loss_fn) - >>> train_net = nn.TrainOneStepCell(loss_net, optim) - """ - - def __init__(self, network, optimizer, sens=1.0): - super(TrainOneStepCell, self).__init__(auto_prefix=False) - self.network = network - self.network.add_flags(defer_inline=True) - self.weights = ParameterTuple(network.trainable_params()) - self.optimizer = optimizer - self.grad = C.GradOperation('grad', get_by_list=True, sens_param=True) - self.sens = sens - - def construct(self): - weights = self.weights - loss = self.network() - sens = P.Fill()(P.DType()(loss), P.Shape()(loss), self.sens) - grads = self.grad(self.network, weights)(sens) - return F.depend(loss, self.optimizer(grads)) - - -class TrainNetWrapper(nn.Cell): - """ - Wraps the GCN model with optimizer. - - Args: - network (Cell): GCN network. - label (numpy.ndarray): Dataset labels. - mask (numpy.ndarray): Mask for training, evaluation or test. - config (ConfigGCN): Configuration for GCN. - """ - - def __init__(self, network, label, mask, config): - super(TrainNetWrapper, self).__init__(auto_prefix=True) - self.network = network - loss_net = LossWrapper(network, label, mask, config.weight_decay) - optimizer = nn.Adam(loss_net.trainable_params(), - learning_rate=config.learning_rate) - self.loss_train_net = TrainOneStepCell(loss_net, optimizer) - self.accuracy = Accuracy(label, mask) - - def construct(self): - loss = self.loss_train_net() - accuracy = self.accuracy(self.network()) - return loss, accuracy diff --git a/model_zoo/gcn/src/metrics.py b/model_zoo/gcn/src/metrics.py index 5930956776..e923f17e39 100644 --- a/model_zoo/gcn/src/metrics.py +++ b/model_zoo/gcn/src/metrics.py @@ -17,6 +17,9 @@ from mindspore import nn from mindspore import Tensor from mindspore.common import dtype as mstype from mindspore.ops import operations as P +from mindspore.common.parameter import ParameterTuple +from mindspore.ops import composite as C +from mindspore.ops import functional as F class Loss(nn.Cell): @@ -68,3 +71,116 @@ class Accuracy(nn.Cell): mask = mask / mask_reduce accuracy_all *= mask return self.mean(accuracy_all) + + +class LossAccuracyWrapper(nn.Cell): + """ + Wraps the GCN model with loss and accuracy cell. + + Args: + network (Cell): GCN network. + label (numpy.ndarray): Dataset labels. + mask (numpy.ndarray): Mask for training, evaluation or test. + weight_decay (float): Weight decay parameter for weight of the first convolution layer. + """ + + def __init__(self, network, label, mask, weight_decay): + super(LossAccuracyWrapper, self).__init__() + self.network = network + self.loss = Loss(label, mask, weight_decay, network.trainable_params()[0]) + self.accuracy = Accuracy(label, mask) + + def construct(self): + preds = self.network() + loss = self.loss(preds) + accuracy = self.accuracy(preds) + return loss, accuracy + + +class LossWrapper(nn.Cell): + """ + Wraps the GCN model with loss. + + Args: + network (Cell): GCN network. + label (numpy.ndarray): Dataset labels. + mask (numpy.ndarray): Mask for training. + weight_decay (float): Weight decay parameter for weight of the first convolution layer. + """ + + def __init__(self, network, label, mask, weight_decay): + super(LossWrapper, self).__init__() + self.network = network + self.loss = Loss(label, mask, weight_decay, network.trainable_params()[0]) + + def construct(self): + preds = self.network() + loss = self.loss(preds) + return loss + + +class TrainOneStepCell(nn.Cell): + r""" + Network training package class. + + Wraps the network with an optimizer. The resulting Cell be trained without inputs. + Backward graph will be created in the construct function to do parameter updating. Different + parallel modes are available to run the training. + + Args: + network (Cell): The training network. + optimizer (Cell): Optimizer for updating the weights. + sens (Number): The scaling number to be filled as the input of backpropagation. Default value is 1.0. + + Outputs: + Tensor, a scalar Tensor with shape :math:`()`. + + Examples: + >>> net = Net() + >>> loss_fn = nn.SoftmaxCrossEntropyWithLogits() + >>> optim = nn.Momentum(net.trainable_params(), learning_rate=0.1, momentum=0.9) + >>> loss_net = nn.WithLossCell(net, loss_fn) + >>> train_net = nn.TrainOneStepCell(loss_net, optim) + """ + + def __init__(self, network, optimizer, sens=1.0): + super(TrainOneStepCell, self).__init__(auto_prefix=False) + self.network = network + self.network.add_flags(defer_inline=True) + self.weights = ParameterTuple(network.trainable_params()) + self.optimizer = optimizer + self.grad = C.GradOperation('grad', get_by_list=True, sens_param=True) + self.sens = sens + + def construct(self): + weights = self.weights + loss = self.network() + sens = P.Fill()(P.DType()(loss), P.Shape()(loss), self.sens) + grads = self.grad(self.network, weights)(sens) + return F.depend(loss, self.optimizer(grads)) + + +class TrainNetWrapper(nn.Cell): + """ + Wraps the GCN model with optimizer. + + Args: + network (Cell): GCN network. + label (numpy.ndarray): Dataset labels. + mask (numpy.ndarray): Mask for training, evaluation or test. + config (ConfigGCN): Configuration for GCN. + """ + + def __init__(self, network, label, mask, config): + super(TrainNetWrapper, self).__init__(auto_prefix=True) + self.network = network + loss_net = LossWrapper(network, label, mask, config.weight_decay) + optimizer = nn.Adam(loss_net.trainable_params(), + learning_rate=config.learning_rate) + self.loss_train_net = TrainOneStepCell(loss_net, optimizer) + self.accuracy = Accuracy(label, mask) + + def construct(self): + loss = self.loss_train_net() + accuracy = self.accuracy(self.network()) + return loss, accuracy diff --git a/model_zoo/gcn/train.py b/model_zoo/gcn/train.py index 220d2ecd6b..a5e8094804 100644 --- a/model_zoo/gcn/train.py +++ b/model_zoo/gcn/train.py @@ -26,9 +26,10 @@ from matplotlib import animation from sklearn import manifold from mindspore import context -from model_zoo.gcn.src.gcn import GCN, LossAccuracyWrapper, TrainNetWrapper -from model_zoo.gcn.src.config import ConfigGCN -from model_zoo.gcn.src.dataset import get_adj_features_labels, get_mask +from src.gcn import GCN +from src.metrics import LossAccuracyWrapper, TrainNetWrapper +from src.config import ConfigGCN +from src.dataset import get_adj_features_labels, get_mask def t_SNE(out_feature, dim): diff --git a/tests/st/gnn/gcn/test_gcn.py b/tests/st/gnn/gcn/test_gcn.py index e5804809aa..afe123cdb0 100644 --- a/tests/st/gnn/gcn/test_gcn.py +++ b/tests/st/gnn/gcn/test_gcn.py @@ -17,7 +17,8 @@ import time import pytest import numpy as np from mindspore import context -from model_zoo.gcn.src.gcn import GCN, LossAccuracyWrapper, TrainNetWrapper +from model_zoo.gcn.src.gcn import GCN +from model_zoo.gcn.src.metrics import LossAccuracyWrapper, TrainNetWrapper from model_zoo.gcn.src.config import ConfigGCN from model_zoo.gcn.src.dataset import get_adj_features_labels, get_mask From ddedc416135523e3ed74f6711ff690cde7f47cab Mon Sep 17 00:00:00 2001 From: liuxiao Date: Wed, 1 Jul 2020 15:12:46 +0800 Subject: [PATCH 214/254] Add TBE operators SparseApplyFtrlV2\SparseApplyAdagradV2 for VM. --- mindspore/ccsrc/kernel/tbe/tbe_adapter.cc | 2 + mindspore/ops/_op_impl/tbe/__init__.py | 2 + .../_op_impl/tbe/sparse_apply_adagrad_v2.py | 48 +++++ .../ops/_op_impl/tbe/sparse_apply_ftrl_v2.py | 52 +++++ mindspore/ops/operations/__init__.py | 4 +- mindspore/ops/operations/nn_ops.py | 177 +++++++++++++++++- tests/ut/python/ops/test_ops.py | 33 ++++ 7 files changed, 316 insertions(+), 2 deletions(-) create mode 100644 mindspore/ops/_op_impl/tbe/sparse_apply_adagrad_v2.py create mode 100644 mindspore/ops/_op_impl/tbe/sparse_apply_ftrl_v2.py diff --git a/mindspore/ccsrc/kernel/tbe/tbe_adapter.cc b/mindspore/ccsrc/kernel/tbe/tbe_adapter.cc index cbc31415ec..c38f48763e 100644 --- a/mindspore/ccsrc/kernel/tbe/tbe_adapter.cc +++ b/mindspore/ccsrc/kernel/tbe/tbe_adapter.cc @@ -70,11 +70,13 @@ static std::map tbe_func_adapter_map = { {"strided_slice", "strided_slice_d"}, {"strided_slice_grad", "strided_slice_grad_d"}, {"sparse_apply_ftrl", "sparse_apply_ftrl_d"}, + {"sparse_apply_ftrl_v2", "sparse_apply_ftrl_v2_d"}, {"apply_ada_max", "apply_ada_max_d"}, {"apply_adadelta", "apply_adadelta_d"}, {"apply_adagrad", "apply_adagrad_d"}, {"apply_adagrad_v2", "apply_adagradv2_d"}, {"sparse_apply_adagrad", "sparse_apply_adagrad_d"}, + {"sparse_apply_adagrad_v2", "sparse_apply_adagrad_v2_d"}, {"apply_proximal_adagrad", "apply_proximal_adagrad_d"}, {"sparse_apply_proximal_adagrad", "sparse_apply_proximal_adagrad_d"}, {"apply_add_sign", "apply_add_sign_d"}, diff --git a/mindspore/ops/_op_impl/tbe/__init__.py b/mindspore/ops/_op_impl/tbe/__init__.py index be3d313554..fa2be6d515 100644 --- a/mindspore/ops/_op_impl/tbe/__init__.py +++ b/mindspore/ops/_op_impl/tbe/__init__.py @@ -38,6 +38,8 @@ from .apply_add_sign import _apply_add_sign_tbe from .apply_power_sign import _apply_power_sign_tbe from .apply_gradient_descent import _apply_gradient_descent_tbe from .apply_proximal_gradient_descent import _apply_proximal_gradient_descent_tbe +from .sparse_apply_ftrl_v2 import _sparse_apply_ftrl_v2_tbe +from .sparse_apply_adagrad_v2 import _sparse_apply_adagrad_v2_tbe from .approximate_equal import _approximate_equal_tbe from .adam_apply_one import _adam_apply_one_tbe from .assign import _assign_tbe diff --git a/mindspore/ops/_op_impl/tbe/sparse_apply_adagrad_v2.py b/mindspore/ops/_op_impl/tbe/sparse_apply_adagrad_v2.py new file mode 100644 index 0000000000..088edb60d3 --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/sparse_apply_adagrad_v2.py @@ -0,0 +1,48 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""SparseApplyAdagradV2D op""" +from mindspore.ops.op_info_register import op_info_register, TBERegOp, DataType + +sparse_apply_adagrad_v2_d_op_info = TBERegOp("SparseApplyAdagradV2") \ + .fusion_type("OPAQUE") \ + .async_flag(False) \ + .binfile_name("sparse_apply_adagrad_v2_d.so") \ + .compute_cost(10) \ + .kernel_name("sparse_apply_adagrad_v2_d") \ + .partial_flag(True) \ + .attr("lr", "required", "float", "all") \ + .attr("epsilon", "required", "float", "all") \ + .attr("use_locking", "optional", "bool", "all") \ + .attr("update_slots", "optional", "bool", "all") \ + .input(0, "var", False, "required", "all") \ + .input(1, "accum", False, "required", "all") \ + .input(2, "grad", False, "required", "all") \ + .input(3, "indices", False, "required", "all") \ + .output(0, "var", False, "required", "all") \ + .output(1, "accum", False, "required", "all") \ + .dtype_format(DataType.F32_NCHW, DataType.F32_NCHW, DataType.F32_NCHW, DataType.I32_NCHW, + DataType.F32_NCHW, DataType.F32_NCHW) \ + .dtype_format(DataType.F32_NHWC, DataType.F32_NHWC, DataType.F32_NHWC, DataType.I32_NHWC, + DataType.F32_NHWC, DataType.F32_NHWC) \ + .dtype_format(DataType.F32_Default, DataType.F32_Default, DataType.F32_Default, DataType.I32_Default, + DataType.F32_Default, DataType.F32_Default) \ + .get_op_info() + + +@op_info_register(sparse_apply_adagrad_v2_d_op_info) +def _sparse_apply_adagrad_v2_tbe(): + """SparseApplyAdagradV2D TBE register""" + return diff --git a/mindspore/ops/_op_impl/tbe/sparse_apply_ftrl_v2.py b/mindspore/ops/_op_impl/tbe/sparse_apply_ftrl_v2.py new file mode 100644 index 0000000000..518c524010 --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/sparse_apply_ftrl_v2.py @@ -0,0 +1,52 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""SparseApplyFtrlV2D op""" +from mindspore.ops.op_info_register import op_info_register, TBERegOp, DataType + +sparse_apply_ftrl_v2_d_op_info = TBERegOp("SparseApplyFtrlV2") \ + .fusion_type("OPAQUE") \ + .async_flag(False) \ + .binfile_name("sparse_apply_ftrl_v2_d.so") \ + .compute_cost(10) \ + .kernel_name("sparse_apply_ftrl_v2_d") \ + .partial_flag(True) \ + .attr("lr", "required", "float", "all") \ + .attr("l1", "required", "float", "all") \ + .attr("l2", "required", "float", "all") \ + .attr("l2_shrinkage", "required", "float", "all") \ + .attr("lr_power", "required", "float", "all") \ + .attr("use_locking", "optional", "bool", "true,false", "false") \ + .input(0, "var", False, "required", "all") \ + .input(1, "accum", False, "required", "all") \ + .input(2, "linear", False, "required", "all") \ + .input(3, "grad", False, "required", "all") \ + .input(4, "indices", False, "required", "all") \ + .output(0, "var", False, "required", "all") \ + .output(1, "accum", False, "required", "all") \ + .output(2, "linear", False, "required", "all") \ + .dtype_format(DataType.F32_NCHW, DataType.F32_NCHW, DataType.F32_NCHW, DataType.F32_NCHW, + DataType.I32_NCHW, DataType.F32_NCHW, DataType.F32_NCHW, DataType.F32_NCHW) \ + .dtype_format(DataType.F32_NHWC, DataType.F32_NHWC, DataType.F32_NHWC, DataType.F32_NHWC, + DataType.I32_NHWC, DataType.F32_NHWC, DataType.F32_NHWC, DataType.F32_NHWC) \ + .dtype_format(DataType.F32_Default, DataType.F32_Default, DataType.F32_Default, DataType.F32_Default, + DataType.I32_Default, DataType.F32_Default, DataType.F32_Default, DataType.F32_Default) \ + .get_op_info() + + +@op_info_register(sparse_apply_ftrl_v2_d_op_info) +def _sparse_apply_ftrl_v2_tbe(): + """SparseApplyFtrlV2D TBE register""" + return diff --git a/mindspore/ops/operations/__init__.py b/mindspore/ops/operations/__init__.py index bc4edce193..06a19d2db7 100644 --- a/mindspore/ops/operations/__init__.py +++ b/mindspore/ops/operations/__init__.py @@ -72,7 +72,7 @@ from .nn_ops import (LSTM, SGD, Adam, SparseApplyAdam, SparseApplyLazyAdam, Appl SoftmaxCrossEntropyWithLogits, ROIAlign, SparseSoftmaxCrossEntropyWithLogits, Tanh, TopK, BinaryCrossEntropy, SparseApplyAdagrad, LARSUpdate, ApplyFtrl, SparseApplyFtrl, - ApplyProximalAdagrad, SparseApplyProximalAdagrad, + ApplyProximalAdagrad, SparseApplyProximalAdagrad, SparseApplyAdagradV2, SparseApplyFtrlV2, ApplyAdaMax, ApplyAdadelta, ApplyAdagrad, ApplyAdagradV2, ApplyAddSign, ApplyPowerSign, ApplyGradientDescent, ApplyProximalGradientDescent, ApplyRMSProp, ApplyCenteredRMSProp, BasicLSTMCell, InTopK) @@ -284,6 +284,7 @@ __all__ = [ "Abs", "BinaryCrossEntropy", "SparseApplyAdagrad", + "SparseApplyAdagradV2", "SpaceToDepth", "DepthToSpace", "Conv2DBackpropInput", @@ -294,6 +295,7 @@ __all__ = [ "ApplyFtrl", "SpaceToBatch", "SparseApplyFtrl", + "SparseApplyFtrlV2", "ApplyProximalAdagrad", "SparseApplyProximalAdagrad", "ApplyAdaMax", diff --git a/mindspore/ops/operations/nn_ops.py b/mindspore/ops/operations/nn_ops.py index c07f072f38..6320e9e011 100644 --- a/mindspore/ops/operations/nn_ops.py +++ b/mindspore/ops/operations/nn_ops.py @@ -3600,6 +3600,88 @@ class SparseApplyAdagrad(PrimitiveWithInfer): return var_type, accum_type +class SparseApplyAdagradV2(PrimitiveWithInfer): + r""" + Update relevant entries according to the adagrad scheme. + + .. math:: + accum += grad * grad + .. math:: + var -= lr * grad * \frac{1}{\sqrt{accum} + \epsilon} + + Args: + lr (float): Learning rate. + epsilon (float): A small value added for numerical stability. + use_locking (bool): If `True`, updating of the var and accum tensors will be protected. Default: False. + update_slots (bool): If `True`, the computation logic will be different to `False`. Default: True. + + Inputs: + - **var** (Parameter) - Variable to be updated. The type must be float32. + - **accum** (Parameter) - Accum to be updated. The shape must be the same as `var`'s shape, + the type must be float32. + - **grad** (Tensor) - Gradient. The shape must be the same as `var`'s shape except first dimension, + the type must be float32. + - **indices** (Tensor) - A vector of indices into the first dimension of `var` and `accum`. + The shape of `indices` must be the same as `grad` in first dimension, the type must be int32. + + Outputs: + Tuple of 2 Tensor, the updated parameters. + + - **var** (Tensor) - The same shape and data type as `var`. + - **accum** (Tensor) - The same shape and data type as `accum`. + + Examples: + >>> import numpy as np + >>> import mindspore.nn as nn + >>> from mindspore import Tensor, Parameter + >>> from mindspore.ops import operations as P + >>> import mindspore.common.dtype as mstype + >>> class Net(nn.Cell): + >>> def __init__(self): + >>> super(Net, self).__init__() + >>> self.sparse_apply_adagrad_v2 = P.SparseApplyAdagradV2(lr=1e-8, epsilon=1e-6) + >>> self.var = Parameter(Tensor(np.ones([3, 3, 3]).astype(np.float32)), name="var") + >>> self.accum = Parameter(Tensor(np.ones([3, 3, 3]).astype(np.float32)), name="accum") + >>> + >>> def construct(self, grad, indices): + >>> out = self.sparse_apply_adagrad_v2(self.var, self.accum, grad, indices) + >>> return out + >>> net = Net() + >>> grad = Tensor(np.random.rand(3, 3, 3).astype(np.float32)) + >>> indices = Tensor([0, 1, 2], mstype.int32) + >>> result = net(grad, indices) + """ + + __mindspore_signature__ = ( + ('var', sig_rw.RW_WRITE, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), + ('accum', sig_rw.RW_WRITE, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), + ('grad', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), + ('indices', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T1) + ) + + @prim_attr_register + def __init__(self, lr, epsilon, use_locking=False, update_slots=True): + self.lr = validator.check_value_type("lr", lr, [float], self.name) + self.epsilon = validator.check_value_type("epsilon", epsilon, [float], self.name) + self.use_locking = validator.check_value_type("update_slots", update_slots, [bool], self.name) + self.update_slots = validator.check_value_type("use_locking", use_locking, [bool], self.name) + + def infer_shape(self, var_shape, accum_shape, grad_shape, indices_shape): + validator.check('var shape', var_shape, 'accum shape', accum_shape, Rel.EQ, self.name) + validator.check('len of var shape', len(var_shape), 'len of grad shape', len(grad_shape), Rel.EQ, self.name) + if len(var_shape) > 1: + validator.check('var_shape[1:]', var_shape[1:], 'grad_shape[1:]', grad_shape[1:], Rel.EQ, self.name) + validator.check_integer("indices rank", len(indices_shape), 1, Rel.EQ, self.name) + validator.check('grad_shape[0]', grad_shape[0], 'indices_shape[0]', indices_shape[0], Rel.EQ, self.name) + return var_shape, accum_shape + + def infer_dtype(self, var_type, accum_type, grad_type, indices_type): + args = {'var': var_type, 'accum': accum_type, 'grad': grad_type} + validator.check_tensor_type_same(args, [mstype.float32], self.name) + validator.check_tensor_type_same({'indices': indices_type}, [mstype.int32], self.name) + return var_type, accum_type + + class ApplyProximalAdagrad(PrimitiveWithInfer): r""" Update relevant entries according to the proximal adagrad algorithm. @@ -3664,7 +3746,8 @@ class ApplyProximalAdagrad(PrimitiveWithInfer): @prim_attr_register def __init__(self, use_locking=False): - self.init_prim_io_names(inputs=['var', 'accum', 'lr', 'l1', 'l2', 'grad'], outputs=['output']) + self.init_prim_io_names(inputs=['var', 'accum', 'lr', 'l1', 'l2', 'grad'], + outputs=['var', 'accum']) self.use_locking = validator.check_value_type("use_locking", use_locking, [bool], self.name) def infer_shape(self, var_shape, accum_shape, lr_shape, l1_shape, l2_shape, grad_shape): @@ -4371,6 +4454,98 @@ class SparseApplyFtrl(PrimitiveWithInfer): return var_dtype, accum_dtype, linear_dtype +class SparseApplyFtrlV2(PrimitiveWithInfer): + """ + Update relevant entries according to the FTRL-proximal scheme. + + Args: + lr (float): The learning rate value, must be positive. + l1 (float): l1 regularization strength, must be greater than or equal to zero. + l2 (float): l2 regularization strength, must be greater than or equal to zero. + l2_shrinkage (float): L2 shrinkage regularization. + lr_power (float): Learning rate power controls how the learning rate decreases during training, + must be less than or equal to zero. Use fixed learning rate if `lr_power` is zero. + use_locking (bool): If `True`, updating of the var and accum tensors will be protected. Default: False. + + Inputs: + - **var** (Parameter) - The variable to be updated. The data type must be float32. + - **accum** (Parameter) - The accum to be updated, must be same type and shape as `var`. + - **linear** (Parameter) - The linear to be updated, must be same type and shape as `var`. + - **grad** (Tensor) - A tensor of the same type as `var`, for the gradient. + - **indices** (Tensor) - A vector of indices into the first dimension of `var` and `accum`. + The shape of `indices` must be the same as `grad` in first dimension. The type must be int32. + + Outputs: + Tuple of 3 Tensor, the updated parameters. + + - **var** (Tensor): Tensor, has the same shape and type as `var`. + - **accum** (Tensor): Tensor, has the same shape and type as `accum`. + - **linear** (Tensor): Tensor, has the same shape and type as `linear`. + + Examples: + >>> import mindspore + >>> import mindspore.nn as nn + >>> import numpy as np + >>> from mindspore import Parameter + >>> from mindspore import Tensor + >>> from mindspore.ops import operations as P + >>> class SparseApplyFtrlV2Net(nn.Cell): + >>> def __init__(self): + >>> super(SparseApplyFtrlV2Net, self).__init__() + >>> self.sparse_apply_ftrl_v2 = P.SparseApplyFtrlV2(lr=0.01, l1=0.0, l2=0.0, + l2_shrinkage=0.0, lr_power=-0.5) + >>> self.var = Parameter(Tensor(np.random.rand(3, 3).astype(np.float32)), name="var") + >>> self.accum = Parameter(Tensor(np.random.rand(3, 3).astype(np.float32)), name="accum") + >>> self.linear = Parameter(Tensor(np.random.rand(3, 3).astype(np.float32)), name="linear") + >>> + >>> def construct(self, grad, indices): + >>> out = self.sparse_apply_ftrl_v2(self.var, self.accum, self.linear, grad, indices) + >>> return out + >>> + >>> net = SparseApplyFtrlV2Net() + >>> grad = Tensor(np.random.rand(3, 3).astype(np.float32)) + >>> indices = Tensor(np.ones([3]), mindspore.int32) + >>> output = net(grad, indices) + """ + + __mindspore_signature__ = ( + ('var', sig_rw.RW_WRITE, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), + ('accum', sig_rw.RW_WRITE, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), + ('linear', sig_rw.RW_WRITE, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), + ('grad', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T), + ('indices', sig_rw.RW_READ, sig_kind.KIND_POSITIONAL_KEYWORD, sig_kind.KIND_EMPTY_DEFAULT_VALUE, sig_dtype.T1) + ) + + @prim_attr_register + def __init__(self, lr, l1, l2, l2_shrinkage, lr_power, use_locking=False): + validator.check_value_type("lr", lr, [float], self.name) + validator.check_value_type("l1", l1, [float], self.name) + validator.check_value_type("l2", l2, [float], self.name) + validator.check_value_type("lr_power", lr_power, [float], self.name) + self.lr = validator.check_number_range("lr", lr, 0.0, float("inf"), Rel.INC_NEITHER, self.name) + self.l1 = validator.check_number_range("l1", l1, 0.0, float("inf"), Rel.INC_LEFT, self.name) + self.l2 = validator.check_number_range("l2", l2, 0.0, float("inf"), Rel.INC_LEFT, self.name) + self.lr_power = validator.check_number("lr_power", lr_power, 0, Rel.LE, self.name) + self.l2_shrinkage = validator.check_value_type("l2_shrinkage", l2_shrinkage, [float], self.name) + self.use_locking = validator.check_value_type("use_locking", use_locking, [bool], self.name) + + def infer_shape(self, var_shape, accum_shape, linear_shape, grad_shape, indices_shape): + validator.check('var shape', var_shape, 'accum shape', accum_shape, Rel.EQ, self.name) + validator.check('var shape', var_shape, 'linear shape', linear_shape, Rel.EQ, self.name) + if len(var_shape) > 1: + validator.check('var_shape[1:]', var_shape[1:], 'grad_shape[1:]', grad_shape[1:], Rel.EQ, self.name) + validator.check_integer("indices rank", len(indices_shape), 1, Rel.EQ, self.name) + validator.check('grad_shape[0]', grad_shape[0], 'indices_shape[0]', indices_shape[0], Rel.EQ, self.name) + return var_shape, accum_shape, linear_shape + + def infer_dtype(self, var_dtype, accum_dtype, linear_dtype, grad_dtype, indices_dtype): + args = {"var_dtype": var_dtype, "accum_dtype": accum_dtype, + "linear_dtype": linear_dtype, "grad_dtype": grad_dtype} + validator.check_tensor_type_same(args, [mstype.float32], self.name) + validator.check_tensor_type_same({"indices_dtype": indices_dtype}, [mstype.int32], self.name) + return var_dtype, accum_dtype, linear_dtype + + class ConfusionMulGrad(PrimitiveWithInfer): """ `output0` is the result of which input0 dot multily input1. diff --git a/tests/ut/python/ops/test_ops.py b/tests/ut/python/ops/test_ops.py index 98a7b766e7..f55d42e28b 100755 --- a/tests/ut/python/ops/test_ops.py +++ b/tests/ut/python/ops/test_ops.py @@ -306,6 +306,19 @@ class SparseApplyFtrlNet(nn.Cell): return out +class SparseApplyFtrlV2Net(nn.Cell): + def __init__(self): + super(SparseApplyFtrlV2Net, self).__init__() + self.sparse_apply_ftrl_v2 = P.SparseApplyFtrlV2(lr=0.001, l1=0.0, l2=0.0, l2_shrinkage=0.0, lr_power=-0.5) + self.var = Parameter(Tensor(np.random.rand(3, 3).astype(np.float32)), name="var") + self.accum = Parameter(Tensor(np.random.rand(3, 3).astype(np.float32)), name="accum") + self.linear = Parameter(Tensor(np.random.rand(3, 3).astype(np.float32)), name="linear") + + def construct(self, grad, indices): + out = self.sparse_apply_ftrl_v2(self.var, self.accum, self.linear, grad, indices) + return out + + class SparseApplyProximalAdagradNet(nn.Cell): def __init__(self): super(SparseApplyProximalAdagradNet, self).__init__() @@ -467,6 +480,18 @@ class SparseApplyAdagradNet(nn.Cell): return out +class SparseApplyAdagradV2Net(nn.Cell): + def __init__(self): + super(SparseApplyAdagradV2Net, self).__init__() + self.sparse_apply_adagrad_v2 = P.SparseApplyAdagradV2(lr=0.01, epsilon=0.001) + self.var = Parameter(Tensor(np.random.rand(3, 3).astype(np.float32)), name="var") + self.accum = Parameter(Tensor(np.random.rand(3, 3).astype(np.float32)), name="accum") + + def construct(self, grad, indices): + out = self.sparse_apply_adagrad_v2(self.var, self.accum, grad, indices) + return out + + class ApplyRMSNet(nn.Cell): def __init__(self): super(ApplyRMSNet, self).__init__() @@ -1376,10 +1401,18 @@ test_case_nn_ops = [ 'desc_inputs': [[3, 3], Tensor(np.ones((3,), np.int32))], 'desc_bprop': [[3, 3], [3, 3]], 'skip': ['backward']}), + ('SparseApplyAdagradV2', { + 'block': SparseApplyAdagradV2Net(), + 'desc_inputs': [[3, 3], Tensor(np.ones((3,), np.int32))], + 'skip': ['backward']}), ('SparseApplyFtrl', { 'block': SparseApplyFtrlNet(), 'desc_inputs': [[3, 3], Tensor(np.ones((3,), np.int32))], 'skip': ['backward']}), + ('SparseApplyFtrlV2', { + 'block': SparseApplyFtrlV2Net(), + 'desc_inputs': [[3, 3], Tensor(np.ones((3,), np.int32))], + 'skip': ['backward']}), ('ApplyProximalAdagrad', { 'block': ApplyProximalAdagradNet(), 'desc_inputs': [[3, 3]], From b26f6b6b6778699c1ecd66a84cb972202bffbc7d Mon Sep 17 00:00:00 2001 From: BowenK Date: Mon, 22 Jun 2020 09:31:42 +0800 Subject: [PATCH 215/254] Add python pass support --- .../optimizer/irpass/arithmetic_simplify.h | 4 - mindspore/ccsrc/optimizer/pass_group.cc | 69 +++++ mindspore/ccsrc/optimizer/pass_group.h | 61 +++++ mindspore/ccsrc/optimizer/py_pass.cc | 236 ++++++++++++++++++ mindspore/ccsrc/optimizer/py_pass.h | 56 +++++ mindspore/ccsrc/optimizer/py_pass_manager.cc | 84 +++++++ mindspore/ccsrc/optimizer/py_pass_manager.h | 66 +++++ mindspore/ccsrc/pipeline/action.cc | 27 ++ mindspore/ccsrc/pipeline/pipeline.cc | 2 + mindspore/common/python_pass_register.py | 80 ++++++ mindspore/nn/layer/basic.py | 3 +- 11 files changed, 683 insertions(+), 5 deletions(-) create mode 100644 mindspore/ccsrc/optimizer/pass_group.cc create mode 100644 mindspore/ccsrc/optimizer/pass_group.h create mode 100644 mindspore/ccsrc/optimizer/py_pass.cc create mode 100644 mindspore/ccsrc/optimizer/py_pass.h create mode 100644 mindspore/ccsrc/optimizer/py_pass_manager.cc create mode 100644 mindspore/ccsrc/optimizer/py_pass_manager.h create mode 100644 mindspore/common/python_pass_register.py diff --git a/mindspore/ccsrc/optimizer/irpass/arithmetic_simplify.h b/mindspore/ccsrc/optimizer/irpass/arithmetic_simplify.h index 1836a88dbc..24dea8f1f0 100644 --- a/mindspore/ccsrc/optimizer/irpass/arithmetic_simplify.h +++ b/mindspore/ccsrc/optimizer/irpass/arithmetic_simplify.h @@ -346,10 +346,6 @@ class TensorAddByZero : public AnfVisitor { } void Visit(const AnfNodePtr &node) override { - if (IsPrimitive(node, prim::kPrimZerosLike)) { - is_zero_ = true; - return; - } if (node->isa() && CheckTensorConstant(0).IsTensorScalarConstant(GetValueNode(node))) { is_zero_ = true; return; diff --git a/mindspore/ccsrc/optimizer/pass_group.cc b/mindspore/ccsrc/optimizer/pass_group.cc new file mode 100644 index 0000000000..2d1ab07f7d --- /dev/null +++ b/mindspore/ccsrc/optimizer/pass_group.cc @@ -0,0 +1,69 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "optimizer/pass_group.h" + +namespace mindspore { +namespace opt { +namespace python_pass { +void PassGroup::AddPass(const PythonPassPtr &pass) { + if (pass != nullptr) { + passes_.push_back(pass); + } +} + +bool PassGroup::DeletePass(const std::string &pass_name) { + for (auto iter = passes_.begin(); iter != passes_.end(); iter++) { + if ((*iter)->name() == pass_name) { + *iter = nullptr; + passes_.erase(iter); + return true; + } + } + return false; +} + +bool PassGroup::Run(const FuncGraphPtr &func_graph, const std::vector &passes) const { + if (func_graph == nullptr) { + return false; + } + bool changed = false; + for (const auto &pass : passes) { + if (pass != nullptr) { + if (pass->Run(func_graph)) { + changed = true; + } + } + } + return changed; +} + +bool PassGroup::Run(const FuncGraphPtr &func_graph) const { + bool changed = false; + // run all passes + bool change = true; + while (change) { + change = Run(func_graph, passes_); + changed = change || changed; + if (run_only_once_) { + break; + } + } + return changed; +} + +} // namespace python_pass +} // namespace opt +} // namespace mindspore diff --git a/mindspore/ccsrc/optimizer/pass_group.h b/mindspore/ccsrc/optimizer/pass_group.h new file mode 100644 index 0000000000..895f5a4128 --- /dev/null +++ b/mindspore/ccsrc/optimizer/pass_group.h @@ -0,0 +1,61 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_OPTIMIZER_PASS_GROUP_H_ +#define MINDSPORE_CCSRC_OPTIMIZER_PASS_GROUP_H_ + +#include +#include +#include +#include + +#include "optimizer/py_pass.h" + +namespace mindspore { +namespace opt { +namespace python_pass { +class PassGroup { + public: + explicit PassGroup(const std::string &name = "pass_group", bool run_only_once = false) + : name_(name), passes_{}, run_only_once_(run_only_once) {} + virtual ~PassGroup() = default; + // Add graph pass, the pass object will be freed when pass manager freed. + void AddPass(const PythonPassPtr &pass); + // Delete graph pass before the pass manager is freed. + bool DeletePass(const std::string &pass_name); + // Run passes added in pass manager on the input graph + // @param [inout] graph The graph to be optimized + // @return true, graph changed + // @return false, graph not changed + bool Run(const FuncGraphPtr &func_graph) const; + // Run the given graph passes on the input graph + // @param [inout] graph The graph to be optimized + // @param [in] passes The given graph passes + // @return true, graph changed + // @return false, graph not changed + bool Run(const FuncGraphPtr &func_graph, const std::vector &passes) const; + std::string name() const { return name_; } + + private: + const std::string name_; + std::vector passes_; + bool run_only_once_; +}; +using PassGroupPtr = std::shared_ptr; +} // namespace python_pass +} // namespace opt +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_OPTIMIZER_PASS_GROUP_H_ diff --git a/mindspore/ccsrc/optimizer/py_pass.cc b/mindspore/ccsrc/optimizer/py_pass.cc new file mode 100644 index 0000000000..8ce348b22e --- /dev/null +++ b/mindspore/ccsrc/optimizer/py_pass.cc @@ -0,0 +1,236 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "optimizer/py_pass.h" +#include +#include +#include +#include +#include + +#include "ir/func_graph.h" +#include "ir/manager.h" +#include "pipeline/parse/parse_base.h" +#include "pipeline/resource.h" + +namespace mindspore { +namespace opt { +namespace python_pass { +namespace internal { +std::string GetNodeRepr(AnfNodePtr node) { + if (node != nullptr) { + if (node->isa()) { + std::string repr = "("; + auto const &inputs = node->cast()->inputs(); + for (auto &input : inputs) { + repr += " "; + repr += GetNodeRepr(input); + repr += " "; + } + repr += ")"; + return repr; + } + if (node->isa()) { + return GetValueNode(node)->ToString(); + } + return node->ToString(); + } + return ""; +} + +void ResolveFuncGraph_(const FuncGraphPtr &fg) { + auto manager = Manage(fg, false); + parse::python_adapter::set_use_signature_in_resolve(false); + parse::ResolveAll(manager); +} + +bool Match(const AnfNodePtr &pattern, const AnfNodePtr &node, const NodeEquivPtr &equiv_ptr) { + if (node == nullptr) { + return false; + } + MS_EXCEPTION_IF_NULL(pattern); + if (pattern->isa()) { + if (!node->isa()) { + return false; + } + if (GetNodeRepr(pattern) == GetNodeRepr(node)) { + // add to equiv_ptr + equiv_ptr->insert(std::make_pair(GetValueNode(pattern)->ToString(), node)); + return true; + } + return false; + } else if (pattern->isa()) { + MS_LOG(DEBUG) << pattern->ToString() + "\n"; + // add to equiv_ptr + equiv_ptr->insert(std::make_pair(pattern->ToString(), node)); + return true; + } else if (pattern->isa()) { + // match every single sub ANode + if (!node->isa()) { + return false; + } + auto pattern_inputs = pattern->cast()->inputs(); + auto node_inputs = node->cast()->inputs(); + if (pattern_inputs.size() != node_inputs.size()) { + return false; + } + for (auto p_item = pattern_inputs.begin(), node_item = node_inputs.begin(); p_item != pattern_inputs.end(); + p_item++, node_item++) { + auto res = Match(*p_item, *node_item, equiv_ptr); + if (!res) { + return false; + } + } + return true; + } + MS_LOG(EXCEPTION) << "Unexpected condition, (" + pattern->ToString() + " , " + node->ToString() + ")\n"; +} + +AnfNodePtr BuildTarget(const FuncGraphPtr &func_graph, const AnfNodePtr cur_raw_dst_node_, + const NodeEquivPtr &equiv_ptr) { + if (cur_raw_dst_node_->isa()) { + auto sub_pair = equiv_ptr->find(cur_raw_dst_node_->ToString()); + if (sub_pair != equiv_ptr->end()) { + return sub_pair->second; + } + MS_LOG(EXCEPTION) << "cur_raw_dst_node_ : " + internal::GetNodeRepr(cur_raw_dst_node_) + "\n"; + } else if (cur_raw_dst_node_->isa()) { + // check primitive ValueNode + auto sub_pair = equiv_ptr->find(cur_raw_dst_node_->cast()->value()->ToString()); + if (sub_pair != equiv_ptr->end()) { + return sub_pair->second; + } + return cur_raw_dst_node_; + } else if (cur_raw_dst_node_->isa()) { + std::vector new_inputs; + auto inputs = cur_raw_dst_node_->cast()->inputs(); + for (auto sub_node = inputs.begin(); sub_node != inputs.end(); sub_node++) { + auto subed = internal::BuildTarget(func_graph, *sub_node, equiv_ptr); + new_inputs.push_back(subed); + } + return func_graph->NewCNode(new_inputs); + } + MS_LOG(EXCEPTION) << "Unexpected node type, got : " + internal::GetNodeRepr(cur_raw_dst_node_); +} + +bool isTraversable(const AnfNodePtr &node) { + if (node == nullptr) { + return false; + } + if (node->isa() || node->isa()) { + return true; + } + if (IsValueNode(node) || IsValueNode(node)) { + return true; + } + return false; +} +} // namespace internal + +void PythonPass::Build(const py::function &src, const py::function &dst) { + // 1. get FuncGraph from py::function + auto src_fg_ = parse::ParsePythonCode(src); + auto dst_fg_ = parse::ParsePythonCode(dst); + if (src_fg_ == nullptr || dst_fg_ == nullptr) { + MS_LOG(EXCEPTION) << "Failed to parse python code.\n"; + } + // 2. Resolve + internal::ResolveFuncGraph_(src_fg_); + internal::ResolveFuncGraph_(dst_fg_); + // 3. from FuncGraphPtr to ValueNode + src_node_ = src_fg_->output(); + dst_node_ = dst_fg_->output(); +} + +PythonPass::PythonPass(const std::string &name, const py::function &src, const py::function &dst, bool run_only_once, + bool multigraph) + : name_(name), run_only_once_(run_only_once), multigraph_(multigraph) { + Build(src, dst); +} + +AnfNodePtr PythonPass::Run(const FuncGraphPtr &func_graph, const AnfNodePtr &node) { + auto equiv_ptr = std::make_shared(); + bool is_a_match = internal::Match(src_node_, node, equiv_ptr); + if (is_a_match) { + auto new_node = internal::BuildTarget(func_graph, dst_node_, equiv_ptr); + MS_LOG(DEBUG) << "To be replaced node: " + internal::GetNodeRepr(new_node) + "\n"; + return new_node; + } + return nullptr; +} + +bool PythonPass::Run(const FuncGraphPtr &func_graph) { + MS_EXCEPTION_IF_NULL(func_graph); + FuncGraphManagerPtr manager = func_graph->manager(); + MS_EXCEPTION_IF_NULL(manager); + manager->AddFuncGraph(func_graph); + auto seen = NewSeenGeneration(); + // 1024 is for the initial capacity of deque + std::deque todo(1024); + todo.push_back(func_graph->output()); + bool changes = false; + + auto &all_nodes = manager->all_nodes(); + while (!todo.empty()) { + AnfNodePtr node = todo.front(); + todo.pop_front(); + + // check whether this node has been matched. + if (node == nullptr || node->seen_ == seen || !internal::isTraversable(node) || !all_nodes.contains(node)) { + continue; + } + node->seen_ = seen; + + // select nodes that this transform can be applied. + AnfNodePtr new_node = Run(func_graph, node); + bool change = (new_node != nullptr); + if (new_node != nullptr && new_node != node) { + (void)manager->Replace(node, new_node); + } else if (new_node == nullptr) { + new_node = node; + } + if (run_only_once_) { + return change; + } + + // find success, and add them to todo list + if (IsValueNode(node)) { + todo.push_back(GetValueNode(node)->output()); + } + + if (node->isa()) { + auto &inputs = node->cast()->inputs(); + (void)std::copy(inputs.begin(), inputs.end(), std::back_inserter(todo)); + } + + auto &node_users = manager->node_users(); + if (change && node_users.find(node) != node_users.end()) { + for (auto &use : node_users[node]) { + auto use_node = use.first; + if (use_node == nullptr) { + continue; + } + todo.push_back(use_node); + if (use_node->seen_ == seen) { + use_node->seen_--; + } + } + } + } + return changes; +} +} // namespace python_pass +} // namespace opt +} // namespace mindspore diff --git a/mindspore/ccsrc/optimizer/py_pass.h b/mindspore/ccsrc/optimizer/py_pass.h new file mode 100644 index 0000000000..b01bf7c942 --- /dev/null +++ b/mindspore/ccsrc/optimizer/py_pass.h @@ -0,0 +1,56 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_OPTIMIZER_PASS_H_ +#define MINDSPORE_CCSRC_OPTIMIZER_PASS_H_ +#include +#include +#include + +#include "ir/anf.h" +#include "pybind_api/api_register.h" +#include "pybind_api/export_flags.h" + +namespace mindspore { +namespace opt { +namespace python_pass { +class PythonPass; +using PythonPassPtr = std::shared_ptr; +using NodeEquiv = std::unordered_map; +using NodeEquivPtr = std::shared_ptr; + +class PythonPass { + public: + explicit PythonPass(const std::string &name, const py::function &src, const py::function &dst, + bool run_only_once = false, bool multigraph = true); + ~PythonPass() = default; + bool Run(const FuncGraphPtr &func_graph); + std::string name() const { return name_; } + AnfNodePtr Run(const FuncGraphPtr &func_graph, const AnfNodePtr &node); + + private: + void Build(const py::function &src, const py::function &dst); + AnfNodePtr src_node_ = nullptr; + AnfNodePtr dst_node_ = nullptr; + const std::string name_; + bool run_only_once_; + bool multigraph_ = true; +}; + +using PythonPassPtr = std::shared_ptr; +} // namespace python_pass +} // namespace opt +} // namespace mindspore +#endif // MINDSPORE_CCSRC_OPTIMIZER_PASS_H_ diff --git a/mindspore/ccsrc/optimizer/py_pass_manager.cc b/mindspore/ccsrc/optimizer/py_pass_manager.cc new file mode 100644 index 0000000000..1c36e93c9a --- /dev/null +++ b/mindspore/ccsrc/optimizer/py_pass_manager.cc @@ -0,0 +1,84 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "optimizer/py_pass_manager.h" + +#include +#include +#include +#include + +#include "ir/manager.h" +#include "optimizer/pass_group.h" + +namespace mindspore { +namespace opt { +namespace python_pass { +PyPassManagerPtr PyPassManager::global_instance = nullptr; +std::unordered_map PyPassManager::phase_to_group_; + +PassGroupPtr PyPassManager::GetPassGroup(Phase phase) { + auto pm = phase_to_group_.find(phase); + if (pm == phase_to_group_.end()) { + return nullptr; + } + return pm->second; +} + +PyPassManagerPtr PyPassManager::GetInstance() { + if (global_instance == nullptr) { + global_instance = std::shared_ptr(new (std::nothrow) PyPassManager()); + } + return global_instance; +} + +PyPassManager::PyPassManager() { + phase_to_group_[Phase::RESOLVE] = std::make_shared(); + phase_to_group_[Phase::OPT] = std::make_shared(); +} + +void PyPassManager::Registe(const std::string &pass_name, const py::function &pattern, const py::function &target, + Phase phase, bool run_only_once, bool multigraph) { + auto cur_pm = GetPassGroup(phase); + MS_EXCEPTION_IF_NULL(cur_pm); + PythonPassPtr new_pass = std::make_shared(pass_name, pattern, target, run_only_once, multigraph); + cur_pm->AddPass(new_pass); +} + +void PyPassManager::Unregiste(const std::string &pass_name, Phase phase) { + auto cur_pm = GetPassGroup(phase); + MS_EXCEPTION_IF_NULL(cur_pm); + if (!cur_pm->DeletePass(pass_name)) { + MS_LOG(WARNING) << "No such pass : " + pass_name + "\n"; + } +} + +void PyPassManager::ClearRes() { + MS_LOG(INFO) << "Clear PyPassManager resources!"; + global_instance = nullptr; + phase_to_group_.clear(); +} + +REGISTER_PYBIND_DEFINE( + PyPassManager_, ([](const py::module *m) { + (void)py::enum_(*m, "phase", py::arithmetic()).value("resolve", Phase::RESOLVE).value("opt", Phase::OPT); + (void)py::class_>(*m, "PyPassManager_") + .def(py::init([]() { return PyPassManager::GetInstance(); })) + .def("registe", &PyPassManager::Registe, "Registe python pass") + .def("unregiste", &PyPassManager::Unregiste, "Delete Python Pass"); + })); +} // namespace python_pass +} // namespace opt +} // namespace mindspore diff --git a/mindspore/ccsrc/optimizer/py_pass_manager.h b/mindspore/ccsrc/optimizer/py_pass_manager.h new file mode 100644 index 0000000000..eaeefce213 --- /dev/null +++ b/mindspore/ccsrc/optimizer/py_pass_manager.h @@ -0,0 +1,66 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CCSRC_OPTIMIZER_PY_PASS_MANAGER_H_ +#define MINDSPORE_CCSRC_OPTIMIZER_PY_PASS_MANAGER_H_ + +#include +#include +#include +#include + +#include "ir/anf.h" +#include "ir/func_graph.h" +#include "ir/primitive.h" +#include "utils/graph_utils.h" +#include "common/utils.h" + +#include "pipeline/parse/resolve.h" +#include "optimizer/py_pass.h" +#include "optimizer/pass_group.h" + +namespace mindspore { +namespace opt { +namespace python_pass { +class PyPassManager; +using PyPassManagerPtr = std::shared_ptr; + +enum Phase { RESOLVE, OPT }; + +class PyPassManager { + protected: + PyPassManager(); + static PyPassManagerPtr global_instance; + + public: + // Singletons should not be cloneable and assignable + PyPassManager(const PyPassManager &other) = delete; + void operator=(const PyPassManager &) = delete; + // Access the only global instance + static PyPassManagerPtr GetInstance(); + virtual ~PyPassManager() = default; + void Registe(const std::string &pass_name, const py::function &pattern, const py::function &target, + Phase phase = Phase::RESOLVE, bool run_only_once = false, bool multigraph = true); + void Unregiste(const std::string &pass_name, Phase phase); + PassGroupPtr GetPassGroup(Phase phase); + void ClearRes(); + + private: + static std::unordered_map phase_to_group_; +}; +} // namespace python_pass +} // namespace opt +} // namespace mindspore +#endif // MINDSPORE_CCSRC_OPTIMIZER_PY_PASS_MANAGER_H_ diff --git a/mindspore/ccsrc/pipeline/action.cc b/mindspore/ccsrc/pipeline/action.cc index 3265905848..4c0bb0f81c 100644 --- a/mindspore/ccsrc/pipeline/action.cc +++ b/mindspore/ccsrc/pipeline/action.cc @@ -39,6 +39,7 @@ #include "optimizer/optimizer.h" #include "vm/transform.h" #include "parse/python_adapter.h" +#include "optimizer/py_pass_manager.h" namespace mindspore { namespace pipeline { @@ -420,6 +421,25 @@ bool RemoveValueNodeDuplicationsAction(const ResourcePtr &res) { bool ValidateAction(const ResourcePtr &res) { return ValidatePass(res); } +void ActionPyStub(const ResourcePtr &res, opt::python_pass::Phase phase) { + MS_EXCEPTION_IF_NULL(res->manager()); + MS_EXCEPTION_IF_NULL(res->func_graph()); + auto ppm = opt::python_pass::PyPassManager::GetInstance(); + if (!ppm->GetPassGroup(phase)->Run(res->func_graph())) { + MS_LOG(DEBUG) << "No match.\n"; + } +} + +bool ResolveActionPyStub(const ResourcePtr &res) { + ActionPyStub(res, opt::python_pass::Phase::RESOLVE); + return true; +} + +bool OptActionPyStub(const ResourcePtr &res) { + ActionPyStub(res, opt::python_pass::Phase::RESOLVE); + return true; +} + static std::vector CommonPipeline() { std::vector actions; @@ -432,6 +452,8 @@ static std::vector CommonPipeline() { if (!multi_graphs) { actions.emplace_back(std::make_pair("combine_like_graphs", CombineLikeGraphs)); } + // Add resolve-stage python pass stub + actions.emplace_back(std::make_pair("py_resolve", ResolveActionPyStub)); actions.emplace_back(std::make_pair("inference_opt_prepare", InferenceOptPrepareAction)); // Evaluate type and shape, and specialize actions.emplace_back(std::make_pair("abstract_specialize", AbstractSpecializeAction)); @@ -443,6 +465,8 @@ std::vector GePipeline() { auto actions = CommonPipeline(); // optimize actions.emplace_back(std::make_pair("optimize", GeOptimizeAction)); + // Add opt-stage python pass stub + actions.emplace_back(std::make_pair("py_opt", OptActionPyStub)); actions.emplace_back(std::make_pair("remove_value_node_duplications", RemoveValueNodeDuplicationsAction)); actions.emplace_back(std::make_pair("validate", ValidateAction)); return actions; @@ -454,6 +478,9 @@ std::vector VmPipeline() { // optimize actions.emplace_back(std::make_pair("optimize", VmOptimizeAction)); + // Add opt-stage python pass stub + actions.emplace_back(std::make_pair("py_opt", OptActionPyStub)); + actions.emplace_back(std::make_pair("validate", ValidateAction)); // compile the ANF graph diff --git a/mindspore/ccsrc/pipeline/pipeline.cc b/mindspore/ccsrc/pipeline/pipeline.cc index bb1f693c6b..47b191fbc2 100644 --- a/mindspore/ccsrc/pipeline/pipeline.cc +++ b/mindspore/ccsrc/pipeline/pipeline.cc @@ -39,6 +39,7 @@ #include "device/kernel_runtime_manager.h" #include "debug/trace.h" #include "pynative/pynative_execute.h" +#include "optimizer/py_pass_manager.h" #if (ENABLE_GE || ENABLE_D) #include "pipeline/pipeline_ge.h" @@ -964,6 +965,7 @@ void ClearResAtexit() { pipeline::ExecutorPy::ClearRes(); pipeline::ReclaimOptimizer(); pynative::PynativeExecutor::GetInstance()->ClearRes(); + opt::python_pass::PyPassManager::GetInstance()->ClearRes(); #ifdef ENABLE_GE transform::DfGraphManager::GetInstance().ClearGraph(); transform::DfGraphConvertor::get_adpt_map().clear(); diff --git a/mindspore/common/python_pass_register.py b/mindspore/common/python_pass_register.py new file mode 100644 index 0000000000..36eb37adc7 --- /dev/null +++ b/mindspore/common/python_pass_register.py @@ -0,0 +1,80 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Python pass register""" +from inspect import isfunction +from mindspore._c_expression import PyPassManager_ +from mindspore._c_expression import phase + +class PyPassManager(PyPassManager_): + r""" + Used to registe and unregiste python passes which can be used to alter graphs. + + Args: + pipeline_phase (phase): Specify the stage in which the pass will run in the pipeline. Default: phase.opt. + run_only_once (bool): Specify whether or not to run pass only once. Default: False. + multigraph (bool): Whether or not the pattern exists across graphs. Default: True. + + Raises: + TypeError: If argument has invalid type. + """ + def __init__(self, pipeline_phase=phase.opt, run_only_once=False, multi_graph=True): + if not isinstance(pipeline_phase, phase): + raise TypeError(f"Expecting phase, got : ({type(pipeline_phase)}){pipeline_phase}") + if not isinstance(run_only_once, bool): + raise TypeError(f"Expecting bool, got : ({type(run_only_once)}){run_only_once}") + if not isinstance(multi_graph, bool): + raise TypeError(f"Expecting bool, got : ({type(multi_graph)}){multi_graph}") + PyPassManager_.__init__(self) + self.phase_ = pipeline_phase + self.run_only_once_ = run_only_once + self.multi_graph_ = multi_graph + + def registe(self, py_pass): + if not isfunction(py_pass): + raise TypeError(f"Expecting function pass, got : ({type(py_pass)}){py_pass}") + pattern, target = py_pass() + pass_name = py_pass.__name__ + if not isfunction(pattern): + raise TypeError(f"Expecting function pattern, got : ({type(pattern)}){pattern}") + if not isfunction(target): + raise TypeError(f"Expecting function target, got : ({type(target)}){target}") + super().registe(pass_name, pattern, target, self.phase_, self.run_only_once_, self.multi_graph_) + + def unregiste(self, py_pass, pipeline_phase=phase.opt): + if not isinstance(pipeline_phase, phase): + raise TypeError(f"Expecting phase, got : ({type(pipeline_phase)}){pipeline_phase}") + if isinstance(py_pass, str): + super().unregiste(py_pass, pipeline_phase) + return + if isfunction(py_pass): + super().unregiste(py_pass.__name__, pipeline_phase) + return + raise TypeError(f"Expecting py_pass to be string or function, got ({type(py_pass)}){py_pass}") + + def __call__(self, py_pass): + self.registe(py_pass) + return py_pass + +def registe_pass(pipeline_phase=phase.opt, run_only_once=False, multi_graph=True): + """ + Examples: + >>> @registe_pass() + >>> def toy_pass(): + >>> def pattern(): + >>> pass + >>> def target(): + >>> pass + """ + return PyPassManager(pipeline_phase, run_only_once, multi_graph) diff --git a/mindspore/nn/layer/basic.py b/mindspore/nn/layer/basic.py index 548fbcec1e..5bc678b003 100644 --- a/mindspore/nn/layer/basic.py +++ b/mindspore/nn/layer/basic.py @@ -170,7 +170,8 @@ class Dense(Cell): bias_init (Union[Tensor, str, Initializer, numbers.Number]): The trainable bias_init parameter. The dtype is same as input x. The values of str refer to the function `initializer`. Default: 'zeros'. has_bias (bool): Specifies whether the layer uses a bias vector. Default: True. - activation (str): Regularizer function applied to the output of the layer, eg. 'relu'. Default: None. + activation (str): activate function applied to the output of the fully connected layer, eg. 'relu'. + Default: None. Raises: ValueError: If weight_init or bias_init shape is incorrect. From 1e43c609e0ef110242d59b573810ff0d42133d49 Mon Sep 17 00:00:00 2001 From: duxiutao Date: Wed, 24 Jun 2020 16:16:35 +0800 Subject: [PATCH 216/254] Add test case and fix two bugs 1. add case to guard precision 2. fix a shape bug 3. fix a funcGraph bug --- .../device/ascend/ascend_stream_assign.cc | 25 +-- .../ccsrc/kernel/akg/akg_kernel_build.cc | 3 +- .../ccsrc/session/anf_runtime_algorithm.cc | 2 +- .../ccsrc/session/anf_runtime_algorithm.h | 11 +- .../models/bert/test_bert_graph_kernel.py | 193 ++++++++++++++++++ tests/st/ops/graph_kernel/test_lamb.py | 130 ++++++++++++ 6 files changed, 342 insertions(+), 22 deletions(-) create mode 100644 tests/st/networks/models/bert/test_bert_graph_kernel.py create mode 100644 tests/st/ops/graph_kernel/test_lamb.py diff --git a/mindspore/ccsrc/device/ascend/ascend_stream_assign.cc b/mindspore/ccsrc/device/ascend/ascend_stream_assign.cc index e3491536ee..736d6203e9 100644 --- a/mindspore/ccsrc/device/ascend/ascend_stream_assign.cc +++ b/mindspore/ccsrc/device/ascend/ascend_stream_assign.cc @@ -348,16 +348,13 @@ void AscendStreamAssign::GetProcessedStream(const NotNull &graph uint32_t cur_stream_id = AnfAlgo::GetStreamId(cur_cnode_ptr); if (AnfAlgo::GetCNodeName(cur_cnode_ptr) == kStreamSwitchOpName) { - auto primitive = AnfAlgo::GetCNodePrimitive(cur_cnode_ptr); - MS_EXCEPTION_IF_NULL(primitive); - auto true_stream_id = GetValue(primitive->GetAttr(kAttrTrueBranchStream)); + auto true_stream_id = AnfAlgo::GetNodeAttr(cur_cnode_ptr, kAttrTrueBranchStream); processed_streams_.emplace(true_stream_id); - auto value_ptr = primitive->GetAttr(kStreamNeedActivedFirst); - if (value_ptr == nullptr) { + if (!AnfAlgo::HasNodeAttr(kStreamNeedActivedFirst, cur_cnode_ptr)) { continue; } - auto need_active = GetValue(value_ptr); + auto need_active = AnfAlgo::GetNodeAttr(cur_cnode_ptr, kStreamNeedActivedFirst); if (need_active) { processed_streams_.emplace(cur_stream_id); } @@ -371,20 +368,17 @@ void AscendStreamAssign::GetProcessedStream(const NotNull &graph void AscendStreamAssign::UpdateStreamSwitch(const NotNull &graph_ptr, const CNodePtr &switch_ptr, vector *orders) { orders->emplace_back(switch_ptr); - auto primitive = AnfAlgo::GetCNodePrimitive(switch_ptr); - MS_EXCEPTION_IF_NULL(primitive); - auto value_ptr = primitive->GetAttr(kStreamNeedActivedFirst); - if (value_ptr == nullptr) { + if (!AnfAlgo::HasNodeAttr(kStreamNeedActivedFirst, switch_ptr)) { return; } - auto need_active = GetValue(value_ptr); + auto need_active = AnfAlgo::GetNodeAttr(switch_ptr, kStreamNeedActivedFirst); if (!need_active) { return; } MS_EXCEPTION_IF_NULL(switch_ptr); - auto true_stream_id = GetValue(primitive->GetAttr(kAttrTrueBranchStream)); + auto true_stream_id = AnfAlgo::GetNodeAttr(switch_ptr, kAttrTrueBranchStream); MS_LOG(INFO) << "Streamswtich stream id:" << AnfAlgo::GetStreamId(switch_ptr) << "; active stream id:" << true_stream_id; @@ -677,14 +671,11 @@ void AscendStreamAssign::GetNeedActiveStreams(const NotNull &gra for (size_t i = 0; i < cnode_ptr_list.size(); ++i) { cur_cnode_ptr = cnode_ptr_list[i]; MS_EXCEPTION_IF_NULL(cur_cnode_ptr); - auto primitive = AnfAlgo::GetCNodePrimitive(cur_cnode_ptr); - MS_EXCEPTION_IF_NULL(primitive); - auto value_ptr = primitive->GetAttr(kStreamNeedActivedFirst); - if (value_ptr == nullptr) { + if (!AnfAlgo::HasNodeAttr(kStreamNeedActivedFirst, cur_cnode_ptr)) { continue; } - auto need_active = GetValue(value_ptr); + auto need_active = AnfAlgo::GetNodeAttr(cur_cnode_ptr, kStreamNeedActivedFirst); if (need_active) { auto stream_id = AnfAlgo::GetStreamId(cur_cnode_ptr); MS_LOG(INFO) << "Stream id:" << stream_id << " is need actived at first"; diff --git a/mindspore/ccsrc/kernel/akg/akg_kernel_build.cc b/mindspore/ccsrc/kernel/akg/akg_kernel_build.cc index 6bd1e7747c..0e8d93d47f 100644 --- a/mindspore/ccsrc/kernel/akg/akg_kernel_build.cc +++ b/mindspore/ccsrc/kernel/akg/akg_kernel_build.cc @@ -276,7 +276,8 @@ bool AkgKernelBuild::CreateInputDescJson(const AnfNodePtr &anf_node, nlohmann::j input_desc_json[kName] = op_input_name; input_desc_json[kTensorName] = "input_" + std::to_string(GetInputTensorIdxInc(anf_node, real_input_index)); auto input_shape = AnfAlgo::GetInputDeviceShape(anf_node, real_input_index); - if (GetInputTensorValue(anf_node, real_input_index, &input_desc_json)) { + if (anf_node->func_graph() != nullptr && anf_node->func_graph()->has_attr(FUNC_GRAPH_ATTR_GRAPH_KERNEL) && + GetInputTensorValue(anf_node, real_input_index, &input_desc_json)) { MS_LOG(WARNING) << "we take input[" << real_input_index << "] of [" << anf_node->DebugString(2) << "] as const tensor, shape: [" << Vector2Str(input_shape) << "], value: " << input_desc_json[kValue]; diff --git a/mindspore/ccsrc/session/anf_runtime_algorithm.cc b/mindspore/ccsrc/session/anf_runtime_algorithm.cc index 56983a4d22..5f896282dc 100644 --- a/mindspore/ccsrc/session/anf_runtime_algorithm.cc +++ b/mindspore/ccsrc/session/anf_runtime_algorithm.cc @@ -291,7 +291,7 @@ bool AnfRuntimeAlgorithm::HasNodeAttr(const std::string &key, const CNodePtr &no // graph kernel cnode. auto fg = AnfAlgo::GetCNodeFuncGraphPtr(node); MS_EXCEPTION_IF_NULL(fg); - return fg->has_flag(key); + return fg->has_attr(key); } size_t AnfRuntimeAlgorithm::GetInputTensorNum(const AnfNodePtr &node) { diff --git a/mindspore/ccsrc/session/anf_runtime_algorithm.h b/mindspore/ccsrc/session/anf_runtime_algorithm.h index c46f0b5955..ae8b450e2f 100644 --- a/mindspore/ccsrc/session/anf_runtime_algorithm.h +++ b/mindspore/ccsrc/session/anf_runtime_algorithm.h @@ -68,9 +68,14 @@ class AnfRuntimeAlgorithm { std::string node_debug_log = node->DebugString(); MS_LOG(EXCEPTION) << "Only cnode has attr, but this anf is " << node_debug_log.c_str(); } - auto primitive = GetCNodePrimitive(node); - MS_EXCEPTION_IF_NULL(primitive); - return GetValue(primitive->GetAttr(key)); + // single op cnode. + if (auto primitive = GetCNodePrimitive(node); primitive != nullptr) { + return GetValue(primitive->GetAttr(key)); + } + // graph kernel cnode. + auto fg = GetCNodeFuncGraphPtr(node); + MS_EXCEPTION_IF_NULL(fg); + return GetValue(fg->get_attr(key)); } static bool IsTupleOutput(const AnfNodePtr &anf); // set attr of anf node diff --git a/tests/st/networks/models/bert/test_bert_graph_kernel.py b/tests/st/networks/models/bert/test_bert_graph_kernel.py new file mode 100644 index 0000000000..ec71cbaa4f --- /dev/null +++ b/tests/st/networks/models/bert/test_bert_graph_kernel.py @@ -0,0 +1,193 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""train bert network without lossscale""" + +import os +import pytest +import numpy as np + +import mindspore.common.dtype as mstype +import mindspore.dataset.engine.datasets as de +import mindspore.dataset.transforms.c_transforms as C +from mindspore import context +from mindspore import log as logger +from mindspore.common.tensor import Tensor +from mindspore.nn.optim import Lamb +from mindspore.train.callback import Callback +from mindspore.train.loss_scale_manager import DynamicLossScaleManager +from mindspore.train.model import Model +from src.bert_for_pre_training import BertNetworkWithLoss, BertTrainOneStepWithLossScaleCell +from src.bert_model import BertConfig + +DATA_DIR = ["/home/workspace/mindspore_dataset/bert/example/examples.tfrecord"] +SCHEMA_DIR = "/home/workspace/mindspore_dataset/bert/example/datasetSchema.json" + +def get_config(version='base', batch_size=1): + """get config""" + if version == 'base': + bert_config = BertConfig( + batch_size=batch_size, + seq_length=128, + vocab_size=21136, + hidden_size=768, + num_hidden_layers=2, + num_attention_heads=12, + intermediate_size=3072, + hidden_act="gelu", + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + max_position_embeddings=512, + type_vocab_size=2, + initializer_range=0.02, + use_relative_positions=True, + input_mask_from_dataset=True, + token_type_ids_from_dataset=True, + dtype=mstype.float32, + compute_type=mstype.float32) + elif version == 'large': + bert_config = BertConfig( + batch_size=batch_size, + seq_length=128, + vocab_size=30522, + hidden_size=1024, + num_hidden_layers=2, + num_attention_heads=16, + intermediate_size=4096, + hidden_act="gelu", + hidden_dropout_prob=0.0, + attention_probs_dropout_prob=0.0, + max_position_embeddings=512, + type_vocab_size=2, + initializer_range=0.02, + use_relative_positions=True, + input_mask_from_dataset=True, + token_type_ids_from_dataset=True, + dtype=mstype.float32, + compute_type=mstype.float16, + enable_fused_layernorm=True) + else: + bert_config = BertConfig(batch_size=batch_size) + return bert_config + + +def me_de_train_dataset(): + """test me de train dataset""" + # apply repeat operations + repeat_count = 1 + ds = de.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["input_ids", "input_mask", "segment_ids", + "next_sentence_labels", "masked_lm_positions", + "masked_lm_ids", "masked_lm_weights"], shuffle=False) + type_cast_op = C.TypeCast(mstype.int32) + ds = ds.map(input_columns="masked_lm_ids", operations=type_cast_op) + ds = ds.map(input_columns="masked_lm_positions", operations=type_cast_op) + ds = ds.map(input_columns="next_sentence_labels", operations=type_cast_op) + ds = ds.map(input_columns="segment_ids", operations=type_cast_op) + ds = ds.map(input_columns="input_mask", operations=type_cast_op) + ds = ds.map(input_columns="input_ids", operations=type_cast_op) + # apply batch operations + batch_size = int(os.getenv('BATCH_SIZE', '16')) + ds = ds.batch(batch_size, drop_remainder=True) + ds = ds.repeat(repeat_count) + return ds + + +def weight_variable(shape): + """weight variable""" + np.random.seed(1) + ones = np.random.uniform(-0.1, 0.1, size=shape).astype(np.float32) + return Tensor(ones) + + +class ModelCallback(Callback): + def __init__(self): + super(ModelCallback, self).__init__() + self.loss_list = [] + self.overflow_list = [] + self.lossscale_list = [] + + def step_end(self, run_context): + cb_params = run_context.original_args() + self.loss_list.append(cb_params.net_outputs[0].asnumpy()[0]) + self.overflow_list.append(cb_params.net_outputs[1].asnumpy()) + self.lossscale_list.append(cb_params.net_outputs[2].asnumpy()) + print("epoch: {}, outputs are: {}".format(cb_params.cur_epoch_num, str(cb_params.net_outputs))) + + +@pytest.mark.level0 +@pytest.mark.platform_arm_ascend_training +@pytest.mark.platform_x86_ascend_training +@pytest.mark.env_onecard +def test_bert_tdt(): + """test bert tdt""" + np.random.seed(0) + context.set_context(mode=context.GRAPH_MODE, device_target="Ascend", reserve_class_name_in_scope=False) + context.set_context(enable_graph_kernel=True) + ds = me_de_train_dataset() + config = get_config(version='large', batch_size=16) + netwithloss = BertNetworkWithLoss(config, True) + optimizer = Lamb(netwithloss.trainable_params(), decay_steps=ds.get_dataset_size()*ds.get_repeat_count(), + start_learning_rate=5e-5, end_learning_rate=1e-9, + power=10.0, warmup_steps=0, weight_decay=0.01) + scale_window = 3 + scale_manager = DynamicLossScaleManager(262144, 2, scale_window) + netwithgrads = BertTrainOneStepWithLossScaleCell(netwithloss, optimizer=optimizer, + scale_update_cell=scale_manager.get_update_cell()) + netwithgrads.set_train(True) + model = Model(netwithgrads) + callback = ModelCallback() + params = netwithloss.trainable_params() + for param in params: + param.init_data() + value = param.default_input + name = param.name + if isinstance(value, Tensor): + if name.split('.')[-1] in ['weight']: + if name.split('.')[-3] in ['cls2']: + logger.info("***************** BERT param name is 1 {}".format(name)) + param.default_input = weight_variable(value.asnumpy().shape) + else: + logger.info("***************** BERT param name is 2 {}".format(name)) + tempshape = value.asnumpy().shape + shape = (tempshape[1], tempshape[0]) + weight_value = weight_variable(shape).asnumpy() + param.default_input = Tensor(np.transpose(weight_value, [1, 0])) + else: + logger.info("***************** BERT param name is 3 {}".format(name)) + param.default_input = weight_variable(value.asnumpy().shape) + model.train(1, ds, callbacks=callback, dataset_sink_mode=False) + + # assertion occurs while the loss value, overflow state or loss_scale value is wrong + loss_value = np.array(callback.loss_list) + expect_loss_value = [12.559319, 12.333815, 12.339806, 12.350235, 12.343947, 12.830965, 12.375336, 12.973715, + 12.57929, 12.7766905] + error = loss_value - expect_loss_value + print("loss value: {}".format(loss_value)) + print("error value: {}".format(error)) + assert np.allclose(loss_value, expect_loss_value, 0, 0.0005) + + overflow = np.array(callback.overflow_list) + expect_overflow = [True, True, True, True, False, False, False, True, False, False] + print("overflow: {}".format(overflow)) + assert (overflow == expect_overflow).all() + + loss_scale = np.array(callback.lossscale_list) + expect_loss_scale = [131072.0, 65536.0, 32768.0, 16384.0, 16384.0, 16384.0, 32768.0, 16384.0, 16384.0, 16384.0] + print("loss scale: {}".format(loss_scale)) + assert np.allclose(loss_scale, expect_loss_scale, 0, 0) + + +if __name__ == '__main__': + test_bert_tdt() diff --git a/tests/st/ops/graph_kernel/test_lamb.py b/tests/st/ops/graph_kernel/test_lamb.py new file mode 100644 index 0000000000..d34c0eea57 --- /dev/null +++ b/tests/st/ops/graph_kernel/test_lamb.py @@ -0,0 +1,130 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +import pytest +import numpy as np +import mindspore.context as context +from mindspore import Tensor, Parameter +from mindspore.nn import Cell +from mindspore.nn.graph_kernels import LambUpdateWithLR, LambNextMV + +context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") + +class LambNet(Cell): + def __init__(self, i2, i5, x6): + super(LambNet, self).__init__() + self.i2 = Parameter(i2, name='i2') + self.i5 = Parameter(i5, name='i5') + self.x6 = Parameter(x6, name='x6') + self.lamb_next = LambNextMV() + self.lamb_update = LambUpdateWithLR() + + def construct(self, i1, i3, i4, i6, i7, i8, i9, ix0, ix1, ix2, ix3, + x1, x2, x3, x4, x5, gy, se, my): + return self.lamb_next(i1, self.i2, i3, i4, self.i5, i6, i7, i8, i9, ix0, + ix1, ix2, ix3), \ + self.lamb_update(x1, x2, x3, x4, x5, self.x6, gy, se, my) + +def LambUpdateNumpy(x1, x2, x3, x4, x5, x6, gy, se, my): + trust_ratio = np.where(np.greater(x2, gy), + np.where(np.greater(x1, gy), np.divide(x2, x3), se), + se) + trust_ratio = np.maximum(np.minimum(trust_ratio, my), gy) + update_with_lr = trust_ratio * x4 * x5 + next_param = x6 - np.reshape(update_with_lr, x6.shape) + return next_param + +def LambNextMVNumpy(i1, i2, i3, i4, i5, i6, i7, i8, i9, x0, x1, x2, x3): + m_fp32 = i5.astype(np.float32) + v_fp32 = i2.astype(np.float32) + next_m = i8 * m_fp32 + i9 * i4 + next_v = x0 * v_fp32 + x1 * i1 + next_mm = next_m / i6 + next_vv = next_v / i3 + update = next_mm / (np.sqrt(next_vv) + x3) + add3 = next_mm / np.sqrt(next_vv + x3) + x2 * i7 + return add3, next_m, next_v, update + + +def tensor_all(*args): + res = [Tensor(a) for a in args] + return res + +@pytest.mark.level0 +@pytest.mark.platform_arm_ascend_training +@pytest.mark.platform_x86_ascend_training +@pytest.mark.env_onecard +def test_graph_kernel_lamb(): + shape = [1, 16] + oshape = [1] + np.random.seed(0) + x1 = np.random.normal(0, 1, oshape).astype(np.float32) + x2 = np.random.normal(0, 1, oshape).astype(np.float32) + x3 = np.random.normal(0, 1, oshape).astype(np.float32) + x4 = np.random.normal(0, 1, oshape).astype(np.float32) + x5 = np.random.normal(0, 1, shape).astype(np.float32) + x6 = np.random.normal(0, 1, shape).astype(np.float32) + gy = np.random.normal(0, 1, oshape).astype(np.float32) + se = np.random.normal(0, 1, oshape).astype(np.float32) + my = np.random.normal(0, 1, oshape).astype(np.float32) + + tx1, tx2, tx3, tx4, tx5, tx6, tgy, tse, tmy = tensor_all( + x1, x2, x3, x4, x5, x6, gy, se, my) + + np.random.seed(1) + i1 = np.abs(np.random.normal(0, 1, shape)).astype(np.float32) + i2 = np.abs(np.random.normal(0, 1, shape)).astype(np.float32) + i3 = np.abs(np.random.normal(0, 1, shape)).astype(np.float32) + i4 = np.random.normal(0, 1, shape).astype(np.float32) + i5 = np.random.normal(0, 1, shape).astype(np.float32) + i6 = np.abs(np.random.normal(0, 1, shape)).astype(np.float32) + i7 = np.random.normal(0, 1, shape).astype(np.float32) + i8 = np.random.normal(0, 1, shape).astype(np.float32) + i9 = np.random.normal(0, 1, shape).astype(np.float32) + ix0 = np.abs(np.random.normal(0, 1, shape)).astype(np.float32) + ix1 = np.abs(np.random.normal(0, 1, shape)).astype(np.float32) + ix2 = np.random.normal(0, 1, shape).astype(np.float32) + ix3 = np.ones(shape).astype(np.float32) * 1e-6 + + ti1, ti2, ti3, ti4, ti5, ti6, ti7, ti8, ti9, tix0, tix1, tix2, tix3 = \ + tensor_all(i1, i2, i3, i4, i5, i6, i7, i8, i9, ix0, ix1, ix2, ix3) + + context.set_context(enable_graph_kernel=True) + + net = LambNet(ti2, ti5, tx6) + (wa3, wup), _ = net(ti1, ti3, ti4, ti6, ti7, ti8, ti9, tix0, tix1, tix2, tix3, + tx1, tx2, tx3, tx4, tx5, tgy, tse, tmy) + + wi2 = net.i2.data.asnumpy().copy() + wi5 = net.i5.data.asnumpy().copy() + ares = net.x6.data.asnumpy().copy() + + context.set_context(enable_graph_kernel=False) + + a3, a0, a1, up = LambNextMVNumpy(i1, i2, i3, i4, i5, i6, i7, i8, i9, ix0, + ix1, ix2, ix3) + + np_res = LambUpdateNumpy(x1, x2, x3, x4, x5, x6, gy, se, my) + + rtol = 0.0001 + atol = 0.0001 + + wres = (wa3.asnumpy().copy(), wi5, wi2, wup.asnumpy().copy()) + bres = (a3, a0, a1, up) + + cmp_res = list(map(lambda x, y: np.allclose(x, y, rtol, atol), + wres, bres)) + + assert all(cmp_res) and np.allclose(ares, np_res, rtol, atol) From 1e51414f1b7418ff823ceb8304f1bd70f15558ac Mon Sep 17 00:00:00 2001 From: panbingao Date: Mon, 29 Jun 2020 11:10:24 +0800 Subject: [PATCH 217/254] Adapting operator Softsign in ME --- mindspore/ops/_grad/grad_nn_ops.py | 15 +++++++++++ mindspore/ops/_op_impl/tbe/__init__.py | 1 + mindspore/ops/_op_impl/tbe/softsign.py | 37 ++++++++++++++++++++++++++ mindspore/ops/operations/__init__.py | 3 ++- mindspore/ops/operations/nn_ops.py | 35 ++++++++++++++++++++++++ tests/ut/python/ops/test_ops.py | 4 +++ 6 files changed, 94 insertions(+), 1 deletion(-) create mode 100644 mindspore/ops/_op_impl/tbe/softsign.py diff --git a/mindspore/ops/_grad/grad_nn_ops.py b/mindspore/ops/_grad/grad_nn_ops.py index 107de1768c..b34d452cba 100755 --- a/mindspore/ops/_grad/grad_nn_ops.py +++ b/mindspore/ops/_grad/grad_nn_ops.py @@ -336,6 +336,21 @@ def get_bprop_softplus(self): return bprop +@bprop_getters.register(P.Softsign) +def get_bprop_softsign(self): + """Grad definition for `Softsign` operation.""" + mul = P.Mul() + absolute = P.Abs() + div = P.Div() + square = P.Square() + + def bprop(x, out, dout): + dx = mul(dout, div(1, square(1 + absolute(x)))) + return (dx,) + + return bprop + + @bprop_getters.register(P.Tanh) def get_bprop_tanh(self): """Grad definition for `Tanh` operation.""" diff --git a/mindspore/ops/_op_impl/tbe/__init__.py b/mindspore/ops/_op_impl/tbe/__init__.py index fa2be6d515..a4f8aa42ed 100644 --- a/mindspore/ops/_op_impl/tbe/__init__.py +++ b/mindspore/ops/_op_impl/tbe/__init__.py @@ -122,6 +122,7 @@ from .round import _round_tbe from .tanh import _tanh_tbe from .tanh_grad import _tanh_grad_tbe from .softmax import _softmax_tbe +from .softsign import _softsign_tbe from .softplus import _softplus_tbe from .softplus_grad import _softplus_grad_tbe from .softmax_grad_ext import _softmax_grad_ext_tbe diff --git a/mindspore/ops/_op_impl/tbe/softsign.py b/mindspore/ops/_op_impl/tbe/softsign.py new file mode 100644 index 0000000000..9f1609bf0a --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/softsign.py @@ -0,0 +1,37 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +"""Softsign op""" +from mindspore.ops.op_info_register import op_info_register, TBERegOp, DataType + +softsign_op_info = TBERegOp("Softsign") \ + .fusion_type("OPAQUE") \ + .async_flag(False) \ + .binfile_name("softsign.so") \ + .compute_cost(10) \ + .kernel_name("softsign") \ + .partial_flag(True) \ + .op_pattern("formatAgnostic") \ + .input(0, "x", False, "required", "all") \ + .output(0, "y", False, "required", "all") \ + .dtype_format(DataType.F16_Default, DataType.F16_Default) \ + .dtype_format(DataType.F32_Default, DataType.F32_Default) \ + .get_op_info() + + +@op_info_register(softsign_op_info) +def _softsign_tbe(): + """Softsign TBE register""" + return diff --git a/mindspore/ops/operations/__init__.py b/mindspore/ops/operations/__init__.py index 06a19d2db7..b2d0fc7382 100644 --- a/mindspore/ops/operations/__init__.py +++ b/mindspore/ops/operations/__init__.py @@ -68,7 +68,7 @@ from .nn_ops import (LSTM, SGD, Adam, SparseApplyAdam, SparseApplyLazyAdam, Appl MaxPoolWithArgmax, OneHot, Pad, MirrorPad, PReLU, ReLU, ReLU6, ReLUV2, HSwish, HSigmoid, ResizeBilinear, Sigmoid, SigmoidCrossEntropyWithLogits, - SmoothL1Loss, Softmax, Softplus, LRN, + SmoothL1Loss, Softmax, Softsign, Softplus, LRN, SoftmaxCrossEntropyWithLogits, ROIAlign, SparseSoftmaxCrossEntropyWithLogits, Tanh, TopK, BinaryCrossEntropy, SparseApplyAdagrad, LARSUpdate, ApplyFtrl, SparseApplyFtrl, @@ -115,6 +115,7 @@ __all__ = [ 'SparseApplyLazyAdam', 'Softplus', 'Softmax', + 'Softsign', 'LogSoftmax', 'SoftmaxCrossEntropyWithLogits', 'ROIAlign', diff --git a/mindspore/ops/operations/nn_ops.py b/mindspore/ops/operations/nn_ops.py index 6320e9e011..a1f5887217 100644 --- a/mindspore/ops/operations/nn_ops.py +++ b/mindspore/ops/operations/nn_ops.py @@ -224,6 +224,41 @@ class Softplus(PrimitiveWithInfer): return input_x +class Softsign(PrimitiveWithInfer): + r""" + Softsign activation function. + + The function is shown as follows: + + .. math:: + \text{output} = \frac{\text{input_x}}{1 + \abs{\text{input_x}}}, + + Inputs: + - **input_x** (Tensor) - The input tensor whose data type should be float. + + Outputs: + Tensor, with the same type and shape as the `input_x`. + + Examples: + >>> input_x = Tensor(np.array([0, -1, 2, 30, -30]), mindspore.float32) + >>> softsign = P.Softsign() + >>> softsign(input_x) + [0. -0.5 0.6666667 0.9677419 -0.9677419] + """ + + @prim_attr_register + def __init__(self): + """init Softsign""" + self.init_prim_io_names(inputs=['x'], outputs=['output']) + + def infer_shape(self, input_x): + return input_x + + def infer_dtype(self, input_x): + validator.check_tensor_type_same({'input_x': input_x}, mstype.float_type, self.name) + return input_x + + class ReLU(PrimitiveWithInfer): r""" Computes ReLU(Rectified Linear Unit) of input tensor element-wise. diff --git a/tests/ut/python/ops/test_ops.py b/tests/ut/python/ops/test_ops.py index f55d42e28b..a99b231fa7 100755 --- a/tests/ut/python/ops/test_ops.py +++ b/tests/ut/python/ops/test_ops.py @@ -1376,6 +1376,10 @@ test_case_nn_ops = [ 'block': P.Softmax(), 'desc_inputs': [[5, 5]], 'desc_bprop': [[5, 5]]}), + ('Softsign', { + 'block': P.Softsign(), + 'desc_inputs': [[5, 5]], + 'desc_bprop': [[5, 5]]}), ('DepthwiseConv2dNative_1', { 'block': P.DepthwiseConv2dNative(3, (3, 3), pad_mode="pad", pad=1, stride=2), 'desc_inputs': [[10, 32, 32, 32], [1, 32, 3, 3]], From 793737ab62b5db47e9b35f4eecec0a6552be4f7c Mon Sep 17 00:00:00 2001 From: duxiutao Date: Thu, 2 Jul 2020 16:03:01 +0800 Subject: [PATCH 218/254] add primitive operator to test_lamb --- tests/st/ops/graph_kernel/test_lamb.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/st/ops/graph_kernel/test_lamb.py b/tests/st/ops/graph_kernel/test_lamb.py index d34c0eea57..dfe975c91a 100644 --- a/tests/st/ops/graph_kernel/test_lamb.py +++ b/tests/st/ops/graph_kernel/test_lamb.py @@ -33,7 +33,8 @@ class LambNet(Cell): def construct(self, i1, i3, i4, i6, i7, i8, i9, ix0, ix1, ix2, ix3, x1, x2, x3, x4, x5, gy, se, my): - return self.lamb_next(i1, self.i2, i3, i4, self.i5, i6, i7, i8, i9, ix0, + i1_ = i1 + i3 + return self.lamb_next(i1_, self.i2, i3, i4, self.i5, i6, i7, i8, i9, ix0, ix1, ix2, ix3), \ self.lamb_update(x1, x2, x3, x4, x5, self.x6, gy, se, my) @@ -113,7 +114,8 @@ def test_graph_kernel_lamb(): context.set_context(enable_graph_kernel=False) - a3, a0, a1, up = LambNextMVNumpy(i1, i2, i3, i4, i5, i6, i7, i8, i9, ix0, + i1_ = i1 + i3 + a3, a0, a1, up = LambNextMVNumpy(i1_, i2, i3, i4, i5, i6, i7, i8, i9, ix0, ix1, ix2, ix3) np_res = LambUpdateNumpy(x1, x2, x3, x4, x5, x6, gy, se, my) From d0c7ece681c7cc6a4724b2e84c2273959fbb6d23 Mon Sep 17 00:00:00 2001 From: jiangjinsheng Date: Thu, 2 Jul 2020 15:33:34 +0800 Subject: [PATCH 219/254] fix HistogramFixedWidth --- mindspore/ops/operations/array_ops.py | 4 +--- mindspore/ops/operations/math_ops.py | 3 ++- mindspore/ops/operations/nn_ops.py | 5 ++++- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/mindspore/ops/operations/array_ops.py b/mindspore/ops/operations/array_ops.py index daefea64c8..0e1456c250 100644 --- a/mindspore/ops/operations/array_ops.py +++ b/mindspore/ops/operations/array_ops.py @@ -1371,11 +1371,9 @@ class UnsortedSegmentMin(PrimitiveWithInfer): """ Computes the minimum along segments of a tensor. - If the given segment_ids is negative, the value will be ignored. - Inputs: - **input_x** (Tensor) - The shape is :math:`(x_1, x_2, ..., x_R)`. - - **segment_ids** (Tensor) - A `1-D` tensor whose shape is :math:`(x_1)`. + - **segment_ids** (Tensor) - A `1-D` tensor whose shape is :math:`(x_1)`, the value should be >= 0. - **num_segments** (int) - The value spcifies the number of distinct `segment_ids`. Outputs: diff --git a/mindspore/ops/operations/math_ops.py b/mindspore/ops/operations/math_ops.py index 5198e912fa..d7fa77787e 100644 --- a/mindspore/ops/operations/math_ops.py +++ b/mindspore/ops/operations/math_ops.py @@ -1356,7 +1356,7 @@ class HistogramFixedWidth(PrimitiveWithInfer): Args: dtype (string): An optional attribute. Must be one of the following types: "int32", "int64". Default: "int32". - nbins (Tensor): Number of histogram bins, the type is int32. + nbins (int): Number of histogram bins, the type is positive integer. Inputs: - **x** (Tensor) - Numeric Tensor. Must be one of the following types: int32, float32, float16. @@ -1377,6 +1377,7 @@ class HistogramFixedWidth(PrimitiveWithInfer): @prim_attr_register def __init__(self, nbins, dtype='int32'): self.nbins = validator.check_value_type("nbins", nbins, [int], self.name) + validator.check_integer("nbins", nbins, 1, Rel.GE, self.name) valid_values = ['int32', 'int64'] self.dtype = validator.check_string("dtype", dtype, valid_values, self.name) self.init_prim_io_names(inputs=['x', 'range'], outputs=['y']) diff --git a/mindspore/ops/operations/nn_ops.py b/mindspore/ops/operations/nn_ops.py index c07f072f38..0247122c7a 100644 --- a/mindspore/ops/operations/nn_ops.py +++ b/mindspore/ops/operations/nn_ops.py @@ -1738,6 +1738,8 @@ class SGD(PrimitiveWithInfer): @prim_attr_register def __init__(self, dampening=0.0, weight_decay=0.0, nesterov=False): validator.check_value_type("nesterov", nesterov, [bool], self.name) + if nesterov and dampening != 0: + raise ValueError(f"Nesterov need zero dampening!") self.init_prim_io_names(inputs=['parameters', 'gradient', 'learning_rate', 'accum', 'momentum', 'stat'], outputs=['output']) @@ -2151,7 +2153,8 @@ class ResizeBilinear(PrimitiveWithInfer): rescale by `new_height / height`. Default: False. Inputs: - - **input** (Tensor) - Image to be resized. Tensor of shape `(N_i, ..., N_n, height, width)`. + - **input** (Tensor) - Image to be resized. Tensor of shape `(N_i, ..., N_n, height, width)`, + with data type of float32 or float16. Outputs: Tensor, resized image. Tensor of shape `(N_i, ..., N_n, new_height, new_width)` in `float32`. From de43c1eefae4ddf0c5b9951316c794d822356979 Mon Sep 17 00:00:00 2001 From: guohongzilong <2713219276@qq.com> Date: Thu, 2 Jul 2020 17:36:12 +0800 Subject: [PATCH 220/254] fix group params order --- mindspore/nn/optim/optimizer.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mindspore/nn/optim/optimizer.py b/mindspore/nn/optim/optimizer.py index 16f252adff..cdf1565f34 100755 --- a/mindspore/nn/optim/optimizer.py +++ b/mindspore/nn/optim/optimizer.py @@ -360,16 +360,18 @@ class Optimizer(Cell): if len(ordered_parameters) != len(self.group_params): raise ValueError(f"The value of 'order_params' should be same with all group parameters.") + ordered_params = [None] * params_length ordered_learning_rate = [None] * params_length ordered_weight_decay = [None] * params_length params_name = [param.name for param in ordered_parameters] for param, lr, wd in zip(self.group_params, self.group_lr, self.group_weight_decay): index = params_name.index(param.name) + ordered_params[index] = param ordered_learning_rate[index] = lr ordered_weight_decay[index] = wd - self.group_params = list(ordered_parameters) + self.group_params = ordered_params self.group_lr = ordered_learning_rate self.group_weight_decay = ordered_weight_decay From e43471ac2d2f3a6e8498f29d880a9ca1be068883 Mon Sep 17 00:00:00 2001 From: yanghaitao Date: Thu, 2 Jul 2020 17:53:24 +0800 Subject: [PATCH 221/254] fix device queue profiling bug --- mindspore/ccsrc/dataset/engine/perf/device_queue_tracing.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mindspore/ccsrc/dataset/engine/perf/device_queue_tracing.h b/mindspore/ccsrc/dataset/engine/perf/device_queue_tracing.h index f7c6da3a04..13ef7121c1 100644 --- a/mindspore/ccsrc/dataset/engine/perf/device_queue_tracing.h +++ b/mindspore/ccsrc/dataset/engine/perf/device_queue_tracing.h @@ -35,7 +35,7 @@ class DeviceQueueTracing : public Tracing { // @return Status - The error code return Status Record(const int32_t type, const int32_t extra_info, const int32_t batch_num, const int32_t value); - std::string Name() const override { return "Device Queue Tracing"; }; + std::string Name() const override { return kDeviceQueueTracingName; }; // Save tracing data to file // @return Status - The error code return From a0623e1528ee5b72b63defbf65db0fb9c5d418bf Mon Sep 17 00:00:00 2001 From: He Wei Date: Thu, 2 Jul 2020 17:08:03 +0800 Subject: [PATCH 222/254] Fix bool type tensor problem Problem: Create 'uint8' type tensor from float data raise RuntimeError; Fix: Use same internal data type for 'uint8' and 'bool' tensor and allows convert float data to bool tensor. --- mindspore/ccsrc/ir/tensor.cc | 19 +------------------ tests/ut/python/ir/test_tensor.py | 18 ++++++++++++++---- 2 files changed, 15 insertions(+), 22 deletions(-) diff --git a/mindspore/ccsrc/ir/tensor.cc b/mindspore/ccsrc/ir/tensor.cc index dccdbb65b8..c06ba2a820 100644 --- a/mindspore/ccsrc/ir/tensor.cc +++ b/mindspore/ccsrc/ir/tensor.cc @@ -30,8 +30,6 @@ namespace mindspore { namespace tensor { -using Bool = unsigned char; - static std::string MakeId() { // Use atomic to make id generator thread safe. static std::atomic last_id{1}; @@ -50,10 +48,7 @@ template std::vector CopyData(const std::vector &shape, void *data, TypeId data_type) { const size_t count = SizeOf(shape); switch (data_type) { - case kNumberTypeBool: { - auto buf = static_cast(data); - return std::vector(buf, buf + count); - } + case kNumberTypeBool: case kNumberTypeUInt8: { auto buf = static_cast(data); return std::vector(buf, buf + count); @@ -104,14 +99,6 @@ std::vector CopyData(const std::vector &shape, void *data, TypeId data_t MS_LOG(EXCEPTION) << "Cannot construct Tensor because of unsupported data type: " << data_type << "."; } -// Convert to bool is not allowed. -template <> -std::vector CopyData(const std::vector &shape, void *data, TypeId data_type) { - MS_LOG(EXCEPTION) << "Cannot convert from " << TypeIdLabel(data_type) << " to " << TypeIdLabel(kNumberTypeBool) - << "."; - return {}; -} - template std::vector CopyData(const std::vector &shape, void *data, size_t data_len) { size_t size = SizeOf(shape); @@ -192,10 +179,6 @@ template TensorDataPtr MakeTensorData(TypeId data_type, const std::vector &shape, Args... args) { switch (data_type) { case kNumberTypeBool: - // std::vector is a specialization of std::vector, - // it may use single bit instead of sizeof(bool) bytes, - // so we use std::vector for bool tensors. - return std::make_shared>(shape, args...); case kNumberTypeUInt8: return std::make_shared>(shape, args...); case kNumberTypeInt8: diff --git a/tests/ut/python/ir/test_tensor.py b/tests/ut/python/ir/test_tensor.py index 357187cddd..72100b2715 100644 --- a/tests/ut/python/ir/test_tensor.py +++ b/tests/ut/python/ir/test_tensor.py @@ -430,10 +430,20 @@ def test_tensor_dtype_np_int64(): def test_tensor_dtype_fp32_to_bool(): - with pytest.raises(RuntimeError): - input_ = np.random.randn(2, 3, 4, 5).astype(np.float32) - input_ = ms.Tensor(input_) - _ = ms.Tensor(input_, dtype=ms.bool_) + input_ = np.random.randn(2, 3, 4, 5).astype(np.float32) + input_ = ms.Tensor(input_) + t = ms.Tensor(input_, dtype=ms.bool_) + assert isinstance(t, ms.Tensor) + assert t.shape == (2, 3, 4, 5) + assert t.dtype == ms.bool_ + + +def test_tensor_dtype_fp64_to_uint8(): + array = np.array([[1, 2, 3], [4, 5, 6]], dtype=np.float64) + t = ms.Tensor(array, ms.uint8) + assert isinstance(t, ms.Tensor) + assert t.shape == (2, 3) + assert t.dtype == ms.uint8 def test_tensor_operation(): From 77b5ae054af6e16c533eb5dea03ae10bbe84be1d Mon Sep 17 00:00:00 2001 From: xutianchun Date: Thu, 2 Jul 2020 20:01:51 +0800 Subject: [PATCH 223/254] fix CropAndResize doc --- mindspore/ops/operations/image_ops.py | 28 +++++++++---------- .../test_aicpu_ops/test_crop_and_reszie.py | 4 +-- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/mindspore/ops/operations/image_ops.py b/mindspore/ops/operations/image_ops.py index 68dae34530..bd26eba906 100644 --- a/mindspore/ops/operations/image_ops.py +++ b/mindspore/ops/operations/image_ops.py @@ -34,21 +34,21 @@ class CropAndResize(PrimitiveWithInfer): Inputs: - **x** (Tensor) - The input image must be a 4-D tensor of shape [batch, image_height, image_width, depth]. - Types allowed: int8, int16, int32, int64, float16, float32, float64, uint8, uint16. + Types allowed: int8, int16, int32, int64, float16, float32, float64, uint8, uint16. - **boxes** (Tensor) - A 2-D tensor of shape [num_boxes, 4]. - The i-th row of the tensor specifies the coordinates of a box in the box_ind[i] image - and is specified in normalized coordinates [y1, x1, y2, x2]. A normalized coordinate value of y is mapped to - the image coordinate at y * (image_height - 1), so as the [0, 1] interval of normalized image height is - mapped to [0, image_height - 1] in image height coordinates. We do allow y1 > y2, in which case the sampled - crop is an up-down flipped version of the original image. The width dimension is treated similarly. - Normalized coordinates outside the [0, 1] range are allowed, in which case we use extrapolation_value to - extrapolate the input image values. Types allowd: float32. + The i-th row of the tensor specifies the coordinates of a box in the box_ind[i] image + and is specified in normalized coordinates [y1, x1, y2, x2]. A normalized coordinate value of y is mapped to + the image coordinate at y * (image_height - 1), so as the [0, 1] interval of normalized image height is + mapped to [0, image_height - 1] in image height coordinates. We do allow y1 > y2, in which case the sampled + crop is an up-down flipped version of the original image. The width dimension is treated similarly. + Normalized coordinates outside the [0, 1] range are allowed, in which case we use extrapolation_value to + extrapolate the input image values. Types allowd: float32. - **box_index** (Tensor) - A 1-D tensor of shape [num_boxes] with int32 values in [0, batch). - The value of box_ind[i] specifies the image that the i-th box refers to. Types allowd: int32. + The value of box_ind[i] specifies the image that the i-th box refers to. Types allowd: int32. - **crop_size** (Tensor) - Only constant value is allowd. Types allowed: int32. - A 1-D tensor of 2 elements, size = [crop_height, crop_width]. - All cropped image patches are resized to this size. The aspect ratio of the image content is not preserved. - Both crop_height and crop_width need to be positive. + A 1-D tensor of 2 elements, size = [crop_height, crop_width]. + All cropped image patches are resized to this size. The aspect ratio of the image content is not preserved. + Both crop_height and crop_width need to be positive. Outputs: A 4-D tensor of shape [num_boxes, crop_height, crop_width, depth] with type: float32. @@ -68,8 +68,8 @@ class CropAndResize(PrimitiveWithInfer): >>> IMAGE_WIDTH = 256 >>> CHANNELS = 3 >>> image = np.random.normal(size=[BATCH_SIZE, IMAGE_HEIGHT, IMAGE_WIDTH, CHANNELS]).astype(np.float32) - >>> boxes = np.random.uniform(shape=[NUM_BOXES, 4]).astype(np.float32) - >>> box_index = np.random.uniform(shape=[NUM_BOXES], low=0, high=BATCH_SIZE).astype(np.int32) + >>> boxes = np.random.uniform(size=[NUM_BOXES, 4]).astype(np.float32) + >>> box_index = np.random.uniform(size=[NUM_BOXES], low=0, high=BATCH_SIZE).astype(np.int32) >>> crop_size = np.array([24, 24]).astype(np.int32) >>> crop_and_resize = CropAndResizeNet(crop_size=Tensor(crop_size)) >>> output = crop_and_resize(Tensor(image), Tensor(boxes), Tensor(box_index)) diff --git a/tests/st/ops/ascend/test_aicpu_ops/test_crop_and_reszie.py b/tests/st/ops/ascend/test_aicpu_ops/test_crop_and_reszie.py index f85751975d..92f7fb6b42 100644 --- a/tests/st/ops/ascend/test_aicpu_ops/test_crop_and_reszie.py +++ b/tests/st/ops/ascend/test_aicpu_ops/test_crop_and_reszie.py @@ -41,8 +41,8 @@ def test_net_float32(): image_width = 256 channels = 3 image = np.random.normal(size=[batch_size, image_height, image_width, channels]).astype(np.float32) - boxes = np.random.uniform(shape=[num_boxes, 4]).astype(np.float32) - box_index = np.random.uniform(shape=[num_boxes], low=0, high=batch_size).astype(np.int32) + boxes = np.random.uniform(size=[num_boxes, 4]).astype(np.float32) + box_index = np.random.uniform(size=[num_boxes], low=0, high=batch_size).astype(np.int32) crop_size = np.array([24, 24]).astype(np.int32) net = Net(crop_size=Tensor(crop_size)) output = net(Tensor(image), Tensor(boxes), Tensor(box_index)) From c5951b060226d2f4f5f3c0d42ed29cccb5f36207 Mon Sep 17 00:00:00 2001 From: buxue Date: Thu, 2 Jul 2020 21:52:47 +0800 Subject: [PATCH 224/254] fix tensor index when there is only one element in tuple --- .../ops/composite/multitype_ops/_compile_utils.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/mindspore/ops/composite/multitype_ops/_compile_utils.py b/mindspore/ops/composite/multitype_ops/_compile_utils.py index 86100818d1..d6ce3ac6f6 100644 --- a/mindspore/ops/composite/multitype_ops/_compile_utils.py +++ b/mindspore/ops/composite/multitype_ops/_compile_utils.py @@ -267,6 +267,8 @@ def _tensor_index_by_tuple_slice(data, t): def tensor_index_by_tuple(data, tuple_index): """Tensor getitem by tuple of various types""" + if len(tuple_index) == 1: + return data[tuple_index[0]] indexes_types = hyper_map(F.typeof, tuple_index) index_elements_type = const_utils.tuple_index_elements_type(indexes_types, const_utils.TENSOR_GETITEM) if index_elements_type == const_utils.NO_TENSOR: @@ -430,6 +432,9 @@ def tensor_setitem_by_slice_with_number(data, input_slice, value): def tensor_setitem_by_tuple_with_number(data, tuple_index, value): """Assigns the tensor by tuple with number value.""" + if len(tuple_index) == 1: + data[tuple_index[0]] = value + return data indexes_types = hyper_map(F.typeof, tuple_index) index_elements_type = const_utils.tuple_index_elements_type(indexes_types, const_utils.TENSOR_SETITEM) @@ -489,6 +494,9 @@ def tensor_setitem_by_slice_with_tensor(data, input_slice, value): def tensor_setitem_by_tuple_with_tensor(data, tuple_index, value): """Assigns the tensor by tuple with tensor value.""" + if len(tuple_index) == 1: + data[tuple_index[0]] = value + return data indexes_types = hyper_map(F.typeof, tuple_index) index_elements_type = const_utils.tuple_index_elements_type(indexes_types, const_utils.TENSOR_SETITEM) @@ -509,6 +517,9 @@ def tensor_setitem_by_tuple_with_tensor(data, tuple_index, value): def tensor_setitem_by_tuple_with_tuple(data, tuple_index, value): """Assigns the tensor by tuple with tuple of value.""" + if len(tuple_index) == 1: + data[tuple_index[0]] = value + return data indexes_types = hyper_map(F.typeof, tuple_index) index_elements_type = const_utils.tuple_index_elements_type(indexes_types, const_utils.TENSOR_SETITEM) From 383564c41721bd6b6fd541b558062a9212363de0 Mon Sep 17 00:00:00 2001 From: ale Date: Tue, 30 Jun 2020 12:05:30 -0400 Subject: [PATCH 225/254] fix perf_data test --- tests/ut/cpp/dataset/perf_data_test.cc | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/tests/ut/cpp/dataset/perf_data_test.cc b/tests/ut/cpp/dataset/perf_data_test.cc index 0eaf74126d..048ee1f21a 100644 --- a/tests/ut/cpp/dataset/perf_data_test.cc +++ b/tests/ut/cpp/dataset/perf_data_test.cc @@ -34,12 +34,15 @@ TEST_F(MindDataTestPerfData, Test1) { std::vector row = {1, 2, 3}; p1.AddSample(row); p2.AddSample(row); + EXPECT_EQ(p1.size(), 1); EXPECT_EQ(p1.size(), p2.size()); p1.AddSample(row); p2.AddSample(row); + EXPECT_EQ(p1.size(), 2); EXPECT_EQ(p1.size(), p2.size()); - row = {4, 5, 6}; - p2.AddSample(row); + std::vector row1 = {4, 5, 6}; + p2.AddSample(row1); + EXPECT_EQ(p2.size(), 2); auto r1 = p2.Row(static_cast(0)); for (auto i = 0; i < 3; i++) { EXPECT_EQ(r1[i], i + 1); @@ -62,9 +65,12 @@ TEST_F(MindDataTestPerfData, Test2) { EXPECT_EQ(pd[0][0], 1); EXPECT_EQ(pd[1][0], 2); EXPECT_EQ(pd[2][0], 3); - row = {4, 5, 6}; - pd.AddSample(row); + auto row1 = {4, 5, 6}; + pd.AddSample(row1); EXPECT_EQ(pd[0][0], 1); EXPECT_EQ(pd[1][0], 2); EXPECT_EQ(pd[2][0], 3); + EXPECT_EQ(pd[0][1], 4); + EXPECT_EQ(pd[1][1], 5); + EXPECT_EQ(pd[2][1], 6); } \ No newline at end of file From 6cfa0891b47cc32624b612e99d64c07833ddf0f0 Mon Sep 17 00:00:00 2001 From: Jesse Lee Date: Thu, 2 Jul 2020 13:24:46 -0400 Subject: [PATCH 226/254] Addressing comments for PR 2602 --- .../dataset/engine/datasetops/dataset_op.cc | 4 +- .../dataset/engine/datasetops/dataset_op.h | 276 +++++++++--------- 2 files changed, 139 insertions(+), 141 deletions(-) diff --git a/mindspore/ccsrc/dataset/engine/datasetops/dataset_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/dataset_op.cc index 170d9a7ce8..3e31f6c017 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/dataset_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/dataset_op.cc @@ -348,7 +348,6 @@ Status DatasetOp::Accept(NodePass *p, bool *modified) { // A helper function with some common code that leaf nodes can use during // prepare phase for checking if they need to assign a sampler to the cache. -// @return - Status Status DatasetOp::SaveSamplerForCache(bool random_access_op) { // If we are a descendant under a cache op and we have a sampler, then save this sampler // to a stack so that the cache can pick it up during it's processing above us. @@ -362,8 +361,7 @@ Status DatasetOp::SaveSamplerForCache(bool random_access_op) { } else if (!random_access_op) { // A sampler exists, but we are not in a caching tree and we are not a random access mappable leaf. // This is an error because that type of leaf does not use sampling unless there's a cache to hook it into. - return Status( - StatusCode::kUnexpectedError, __LINE__, __FILE__, + RETURN_STATUS_UNEXPECTED( "Non-mappable leaf op has a sampler, but it only supports sampling if there is a cache after it in the tree"); } } diff --git a/mindspore/ccsrc/dataset/engine/datasetops/dataset_op.h b/mindspore/ccsrc/dataset/engine/datasetops/dataset_op.h index 9ee287d050..ab5cb90357 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/dataset_op.h +++ b/mindspore/ccsrc/dataset/engine/datasetops/dataset_op.h @@ -36,8 +36,8 @@ class NodePass; class Sampler; -// The base class DatasetOp is the main tree node. It is an abstract class, so -// the actual implementation of the operators will be derived from here. +/// \brief The base class DatasetOp is the main tree node. It is an abstract class, so +/// the actual implementation of the operators will be derived from here. class DatasetOp : public std::enable_shared_from_this { // Allow execution tree to access internal members friend class ExecutionTree; @@ -55,114 +55,114 @@ class DatasetOp : public std::enable_shared_from_this { // Flags that control operator runtime behaviours enum OpState { kDeOpRunning = 0, kDeOpIdle = 1, kDeOpTerminated }; - // Constructor - // @param op_connector_size - The size for the output connector of this operator. - // @param sampler - The sampler for the op + /// Constructor + /// \param op_connector_size - The size for the output connector of this operator. + /// \param sampler - The sampler for the op explicit DatasetOp(int32_t op_connector_size, std::shared_ptr sampler); - // Destructor + /// Destructor virtual ~DatasetOp() { tree_ = nullptr; } - // Adds a operator to become our child. - // @param child - shared pointer to the child to add. + /// Adds a operator to become our child. + /// \param child - shared pointer to the child to add. Status AddChild(std::shared_ptr child); - // Remove a operator from our children. - // @param child - shared pointer to the child to remove. + /// Remove a operator from our children. + /// \param child - shared pointer to the child to remove. Status RemoveChild(std::shared_ptr child); /// \brief Removes this node from the tree and connects it's parent/child together. /// \return Status eerror code returned Status Remove(); - // Getter function to get a shared pointer to our child - // @param child_index - An operator can have n children. Indicates choose which child to return. + /// \brief Getter function to get a shared pointer to our child + /// \param child_index - An operator can have n children. Indicates choose which child to return. std::shared_ptr child(int32_t child_index) const; - // Inserts a operator as the parent current op. - // Inserted op will become the sole parent of the current op. - // The existing parent of the current op will be transferred to the inserted op. + /// \brief Inserts a operator as the parent current op. + /// Inserted op will become the sole parent of the current op. + /// The existing parent of the current op will be transferred to the inserted op. Status InsertAsParent(std::shared_ptr to_add); - // Creates the connector within this operator - // @param num_producers - number of threads that write into this connector - // @param num_consumers - number of threads that read from this connector + /// \brief Creates the connector within this operator + /// \param num_producers - number of threads that write into this connector + /// \param num_consumers - number of threads that read from this connector void CreateConnector(int32_t num_producers, int32_t num_consumers); - // A print method typically used for debugging - // @param out - The output stream to write output to - // @param show_all - A bool to control if you want to show all info or just a summary + /// \brief A print method typically used for debugging + /// \param out - The output stream to write output to + /// \param show_all - A bool to control if you want to show all info or just a summary virtual void Print(std::ostream &out, bool show_all) const; - // << Stream output operator overload - // @notes This allows you to write the debug print info using stream operators - // @param out - reference to the output stream being overloaded - // @param dO - reference to the DatasetOp to display - // @return - the output stream must be returned + /// \brief << Stream output operator overload + /// \notes This allows you to write the debug print info using stream operators + /// \param out - reference to the output stream being overloaded + /// \param dO - reference to the DatasetOp to display + /// \return - the output stream must be returned friend std::ostream &operator<<(std::ostream &out, const DatasetOp &dO) { dO.Print(out, false); return out; } - // Class functor operator (). - // DatasetOps operate by launching a thread (see ExecutionTree). - // This pure virtual version makes the requirement that derived classes must provide a functor - // that will execute their main runtime loop code. - // @return Status - The error code return + /// \brief Class functor operator (). + /// DatasetOps operate by launching a thread (see ExecutionTree). + /// This pure virtual version makes the requirement that derived classes must provide a functor + /// that will execute their main runtime loop code. + /// \return Status - The error code return virtual Status operator()() = 0; - // Gets the next buffer from the given child - // @notes See GetNextInput for similar function that has built-in message handling - // @param p_buffer - The shared pointer for the fetched buffer to return (by reference) - // @param worker_id - The worker id - // @return Status - The error code return + /// \brief Gets the next buffer from the given child + /// \notes See GetNextInput for similar function that has built-in message handling + /// \param p_buffer - The shared pointer for the fetched buffer to return (by reference) + /// \param worker_id - The worker id + /// \return Status - The error code return virtual Status GetNextBuffer(std::unique_ptr *p_buffer, int32_t worker_id) { return GetNextBuffer(p_buffer, worker_id, false); } - // Gets the next buffer from the given child - // @notes See GetNextInput for similar function that has built-in message handling - // @param p_buffer - The shared pointer for the fetched buffer to return (by reference) - // @return Status - The error code return + /// \brief Gets the next buffer from the given child + /// \notes See GetNextInput for similar function that has built-in message handling + /// \param p_buffer - The shared pointer for the fetched buffer to return (by reference) + /// \return Status - The error code return virtual Status GetNextBuffer(std::unique_ptr *p_buffer) { return GetNextBuffer(p_buffer, 0, false); } - // Gets the next buffer from the given child - // @notes See GetNextInput for similar function that has built-in message handling - // @param p_buffer - The shared pointer for the fetched buffer to return (by reference) - // @param worker_id - The worker id - // @param retry_if_eoe Set this flag to true to allow calling pop() again after the first pop() returns EOE. - // @return Status - The error code return + /// \brief Gets the next buffer from the given child + /// \notes See GetNextInput for similar function that has built-in message handling + /// \param p_buffer - The shared pointer for the fetched buffer to return (by reference) + /// \param worker_id - The worker id + /// \param retry_if_eoe Set this flag to true to allow calling pop() again after the first pop() returns EOE. + /// \return Status - The error code return virtual Status GetNextBuffer(std::unique_ptr *p_buffer, int32_t worker_id, bool retry_if_eoe); - // Gets the next buffer from the given child . This function also has built-in eoe and eof - // message handling so that child classes don't have to manually code pass-through logic when - // those messages are received. - // @param p_buffer - The shared pointer for the fetched buffer to return (by reference) - // @param worker_id - The worker id - // @return Status - The error code return + /// \brief Gets the next buffer from the given child . This function also has built-in eoe and eof + /// message handling so that child classes don't have to manually code pass-through logic when + /// those messages are received. + /// \param p_buffer - The shared pointer for the fetched buffer to return (by reference) + /// \param worker_id - The worker id + /// \return Status - The error code return Status GetNextInput(std::unique_ptr *p_buffer, int32_t worker_id = 0, int32_t child_index = 0); - // Performs handling for when an eoe message is received. - // The base class implementation simply flows the eoe message to output. Derived classes - // may override if they need to perform special eoe handling. - // @param worker_id - The worker id - // @return Status - The error code return + /// \brief Performs handling for when an eoe message is received. + /// The base class implementation simply flows the eoe message to output. Derived classes + /// may override if they need to perform special eoe handling. + /// \param worker_id - The worker id + /// \return Status - The error code return virtual Status EoeReceived(int32_t worker_id); - // Performs handling for when an eof message is received. - // The base class implementation simply flows the eof message to output. Derived classes - // may override if they need to perform special eof handling. - // @param worker_id - The worker id - // @return Status - The error code return + /// \brief Performs handling for when an eof message is received. + /// The base class implementation simply flows the eof message to output. Derived classes + /// may override if they need to perform special eof handling. + /// \param worker_id - The worker id + /// \return Status - The error code return virtual Status EofReceived(int32_t worker_id); - // Derived classes may implement the reset function if the operator is stateful and needs - // specific reset handling that is not contained in this common code version of the reset - // @return Status - The error code return + /// \brief Derived classes may implement the reset function if the operator is stateful and needs + /// specific reset handling that is not contained in this common code version of the reset + /// \return Status - The error code return virtual Status Reset(); - // This calls the reset function on this subtree in pre-order - // @return Status - The error code return + /// \brief This calls the reset function on this subtree in pre-order + /// \return Status - The error code return virtual Status ResetSubtree() { RETURN_IF_NOT_OK(Reset()); for (const auto &c : child_) { @@ -171,69 +171,68 @@ class DatasetOp : public std::enable_shared_from_this { return Status::OK(); } - // During tree prepare phase, operators may have specific pre-operations to perform depending on - // their role. - // @notes Derived versions of this function should always call it's superclass version first - // before providing their own implementations. + /// \brief During tree prepare phase, operators may have specific pre-operations to perform depending on + /// their role. + /// \notes Derived versions of this function should always call it's superclass version first + /// before providing their own implementations. virtual Status PrepareNodePreAction(); - // During tree prepare phase, operators may have specific post-operations to perform depending on - // their role. - // @notes Derived versions of this function should always call it's superclass version first - // before providing their own implementations. + /// \brief During tree prepare phase, operators may have specific post-operations to perform depending on + /// their role. + /// \notes Derived versions of this function should always call it's superclass version first + /// before providing their own implementations. virtual Status PrepareNodePostAction(); - // Getter function - // @return The operator id + /// \brief Getter function + /// \return The operator id int32_t id() const { return operator_id_; } - // Getter function - // @return The prepare flags + /// \brief Getter function + /// \return The prepare flags virtual uint32_t PrepareFlags() const; - // Getter function - // @return The number of workers in this op + /// \brief Getter function + /// \return The number of workers in this op virtual int32_t num_workers() const = 0; - // Getter function - // @return The number of threads consuming from previous op. + /// \brief Getter function + /// \return The number of threads consuming from previous op. virtual int32_t num_consumers() const = 0; - // Getter function - // @return The number of threads producing to the output connector. + /// \brief Getter function + /// \return The number of threads producing to the output connector. virtual int32_t num_producers() const = 0; - // Getter function - // @return T/F if this is an inlined operator + /// \brief Getter function + /// \return T/F if this is an inlined operator bool inlined() const { return (oc_queue_size_ == 0); } - // Setter function - // @return Sets the control flags + /// \brief Setter function + /// \return Sets the control flags void set_control_flag(uint64_t flag) { BitSet(&op_ctrl_flags_, flag); } - // Setter function - // @return Sets the control flags + /// \brief Setter function + /// \return Sets the control flags void ClearControlFlag(uint64_t flag) { BitClear(&op_ctrl_flags_, flag); } - // Register the internal worker connectors. No op unless it is a parallel op - // @return Status + /// \brief Register the internal worker connectors. No op unless it is a parallel op + /// \return Status virtual Status RegisterWorkerConnectors() { return Status::OK(); } - // Getter for the column name mapping - // @return The returned map + /// \brief Getter for the column name mapping + /// \return The returned map std::unordered_map column_name_id_map() const { return column_name_id_map_; } - // Checks if the column name map has been set up yet for this op - // @return - T/F if the operator has the map set up + /// \brief Checks if the column name map has been set up yet for this op + /// \return - T/F if the operator has the map set up bool HasColumnNameMap() const { return (column_name_id_map_.empty()); } - // gives a string output for the column map for handy debug printing - // @return - the column name map as a string + /// \brief gives a string output for the column map for handy debug printing + /// \return - the column name map as a string std::string ColumnNameMapAsString() const; - // Getter function - // @return connector size of current op - + /// \brief Getter function + /// \return connector size of current op int32_t ConnectorSize() const { if (!inlined()) { return out_connector_->size(); @@ -242,12 +241,13 @@ class DatasetOp : public std::enable_shared_from_this { return ChildOpConnectorSize(); } + /// \brief Counting number of buffer sent out by a connector int64_t ConnectorOutBufferCount() const { return out_connector_ == nullptr ? int64_t(-1) : static_cast(out_connector_->out_buffers_count()); } - // Getter function - // @return connector size of current op + /// \brief Getter function + /// \return connector size of current op int32_t ConnectorCapacity() const { if (!inlined()) { return out_connector_->capacity(); @@ -256,16 +256,16 @@ class DatasetOp : public std::enable_shared_from_this { return ChildOpConnectorCapacity(); } - // Getter function - // @return connector size of child op + /// \brief Getter function + /// \return connector size of child op int32_t ChildOpConnectorSize(int32_t child_index = 0) const { return child_[child_index]->ConnectorSize(); } - // Getter function - // @return connector capacity of child op + /// \brief Getter function + /// \return connector capacity of child op int32_t ChildOpConnectorCapacity(int32_t child_index = 0) const { return child_[child_index]->ConnectorCapacity(); } - // Children Getter - // @return Vector of Children + /// \brief Children Getter + /// \return Vector of Children std::vector> Children() const { return child_; } /// \brief Base method for NodePass pre-visit. A tree walk consists of walking down the tree and also walking back up @@ -284,19 +284,19 @@ class DatasetOp : public std::enable_shared_from_this { /// \return Status of the node visit virtual Status Accept(NodePass *p, bool *modified); - // Op name getter - // @return Name of the current Op + /// Op name getter + /// \return Name of the current Op virtual std::string Name() const { return "DatasetOp"; } - // Execution Tree getter - // @return Pointer to the ExecutionTree the current op belongs to, no ownership + /// Execution Tree getter + /// \return Pointer to the ExecutionTree the current op belongs to, no ownership ExecutionTree *Tree() { return tree_; } - // Getter for the sampler - // @return Shared pointer to the sampler (may return nullptr) + /// Getter for the sampler + /// \return Shared pointer to the sampler (may return nullptr) std::shared_ptr sampler() { return sampler_; } - // Computes a CRC value for the operator + /// Computes a CRC value for the operator static uint32_t GenerateCRC(const std::shared_ptr &op); /// \brief A helper templated function for casting "this" pointer to shared_ptr @@ -308,27 +308,27 @@ class DatasetOp : public std::enable_shared_from_this { } protected: - // Adds a parent operator to this operator - // @notes External callers do not have access to this function. - // @param parent - The parent node to add + /// Adds a parent operator to this operator + /// \notes External callers do not have access to this function. + /// \param parent - The parent node to add void AddParent(DatasetOp *parent); - // Removes a parent operator from this operator - // @notes External callers do not have access to this function. - // @param parent - The parent node to remove + /// Removes a parent operator from this operator + /// \notes External callers do not have access to this function. + /// \param parent - The parent node to remove void RemoveParent(const DatasetOp *parent); - // Compute the current op's column map using its child's column map. - // Get called during the tree post-prepare phase in PrepareNodePostAction. - // This base implementation just inherits the map from child 0, and can only be used if the number of children is 1. - // Operations changing the column map it inherits from the child must overwrite this function. - // @return - Status + /// Compute the current op's column map using its child's column map. + /// Get called during the tree post-prepare phase in PrepareNodePostAction. + /// This base implementation just inherits the map from child 0, and can only be used if the number of children is 1. + /// Operations changing the column map it inherits from the child must overwrite this function. + /// \return - Status virtual Status ComputeColMap(); - // A helper function with some common code that leaf nodes can use during - // prepare phase for checking if they need to assign a sampler to the cache. - // @param random_access_op - indicate if this is a mappable random access leaf or not - // @return - Status + /// A helper function with some common code that leaf nodes can use during + /// pre/pare phase for checking if they need to assign a sampler to the cache. + /// \param random_access_op - indicate if this is a mappable random access leaf or not + /// \return - Status Status SaveSamplerForCache(bool random_access_op); std::vector> child_; // Child nodes @@ -344,14 +344,14 @@ class DatasetOp : public std::enable_shared_from_this { std::mutex column_name_map_mutex_; // For protecting shared access to the column map private: - // Sets the operator id. - // @notes No public interface. Only the class itself, or it's friend the execution tree can set - // this - // @param op_id - the Id value to set into the operator + /// Sets the operator id. + /// \notes No public interface. Only the class itself, or it's friend the execution tree can set + /// this + /// \param op_id - the Id value to set into the operator void set_id(int32_t op_id) { operator_id_ = op_id; } - // Sets the tree into the op so that the operator has a back pointer to the tree. - // @param tree - the tree to assign to the op. + /// Sets the tree into the op so that the operator has a back pointer to the tree. + /// \param tree - the tree to assign to the op. void set_tree(ExecutionTree *tree) { tree_ = tree; } }; } // namespace dataset From 90650658faa8715b0e72152e9b855ea7f1832b45 Mon Sep 17 00:00:00 2001 From: islam_amin Date: Thu, 2 Jul 2020 16:14:21 -0400 Subject: [PATCH 227/254] Unify Python Docs for C++ Detection Ops --- .../dataset/transforms/vision/c_transforms.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/mindspore/dataset/transforms/vision/c_transforms.py b/mindspore/dataset/transforms/vision/c_transforms.py index 3fdf7795d0..43ac037541 100644 --- a/mindspore/dataset/transforms/vision/c_transforms.py +++ b/mindspore/dataset/transforms/vision/c_transforms.py @@ -151,7 +151,7 @@ class RandomCrop(cde.RandomCropOp): class RandomCropWithBBox(cde.RandomCropWithBBoxOp): """ - Crop the input image at a random location and adjust bounding boxes for crop area + Crop the input image at a random location and adjust bounding boxes accordingly. Args: size (int or sequence): The output size of the cropped image. @@ -213,8 +213,7 @@ class RandomHorizontalFlip(cde.RandomHorizontalFlipOp): class RandomHorizontalFlipWithBBox(cde.RandomHorizontalFlipWithBBoxOp): """ - Flip the input image horizontally, randomly with a given probability. - Maintains data integrity by also flipping bounding boxes in an object detection pipeline. + Flip the input image horizontally, randomly with a given probability and adjust bounding boxes accordingly. Args: prob (float, optional): Probability of the image being flipped (default=0.5). @@ -242,7 +241,7 @@ class RandomVerticalFlip(cde.RandomVerticalFlipOp): class RandomVerticalFlipWithBBox(cde.RandomVerticalFlipWithBBoxOp): """ - Flip the input image vertically, randomly with a given probability and adjust bounding boxes as well + Flip the input image vertically, randomly with a given probability and adjust bounding boxes accordingly. Args: prob (float, optional): Probability of the image being flipped (default=0.5). @@ -256,8 +255,7 @@ class RandomVerticalFlipWithBBox(cde.RandomVerticalFlipWithBBoxOp): class BoundingBoxAugment(cde.BoundingBoxAugmentOp): """ - Apply a given image transform on a random selection of bounding box regions - of a given image. + Apply a given image transform on a random selection of bounding box regions of a given image. Args: transform: C++ transformation function to be applied on random selection @@ -305,7 +303,7 @@ class Resize(cde.ResizeOp): class ResizeWithBBox(cde.ResizeWithBBoxOp): """ - Resize the input image to the given size and adjust the bounding boxes accordingly. + Resize the input image to the given size and adjust bounding boxes accordingly. Args: size (int or sequence): The output size of the resized image. @@ -335,7 +333,7 @@ class ResizeWithBBox(cde.ResizeWithBBoxOp): class RandomResizedCropWithBBox(cde.RandomCropAndResizeWithBBoxOp): """ - Crop the input image to a random size and aspect ratio and adjust the Bounding Boxes accordingly + Crop the input image to a random size and aspect ratio and adjust bounding boxes accordingly. Args: size (int or sequence): The size of the output image. @@ -534,7 +532,7 @@ class RandomResize(cde.RandomResizeOp): class RandomResizeWithBBox(cde.RandomResizeWithBBoxOp): """ Tensor operation to resize the input image using a randomly selected interpolation mode and adjust - the bounding boxes accordingly. + bounding boxes accordingly. Args: size (int or sequence): The output size of the resized image. From 29aa589972070f3cd0b43e60ff01e25d6ad1190d Mon Sep 17 00:00:00 2001 From: peilin-wang Date: Thu, 2 Jul 2020 16:49:09 -0400 Subject: [PATCH 228/254] added check for invalid type for boolean args --- mindspore/dataset/engine/validators.py | 7 +++++++ .../python/dataset/test_bucket_batch_by_length.py | 13 +++++++++++++ 2 files changed, 20 insertions(+) diff --git a/mindspore/dataset/engine/validators.py b/mindspore/dataset/engine/validators.py index 2a0bef3b42..744a9b94be 100644 --- a/mindspore/dataset/engine/validators.py +++ b/mindspore/dataset/engine/validators.py @@ -606,8 +606,15 @@ def check_bucket_batch_by_length(method): nreq_param_list = ['column_names', 'bucket_boundaries', 'bucket_batch_sizes'] check_param_type(nreq_param_list, param_dict, list) + nbool_param_list = ['pad_to_bucket_boundary', 'drop_remainder'] + check_param_type(nbool_param_list, param_dict, bool) + # check column_names: must be list of string. column_names = param_dict.get("column_names") + + if not column_names: + raise ValueError("column_names cannot be empty") + all_string = all(isinstance(item, str) for item in column_names) if not all_string: raise TypeError("column_names should be a list of str.") diff --git a/tests/ut/python/dataset/test_bucket_batch_by_length.py b/tests/ut/python/dataset/test_bucket_batch_by_length.py index 4436f98e53..febcc6483f 100644 --- a/tests/ut/python/dataset/test_bucket_batch_by_length.py +++ b/tests/ut/python/dataset/test_bucket_batch_by_length.py @@ -53,6 +53,9 @@ def test_bucket_batch_invalid_input(): negative_bucket_batch_sizes = [1, 2, 3, -4] zero_bucket_batch_sizes = [0, 1, 2, 3] + invalid_type_pad_to_bucket_boundary = "" + invalid_type_drop_remainder = "" + with pytest.raises(TypeError) as info: _ = dataset.bucket_batch_by_length(invalid_column_names, bucket_boundaries, bucket_batch_sizes) assert "column_names should be a list of str" in str(info.value) @@ -93,6 +96,16 @@ def test_bucket_batch_invalid_input(): _ = dataset.bucket_batch_by_length(column_names, bucket_boundaries, bucket_boundaries) assert "bucket_batch_sizes must contain one element more than bucket_boundaries" in str(info.value) + with pytest.raises(TypeError) as info: + _ = dataset.bucket_batch_by_length(column_names, bucket_boundaries, bucket_batch_sizes, + None, None, invalid_type_pad_to_bucket_boundary) + assert "Wrong input type for pad_to_bucket_boundary, should be " in str(info.value) + + with pytest.raises(TypeError) as info: + _ = dataset.bucket_batch_by_length(column_names, bucket_boundaries, bucket_batch_sizes, + None, None, False, invalid_type_drop_remainder) + assert "Wrong input type for drop_remainder, should be " in str(info.value) + def test_bucket_batch_multi_bucket_no_padding(): dataset = ds.GeneratorDataset((lambda: generate_sequential_same_shape(10)), ["col1"]) From 6bd66e79531ce560b50f3a80fbd2a32226054b50 Mon Sep 17 00:00:00 2001 From: Hoai Linh Tran Date: Thu, 2 Jul 2020 15:29:12 -0400 Subject: [PATCH 229/254] Fix memcpy calls. Add ut tests for arithmetic_simplify. Split long arithmetic_simplify.h to arithmetic_simplify.cc Code checking --- .../optimizer/irpass/arithmetic_simplify.cc | 680 ++++++++++++++++++ .../optimizer/irpass/arithmetic_simplify.h | 639 ++-------------- .../python/pynative_mode/test_framstruct.py | 116 +++ 3 files changed, 841 insertions(+), 594 deletions(-) create mode 100644 mindspore/ccsrc/optimizer/irpass/arithmetic_simplify.cc diff --git a/mindspore/ccsrc/optimizer/irpass/arithmetic_simplify.cc b/mindspore/ccsrc/optimizer/irpass/arithmetic_simplify.cc new file mode 100644 index 0000000000..b111a6b67a --- /dev/null +++ b/mindspore/ccsrc/optimizer/irpass/arithmetic_simplify.cc @@ -0,0 +1,680 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include + +#include "optimizer/irpass/arithmetic_simplify.h" +#include "ir/optimizer_caller.h" +#include "ir/visitor.h" +#include "operator/ops.h" +#include "optimizer/irpass.h" +#include "optimizer/irpass/prim_eliminate.h" +#include "optimizer/optimizer.h" + +namespace mindspore { +namespace opt { +namespace irpass { +// {prim::kPrimScalarMul, 0, X}, {prim::kPrimScalarMul, X, 0} +// {prim::kPrimScalarMul, 1, X}, {prim::kPrimScalarMul, X, 1} +AnfNodePtr MultiplyByZeroOrOne::operator()(const OptimizerPtr &, const AnfNodePtr &node) { + Reset(); + AnfVisitor::Match(prim::kPrimScalarMul)(node); + + if (is_zero_) { + return NewValueNode(zero_); + } + if (is_one_) { + return x_; + } + return nullptr; +} + +void MultiplyByZeroOrOne::Visit(const AnfNodePtr &node) { + if (is_one_ || node->isa()) { + x_ = node; + return; + } + + AnfVisitor::Visit(node); + if (!is_one_) { + x_ = node; + } +} + +void MultiplyByZeroOrOne::Visit(const ValueNodePtr &vnode) { + auto value = vnode->value(); + if (*value == *zero_) { + is_zero_ = true; + } else if (*value == *one_) { + is_one_ = true; + } +} + +void MultiplyByZeroOrOne::Reset() { + x_ = nullptr; + is_one_ = false; + is_zero_ = false; +} + +// Support class used for checking if all values of a Tensor are equal `check_value_` +// Supported data types: double, float/float32, int/int32 +bool CheckTensorConstant::IsTensorConstant(const ValuePtr &value) { + if (!value->isa()) { + return false; + } + auto tensor_ptr = dyn_cast(value); + TypeId tensor_type = tensor_ptr->Dtype()->type_id(); + if ((tensor_type == TypeId::kNumberTypeFloat32) || (tensor_type == TypeId::kNumberTypeFloat)) { + float *data2 = reinterpret_cast(tensor_ptr->data_c()); + for (int i = 0; i < tensor_ptr->DataSize(); i++) { + if (fabs(data2[i] - check_value_) > FLT_EPSILON) { + return false; + } + } + return true; + } else if (tensor_type == TypeId::kNumberTypeFloat64) { + double *data2 = reinterpret_cast(tensor_ptr->data_c()); + for (int i = 0; i < tensor_ptr->DataSize(); i++) { + if (fabs(data2[i] - check_value_) > DBL_EPSILON) { + return false; + } + } + return true; + } else if ((tensor_type == TypeId::kNumberTypeInt32) || (tensor_type == TypeId::kNumberTypeInt)) { + int *data2 = reinterpret_cast(tensor_ptr->data_c()); + for (int i = 0; i < tensor_ptr->DataSize(); i++) { + if (data2[i] != check_value_) { + return false; + } + } + return true; + } + // input Data Types is not supported + return false; +} + +bool CheckTensorConstant::IsTensorScalarConstant(const ValuePtr &value) { + if (!value->isa()) { + return false; + } + auto tensor_ptr = dyn_cast(value); + if ((tensor_ptr->DataSize() > 1) || (tensor_ptr->DataDim() > 0)) { + return false; + } + return IsTensorConstant(value); +} + +void *TensorMultiplyBase::GetPointerToTensorData(const AnfNodePtr &node, bool writable) { + if (!node->isa()) { + return nullptr; + } + + auto value = node->cast()->value(); + + if (!value->isa()) { + return nullptr; + } + + tensor::TensorPtr tensor_ptr = dyn_cast(value); + return tensor_ptr->data_c(); +} + +// Make a new tensor (when possible) with the same shape as of `node` +// If x is nullptr then fill new tensor will "0" +// If x is a tensor with empty shape then fill new tensor with the single value of x +// If x is a tensor with same shape as `node` then return x as result +AnfNodePtr TensorMultiplyBase::NewTensorFilledWithData(const AnfNodePtr &node, const AnfNodePtr &x) { + if ((node->abstract() == nullptr) || !node->abstract()->isa()) { + return nullptr; + } + + auto tensor_abstract = node->abstract()->cast(); + TypePtr tensor_type_ptr = tensor_abstract->element()->BuildType(); + std::vector tensor_shape = tensor_abstract->shape()->shape(); + + auto new_tensor_ptr = std::make_shared(tensor_type_ptr->type_id(), tensor_shape); + size_t mem_size = GetTypeByte(tensor_type_ptr) * IntToSize(new_tensor_ptr->ElementsNum()); + char *data = reinterpret_cast(new_tensor_ptr->data_c()); + + if (x == nullptr) { + std::memset(data, 0, mem_size); + auto new_vnode = NewValueNode(new_tensor_ptr); + new_vnode->set_abstract(new_tensor_ptr->ToAbstract()); + return new_vnode; + } + // x is not nullptr + if (x->isa()) { + if ((x->abstract() == nullptr) || !x->abstract()->isa()) { + return nullptr; + } + auto x_abstract = x->abstract()->cast(); + std::vector x_shape = x_abstract->shape()->shape(); + + if (x_shape != tensor_shape) { + return nullptr; + } + return x; + } + + if (!x->isa()) { + return nullptr; + } + auto x_value = x->cast()->value(); + if (!x_value->isa()) { + return nullptr; + } + + auto x_tensor_ptr = dyn_cast(x_value); + + if ((x_tensor_ptr->DataSize() > 1) && (x_tensor_ptr->DataSize() != new_tensor_ptr->DataSize())) { + return nullptr; + } + char *source_data = reinterpret_cast(GetPointerToTensorData(x)); + if (x_tensor_ptr->DataSize() == 1) { + for (int i = 0; i < new_tensor_ptr->ElementsNum(); i++) { + memcpy(data + i * GetTypeByte(tensor_type_ptr), source_data, GetTypeByte(tensor_type_ptr)); + } + } else { + memcpy(data, source_data, mem_size); + } + auto new_vnode = NewValueNode(new_tensor_ptr); + new_vnode->set_abstract(new_tensor_ptr->ToAbstract()); + return new_vnode; +} + +// {prim::kPrimMul, 0, X}, {prim::kPrimMul, X, 0} +AnfNodePtr TensorMultiplyByZero::operator()(const OptimizerPtr &, const AnfNodePtr &node) { + Reset(); + AnfVisitor::Match(prim::kPrimMul)(node); + + if (is_zero_) { + if (x_->func_graph() != node->func_graph()) { + return nullptr; + } + return NewTensorFilledWithData(node); + } + return nullptr; +} + +void TensorMultiplyByZero::Visit(const AnfNodePtr &node) { + if (is_zero_) { + x_ = node; + return; + } + + if (IsParam(node)) { + x_ = node; + return; + } + + if (IsCNode(node)) { + CNodePtr cnode = node->cast(); + if (IsPrimitive(cnode->input(0), prim::kPrimZerosLike)) { + is_zero_ = true; + return; + } + x_ = node; + return; + } + auto value = node->cast()->value(); + if (CheckTensorConstant(0).IsTensorConstant(value)) { + is_zero_ = true; + return; + } + x_ = node; +} + +void TensorMultiplyByZero::Visit(const ValueNodePtr &vnode) { + auto value = vnode->value(); + if (CheckTensorConstant(0).IsTensorConstant(value)) { + is_zero_ = true; + return; + } + x_ = vnode; +} +void TensorMultiplyByZero::Reset() { + x_ = nullptr; + is_zero_ = false; +} + +// {prim::kPrimMul, 1, X}, {prim::kPrimMul, X, 1} +AnfNodePtr TensorMultiplyByOne::operator()(const OptimizerPtr &, const AnfNodePtr &node) { + Reset(); + AnfVisitor::Match(prim::kPrimMul)(node); + + if (is_one_) { + return NewTensorFilledWithData(node, x_); + } + return nullptr; +} + +void TensorMultiplyByOne::Visit(const AnfNodePtr &node) { + if (is_one_) { + x_ = node; + return; + } + + if (IsParam(node) || IsCNode(node)) { + x_ = node; + return; + } + + auto value = node->cast()->value(); + if (CheckTensorConstant(1).IsTensorConstant(value)) { + is_one_ = true; + return; + } + x_ = node; +} + +void TensorMultiplyByOne::Visit(const ValueNodePtr &vnode) { + auto value = vnode->value(); + if (CheckTensorConstant(1).IsTensorConstant(value)) { + is_one_ = true; + return; + } + x_ = vnode; +} +void TensorMultiplyByOne::Reset() { + x_ = nullptr; + is_one_ = false; +} + +// {prim::kPrimScalarAdd, X, 0} +// {prim::kPrimScalarAdd, 0, X} +AnfNodePtr AddByZero::operator()(const OptimizerPtr &, const AnfNodePtr &node) { + Reset(); + AnfVisitor::Match(prim::kPrimScalarAdd)(node); + + if (is_zero_) { + return x_; + } + return nullptr; +} + +void AddByZero::Visit(const AnfNodePtr &node) { + if (node->isa() && + ((*GetValueNode(node) == *zero_) || CheckTensorConstant(0).IsTensorScalarConstant(GetValueNode(node)))) { + is_zero_ = true; + return; + } + + x_ = node; +} + +void AddByZero::Reset() { + x_ = nullptr; + is_zero_ = false; +} + +// {prim::kPrimTensorAdd, {kPrimZerosLike, Y}, X}, +// {prim::kPrimTensorAdd, X, {kPrimZerosLike, Y}} +AnfNodePtr TensorAddByZero::operator()(const OptimizerPtr &, const AnfNodePtr &node) { + Reset(); + AnfVisitor::Match(prim::kPrimTensorAdd)(node); + + if (is_zero_) { + return x_; + } + return nullptr; +} + +void TensorAddByZero::Visit(const AnfNodePtr &node) { + if (node->isa() && CheckTensorConstant(0).IsTensorScalarConstant(GetValueNode(node))) { + is_zero_ = true; + return; + } + + x_ = node; +} + +void TensorAddByZero::Visit(const ValueNodePtr &vnode) { + auto value = vnode->value(); + if (CheckTensorConstant(0).IsTensorConstant(value)) { + is_zero_ = true; + return; + } +} + +void TensorAddByZero::Reset() { + x_ = nullptr; + is_zero_ = false; +} + +// {PrimMomentum, {kPrimZerosLike, X}, Y, Z, Xs} -> {prim::kPrimMakeTuple, Z, Y} +AnfNodePtr OptUpdateZeroTensor::operator()(const OptimizerPtr &, const AnfNodePtr &node) { + if (!IsPrimitiveCNode(node, prim::kPrimMomentum) || node->func_graph() == nullptr) { + return nullptr; + } + + // {PrimMomentum, {...}, Y, Z, Xs} + auto &inputs = node->cast()->inputs(); + if (inputs.size() < 4 || !IsPrimitiveCNode(inputs[1], prim::kPrimZerosLike)) { + return nullptr; + } + auto y = inputs[2]; + auto z = inputs[3]; + + // {kPrimZerosLike, X} + if (inputs[1]->cast()->size() != 2) { + return nullptr; + } + + // {prim::kPrimMakeTuple, Z, Y} + return node->func_graph()->NewCNode({NewValueNode(prim::kPrimMakeTuple), z, y}); +} + +// {prim::kPrimMul, Tensor1, {prim::kPrimMul, Tensor2, {...}}} -> +// {prim::kPrimMul, {...}, {prim::kPrimMul, Tensor1, Tensor2}} +// Support function to multiply two constant tensors: partially support broadcasting shapes +template +void ConstantDuplicateMul::Multiply(void *in_data_1, int in_data_1_size, void *in_data_2, int in_data_2_size, + void **out_data, int out_data_size) { + T *data_1 = reinterpret_cast(in_data_1); + T *data_2 = reinterpret_cast(in_data_2); + T *data_out = new T[out_data_size]; + + if (in_data_1_size == 1) { + for (int i = 0; i < out_data_size; i++) { + data_out[i] = data_1[0]; + } + } else { + for (int i = 0; i < out_data_size; i++) { + data_out[i] = data_1[i]; + } + } + if (in_data_2_size == 1) { + for (int i = 0; i < out_data_size; i++) { + data_out[i] *= data_2[0]; + } + } else { + for (int i = 0; i < out_data_size; i++) { + data_out[i] *= data_2[i]; + } + } + *out_data = reinterpret_cast(data_out); + return; +} + +AnfNodePtr ConstantDuplicateMul::MulConstantTensors(const AnfNodePtr &vnode_1, const AnfNodePtr &vnode_2, + const AnfNodePtr &node_3) { + if (!vnode_1->isa() || !vnode_2->isa() || (vnode_1->abstract() == nullptr) || + (vnode_2->abstract() == nullptr) || (node_3->abstract() == nullptr)) { + return nullptr; + } + + auto value_1 = GetValueNode(vnode_1); + auto value_2 = GetValueNode(vnode_2); + + if (!value_1->isa() || !value_2->isa()) { + return nullptr; + } + + auto tensor_ptr_1 = dyn_cast(value_1); + auto tensor_ptr_2 = dyn_cast(value_2); + + auto tensor_1_abstract = vnode_1->abstract()->cast(); + auto tensor_2_abstract = vnode_1->abstract()->cast(); + auto tensor_3_abstract = node_3->abstract()->cast(); + + TypePtr tensor_1_type_ptr = tensor_1_abstract->element()->BuildType(); + TypePtr tensor_2_type_ptr = tensor_2_abstract->element()->BuildType(); + TypePtr tensor_3_type_ptr = tensor_3_abstract->element()->BuildType(); + + if ((tensor_1_type_ptr->type_id() != tensor_3_type_ptr->type_id()) || + (tensor_2_type_ptr->type_id() != tensor_3_type_ptr->type_id())) { + return nullptr; + } + + std::vector tensor_out_shape = tensor_3_abstract->shape()->shape(); + + int data_out_size = std::accumulate(tensor_out_shape.begin(), tensor_out_shape.end(), 1, std::multiplies()); + + if ((tensor_ptr_1->DataSize() > 1) && (tensor_ptr_1->DataSize() != data_out_size)) { + return nullptr; + } + if ((tensor_ptr_2->DataSize() > 1) && (tensor_ptr_2->DataSize() != data_out_size)) { + return nullptr; + } + + void *data_out; + + if ((tensor_3_type_ptr->type_id() == TypeId::kNumberTypeFloat32) || + (tensor_3_type_ptr->type_id() == TypeId::kNumberTypeFloat)) { + Multiply(tensor_ptr_1->data_c(), tensor_ptr_1->DataSize(), tensor_ptr_2->data_c(), tensor_ptr_2->DataSize(), + &data_out, data_out_size); + } else { + if (tensor_3_type_ptr->type_id() == TypeId::kNumberTypeFloat64) { + Multiply(tensor_ptr_1->data_c(), tensor_ptr_1->DataSize(), tensor_ptr_2->data_c(), + tensor_ptr_2->DataSize(), &data_out, data_out_size); + } else { + if ((tensor_3_type_ptr->type_id() == TypeId::kNumberTypeInt32) || + (tensor_3_type_ptr->type_id() == TypeId::kNumberTypeInt)) { + Multiply(tensor_ptr_1->data_c(), tensor_ptr_1->DataSize(), tensor_ptr_2->data_c(), + tensor_ptr_2->DataSize(), &data_out, data_out_size); + } else { + // Un-support data types + return nullptr; + } + } + } + + auto new_tensor_ptr = std::make_shared(tensor_3_type_ptr->type_id(), tensor_out_shape); + size_t mem_size = GetTypeByte(tensor_3_type_ptr) * IntToSize(new_tensor_ptr->ElementsNum()); + char *data = reinterpret_cast(new_tensor_ptr->data_c()); + memcpy(data, data_out, mem_size); + + auto new_vnode = NewValueNode(new_tensor_ptr); + new_vnode->set_abstract(new_tensor_ptr->ToAbstract()); + return new_vnode; +} + +AnfNodePtr ConstantDuplicateMul::operator()(const OptimizerPtr &, const AnfNodePtr &node) { + Reset(); + // {prim::kPrimMul, Tensor1, {...}} + AnfVisitor::Match(prim::kPrimMul, {IsNode, IsNode})(node); + if (vnode_ == nullptr || c_p_node_ == nullptr) { + return nullptr; + } + + if (!IsCNode(c_p_node_)) { + return nullptr; + } + + auto tensor1 = vnode_; + auto mul = c_p_node_->cast(); + + Reset(); + // {prim::kPrimMul, Tensor2, {...}} + AnfVisitor::Match(prim::kPrimMul, {IsNode, IsNode})(mul); + if (vnode_ == nullptr || c_p_node_ == nullptr) { + return nullptr; + } + auto tensor2 = vnode_; + auto c_p_node = c_p_node_; + + auto PrimMul = GetValueNode(mul->input(0)); + auto fg = node->func_graph(); + + auto new_mul_tensor = MulConstantTensors(tensor1, tensor2, c_p_node); + if (new_mul_tensor == nullptr) { + auto ttmul = NewCNode({NewValueNode(PrimMul), tensor1, tensor2}, fg); + return NewCNode({NewValueNode(PrimMul), c_p_node, ttmul}, fg); + } + return NewCNode({NewValueNode(PrimMul), c_p_node, new_mul_tensor}, fg); +} + +void ConstantDuplicateMul::Visit(const AnfNodePtr &node) { + if (IsValueNode(node)) { + vnode_ = node; + } + + if (IsCNode(node) || IsParam(node)) { + c_p_node_ = node; + } +} + +void ConstantDuplicateMul::Reset() { + vnode_ = nullptr; + c_p_node_ = nullptr; +} + +AnfNodePtr PowerOneEliminate::operator()(const OptimizerPtr &, const AnfNodePtr &node) { + if (!IsPrimitiveCNode(node, prim::kPrimPow) || node->func_graph() == nullptr) { + return nullptr; + } + + auto &inputs = node->cast()->inputs(); + if (!IsValueNode(inputs[2])) { + return nullptr; + } + auto scalar = GetValueNode(inputs[2]); + if (scalar->isa() && GetValue(scalar) == 1.0) { + return inputs[1]; + } else if (scalar->isa() && GetValue(scalar) == 1) { + return inputs[1]; + } + return nullptr; +} + +// grad = AllReduce(grad) / worker_number +// grad = grad + weight * decy +// -> +// grad = grad + weight * decy +// grad = AllReduce(grad) / worker_number +// {prim::kPrimAddN, {prim::kPrimMakeTuple, {prim::kPrimMul, {prim::kPrimAllReduce, X}, Y}, Z}} -> +// {prim::kPrimMul, {prim::kPrimAllReduce, {prim::kPrimAddN,{prim::kPrimMakeTuple, Z, X}}}, Y} +AnfNodePtr AdjustAllReduceMulAdd::operator()(const OptimizerPtr &, const AnfNodePtr &node) { + Reset(); + // {prim::kPrimAddN, Zs} + if (!IsPrimitiveCNode(node, prim::kPrimAddN)) { + return nullptr; + } + auto addn = node->cast(); + if (addn->size() != 2) { + return nullptr; + } + AnfVisitor::Match(prim::kPrimMakeTuple, {IsNode, IsNode})(addn->input(1)); + if (x_ == nullptr || y_ == nullptr || z_ == nullptr || all_reduce_fg_ == nullptr) { + return nullptr; + } + auto addn_maketuple = addn->input(1); + + auto fg = all_reduce_fg_; + // addn inputs cross the graph, make the inputs same as allreduce node. + if (z_->isa() && fg != z_->func_graph()) { + auto cnode_z = z_->cast(); + z_ = NewCNode(cnode_z->inputs(), fg); + } + + auto addn_op_node = addn->input(0); + auto make_tuple_op_node = addn->input(1)->cast()->input(0); + + AnfNodePtr tuple = NewCNode({make_tuple_op_node, z_, x_}, fg); + AnfNodePtr add = NewCNode({addn_op_node, tuple}, fg); + AnfNodePtr all_reduce = NewCNode({all_reduce_, add}, fg); + AnfNodePtr mul = NewCNode({mul_, all_reduce, y_}, fg); + ProcessDependEdge(fg, addn_maketuple, all_reduce); + return mul; +} + +void AdjustAllReduceMulAdd::ProcessDependEdge(const FuncGraphPtr &fg, const AnfNodePtr &addn_maketuple, + const AnfNodePtr &new_node) { + // If has dynamic loss scale. + auto &users_map = fg->manager()->node_users(); + auto it = users_map.find(mul_cnode_); + if (it != users_map.end()) { + auto users = it->second; + for (auto &user_pair : users) { + auto node = user_pair.first; + if (node != addn_maketuple) { + if (IsPrimitiveCNode(node, prim::kPrimMakeTuple)) { + fg->manager()->SetEdge(node, user_pair.second, new_node); + } + } + } + } +} + +void AdjustAllReduceMulAdd::Visit(const AnfNodePtr &node) { + if (level_ == 0) { + level_ = 1; + is_reduce_match_ = false; + // {prim::kPrimMul, {prim::kPrimAllReduce, X}, Y} + AnfVisitor::Match(prim::kPrimMul)(node); + level_ = 0; + if (is_reduce_match_) { + mul_ = node->cast()->input(0); + mul_cnode_ = node->cast(); + y_ = tmp_; + } else { + z_ = node; + } + } + + if (level_ == 1) { + // {prim::kPrimAllReduce, X} + if (IsPrimitiveCNode(node, prim::kPrimAllReduce)) { + auto cnode = node->cast(); + if (cnode->size() > 1) { + all_reduce_ = cnode->input(0); + x_ = cnode->input(1); + is_reduce_match_ = true; + all_reduce_fg_ = cnode->func_graph(); + } + } else { + tmp_ = node; + } + } +} + +void AdjustAllReduceMulAdd::Reset() { + level_ = 0; + is_reduce_match_ = false; + x_ = nullptr; + y_ = nullptr; + z_ = nullptr; + tmp_ = nullptr; + all_reduce_fg_ = nullptr; +} + +AnfNodePtr ArithmeticSimplify::operator()(const OptimizerPtr &optimizer, const AnfNodePtr &node) { + AnfNodePtr new_node; + for (auto &eliminater : eliminaters_) { + new_node = (*eliminater)(optimizer, node); + if (new_node != nullptr) { + return new_node; + } + } + return nullptr; +} + +AnfNodePtr ArithmeticSimplify2::operator()(const OptimizerPtr &optimizer, const AnfNodePtr &node) { + AnfNodePtr new_node; + for (auto &eliminater : eliminaters_) { + new_node = (*eliminater)(optimizer, node); + if (new_node != nullptr) { + return new_node; + } + } + return nullptr; +} +} // namespace irpass +} // namespace opt +} // namespace mindspore diff --git a/mindspore/ccsrc/optimizer/irpass/arithmetic_simplify.h b/mindspore/ccsrc/optimizer/irpass/arithmetic_simplify.h index 940cdde6d2..f4bdb0d655 100644 --- a/mindspore/ccsrc/optimizer/irpass/arithmetic_simplify.h +++ b/mindspore/ccsrc/optimizer/irpass/arithmetic_simplify.h @@ -38,45 +38,11 @@ class MultiplyByZeroOrOne : public AnfVisitor { MultiplyByZeroOrOne() : zero_(MakeValue(0)), one_(MakeValue(1)) {} ~MultiplyByZeroOrOne() override = default; - AnfNodePtr operator()(const OptimizerPtr &, const AnfNodePtr &node) override { - Reset(); - AnfVisitor::Match(prim::kPrimScalarMul)(node); - - if (is_zero_) { - return NewValueNode(zero_); - } - if (is_one_) { - return x_; - } - return nullptr; - } - - void Visit(const AnfNodePtr &node) override { - if (is_one_ || node->isa()) { - x_ = node; - return; - } - - AnfVisitor::Visit(node); - if (!is_one_) { - x_ = node; - } - } + AnfNodePtr operator()(const OptimizerPtr &, const AnfNodePtr &node) override; - void Visit(const ValueNodePtr &vnode) override { - auto value = vnode->value(); - if (*value == *zero_) { - is_zero_ = true; - } else if (*value == *one_) { - is_one_ = true; - } - } - - void Reset() { - x_ = nullptr; - is_one_ = false; - is_zero_ = false; - } + void Visit(const AnfNodePtr &node) override; + void Visit(const ValueNodePtr &vnode) override; + void Reset(); private: bool is_zero_{false}, is_one_{false}; @@ -90,51 +56,9 @@ class CheckTensorConstant { public: explicit CheckTensorConstant(int _check_value = 0) : check_value_(_check_value) {} ~CheckTensorConstant() = default; - bool IsTensorConstant(const ValuePtr &value) { - if (!value->isa()) { - return false; - } - auto tensor_ptr = dyn_cast(value); - TypeId tensor_type = tensor_ptr->Dtype()->type_id(); - if ((tensor_type == TypeId::kNumberTypeFloat32) || (tensor_type == TypeId::kNumberTypeFloat)) { - float *data2 = reinterpret_cast(tensor_ptr->data_c()); - for (int i = 0; i < tensor_ptr->DataSize(); i++) { - if (fabs(data2[i] - check_value_) > FLT_EPSILON) { - return false; - } - } - return true; - } else if (tensor_type == TypeId::kNumberTypeFloat64) { - double *data2 = reinterpret_cast(tensor_ptr->data_c()); - for (int i = 0; i < tensor_ptr->DataSize(); i++) { - if (fabs(data2[i] - check_value_) > DBL_EPSILON) { - return false; - } - } - return true; - } else if ((tensor_type == TypeId::kNumberTypeInt32) || (tensor_type == TypeId::kNumberTypeInt)) { - int *data2 = reinterpret_cast(tensor_ptr->data_c()); - for (int i = 0; i < tensor_ptr->DataSize(); i++) { - if (data2[i] != check_value_) { - return false; - } - } - return true; - } - // Un-support Data Types - return false; - } - bool IsTensorScalarConstant(const ValuePtr &value) { - if (!value->isa()) { - return false; - } - auto tensor_ptr = dyn_cast(value); - if ((tensor_ptr->DataSize() > 1) || (tensor_ptr->DataDim() > 0)) { - return false; - } - return IsTensorConstant(value); - } + bool IsTensorConstant(const ValuePtr &value); + bool IsTensorScalarConstant(const ValuePtr &value); private: int check_value_; @@ -142,83 +66,13 @@ class CheckTensorConstant { class TensorMultiplyBase : public AnfVisitor { protected: - void *GetPointerToTensorData(const AnfNodePtr &node, bool writable = false) { - if (!node->isa()) { - return nullptr; - } - - auto value = node->cast()->value(); - - if (!value->isa()) { - return nullptr; - } - - tensor::TensorPtr tensor_ptr = dyn_cast(value); - return tensor_ptr->data_c(); - } + void *GetPointerToTensorData(const AnfNodePtr &node, bool writable = false); // Make a new tensor (when possible) with the same shape as of `node` // If x is nullptr then fill new tensor will "0" // If x is a tensor with empty shape then fill new tensor with the single value of x // If x is a tensor with same shape as `node` then return x as result - AnfNodePtr NewTensorFilledWithData(const AnfNodePtr &node, const AnfNodePtr &x = nullptr) { - if ((node->abstract() == nullptr) || !node->abstract()->isa()) { - return nullptr; - } - - auto tensor_abstract = node->abstract()->cast(); - TypePtr tensor_type_ptr = tensor_abstract->element()->BuildType(); - std::vector tensor_shape = tensor_abstract->shape()->shape(); - - auto new_tensor_ptr = std::make_shared(tensor_type_ptr->type_id(), tensor_shape); - size_t mem_size = GetTypeByte(tensor_type_ptr) * IntToSize(new_tensor_ptr->ElementsNum()); - char *data = reinterpret_cast(new_tensor_ptr->data_c()); - - if (x == nullptr) { - std::memset(data, 0, mem_size); - auto new_vnode = NewValueNode(new_tensor_ptr); - new_vnode->set_abstract(new_tensor_ptr->ToAbstract()); - return new_vnode; - } - // x is not nullptr - if (x->isa()) { - if ((x->abstract() == nullptr) || !x->abstract()->isa()) { - return nullptr; - } - auto x_abstract = x->abstract()->cast(); - std::vector x_shape = x_abstract->shape()->shape(); - - if (x_shape != tensor_shape) { - return nullptr; - } - return x; - } - - if (!x->isa()) { - return nullptr; - } - auto x_value = x->cast()->value(); - if (!x_value->isa()) { - return nullptr; - } - - auto x_tensor_ptr = dyn_cast(x_value); - - if ((x_tensor_ptr->DataSize() > 1) && (x_tensor_ptr->DataSize() != new_tensor_ptr->DataSize())) { - return nullptr; - } - char *source_data = reinterpret_cast(GetPointerToTensorData(x)); - if (x_tensor_ptr->DataSize() == 1) { - for (int i = 0; i < new_tensor_ptr->ElementsNum(); i++) { - memcpy(source_data, data + i * GetTypeByte(tensor_type_ptr), GetTypeByte(tensor_type_ptr)); - } - } else { - memcpy(source_data, data, mem_size); - } - auto new_vnode = NewValueNode(new_tensor_ptr); - new_vnode->set_abstract(new_tensor_ptr->ToAbstract()); - return new_vnode; - } + AnfNodePtr NewTensorFilledWithData(const AnfNodePtr &node, const AnfNodePtr &x = nullptr); AnfNodePtr x_{nullptr}; }; @@ -228,59 +82,12 @@ class TensorMultiplyByZero : public TensorMultiplyBase { public: TensorMultiplyByZero() : zero_(MakeValue(0)) {} ~TensorMultiplyByZero() override = default; - AnfNodePtr operator()(const OptimizerPtr &, const AnfNodePtr &node) override { - Reset(); - AnfVisitor::Match(prim::kPrimMul)(node); - - if (is_zero_) { - if (x_->func_graph() != node->func_graph()) { - return nullptr; - } - return NewTensorFilledWithData(node); - } - return nullptr; - } - void Visit(const AnfNodePtr &node) override { - if (is_zero_) { - x_ = node; - return; - } - - if (IsParam(node)) { - x_ = node; - return; - } - - if (IsCNode(node)) { - CNodePtr cnode = node->cast(); - if (IsPrimitive(cnode->input(0), prim::kPrimZerosLike)) { - is_zero_ = true; - return; - } - x_ = node; - return; - } - auto value = node->cast()->value(); - if (CheckTensorConstant(0).IsTensorConstant(value)) { - is_zero_ = true; - return; - } - x_ = node; - } + AnfNodePtr operator()(const OptimizerPtr &, const AnfNodePtr &node) override; - void Visit(const ValueNodePtr &vnode) override { - auto value = vnode->value(); - if (CheckTensorConstant(0).IsTensorConstant(value)) { - is_zero_ = true; - return; - } - x_ = vnode; - } - void Reset() { - x_ = nullptr; - is_zero_ = false; - } + void Visit(const AnfNodePtr &node) override; + void Visit(const ValueNodePtr &vnode) override; + void Reset(); private: bool is_zero_{false}; @@ -292,47 +99,11 @@ class TensorMultiplyByOne : public TensorMultiplyBase { public: TensorMultiplyByOne() {} ~TensorMultiplyByOne() override = default; - AnfNodePtr operator()(const OptimizerPtr &, const AnfNodePtr &node) override { - Reset(); - AnfVisitor::Match(prim::kPrimMul)(node); - - if (is_one_) { - return NewTensorFilledWithData(node, x_); - } - return nullptr; - } + AnfNodePtr operator()(const OptimizerPtr &, const AnfNodePtr &node) override; - void Visit(const AnfNodePtr &node) override { - if (is_one_) { - x_ = node; - return; - } - - if (IsParam(node) || IsCNode(node)) { - x_ = node; - return; - } - - auto value = node->cast()->value(); - if (CheckTensorConstant(1).IsTensorConstant(value)) { - is_one_ = true; - return; - } - x_ = node; - } - - void Visit(const ValueNodePtr &vnode) override { - auto value = vnode->value(); - if (CheckTensorConstant(1).IsTensorConstant(value)) { - is_one_ = true; - return; - } - x_ = vnode; - } - void Reset() { - x_ = nullptr; - is_one_ = false; - } + void Visit(const AnfNodePtr &node) override; + void Visit(const ValueNodePtr &vnode) override; + void Reset(); private: bool is_one_{false}; @@ -345,30 +116,10 @@ class AddByZero : public AnfVisitor { AddByZero() : zero_(MakeValue(0)) {} ~AddByZero() override = default; - AnfNodePtr operator()(const OptimizerPtr &, const AnfNodePtr &node) override { - Reset(); - AnfVisitor::Match(prim::kPrimScalarAdd)(node); - - if (is_zero_) { - return x_; - } - return nullptr; - } - - void Visit(const AnfNodePtr &node) override { - if (node->isa() && - ((*GetValueNode(node) == *zero_) || CheckTensorConstant(0).IsTensorScalarConstant(GetValueNode(node)))) { - is_zero_ = true; - return; - } + AnfNodePtr operator()(const OptimizerPtr &, const AnfNodePtr &node) override; - x_ = node; - } - - void Reset() { - x_ = nullptr; - is_zero_ = false; - } + void Visit(const AnfNodePtr &node) override; + void Reset(); private: bool is_zero_{false}; @@ -380,37 +131,11 @@ class AddByZero : public AnfVisitor { // {prim::kPrimTensorAdd, X, {kPrimZerosLike, Y}} class TensorAddByZero : public AnfVisitor { public: - AnfNodePtr operator()(const OptimizerPtr &, const AnfNodePtr &node) override { - Reset(); - AnfVisitor::Match(prim::kPrimTensorAdd)(node); - - if (is_zero_) { - return x_; - } - return nullptr; - } + AnfNodePtr operator()(const OptimizerPtr &, const AnfNodePtr &node) override; - void Visit(const AnfNodePtr &node) override { - if (node->isa() && CheckTensorConstant(0).IsTensorScalarConstant(GetValueNode(node))) { - is_zero_ = true; - return; - } - - x_ = node; - } - - void Visit(const ValueNodePtr &vnode) override { - auto value = vnode->value(); - if (CheckTensorConstant(0).IsTensorConstant(value)) { - is_zero_ = true; - return; - } - } - - void Reset() { - x_ = nullptr; - is_zero_ = false; - } + void Visit(const AnfNodePtr &node) override; + void Visit(const ValueNodePtr &vnode) override; + void Reset(); private: bool is_zero_{false}; @@ -420,27 +145,7 @@ class TensorAddByZero : public AnfVisitor { // {PrimMomentum, {kPrimZerosLike, X}, Y, Z, Xs} -> {prim::kPrimMakeTuple, Z, Y} class OptUpdateZeroTensor : public AnfVisitor { public: - AnfNodePtr operator()(const OptimizerPtr &, const AnfNodePtr &node) override { - if (!IsPrimitiveCNode(node, prim::kPrimMomentum) || node->func_graph() == nullptr) { - return nullptr; - } - - // {PrimMomentum, {...}, Y, Z, Xs} - auto &inputs = node->cast()->inputs(); - if (inputs.size() < 4 || !IsPrimitiveCNode(inputs[1], prim::kPrimZerosLike)) { - return nullptr; - } - auto y = inputs[2]; - auto z = inputs[3]; - - // {kPrimZerosLike, X} - if (inputs[1]->cast()->size() != 2) { - return nullptr; - } - - // {prim::kPrimMakeTuple, Z, Y} - return node->func_graph()->NewCNode({NewValueNode(prim::kPrimMakeTuple), z, y}); - } + AnfNodePtr operator()(const OptimizerPtr &, const AnfNodePtr &node) override; }; // {prim::kPrimMul, Tensor1, {orim::kPrimMul, Tensor2, {...}}} -> @@ -450,156 +155,14 @@ class ConstantDuplicateMul : public AnfVisitor { // Support function to multiply two constant tensors: partially support broadcasting shapes template void Multiply(void *in_data_1, int in_data_1_size, void *in_data_2, int in_data_2_size, void **out_data, - int out_data_size) { - T *data_1 = reinterpret_cast(in_data_1); - T *data_2 = reinterpret_cast(in_data_2); - T *data_out = new T[out_data_size]; - - if (in_data_1_size == 1) { - for (int i = 0; i < out_data_size; i++) { - data_out[i] = data_1[0]; - } - } else { - for (int i = 0; i < out_data_size; i++) { - data_out[i] = data_1[i]; - } - } - if (in_data_2_size == 1) { - for (int i = 0; i < out_data_size; i++) { - data_out[i] *= data_2[0]; - } - } else { - for (int i = 0; i < out_data_size; i++) { - data_out[i] *= data_2[i]; - } - } - *out_data = reinterpret_cast(data_out); - return; - } - - AnfNodePtr MulConstantTensors(const AnfNodePtr &vnode_1, const AnfNodePtr &vnode_2, const AnfNodePtr &node_3) { - if (!vnode_1->isa() || !vnode_2->isa() || (vnode_1->abstract() == nullptr) || - (vnode_2->abstract() == nullptr) || (node_3->abstract() == nullptr)) { - return nullptr; - } - - auto value_1 = GetValueNode(vnode_1); - auto value_2 = GetValueNode(vnode_2); - - if (!value_1->isa() || !value_2->isa()) { - return nullptr; - } - - auto tensor_ptr_1 = dyn_cast(value_1); - auto tensor_ptr_2 = dyn_cast(value_2); - - auto tensor_1_abstract = vnode_1->abstract()->cast(); - auto tensor_2_abstract = vnode_1->abstract()->cast(); - auto tensor_3_abstract = node_3->abstract()->cast(); - - TypePtr tensor_1_type_ptr = tensor_1_abstract->element()->BuildType(); - TypePtr tensor_2_type_ptr = tensor_2_abstract->element()->BuildType(); - TypePtr tensor_3_type_ptr = tensor_3_abstract->element()->BuildType(); - - if ((tensor_1_type_ptr->type_id() != tensor_3_type_ptr->type_id()) || - (tensor_2_type_ptr->type_id() != tensor_3_type_ptr->type_id())) { - return nullptr; - } - - std::vector tensor_out_shape = tensor_3_abstract->shape()->shape(); - - int data_out_size = 1; - for (auto it : tensor_out_shape) { - data_out_size *= it; - } - if ((tensor_ptr_1->DataSize() > 1) && (tensor_ptr_1->DataSize() != data_out_size)) { - return nullptr; - } - if ((tensor_ptr_2->DataSize() > 1) && (tensor_ptr_2->DataSize() != data_out_size)) { - return nullptr; - } - - void *data_out; - - if ((tensor_3_type_ptr->type_id() == TypeId::kNumberTypeFloat32) || - (tensor_3_type_ptr->type_id() == TypeId::kNumberTypeFloat)) { - Multiply(tensor_ptr_1->data_c(), tensor_ptr_1->DataSize(), tensor_ptr_2->data_c(), - tensor_ptr_2->DataSize(), &data_out, data_out_size); - } else { - if (tensor_3_type_ptr->type_id() == TypeId::kNumberTypeFloat64) { - Multiply(tensor_ptr_1->data_c(), tensor_ptr_1->DataSize(), tensor_ptr_2->data_c(), - tensor_ptr_2->DataSize(), &data_out, data_out_size); - } else { - if ((tensor_3_type_ptr->type_id() == TypeId::kNumberTypeInt32) || - (tensor_3_type_ptr->type_id() == TypeId::kNumberTypeInt)) { - Multiply(tensor_ptr_1->data_c(), tensor_ptr_1->DataSize(), tensor_ptr_2->data_c(), - tensor_ptr_2->DataSize(), &data_out, data_out_size); - } else { - // Un-support data types - return nullptr; - } - } - } - - auto new_tensor_ptr = std::make_shared(tensor_3_type_ptr->type_id(), tensor_out_shape); - size_t mem_size = GetTypeByte(tensor_3_type_ptr) * IntToSize(new_tensor_ptr->ElementsNum()); - char *data = reinterpret_cast(new_tensor_ptr->data_c()); - memcpy(data, data_out, mem_size); - - auto new_vnode = NewValueNode(new_tensor_ptr); - new_vnode->set_abstract(new_tensor_ptr->ToAbstract()); - return new_vnode; - } + int out_data_size); - AnfNodePtr operator()(const OptimizerPtr &, const AnfNodePtr &node) override { - Reset(); - // {prim::kPrimMul, Tensor1, {...}} - AnfVisitor::Match(prim::kPrimMul, {IsNode, IsNode})(node); - if (vnode_ == nullptr || c_p_node_ == nullptr) { - return nullptr; - } - - if (!IsCNode(c_p_node_)) { - return nullptr; - } - - auto tensor1 = vnode_; - auto mul = c_p_node_->cast(); - - Reset(); - // {prim::kPrimMul, Tensor2, {...}} - AnfVisitor::Match(prim::kPrimMul, {IsNode, IsNode})(mul); - if (vnode_ == nullptr || c_p_node_ == nullptr) { - return nullptr; - } - auto tensor2 = vnode_; - auto c_p_node = c_p_node_; - - auto PrimMul = GetValueNode(mul->input(0)); - auto fg = node->func_graph(); - - auto new_mul_tensor = MulConstantTensors(tensor1, tensor2, c_p_node); - if (new_mul_tensor == nullptr) { - auto ttmul = NewCNode({NewValueNode(PrimMul), tensor1, tensor2}, fg); - return NewCNode({NewValueNode(PrimMul), c_p_node, ttmul}, fg); - } - return NewCNode({NewValueNode(PrimMul), c_p_node, new_mul_tensor}, fg); - } + AnfNodePtr MulConstantTensors(const AnfNodePtr &vnode_1, const AnfNodePtr &vnode_2, const AnfNodePtr &node_3); - void Visit(const AnfNodePtr &node) override { - if (IsValueNode(node)) { - vnode_ = node; - } + AnfNodePtr operator()(const OptimizerPtr &, const AnfNodePtr &node) override; - if (IsCNode(node) || IsParam(node)) { - c_p_node_ = node; - } - } - - void Reset() { - vnode_ = nullptr; - c_p_node_ = nullptr; - } + void Visit(const AnfNodePtr &node) override; + void Reset(); private: AnfNodePtr vnode_; @@ -608,23 +171,7 @@ class ConstantDuplicateMul : public AnfVisitor { class PowerOneEliminate : public AnfVisitor { public: - AnfNodePtr operator()(const OptimizerPtr &, const AnfNodePtr &node) override { - if (!IsPrimitiveCNode(node, prim::kPrimPow) || node->func_graph() == nullptr) { - return nullptr; - } - - auto &inputs = node->cast()->inputs(); - if (!IsValueNode(inputs[2])) { - return nullptr; - } - auto scalar = GetValueNode(inputs[2]); - if (scalar->isa() && GetValue(scalar) == 1.0) { - return inputs[1]; - } else if (scalar->isa() && GetValue(scalar) == 1) { - return inputs[1]; - } - return nullptr; - } + AnfNodePtr operator()(const OptimizerPtr &, const AnfNodePtr &node) override; }; // grad = AllReduce(grad) / worker_number @@ -637,96 +184,11 @@ class PowerOneEliminate : public AnfVisitor { // {prim::kPrimMul, {prim::kPrimAllReduce, {prim::kPrimAddN,{prim::kPrimMakeTuple, Z, X}}}, Y} class AdjustAllReduceMulAdd : public AnfVisitor { public: - AnfNodePtr operator()(const OptimizerPtr &, const AnfNodePtr &node) override { - Reset(); - // {prim::kPrimAddN, Zs} - if (!IsPrimitiveCNode(node, prim::kPrimAddN)) { - return nullptr; - } - auto addn = node->cast(); - if (addn->size() != 2) { - return nullptr; - } - AnfVisitor::Match(prim::kPrimMakeTuple, {IsNode, IsNode})(addn->input(1)); - if (x_ == nullptr || y_ == nullptr || z_ == nullptr || all_reduce_fg_ == nullptr) { - return nullptr; - } - auto addn_maketuple = addn->input(1); - - auto fg = all_reduce_fg_; - // addn inputs cross the graph, make the inputs same as allreduce node. - if (z_->isa() && fg != z_->func_graph()) { - auto cnode_z = z_->cast(); - z_ = NewCNode(cnode_z->inputs(), fg); - } - - auto addn_op_node = addn->input(0); - auto make_tuple_op_node = addn->input(1)->cast()->input(0); - - AnfNodePtr tuple = NewCNode({make_tuple_op_node, z_, x_}, fg); - AnfNodePtr add = NewCNode({addn_op_node, tuple}, fg); - AnfNodePtr all_reduce = NewCNode({all_reduce_, add}, fg); - AnfNodePtr mul = NewCNode({mul_, all_reduce, y_}, fg); - ProcessDependEdge(fg, addn_maketuple, all_reduce); - return mul; - } - void ProcessDependEdge(const FuncGraphPtr &fg, const AnfNodePtr &addn_maketuple, const AnfNodePtr &new_node) { - // If has dynamic loss scale. - auto &users_map = fg->manager()->node_users(); - auto it = users_map.find(mul_cnode_); - if (it != users_map.end()) { - auto users = it->second; - for (auto &user_pair : users) { - auto node = user_pair.first; - if (node != addn_maketuple) { - if (IsPrimitiveCNode(node, prim::kPrimMakeTuple)) { - fg->manager()->SetEdge(node, user_pair.second, new_node); - } - } - } - } - } - void Visit(const AnfNodePtr &node) override { - if (level_ == 0) { - level_ = 1; - is_reduce_match_ = false; - // {prim::kPrimMul, {prim::kPrimAllReduce, X}, Y} - AnfVisitor::Match(prim::kPrimMul)(node); - level_ = 0; - if (is_reduce_match_) { - mul_ = node->cast()->input(0); - mul_cnode_ = node->cast(); - y_ = tmp_; - } else { - z_ = node; - } - } - - if (level_ == 1) { - // {prim::kPrimAllReduce, X} - if (IsPrimitiveCNode(node, prim::kPrimAllReduce)) { - auto cnode = node->cast(); - if (cnode->size() > 1) { - all_reduce_ = cnode->input(0); - x_ = cnode->input(1); - is_reduce_match_ = true; - all_reduce_fg_ = cnode->func_graph(); - } - } else { - tmp_ = node; - } - } - } + AnfNodePtr operator()(const OptimizerPtr &, const AnfNodePtr &node) override; - void Reset() { - level_ = 0; - is_reduce_match_ = false; - x_ = nullptr; - y_ = nullptr; - z_ = nullptr; - tmp_ = nullptr; - all_reduce_fg_ = nullptr; - } + void ProcessDependEdge(const FuncGraphPtr &fg, const AnfNodePtr &addn_maketuple, const AnfNodePtr &new_node); + void Visit(const AnfNodePtr &node) override; + void Reset(); private: int level_{0}; @@ -758,20 +220,18 @@ class ArithmeticSimplify : public OptimizerCaller { } ~ArithmeticSimplify() = default; - AnfNodePtr operator()(const OptimizerPtr &optimizer, const AnfNodePtr &node) override { - AnfNodePtr new_node; - for (auto &eliminater : eliminaters_) { - new_node = (*eliminater)(optimizer, node); - if (new_node != nullptr) { - return new_node; - } - } - return nullptr; - } + AnfNodePtr operator()(const OptimizerPtr &optimizer, const AnfNodePtr &node) override; private: - OptimizerCallerPtr multiply_by_zero_or_one_, tensor_multiply_by_one_, add_by_zero_, tensor_add_by_zero_, identity_, - opt_update_zero_tensor_, constant_duplicate_mul_, power_one_; + OptimizerCallerPtr multiply_by_zero_or_one_; + OptimizerCallerPtr tensor_multiply_by_one_; + OptimizerCallerPtr add_by_zero_; + OptimizerCallerPtr tensor_add_by_zero_; + OptimizerCallerPtr identity_; + OptimizerCallerPtr opt_update_zero_tensor_; + OptimizerCallerPtr constant_duplicate_mul_; + OptimizerCallerPtr power_one_; + std::vector eliminaters_{}; }; @@ -787,16 +247,7 @@ class ArithmeticSimplify2 : public OptimizerCaller { } ~ArithmeticSimplify2() = default; - AnfNodePtr operator()(const OptimizerPtr &optimizer, const AnfNodePtr &node) override { - AnfNodePtr new_node; - for (auto &eliminater : eliminaters_) { - new_node = (*eliminater)(optimizer, node); - if (new_node != nullptr) { - return new_node; - } - } - return nullptr; - } + AnfNodePtr operator()(const OptimizerPtr &optimizer, const AnfNodePtr &node) override; private: OptimizerCallerPtr tensor_multiply_by_zero_; diff --git a/tests/ut/python/pynative_mode/test_framstruct.py b/tests/ut/python/pynative_mode/test_framstruct.py index 99d792671a..39a4c97ab9 100644 --- a/tests/ut/python/pynative_mode/test_framstruct.py +++ b/tests/ut/python/pynative_mode/test_framstruct.py @@ -549,6 +549,122 @@ def test_zeros(): assert res == Tensor(np.zeros([2, 3]).astype(np.int32)) +@ms_function +def arithmetic_simplify_01(x, y): + """ arithmetic_simplify_01 """ + return C.zeros_like(x) * y + + +def test_arithmetic_simplify_01(): + """ test_arithmetic_simplify_01 """ + x = Tensor(np.ones([2, 3]).astype(np.int32)) + y = Tensor(np.array([[1, 2, 3], [4, 5, 6]]).astype(np.int32)) + res = arithmetic_simplify_01(x, y) + expect = np.zeros([2, 3]).astype(np.int32) + assert np.all(res.asnumpy() == expect) + + +@ms_function +def arithmetic_simplify_02(x, y): + """ arithmetic_simplify_02 """ + return C.ones_like(x) * y + + +def test_arithmetic_simplify_02(): + """ test_arithmetic_simplify_02 """ + x = Tensor(np.ones([2, 3]).astype(np.int32)) + y = Tensor(np.array([[1, 2, 3], [4, 5, 6]]).astype(np.int32)) + res = arithmetic_simplify_02(x, y) + expect = np.array([[1, 2, 3], [4, 5, 6]]).astype(np.int32) + assert np.all(res.asnumpy() == expect) + + +@ms_function +def arithmetic_simplify_03(x, y): + """ arithmetic_simplify_03 """ + return x * C.ones_like(y) + + +def test_arithmetic_simplify_03(): + """ test_arithmetic_simplify_03 """ + x = Tensor(np.ones([2, 3]).astype(np.int32)) + y = Tensor(np.array([[1, 2, 3], [4, 5, 6]]).astype(np.int32)) + res = arithmetic_simplify_03(x, y) + expect = np.ones([2, 3]).astype(np.int32) + assert np.all(res.asnumpy() == expect) + + +@ms_function +def arithmetic_simplify_04(x): + """ arithmetic_simplify_04 """ + return x + 0 + + +def test_arithmetic_simplify_04(): + """ test_arithmetic_simplify_04 """ + x = Tensor(np.array([[1, 2, 3], [4, 5, 6]]).astype(np.int32)) + res = arithmetic_simplify_04(x) + expect = np.array([[1, 2, 3], [4, 5, 6]]).astype(np.int32) + assert np.all(res.asnumpy() == expect) + + +@ms_function +def arithmetic_simplify_05(x): + """ arithmetic_simplify_05 """ + return x * 1 + + +def test_arithmetic_simplify_05(): + """ test_arithmetic_simplify_05 """ + x = Tensor(np.array([[1, 2, 3], [4, 5, 6]]).astype(np.int32)) + res = arithmetic_simplify_05(x) + expect = np.array([[1, 2, 3], [4, 5, 6]]).astype(np.int32) + assert np.all(res.asnumpy() == expect) + + +@ms_function +def arithmetic_simplify_06(x): + """ arithmetic_simplify_06 """ + return x * 2 * 5 + + +def test_arithmetic_simplify_06(): + """ test_arithmetic_simplify_06 """ + x = Tensor(np.array([[1, 2, 3], [4, 5, 6]]).astype(np.int32)) + res = arithmetic_simplify_06(x) + expect = np.array([[10, 20, 30], [40, 50, 60]]).astype(np.int32) + assert np.all(res.asnumpy() == expect) + + +@ms_function +def arithmetic_simplify_07(x): + """ arithmetic_simplify_07 """ + return (x + 1) * 2 * 5 + + +def test_arithmetic_simplify_07(): + """ test_arithmetic_simplify_07 """ + x = Tensor(np.array([[1, 2, 3], [4, 5, 6]]).astype(np.int32)) + res = arithmetic_simplify_07(x) + expect = np.array([[20, 30, 40], [50, 60, 70]]).astype(np.int32) + assert np.all(res.asnumpy() == expect) + + +@ms_function +def arithmetic_simplify_08(x, y): + """ arithmetic_simplify_08 """ + return 1 * x * 1 * 1 + 1 * 0 * 1 + 0 + y * 1 + + +def test_arithmetic_simplify_08(): + """ test_arithmetic_simplify_08 """ + x = Tensor(np.array([[1, 2, 3], [4, 5, 6]]).astype(np.int32)) + y = Tensor(np.ones([2, 3]).astype(np.int32)) + res = arithmetic_simplify_08(x, y) + expect = np.array([[2, 3, 4], [5, 6, 7]]).astype(np.int32) + assert np.all(res.asnumpy() == expect) + + def test_ScalarGradChecker(): """ test_ScalarGradChecker """ From e9067b4a10f9ffb3e7bfe03497830f71bf84c3b9 Mon Sep 17 00:00:00 2001 From: kswang Date: Thu, 2 Jul 2020 23:24:04 +0800 Subject: [PATCH 230/254] add internal output --- mindspore/ccsrc/device/kernel_runtime.cc | 3 +- mindspore/ccsrc/device/kernel_runtime.h | 2 +- mindspore/ccsrc/ir/anf.cc | 41 ++++- mindspore/ccsrc/ir/anf.h | 2 +- .../ascend/format_type/insert_trans_op.cc | 12 +- mindspore/ccsrc/session/ascend_session.cc | 9 - mindspore/ccsrc/session/ascend_session.h | 2 - mindspore/ccsrc/session/kernel_graph.cc | 73 ++++++++ mindspore/ccsrc/session/kernel_graph.h | 10 ++ mindspore/ccsrc/session/session_basic.cc | 169 +++++++++++++----- mindspore/ccsrc/session/session_basic.h | 4 + mindspore/ccsrc/vm/transform.cc | 39 ---- 12 files changed, 269 insertions(+), 97 deletions(-) diff --git a/mindspore/ccsrc/device/kernel_runtime.cc b/mindspore/ccsrc/device/kernel_runtime.cc index 4b96d90236..8c5654ec4c 100644 --- a/mindspore/ccsrc/device/kernel_runtime.cc +++ b/mindspore/ccsrc/device/kernel_runtime.cc @@ -339,7 +339,7 @@ void KernelRuntime::AssignStaticMemoryInput(const session::KernelGraph *graph) { } } -void KernelRuntime::AssignStaticMemoryOutput(const session::KernelGraph *graph) { +void KernelRuntime::AssignStaticMemoryOutput(session::KernelGraph *graph) { MS_EXCEPTION_IF_NULL(graph); auto nodes = AnfAlgo::GetAllOutput(graph->output(), {prim::kPrimTupleGetItem}); std::vector non_communication_op; @@ -350,6 +350,7 @@ void KernelRuntime::AssignStaticMemoryOutput(const session::KernelGraph *graph) if (!item_with_index.first->isa() || !AnfAlgo::IsRealKernel(item_with_index.first)) { continue; } + graph->AddFinalOutputKernel(item_with_index.first); if (AnfAlgo::IsCommunicationOp(item_with_index.first)) { AssignCommunicationNodeMem(kStaticMem, item_with_index.first); } else { diff --git a/mindspore/ccsrc/device/kernel_runtime.h b/mindspore/ccsrc/device/kernel_runtime.h index 0630b6dede..656ef8e2e6 100644 --- a/mindspore/ccsrc/device/kernel_runtime.h +++ b/mindspore/ccsrc/device/kernel_runtime.h @@ -95,7 +95,7 @@ class KernelRuntime { #endif private: - void AssignStaticMemoryOutput(const session::KernelGraph *graph); + void AssignStaticMemoryOutput(session::KernelGraph *graph); void GenLaunchArgs(const session::KernelGraph &graph, const AnfNodePtr &kernel, AddressPtrList *kernel_inputs, AddressPtrList *kernel_workspaces, AddressPtrList *kernel_outputs); bool LaunchKernelMod(const session::KernelGraph &graph); diff --git a/mindspore/ccsrc/ir/anf.cc b/mindspore/ccsrc/ir/anf.cc index 3b2402172b..4c1d2bf50d 100644 --- a/mindspore/ccsrc/ir/anf.cc +++ b/mindspore/ccsrc/ir/anf.cc @@ -25,7 +25,7 @@ #include "ir/func_graph.h" #include "ir/primitive_base.h" - +#include "utils/context/ms_context.h" #include "operator/ops.h" namespace mindspore { @@ -179,4 +179,43 @@ std::string get_id(const AnfNodePtr &node) { void reset_id() { node_ids.clear(); } } // namespace id_generator + +std::string GetCNodeTarget(const AnfNodePtr &node) { + auto context_ptr = MsContext::GetInstance(); + MS_EXCEPTION_IF_NULL(context_ptr); + std::string default_target = context_ptr->device_target(); + if (!node->isa()) { + return default_target; + } + auto cnode = node->cast(); + MS_EXCEPTION_IF_NULL(cnode); + auto attr_input = cnode->input(0); + if (attr_input == nullptr) { + return default_target; + } + auto value_node = attr_input->cast(); + if (value_node == nullptr) { + return default_target; + } + auto value = value_node->value(); + if (value == nullptr) { + return default_target; + } + if (!value->isa()) { + return default_target; + } + auto primitive = value->cast(); + auto att_target = primitive->GetAttr("primitive_target"); + if (att_target != nullptr) { + if (!att_target->isa()) { + MS_LOG(EXCEPTION) << "Only support string CPU|GPU|Ascend for primitive_target"; + } + auto target = GetValue(att_target); + if (kTargetSet.find(target) == kTargetSet.end()) { + MS_LOG(EXCEPTION) << "Only support string CPU|GPU|Ascend for primitive_target"; + } + return target; + } + return default_target; +} } // namespace mindspore diff --git a/mindspore/ccsrc/ir/anf.h b/mindspore/ccsrc/ir/anf.h index 95a018af06..8a44627885 100644 --- a/mindspore/ccsrc/ir/anf.h +++ b/mindspore/ccsrc/ir/anf.h @@ -448,7 +448,7 @@ void reset_id(); } // namespace id_generator using TaggedNodeMap = std::unordered_map; using TaggedGraph = std::pair; - +std::string GetCNodeTarget(const AnfNodePtr &node); } // namespace mindspore #endif // MINDSPORE_CCSRC_IR_ANF_H_ diff --git a/mindspore/ccsrc/pre_activate/ascend/format_type/insert_trans_op.cc b/mindspore/ccsrc/pre_activate/ascend/format_type/insert_trans_op.cc index 953f464431..3f77c68f86 100644 --- a/mindspore/ccsrc/pre_activate/ascend/format_type/insert_trans_op.cc +++ b/mindspore/ccsrc/pre_activate/ascend/format_type/insert_trans_op.cc @@ -46,6 +46,11 @@ const AnfNodePtr InsertTransOp::Process(const FuncGraphPtr &func_graph, const An if (node == nullptr || !AnfAlgo::IsRealKernel(node)) { return nullptr; } + AnfNodePtr front_node; + auto kernel_graph = func_graph->cast>(); + if (kernel_graph != nullptr && kernel_graph->IsInternalOutput(node)) { + front_node = kernel_graph->GetFrontNodeByInternalOutput(node); + } AnfAlgo::SetNodeAttr(kAttrVisited, MakeValue(true), node); MS_LOG(DEBUG) << "====process op: " << node->DebugString(); AnfNodePtr new_node = InsertTransOpForInput(func_graph, node, kernel_select_); @@ -56,7 +61,12 @@ const AnfNodePtr InsertTransOp::Process(const FuncGraphPtr &func_graph, const An return new_node; } } - return InsertTransOpForOutput(func_graph, new_node, kernel_select_); + auto final_node = InsertTransOpForOutput(func_graph, new_node, kernel_select_); + if (kernel_graph != nullptr && front_node != nullptr) { + auto old_node = kernel_graph->GetInternalOutputByFrontNode(front_node); + kernel_graph->ReplaceInternalOutput(old_node, final_node); + } + return final_node; } } // namespace opt } // namespace mindspore diff --git a/mindspore/ccsrc/session/ascend_session.cc b/mindspore/ccsrc/session/ascend_session.cc index a8626a202b..9ec2b2bafa 100644 --- a/mindspore/ccsrc/session/ascend_session.cc +++ b/mindspore/ccsrc/session/ascend_session.cc @@ -976,15 +976,6 @@ void AscendSession::SetFinalGraphOutput(const BaseRef &output) { } } -KernelGraphPtr AscendSession::GetGraph(mindspore::GraphId graph_id) { - auto it = graphs_.find(graph_id); - if (it == graphs_.end()) { - MS_LOG(WARNING) << "Can't find graph " << graph_id; - return nullptr; - } - return it->second; -} - void AscendSession::InsertSwitchToGraph(GraphId condition_graph_id, GraphId true_graph_id) { MS_LOG(INFO) << "Start!"; MS_LOG(INFO) << "Condition graph id[" << condition_graph_id << "],true graph id[" << true_graph_id << "]"; diff --git a/mindspore/ccsrc/session/ascend_session.h b/mindspore/ccsrc/session/ascend_session.h index 904b011077..4774015457 100755 --- a/mindspore/ccsrc/session/ascend_session.h +++ b/mindspore/ccsrc/session/ascend_session.h @@ -128,8 +128,6 @@ class AscendSession : public SessionBasic { void InsertDependToGraph(GraphId graph_id, const AnfNodePtr &attch_node); // insert depend to graph, used to attch control nodes to graph void InsertControlDependToGraph(GraphId graph_id, const AnfNodePtr &first_node, const AnfNodePtr &second_node); - // Get graph by graph id ,if not exist return null ptr - KernelGraphPtr GetGraph(GraphId graph_id); // set child graph parameter if front arg is a anf void SetChildGraphParameter(const AnfNodePtr &front_anf, GraphId to_graph_id, size_t input_idx); // set child graph parameter if front arg is a tensor diff --git a/mindspore/ccsrc/session/kernel_graph.cc b/mindspore/ccsrc/session/kernel_graph.cc index 306e3351e3..7b53afac2a 100644 --- a/mindspore/ccsrc/session/kernel_graph.cc +++ b/mindspore/ccsrc/session/kernel_graph.cc @@ -329,6 +329,9 @@ CNodePtr KernelGraph::NewCNode(const CNodePtr &cnode) { FrontBackendlMapUpdate(cnode, new_cnode); } AnfAlgo::SetGraphId(graph_id_, cnode.get()); + if (IsInternalOutput(cnode)) { + ReplaceInternalOutput(cnode, new_cnode); + } return new_cnode; } @@ -872,6 +875,76 @@ void KernelGraph::PrintGraphExecuteOrder() const { } } +void KernelGraph::AddInternalOutput(const AnfNodePtr &front_node, const AnfNodePtr &node) { + if (front_node == nullptr || node == nullptr) { + MS_LOG(INFO) << "Front node or node is nullptr"; + return; + } + MS_LOG(INFO) << "Add internal node " << node->DebugString() << " with front node " << front_node->DebugString(); + front_to_internal_outputs_map_[front_node] = node; + internal_outputs_to_front_map_[node] = front_node; +} + +void KernelGraph::ReplaceInternalOutput(const AnfNodePtr &node, const AnfNodePtr &new_node) { + if (new_node == nullptr || node == nullptr) { + MS_LOG(INFO) << "New node or node is nullptr"; + return; + } + if (node == new_node) { + MS_LOG(INFO) << "New node and node is the same"; + return; + } + auto iter = internal_outputs_to_front_map_.find(node); + if (iter == internal_outputs_to_front_map_.end()) { + MS_LOG(INFO) << "Node is not internal output"; + return; + } + MS_LOG(INFO) << "Replace internal node " << node->DebugString() << " To " << new_node->DebugString(); + internal_outputs_to_front_map_[new_node] = iter->second; + front_to_internal_outputs_map_[iter->second] = new_node; + internal_outputs_to_front_map_.erase(iter); +} + +AnfNodePtr KernelGraph::GetInternalOutputByFrontNode(const AnfNodePtr &front_node) const { + auto iter = front_to_internal_outputs_map_.find(front_node); + if (iter != front_to_internal_outputs_map_.end()) { + return iter->second; + } + return nullptr; +} + +bool KernelGraph::IsInternalOutput(const AnfNodePtr &node) const { + if (internal_outputs_to_front_map_.find(node) != internal_outputs_to_front_map_.end()) { + return true; + } + return false; +} + +AnfNodePtr KernelGraph::GetFrontNodeByInternalOutput(const AnfNodePtr &node) const { + auto iter = internal_outputs_to_front_map_.find(node); + if (iter != internal_outputs_to_front_map_.end()) { + return iter->second; + } + return nullptr; +} + +void KernelGraph::AddFinalOutputKernel(const AnfNodePtr &node) { + if (node == nullptr) { + return; + } + (void)final_output_kernels_.insert(node); +} + +bool KernelGraph::IsFinalOutputKernel(const AnfNodePtr &node) const { + if (node == nullptr) { + return false; + } + if (final_output_kernels_.find(node) != final_output_kernels_.end()) { + return true; + } + return false; +} + std::string KernelGraph::ToString() const { return std::string("kernel_graph_").append(std::to_string(graph_id_)); } KernelGraph::~KernelGraph() { device::KernelRuntimeManager::Instance().ClearGraphResource(graph_id_); } diff --git a/mindspore/ccsrc/session/kernel_graph.h b/mindspore/ccsrc/session/kernel_graph.h index c7a826e5fe..d6a67f3f02 100644 --- a/mindspore/ccsrc/session/kernel_graph.h +++ b/mindspore/ccsrc/session/kernel_graph.h @@ -144,6 +144,13 @@ class KernelGraph : public FuncGraph { void PrintGraphExecuteOrder() const; const std::map> &summary_nodes() const { return summary_nodes_; } void set_summary_nodes(const std::map> &nodes) { summary_nodes_ = nodes; } + void AddInternalOutput(const AnfNodePtr &front_node, const AnfNodePtr &node); + void ReplaceInternalOutput(const AnfNodePtr &node, const AnfNodePtr &new_node); + AnfNodePtr GetInternalOutputByFrontNode(const AnfNodePtr &front_node) const; + bool IsInternalOutput(const AnfNodePtr &node) const; + AnfNodePtr GetFrontNodeByInternalOutput(const AnfNodePtr &node) const; + void AddFinalOutputKernel(const AnfNodePtr &node); + bool IsFinalOutputKernel(const AnfNodePtr &node) const; private: // remove value node form graph @@ -202,6 +209,9 @@ class KernelGraph : public FuncGraph { CNodePtr start_label_; CNodePtr end_goto_; bool null_output_; + std::unordered_map front_to_internal_outputs_map_; + std::unordered_map internal_outputs_to_front_map_; + std::set final_output_kernels_; }; } // namespace session using KernelGraphPtr = std::shared_ptr; diff --git a/mindspore/ccsrc/session/session_basic.cc b/mindspore/ccsrc/session/session_basic.cc index 8935d3df2f..72c2bfecc1 100644 --- a/mindspore/ccsrc/session/session_basic.cc +++ b/mindspore/ccsrc/session/session_basic.cc @@ -95,6 +95,13 @@ BaseRef CreateOneTensor(const AnfNodePtr &node, size_t output_index, const Kerne TypeId type_id = kNumberTypeFloat32; type_id = AnfAlgo::GetOutputInferDataType(node, output_index); std::vector temp_shape; + if (graph.IsInternalOutput(node)) { + temp_shape.emplace_back(1); + tensor::TensorPtr tensor = std::make_shared(type_id, temp_shape); + tensor->set_device_address(address); + tensor->set_dirty(false); + return tensor; + } (void)std::copy(shape.begin(), shape.end(), std::back_inserter(temp_shape)); tensor::TensorPtr tensor = std::make_shared(type_id, temp_shape); // if in paynative mode,data only copyed to host when user want to print data @@ -172,48 +179,6 @@ ValueNodePtr CreateNewValueNode(const AnfNodePtr &anf, KernelGraph *graph) { return new_value_node; } -std::vector CreateParameterFromTuple(const AnfNodePtr &node, bool valid_input, KernelGraph *graph) { - MS_EXCEPTION_IF_NULL(node); - MS_EXCEPTION_IF_NULL(graph); - std::vector parameters; - std::vector pre_graph_out = {node}; - // If a cnode is a call, it's input0 is a cnode too, so it doesn't have primitive - if (!AnfAlgo::IsRealKernel(node)) { - pre_graph_out = AnfAlgo::GetAllOutput(node, {prim::kPrimTupleGetItem}); - } - auto valid_inputs = graph->MutableValidInputs(); - MS_EXCEPTION_IF_NULL(valid_inputs); - auto graph_inputs = graph->MutableInputs(); - MS_EXCEPTION_IF_NULL(graph_inputs); - auto create_parameter = [&](const AbstractBasePtr &abstract) -> void { - auto parameter = graph->NewParameter(); - MS_EXCEPTION_IF_NULL(parameter); - parameter->set_abstract(abstract); - auto new_parameter = graph->NewParameter(parameter); - parameters.push_back(new_parameter); - valid_inputs->push_back(valid_input); - graph_inputs->push_back(new_parameter); - }; - for (const auto &out_node : pre_graph_out) { - MS_EXCEPTION_IF_NULL(out_node); - auto abstract = out_node->abstract(); - MS_EXCEPTION_IF_NULL(abstract); - // create multiple parameters if is a tuple output real kernel - if (abstract->isa() && !AnfAlgo::CheckPrimitiveType(out_node, prim::kPrimTupleGetItem)) { - auto tuple_abstract = abstract->cast(); - MS_EXCEPTION_IF_NULL(tuple_abstract); - MS_LOG(INFO) << "Tuple_size [" << tuple_abstract->size() << "]"; - for (size_t output_idx = 0; output_idx < tuple_abstract->size(); output_idx++) { - create_parameter((*tuple_abstract)[output_idx]); - } - continue; - } - // create single parameter if is a abstract real kernel - create_parameter(out_node->abstract()); - } - return parameters; -} - size_t LoadCtrlInputTensor(const std::shared_ptr &graph, std::vector *inputs) { MS_EXCEPTION_IF_NULL(graph); MS_LOG(INFO) << "Load kInputCtrlTensors"; @@ -323,6 +288,103 @@ bool ExistSummaryNode(const KernelGraph *graph) { } // namespace GraphId SessionBasic::graph_sum_ = 0; + +KernelGraphPtr SessionBasic::GetGraph(mindspore::GraphId graph_id) { + auto it = graphs_.find(graph_id); + if (it == graphs_.end()) { + MS_LOG(WARNING) << "Can't find graph " << graph_id; + return nullptr; + } + return it->second; +} + +void SessionBasic::InitInternalOutputParameter(const AnfNodePtr &out_node, const AnfNodePtr ¶meter) { + auto graph_id = GetGraphIdByNode(out_node); + if (graph_id == kInvalidGraphId) { + return; + } + auto node_graph = GetGraph(graph_id); + if (node_graph == nullptr) { + return; + } + MS_LOG(INFO) << "Init parameter with pre graph output node: " << out_node->DebugString(); + auto ref_node = node_graph->GetInternalOutputByFrontNode(out_node); + if (ref_node == nullptr) { + MS_LOG(INFO) << "No corresponding internal output for output node"; + return; + } + auto real_kernel = AnfAlgo::VisitKernel(ref_node, 0); + auto ref_real_node = real_kernel.first; + auto ref_real_node_index = real_kernel.second; + if (ref_real_node->isa() && node_graph->IsInternalOutput(ref_real_node) && + node_graph->IsFinalOutputKernel(ref_real_node)) { + auto kernel_info = ref_real_node->kernel_info(); + if (kernel_info == nullptr || kernel_info->select_kernel_build_info() == nullptr) { + MS_LOG(INFO) << "No kernel info"; + return; + } + auto address = AnfAlgo::GetMutableOutputAddr(ref_real_node, ref_real_node_index); + if (address == nullptr) { + MS_LOG(INFO) << "No kernel address"; + return; + } + auto format = AnfAlgo::GetOutputFormat(ref_real_node, ref_real_node_index); + auto type = AnfAlgo::GetOutputDeviceDataType(ref_real_node, ref_real_node_index); + parameter->set_kernel_info(std::make_shared()); + auto d_kernel_info = parameter->kernel_info(); + MS_EXCEPTION_IF_NULL(d_kernel_info); + kernel::KernelBuildInfo::KernelBuildInfoBuilder builder; + builder.SetOutputsDeviceType({type}); + builder.SetOutputsFormat({format}); + d_kernel_info->set_select_kernel_build_info(builder.Build()); + AnfAlgo::SetOutputAddr(address, 0, parameter.get()); + } +} + +std::vector SessionBasic::CreateParameterFromTuple(const AnfNodePtr &node, bool valid_input, + KernelGraph *graph) { + MS_EXCEPTION_IF_NULL(node); + MS_EXCEPTION_IF_NULL(graph); + std::vector parameters; + std::vector pre_graph_out = {node}; + // If a cnode is a call, it's input0 is a cnode too, so it doesn't have primitive + if (!AnfAlgo::IsRealKernel(node)) { + pre_graph_out = AnfAlgo::GetAllOutput(node, {prim::kPrimTupleGetItem}); + } + auto valid_inputs = graph->MutableValidInputs(); + MS_EXCEPTION_IF_NULL(valid_inputs); + auto graph_inputs = graph->MutableInputs(); + MS_EXCEPTION_IF_NULL(graph_inputs); + auto create_parameter = [&](const AbstractBasePtr &abstract) -> void { + auto parameter = graph->NewParameter(); + MS_EXCEPTION_IF_NULL(parameter); + parameter->set_abstract(abstract); + auto new_parameter = graph->NewParameter(parameter); + parameters.push_back(new_parameter); + valid_inputs->push_back(valid_input); + graph_inputs->push_back(new_parameter); + }; + for (const auto &out_node : pre_graph_out) { + MS_EXCEPTION_IF_NULL(out_node); + auto abstract = out_node->abstract(); + MS_EXCEPTION_IF_NULL(abstract); + // create multiple parameters if is a tuple output real kernel + if (abstract->isa() && !AnfAlgo::CheckPrimitiveType(out_node, prim::kPrimTupleGetItem)) { + auto tuple_abstract = abstract->cast(); + MS_EXCEPTION_IF_NULL(tuple_abstract); + MS_LOG(INFO) << "Tuple_size [" << tuple_abstract->size() << "]"; + for (size_t output_idx = 0; output_idx < tuple_abstract->size(); output_idx++) { + create_parameter((*tuple_abstract)[output_idx]); + } + continue; + } + // create single parameter if is a abstract real kernel + create_parameter(out_node->abstract()); + InitInternalOutputParameter(out_node, parameters[parameters.size() - 1]); + } + return parameters; +} + ParameterPtr SessionBasic::CreateNewParameterFromParameter(const AnfNodePtr &anf, bool valid_input, KernelGraph *graph) { MS_EXCEPTION_IF_NULL(anf); @@ -857,6 +919,29 @@ CNodePtr SessionBasic::ConstructOutput(const AnfNodePtrList &outputs, const std: auto FindEqu = [graph, outputs](const AnfNodePtr &out) -> AnfNodePtr { auto backend_anf = graph->GetBackendAnfByFrontAnf(out); if (backend_anf != nullptr) { + auto front_real_kernel = AnfAlgo::VisitKernel(out, 0); + auto backend_real_kernel = AnfAlgo::VisitKernel(backend_anf, 0); + MS_EXCEPTION_IF_NULL(out); + auto out_func_graph = out->func_graph(); + MS_EXCEPTION_IF_NULL(out_func_graph); + auto out_func_graph_manager = out_func_graph->manager(); + if (out_func_graph_manager == nullptr) { + return backend_anf; + } + auto node_users = out_func_graph_manager->node_users(); + auto users = node_users[out]; + bool internal_output = true; + std::string kernel_target = GetCNodeTarget(front_real_kernel.first); + for (auto user : users) { + if (!AnfAlgo::IsRealKernel(user.first) || kernel_target != GetCNodeTarget(user.first)) { + internal_output = false; + break; + } + } + if (internal_output) { + MS_LOG(INFO) << "Internal output1: " << out->DebugString() << "To " << backend_real_kernel.first->DebugString(); + graph->AddInternalOutput(out, backend_real_kernel.first); + } return backend_anf; } MS_LOG(EXCEPTION) << "Can't find the node in the equiv map!"; diff --git a/mindspore/ccsrc/session/session_basic.h b/mindspore/ccsrc/session/session_basic.h index b9b966d90f..cf85dd0225 100755 --- a/mindspore/ccsrc/session/session_basic.h +++ b/mindspore/ccsrc/session/session_basic.h @@ -110,6 +110,8 @@ class SessionBasic { #endif protected: + // Get graph by graph id ,if not exist return null ptr + KernelGraphPtr GetGraph(GraphId graph_id); virtual void LoadInputData(const std::shared_ptr &kernel_graph, const std::vector &inputs_const) const; void UpdateOutputs(const std::shared_ptr &kernel_graph, VectorRef *const outputs, @@ -127,11 +129,13 @@ class SessionBasic { BaseRef TransformBaseRefListToTuple(const BaseRef &base_ref); // create a new kernel graph and update the graph sum KernelGraphPtr NewKernelGraph(); + std::vector CreateParameterFromTuple(const AnfNodePtr &node, bool valid_input, KernelGraph *graph); virtual ParameterPtr CreateNewParameterFromParameter(const AnfNodePtr &anf, bool valid_input, KernelGraph *graph); ValueNodePtr CreateValueNodeKernelGraph(const AnfNodePtr &anf, KernelGraph *graph); ParameterPtr CreateNewParameter(const AnfNodePtr &anf, KernelGraph *graph); AnfNodePtr CreateNewParameterFromCNode(const AnfNodePtr &anf, bool valid_input, KernelGraph *graph); void AddParameterToGraphInputs(const std::vector ¶meters, KernelGraph *graph); + void InitInternalOutputParameter(const AnfNodePtr &out_node, const AnfNodePtr ¶meter); std::unordered_map> graphs_; std::unordered_map> run_op_graphs_; diff --git a/mindspore/ccsrc/vm/transform.cc b/mindspore/ccsrc/vm/transform.cc index 3876f6279c..80d2fc9df9 100644 --- a/mindspore/ccsrc/vm/transform.cc +++ b/mindspore/ccsrc/vm/transform.cc @@ -52,45 +52,6 @@ const std::vector &GetMsNonlinearOps() { } namespace { -std::string GetCNodeTarget(const AnfNodePtr &node) { - auto context_ptr = MsContext::GetInstance(); - MS_EXCEPTION_IF_NULL(context_ptr); - std::string default_target = context_ptr->device_target(); - if (!node->isa()) { - return default_target; - } - auto cnode = node->cast(); - MS_EXCEPTION_IF_NULL(cnode); - auto attr_input = cnode->input(kAnfPrimitiveIndex); - if (attr_input == nullptr) { - return default_target; - } - auto value_node = attr_input->cast(); - if (value_node == nullptr) { - return default_target; - } - auto value = value_node->value(); - if (value == nullptr) { - return default_target; - } - if (!value->isa()) { - return default_target; - } - auto primitive = value->cast(); - auto att_target = primitive->GetAttr("primitive_target"); - if (att_target != nullptr) { - if (!att_target->isa()) { - MS_LOG(EXCEPTION) << "Only support string CPU|GPU|Ascend for primitive_target"; - } - auto target = GetValue(att_target); - if (kTargetSet.find(target) == kTargetSet.end()) { - MS_LOG(EXCEPTION) << "Only support string CPU|GPU|Ascend for primitive_target"; - } - return target; - } - return default_target; -} - bool ContainMultiTarget(const std::vector &nodes) { auto context_ptr = MsContext::GetInstance(); MS_EXCEPTION_IF_NULL(context_ptr); From 1b4a7cdeb714deb91994c8dce9a28a4667de408e Mon Sep 17 00:00:00 2001 From: lizhenyu Date: Thu, 2 Jul 2020 19:01:46 +0800 Subject: [PATCH 231/254] fix mem swap bug --- .../ccsrc/device/gpu/gpu_kernel_runtime.cc | 2 +- .../mem_reuse/mem_swap_manager.cc | 43 +++++++++++++++---- .../pre_activate/mem_reuse/mem_swap_manager.h | 7 ++- 3 files changed, 40 insertions(+), 12 deletions(-) diff --git a/mindspore/ccsrc/device/gpu/gpu_kernel_runtime.cc b/mindspore/ccsrc/device/gpu/gpu_kernel_runtime.cc index 8095a503e3..ad0e093d7f 100644 --- a/mindspore/ccsrc/device/gpu/gpu_kernel_runtime.cc +++ b/mindspore/ccsrc/device/gpu/gpu_kernel_runtime.cc @@ -565,7 +565,7 @@ void GPUKernelRuntime::FreeKernelDynamicRes(const mindspore::AnfNodePtr &kernel, MS_EXCEPTION_IF_NULL(mem_reuse_util_ptr); auto cnode = kernel->cast(); MS_EXCEPTION_IF_NULL(cnode); - if (AnfAlgo::GetCNodeName(kernel) == kAllReduceOpName) { + if (AnfAlgo::IsCommunicationOp(kernel)) { return; } // Free the input of kernel by reference count. diff --git a/mindspore/ccsrc/pre_activate/mem_reuse/mem_swap_manager.cc b/mindspore/ccsrc/pre_activate/mem_reuse/mem_swap_manager.cc index d81364edfb..14073bfbc9 100644 --- a/mindspore/ccsrc/pre_activate/mem_reuse/mem_swap_manager.cc +++ b/mindspore/ccsrc/pre_activate/mem_reuse/mem_swap_manager.cc @@ -24,7 +24,15 @@ namespace device { namespace memswap { void MemSwapManager::Init(const mindspore::session::KernelGraph *kernel_graph) { MS_EXCEPTION_IF_NULL(kernel_graph); - execution_order_ = kernel_graph->execution_order(); + graph_manager_ = kernel_graph->manager(); + MS_EXCEPTION_IF_NULL(graph_manager_); + auto &kernels = kernel_graph->execution_order(); + for (const auto &kernel : kernels) { + if (AnfAlgo::IsRealCNodeKernel(kernel) && (!opt::IsNopNode(kernel))) { + execution_order_.push_back(kernel); + } + } + size_t kernel_index = 0; for (const auto &kernel : execution_order_) { // parse topo order of kernel @@ -41,7 +49,7 @@ void MemSwapManager::Init(const mindspore::session::KernelGraph *kernel_graph) { } // parse topo order of user kernel - SaveUserKernelTopoOrder(kernel_graph); + SaveUserKernelTopoOrder(); sort(ordered_tensors_.begin(), ordered_tensors_.end(), [](const TensorInfo &a, const TensorInfo &b) { return a.tensor_size_ > b.tensor_size_; }); @@ -62,11 +70,22 @@ void MemSwapManager::Init(const mindspore::session::KernelGraph *kernel_graph) { mem_copy_manager_->Init(); } -void MemSwapManager::SaveUserKernelTopoOrder(const mindspore::session::KernelGraph *kernel_graph) { - MS_EXCEPTION_IF_NULL(kernel_graph); - FuncGraphManagerPtr manager = kernel_graph->manager(); - MS_EXCEPTION_IF_NULL(manager); - NodeUsersMap user_map = manager->node_users(); +bool MemSwapManager::IsCommunicationRelevantOp(const AnfNodePtr &kernel) const { + MS_EXCEPTION_IF_NULL(kernel); + NodeUsersMap &user_map = graph_manager_->node_users(); + auto iter = user_map.find(kernel); + bool adjacent_with_communication_op = false; + if (iter != user_map.end()) { + AnfNodeIndexSet node_set = iter->second; + adjacent_with_communication_op = std::any_of( + node_set.begin(), node_set.end(), + [](const std::pair &node_pair) { return AnfAlgo::IsCommunicationOp(node_pair.first); }); + } + return (AnfAlgo::IsCommunicationOp(kernel)) || adjacent_with_communication_op; +} + +void MemSwapManager::SaveUserKernelTopoOrder() { + NodeUsersMap &user_map = graph_manager_->node_users(); for (const auto &kernel : execution_order_) { auto iter = user_map.find(kernel); if (iter == user_map.end()) { @@ -76,13 +95,16 @@ void MemSwapManager::SaveUserKernelTopoOrder(const mindspore::session::KernelGra auto &kernel_exec_info = SearchKernelExecutionInfo(kernel); for (auto &node_pair : node_set) { auto user_kernel = node_pair.first; - if (!AnfAlgo::IsRealCNodeKernel(user_kernel)) { + if (!AnfAlgo::IsRealCNodeKernel(user_kernel) || opt::IsNopNode(user_kernel)) { continue; } size_t user_kernel_topo_sort = SearchKernelExecutionInfo(user_kernel).topo_order_; auto kernel_with_index = AnfAlgo::GetPrevNodeOutput(user_kernel, node_pair.second - 1); auto &output_idx = kernel_with_index.second; + if (kernel_with_index.first.get() != kernel.get()) { + MS_LOG(EXCEPTION) << "Save user kernel topo order failed for op[" << AnfAlgo::GetCNodeName(kernel) << "]"; + } kernel_exec_info.node_users_map_[output_idx].push_back(user_kernel_topo_sort); } for (auto &node_user_pair : kernel_exec_info.node_users_map_) { @@ -100,6 +122,9 @@ void MemSwapManager::AddSwapInfo() { size_t output_idx = tensor.output_idx_; const AnfNodePtr &kernel = tensor.kernel_; + if (IsCommunicationRelevantOp(kernel)) { + continue; + } auto &kernel_exec_info = SearchKernelExecutionInfo(kernel); auto &node_users_map = kernel_exec_info.node_users_map_; @@ -178,7 +203,7 @@ bool MemSwapManager::RetreatSwapInfo() { while (tensor_size_threshold_idx_ < ordered_tensors_.size() - 1) { ++tensor_size_threshold_idx_; - if (tensor_size_threshold_idx_ > ordered_tensors_[tensor_size_threshold_idx_].tensor_size_) { + if (tensor_size_threshold_ > ordered_tensors_[tensor_size_threshold_idx_].tensor_size_) { tensor_size_threshold_ = ordered_tensors_[tensor_size_threshold_idx_].tensor_size_; break; } diff --git a/mindspore/ccsrc/pre_activate/mem_reuse/mem_swap_manager.h b/mindspore/ccsrc/pre_activate/mem_reuse/mem_swap_manager.h index 7e2823d27c..1969dadb54 100644 --- a/mindspore/ccsrc/pre_activate/mem_reuse/mem_swap_manager.h +++ b/mindspore/ccsrc/pre_activate/mem_reuse/mem_swap_manager.h @@ -91,7 +91,7 @@ class MemSwapManager { void ResetSwapInfo(); - void SaveUserKernelTopoOrder(const mindspore::session::KernelGraph *kernel_graph); + void SaveUserKernelTopoOrder(); void AddKernelTriggerSwap(const AnfNodePtr &kernel, bool trigger_swap); @@ -99,6 +99,8 @@ class MemSwapManager { void AddKernelMemSwapInfo(const AnfNodePtr &kernel, const MemSwapInfo &mem_swap_info); + bool IsCommunicationRelevantOp(const AnfNodePtr &kernel) const; + std::vector execution_order_; std::vector ordered_tensors_; std::unordered_map kernel_execution_info_; @@ -113,7 +115,8 @@ class MemSwapManager { size_t tensor_size_num_; size_t distance_threshold_; - MemCopyManagerPtr mem_copy_manager_; + MemCopyManagerPtr mem_copy_manager_{nullptr}; + FuncGraphManagerPtr graph_manager_{nullptr}; bool mem_swap_initialized_{false}; bool swap_info_already_set_{false}; bool trigger_swap_{false}; From f9b80afdc07b82846be328ad19da2e2a7f4c3b21 Mon Sep 17 00:00:00 2001 From: yuchaojie Date: Fri, 3 Jul 2020 11:01:02 +0800 Subject: [PATCH 232/254] add transformer in model_zoo/README --- model_zoo/README.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/model_zoo/README.md b/model_zoo/README.md index 24be683b22..2dde985679 100644 --- a/model_zoo/README.md +++ b/model_zoo/README.md @@ -32,6 +32,7 @@ In order to facilitate developers to enjoy the benefits of MindSpore framework a - [Natural Language Processing](#natural-language-processing) - [BERT](#bert) - [MASS](#mass) + - [Transformer](#transformer) # Announcements @@ -301,6 +302,26 @@ In order to facilitate developers to enjoy the benefits of MindSpore framework a | Model for inference | | | Scripts | | +#### [Transformer](#table-of-contents) + +| Parameters | Transformer | +| -------------------------- | -------------------------------------------------------------- | +| Published Year | 2017 | +| Paper | [Attention Is All You Need ](https://arxiv.org/abs/1706.03762) | +| Resource | Ascend 910 | +| Features | • Multi-GPU training support with Ascend | +| MindSpore Version | 0.5.0-beta | +| Dataset | WMT Englis-German | +| Training Parameters | epoch=52, batch_size=96 | +| Optimizer | Adam | +| Loss Function | Softmax Cross Entropy | +| BLEU Score | 28.7 | +| Speed | 410ms/step (8pcs) | +| Loss | 2.8 | +| Params (M) | 213.7 | +| Checkpoint for inference | 2.4G (.ckpt file) | +| Scripts | https://gitee.com/mindspore/mindspore/tree/master/model_zoo/Transformer | + #### License [Apache License 2.0](https://github.com/mindspore-ai/mindspore/blob/master/LICENSE) From b29d260be3503105ea3861a860bb6bf787eb3c6c Mon Sep 17 00:00:00 2001 From: zhousiyi Date: Fri, 19 Jun 2020 02:57:03 +0000 Subject: [PATCH 233/254] reuse AddN primitive in opt as AddN will replicated by program_specialize --- mindspore/ccsrc/optimizer/irpass/merge_addn.h | 19 ++++++------------- .../gtest_input/optimizer/opt_test.py | 6 ++---- 2 files changed, 8 insertions(+), 17 deletions(-) diff --git a/mindspore/ccsrc/optimizer/irpass/merge_addn.h b/mindspore/ccsrc/optimizer/irpass/merge_addn.h index c96c35407f..e1e4b8878b 100644 --- a/mindspore/ccsrc/optimizer/irpass/merge_addn.h +++ b/mindspore/ccsrc/optimizer/irpass/merge_addn.h @@ -35,9 +35,6 @@ namespace irpass { // {{PrimAddNClass}, {prim::kPrimMakeTuple, Ys, Xs}} class MergeAddN : public AnfVisitor { public: - MergeAddN() : PrimAddN_(prim::GetPythonOps("AddN", "mindspore.ops.operations")) {} - ~MergeAddN() override = default; - AnfNodePtr operator()(const OptimizerPtr &optimizer, const AnfNodePtr &node) override { Reset(); optimizer_ = optimizer; @@ -47,15 +44,15 @@ class MergeAddN : public AnfVisitor { return nullptr; } - auto fg = node->func_graph(); - // {PrimAddNClass} - auto addn_node = fg->NewCNode({NewValueNode(PrimAddN_)}); + auto cnode = node->cast(); + auto addn = NewValueNode(GetValueNode(cnode->input(0))); // {prim::kPrimMakeTuple, Xs, Ys}, {prim::kPrimMakeTuple, Ys, Xs} (void)args_.insert(args_.begin(), NewValueNode(prim::kPrimMakeTuple)); + auto fg = node->func_graph(); auto make_node = fg->NewCNode(args_); - return fg->NewCNode({addn_node, make_node}); + return fg->NewCNode({addn, make_node}); } void Visit(const CNodePtr &cnode) override { @@ -127,7 +124,6 @@ class MergeAddN : public AnfVisitor { } private: - ValuePtr PrimAddN_; OptimizerPtr optimizer_{nullptr}; std::vector Xs_{}, Ys_{}, args_{}; bool is_inner_{false}, is_outer_{false}, is_match_{false}; @@ -136,9 +132,6 @@ class MergeAddN : public AnfVisitor { // {PrimAddN, {kPrimMakeTuple, Xs}} class AddNZeroFilter : public AnfVisitor { public: - AddNZeroFilter() : PrimAddN_(prim::GetPythonOps("AddN", "mindspore.ops.operations")) {} - ~AddNZeroFilter() override = default; - AnfNodePtr operator()(const OptimizerPtr &, const AnfNodePtr &node) override { Reset(); AnfVisitor::Match(prim::kPrimAddN, {IsCNode})(node); @@ -161,8 +154,9 @@ class AddNZeroFilter : public AnfVisitor { return nullptr; } + auto cnode = node->cast(); + auto addn = NewValueNode(GetValueNode(cnode->input(0))); auto fg = node->func_graph(); - auto addn = fg->NewCNode({NewValueNode(PrimAddN_)}); auto make_tuple = fg->NewCNode(filtered_Xs_); return fg->NewCNode({addn, make_tuple}); } @@ -193,7 +187,6 @@ class AddNZeroFilter : public AnfVisitor { } private: - ValuePtr PrimAddN_; std::vector filtered_Xs_{}, Xs_{}; bool has_zero_like_{false}; }; diff --git a/tests/ut/cpp/python_input/gtest_input/optimizer/opt_test.py b/tests/ut/cpp/python_input/gtest_input/optimizer/opt_test.py index 22e2535819..16c557adbe 100644 --- a/tests/ut/cpp/python_input/gtest_input/optimizer/opt_test.py +++ b/tests/ut/cpp/python_input/gtest_input/optimizer/opt_test.py @@ -875,7 +875,6 @@ def test_merge_addn(tag): """ test_merge_addn """ fns = FnDict() addn = P.AddN() - AddN = P.AddN @fns def before(x, y, z, a): @@ -883,7 +882,7 @@ def test_merge_addn(tag): @fns def after(x, y, z, a): - return AddN()((a, x, y, z)) + return addn((a, x, y, z)) return fns[tag] @@ -892,7 +891,6 @@ def test_addn_zero(tag): """ test_addn_zero """ fns = FnDict() addn = P.AddN() - AddN = P.AddN zero_tensor = Primitive('ZerosLike') @fns @@ -901,7 +899,7 @@ def test_addn_zero(tag): @fns def after(x, y, z, a): - return AddN()((a, z)) + return addn((a, z)) @fns def before_2(x, y, z, a): From 3df68458038f9f4b5bccc19e116803f674f79312 Mon Sep 17 00:00:00 2001 From: yujianfeng Date: Fri, 3 Jul 2020 11:21:19 +0800 Subject: [PATCH 234/254] Fix fusion condition of reshape and transpose --- .../pre_activate/ascend/ir_fusion/reshape_transpose_fusion.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mindspore/ccsrc/pre_activate/ascend/ir_fusion/reshape_transpose_fusion.cc b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/reshape_transpose_fusion.cc index 45bf640abe..9b13002798 100644 --- a/mindspore/ccsrc/pre_activate/ascend/ir_fusion/reshape_transpose_fusion.cc +++ b/mindspore/ccsrc/pre_activate/ascend/ir_fusion/reshape_transpose_fusion.cc @@ -51,8 +51,8 @@ const AnfNodePtr ReshapeTransposeFusion::Process(const FuncGraphPtr &func_graph, auto reshape_cnode = CheckAnfNodeIfCNodeAndInputSize(transpose_cnode->input(1), kBackendReshapeInputNum); MS_EXCEPTION_IF_NULL(reshape_cnode); std::vector reshape_input0_shape = AnfAlgo::GetPrevNodeOutputInferShape(reshape_cnode, 0); - std::vector transpose_input0_shape = AnfAlgo::GetPrevNodeOutputInferShape(transpose_cnode, 0); - if (!CheckShapeDimInfo(reshape_input0_shape) || !CheckShapeDimInfo(transpose_input0_shape)) { + std::vector transpose_output0_shape = AnfAlgo::GetOutputInferShape(transpose_cnode, 0); + if (!CheckShapeDimInfo(reshape_input0_shape) || !CheckShapeDimInfo(transpose_output0_shape)) { return nullptr; } auto prim = std::make_shared(kConfusionTransposeDOpName); From 4419881f2fc01b5f27ebb6a32edd0a5d2622630f Mon Sep 17 00:00:00 2001 From: wuyongkang Date: Fri, 3 Jul 2020 11:54:04 +0800 Subject: [PATCH 235/254] Fix optimizer step counter bug --- mindspore/ccsrc/optimizer/optimizer.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mindspore/ccsrc/optimizer/optimizer.h b/mindspore/ccsrc/optimizer/optimizer.h index 5f21de43cd..77d4589c5e 100644 --- a/mindspore/ccsrc/optimizer/optimizer.h +++ b/mindspore/ccsrc/optimizer/optimizer.h @@ -184,7 +184,8 @@ class Optimizer : public std::enable_shared_from_this { } } }; - use_profile ? (WITH(MsProfile::GetProfile()->Lap(counter++)) run_runc) : run_runc(); + use_profile ? (WITH(MsProfile::GetProfile()->Lap(counter)) run_runc) : run_runc(); + counter++; if (run_only_once_) { break; From cfa41733d639e14a4f8eba395691aada14609aff Mon Sep 17 00:00:00 2001 From: buxue Date: Thu, 2 Jul 2020 11:25:59 +0800 Subject: [PATCH 236/254] support Python built-in function 'enumerate' --- mindspore/_extends/parse/resources.py | 1 + mindspore/_extends/parse/standard_method.py | 16 ++ mindspore/_extends/parse/trope.py | 4 +- mindspore/ccsrc/operator/composite/map.cc | 2 +- .../ccsrc/operator/composite/zip_operation.cc | 40 ++-- mindspore/ccsrc/operator/prim_statement.cc | 1 + mindspore/ccsrc/pipeline/parse/parse.cc | 5 +- .../static_analysis/param_validator.h | 1 + mindspore/ccsrc/pynative/pynative_execute.cc | 13 +- mindspore/common/dtype.py | 2 + .../composite/multitype_ops/_compile_utils.py | 4 +- mindspore/ops/operations/array_ops.py | 2 +- tests/ut/python/ops/test_ops.py | 8 +- .../python/pipeline/parse/test_enumerate.py | 181 ++++++++++++++++++ 14 files changed, 240 insertions(+), 40 deletions(-) create mode 100644 tests/ut/python/pipeline/parse/test_enumerate.py diff --git a/mindspore/_extends/parse/resources.py b/mindspore/_extends/parse/resources.py index 3af574caf9..e60b70efac 100644 --- a/mindspore/_extends/parse/resources.py +++ b/mindspore/_extends/parse/resources.py @@ -116,6 +116,7 @@ convert_object_map = { T.partial: F.partial, T.zip: C.zip_operation, T.print: F.print_, + T.enumerate: M.enumerate_, # custom define operation T.iter: M.ms_iter, diff --git a/mindspore/_extends/parse/standard_method.py b/mindspore/_extends/parse/standard_method.py index 0f3f843b63..936099a4fb 100644 --- a/mindspore/_extends/parse/standard_method.py +++ b/mindspore/_extends/parse/standard_method.py @@ -104,6 +104,15 @@ def bool_(x): return x.__bool__() +def enumerate_(x, start=0): + """Enumerate list or tuple.""" + x_type = F.typeof(x) + ret = () + if check_is_tuple_or_list(x_type, "enumerate"): + ret = zip(range(start, start + len(x)), x) + return ret + + def while_cond(x): """For while condtion, if the condition is a tensor, the loop will not be unrolled""" if F.issubclass_(F.typeof(x), F.typeof(mstype.tensor)): @@ -113,6 +122,13 @@ def while_cond(x): return x +@constexpr +def check_is_tuple_or_list(x, op_name): + """check whether x is list or tuple.""" + if isinstance(x, (mstype.list_type, mstype.tuple_type)): + return True + raise TypeError(f"For '{op_name}', the input parameter should be tuple or list, but got {x}.") + @constexpr def check_is_tensor_bool_cond(shp): """check if tensor is a bool condition""" diff --git a/mindspore/_extends/parse/trope.py b/mindspore/_extends/parse/trope.py index f169c58fb9..28f3196975 100644 --- a/mindspore/_extends/parse/trope.py +++ b/mindspore/_extends/parse/trope.py @@ -27,7 +27,7 @@ from operator import ( # noqa # support system function call from builtins import ( # noqa - bool, getattr, setattr, len, iter, next, pow, range, map, zip, print + bool, getattr, setattr, len, iter, next, pow, range, map, zip, print, enumerate ) # support functools @@ -44,7 +44,7 @@ __all__ = ['add', 'sub', 'mul', 'truediv', 'floordiv', 'mod', 'eq', 'ne', 'lt', 'not_', 'and_', 'or_', 'xor', 'lshift', 'rshift', 'invert', 'is_', 'is_not', 'contains', 'matmul', 'getitem', 'setitem', 'bool', 'getattr', 'setattr', 'len', 'iter', 'next', 'pow', 'range', 'map', 'zip', - 'partial', 'print', + 'partial', 'print', 'enumerate', 'exp', 'log', 'sin', 'cos', 'tan'] diff --git a/mindspore/ccsrc/operator/composite/map.cc b/mindspore/ccsrc/operator/composite/map.cc index 6062f0f5af..2149285323 100644 --- a/mindspore/ccsrc/operator/composite/map.cc +++ b/mindspore/ccsrc/operator/composite/map.cc @@ -181,7 +181,7 @@ AnfNodePtr Map::FullMakeClass(const std::shared_ptr &type, const FuncGrap } AnfNodePtr Map::Make(const FuncGraphPtr &func_graph, const AnfNodePtr &fn_arg, const ArgsPairList &arg_pairs) { - if (arg_pairs.size() < 1) { + if (arg_pairs.empty()) { MS_EXCEPTION(TypeError) << "map() must have at least two arguments"; } bool found = false; diff --git a/mindspore/ccsrc/operator/composite/zip_operation.cc b/mindspore/ccsrc/operator/composite/zip_operation.cc index 33e21da044..38f2b51614 100644 --- a/mindspore/ccsrc/operator/composite/zip_operation.cc +++ b/mindspore/ccsrc/operator/composite/zip_operation.cc @@ -18,44 +18,44 @@ #include "operator/composite/zip_operation.h" #include -#include #include "pipeline/static_analysis/abstract_value.h" #include "ir/anf.h" #include "pipeline/static_analysis/dshape.h" -#include "pipeline/static_analysis/param_validator.h" #include "operator/cc_implementations.h" #include "optimizer/opt.h" -#include "utils/symbolic.h" -#include "./common.h" #include "pybind_api/api_register.h" namespace mindspore { // namespace to support composite operators definition namespace prim { using mindspore::abstract::AbstractBase; +using mindspore::abstract::AbstractList; +using mindspore::abstract::AbstractSequeue; +using mindspore::abstract::AbstractSequeuePtr; using mindspore::abstract::AbstractTuple; FuncGraphPtr ZipOperation::GenerateFuncGraph(const AbstractBasePtrList &args_spec_list) { // zip operation: // input: tuple arguments // output: tuple of items of input iterated on every input - if (args_spec_list.size() == 0) { - MS_LOG(EXCEPTION) << "zip arguments input should not be empty"; + if (args_spec_list.empty()) { + MS_LOG(EXCEPTION) << "For 'zip', there is at least one input."; } - auto is_all_tuple = std::all_of(args_spec_list.begin(), args_spec_list.end(), [](const AbstractBasePtr &abs) -> bool { - MS_EXCEPTION_IF_NULL(abs); - return abs->isa(); - }); - if (!is_all_tuple) { - MS_LOG(EXCEPTION) << "zip input args should be tuple"; + auto is_all_sequeue = + std::all_of(args_spec_list.begin(), args_spec_list.end(), [](const AbstractBasePtr &abs) -> bool { + MS_EXCEPTION_IF_NULL(abs); + return abs->isa(); + }); + if (!is_all_sequeue) { + MS_LOG(EXCEPTION) << "For 'zip', all inputs must be sequence."; } - auto min_abs = std::min_element(args_spec_list.begin(), args_spec_list.end(), - [](const AbstractBasePtr &x, const AbstractBasePtr &y) { - return (x->cast()->size() < y->cast()->size()); - }); + auto min_abs = std::min_element( + args_spec_list.begin(), args_spec_list.end(), [](const AbstractBasePtr &x, const AbstractBasePtr &y) { + return (x->cast()->size() < y->cast()->size()); + }); FuncGraphPtr ret_graph = std::make_shared(); ret_graph->set_flag(FUNC_GRAPH_FLAG_CORE, true); for (size_t idx = 0; idx < args_spec_list.size(); idx++) { @@ -65,12 +65,14 @@ FuncGraphPtr ZipOperation::GenerateFuncGraph(const AbstractBasePtrList &args_spe // generate tuple output of ziped arguments input std::vector make_tuple_nodes; make_tuple_nodes.push_back(NewValueNode(prim::kPrimMakeTuple)); - for (size_t idx = 0; idx < (*min_abs)->cast()->size(); idx++) { + for (size_t idx = 0; idx < (*min_abs)->cast()->size(); idx++) { std::vector make_tuple_zip_nodes; make_tuple_zip_nodes.push_back(NewValueNode(prim::kPrimMakeTuple)); + std::string module_name = "mindspore.ops.composite.multitype_ops.getitem_impl"; + ValuePtr op = prim::GetPythonOps("getitem", module_name); for (size_t arg_idx = 0; arg_idx < args_spec_list.size(); arg_idx++) { - std::vector tuple_get_item_nodes{NewValueNode(prim::kPrimTupleGetItem), - ret_graph->parameters()[arg_idx], NewValueNode(SizeToInt(idx))}; + std::vector tuple_get_item_nodes{NewValueNode(op), ret_graph->parameters()[arg_idx], + NewValueNode(SizeToInt(idx))}; auto tuple_get_item_op = ret_graph->NewCNode(tuple_get_item_nodes); make_tuple_zip_nodes.push_back(tuple_get_item_op); } diff --git a/mindspore/ccsrc/operator/prim_statement.cc b/mindspore/ccsrc/operator/prim_statement.cc index 5eb8d39996..fc40e511e1 100644 --- a/mindspore/ccsrc/operator/prim_statement.cc +++ b/mindspore/ccsrc/operator/prim_statement.cc @@ -229,6 +229,7 @@ AbstractBasePtr InferImplNotInDict(const AnalysisEnginePtr &, const PrimitivePtr // Inputs: x, t return std::make_shared(!IsInDict(primitive, args_spec_list)); } + AbstractBasePtr InferImplIsConstant(const AnalysisEnginePtr &, const PrimitivePtr &primitive, const AbstractBasePtrList &args_spec_list) { // statement: isconstant(x) diff --git a/mindspore/ccsrc/pipeline/parse/parse.cc b/mindspore/ccsrc/pipeline/parse/parse.cc index 66908240cb..77e865cee9 100644 --- a/mindspore/ccsrc/pipeline/parse/parse.cc +++ b/mindspore/ccsrc/pipeline/parse/parse.cc @@ -1048,11 +1048,10 @@ FunctionBlockPtr Parser::ParseFor(const FunctionBlockPtr &block, const py::objec CNodePtr app = body_block->func_graph()->NewCNode({op_next, iter_param}); CNodePtr target_app = body_block->func_graph()->NewCNode({op_getitem, app, NewValueNode(0)}); py::object target_node = python_adapter::GetPyObjAttr(node, "target"); - auto name_id = py::cast(python_adapter::GetPyObjAttr(target_node, "id")); - target_app->debug_info()->set_name(name_id); CNodePtr iter2_app = body_block->func_graph()->NewCNode({op_getitem, app, NewValueNode(1)}); - body_block->WriteVariable(name_id, target_app); + WriteAssignVars(body_block, target_node, target_app); + // link the variable name with the target auto it_info = std::make_shared(target_app->debug_info()); iter_param->debug_info()->set_trace_info(it_info); diff --git a/mindspore/ccsrc/pipeline/static_analysis/param_validator.h b/mindspore/ccsrc/pipeline/static_analysis/param_validator.h index 2f5729aa73..daa436d66d 100644 --- a/mindspore/ccsrc/pipeline/static_analysis/param_validator.h +++ b/mindspore/ccsrc/pipeline/static_analysis/param_validator.h @@ -67,6 +67,7 @@ ABSTRACT_REPORT_NAME_TRAITS(Type) ABSTRACT_REPORT_NAME_TRAITS(KeywordArg) ABSTRACT_REPORT_NAME_TRAITS(Class) ABSTRACT_REPORT_NAME_TRAITS(IndexedSlices) +ABSTRACT_REPORT_NAME_TRAITS(Sequeue) template std::shared_ptr CheckArg(const std::string &op, const AbstractBasePtrList &args_spec_list, size_t index) { diff --git a/mindspore/ccsrc/pynative/pynative_execute.cc b/mindspore/ccsrc/pynative/pynative_execute.cc index 2d92e87996..f477bfbdcd 100644 --- a/mindspore/ccsrc/pynative/pynative_execute.cc +++ b/mindspore/ccsrc/pynative/pynative_execute.cc @@ -226,11 +226,8 @@ void PynativeInfer(const PrimitivePyPtr &prim, const py::list &py_args, OpExecIn AbstractBasePtrList args_spec_list; for (size_t i = 0; i < size; i++) { ValuePtr input_value = PyAttrValue(py_args[i]); - if (!py::hasattr(prim->GetPyObj(), "const_value") && input_value->isa()) { - args_spec_list.emplace_back(abstract::FromValueInside(input_value, true)); - } else { - args_spec_list.emplace_back(abstract::FromValueInside(input_value, false)); - } + args_spec_list.emplace_back(abstract::FromValueInside( + input_value, !py::hasattr(prim->GetPyObj(), "const_value") && input_value->isa())); } AbstractBasePtr infer_res = EvalOnePrim(prim, args_spec_list)->abstract(); op_exec_info->abstract = infer_res; @@ -512,7 +509,7 @@ py::object RunOpInMs(const OpExecInfoPtr &op_exec_info, PynativeStatusCode *stat return result; } -py::object RunOpWithBackendPolicy(MsBackendPolicy backend_policy, const OpExecInfoPtr op_exec_info, +py::object RunOpWithBackendPolicy(MsBackendPolicy backend_policy, const OpExecInfoPtr &op_exec_info, PynativeStatusCode *const status) { MS_EXCEPTION_IF_NULL(status); py::object result; @@ -550,7 +547,7 @@ py::object RunOpWithBackendPolicy(MsBackendPolicy backend_policy, const OpExecIn } AnfNodePtr PynativeExecutor::MakeCNode(const OpExecInfoPtr &op_exec_info, const py::args &args, const py::tuple &out) { - if (!grad_flag_ || graph_info_map_.size() == 0) { + if (!grad_flag_ || graph_info_map_.empty()) { return nullptr; } std::vector inputs; @@ -753,7 +750,7 @@ AnfNodePtr PynativeExecutor::GetInput(const py::object &obj, const py::object &o if (py::isinstance(name_attr)) { MS_LOG(EXCEPTION) << "Parameter object should have name attribute"; } - std::string param_name = py::cast(name_attr); + auto param_name = py::cast(name_attr); if (graph_info_map_[df_builder_].param_map.count(obj_id) == 0) { auto free_param = df_builder_->add_parameter(); free_param->set_name(param_name); diff --git a/mindspore/common/dtype.py b/mindspore/common/dtype.py index 73aa67f67a..85bb1c52d6 100644 --- a/mindspore/common/dtype.py +++ b/mindspore/common/dtype.py @@ -97,6 +97,8 @@ tensor_type = typing.TensorType anything_type = typing.TypeAnything slice_type = typing.Slice ellipsis_type = typing.TypeEllipsis +list_type = typing.List +tuple_type = typing.Tuple number_type = (int8, int16, diff --git a/mindspore/ops/composite/multitype_ops/_compile_utils.py b/mindspore/ops/composite/multitype_ops/_compile_utils.py index d6ce3ac6f6..a02a8c8809 100644 --- a/mindspore/ops/composite/multitype_ops/_compile_utils.py +++ b/mindspore/ops/composite/multitype_ops/_compile_utils.py @@ -65,9 +65,9 @@ def _generate_indices_from_tuple_of_mixed_tensors(data, tuple_index, op_name): tuple_len = len(tuple_index) for i in range(tuple_len): if i in int_positions: - tuple_index_new = tuple_index_new + (F.scalar_to_tensor(tuple_index[i], mstype.int32),) + tuple_index_new += (F.scalar_to_tensor(tuple_index[i], mstype.int32),) else: - tuple_index_new = tuple_index_new + (tuple_index[i],) + tuple_index_new += (tuple_index[i],) indexes_types = hyper_map(F.typeof, tuple_index_new) tensor_positions, slice_positions, ellipsis_position = \ const_utils.separate_mixed_tensors_index(indexes_types, op_name) diff --git a/mindspore/ops/operations/array_ops.py b/mindspore/ops/operations/array_ops.py index 0e1456c250..1d72566c44 100644 --- a/mindspore/ops/operations/array_ops.py +++ b/mindspore/ops/operations/array_ops.py @@ -1469,7 +1469,7 @@ class Concat(PrimitiveWithInfer): def _get_pack_shape(x_shape, x_type, axis, prim_name): """for pack output shape""" validator.check_value_type("shape", x_shape, [tuple, list], prim_name) - validator.check_integer("len of input_x", len(x_shape), 1, Rel.GT, prim_name) + validator.check_integer("len of input_x", len(x_shape), 1, Rel.GE, prim_name) validator.check_subclass("input_x[0]", x_type[0], mstype.tensor, prim_name) rank_base = len(x_shape[0]) N = len(x_shape) diff --git a/tests/ut/python/ops/test_ops.py b/tests/ut/python/ops/test_ops.py index a99b231fa7..45d9349f88 100755 --- a/tests/ut/python/ops/test_ops.py +++ b/tests/ut/python/ops/test_ops.py @@ -1747,6 +1747,10 @@ test_case_array_ops = [ 'desc_inputs': [[128, 128], [128, 128]], 'desc_bprop': [[2, 128, 128]], }), + ('Pack_3', { + 'block': NetForPackInput(P.Pack()), + 'desc_inputs': [[2, 2]], + 'desc_bprop': [[1, 2, 2]]}), ('Unpack_0', { 'block': NetForUnpackInput(P.Unpack(axis=0)), 'desc_inputs': [[2, 4]], @@ -2206,10 +2210,6 @@ raise_set = [ Tensor(np.ones((2, 2), np.float32)), Tensor(np.ones((2,), np.float32))), 'desc_bprop': [[2, 3]]}), - ('Pack', { - 'block': (NetForPackInput(P.Pack()), {'exception': ValueError}), - 'desc_inputs': [[2, 2]], - 'desc_bprop': [[1, 2, 2]]}), ('PReLU', { 'block': (P.PReLU(), {'exception': ValueError}), 'desc_inputs': [[2], [1]], diff --git a/tests/ut/python/pipeline/parse/test_enumerate.py b/tests/ut/python/pipeline/parse/test_enumerate.py new file mode 100644 index 0000000000..cd808696f1 --- /dev/null +++ b/tests/ut/python/pipeline/parse/test_enumerate.py @@ -0,0 +1,181 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" test enumerate""" +import numpy as np +import pytest + +import mindspore.nn as nn +from mindspore import Tensor +from mindspore import context + +context.set_context(mode=context.GRAPH_MODE) + + +def test_enumerate_list_const(): + class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.value = [11, 22, 33, 44] + + def construct(self): + index_sum = 0 + value_sum = 0 + for i, j in enumerate(self.value): + index_sum += i + value_sum += j + return index_sum, value_sum + + net = Net() + assert net() == (6, 110) + + +def test_enumerate_tuple_const(): + class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.value = (11, 22, 33, 44) + + def construct(self): + index_sum = 0 + value_sum = 0 + for i, j in enumerate(self.value): + index_sum += i + value_sum += j + return index_sum, value_sum + + net = Net() + assert net() == (6, 110) + + +def test_enumerate_list_parameter(): + class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + + def construct(self, x, y, z): + index_sum = 0 + value = [x, y, z] + ret = () + for i, j in enumerate(value): + index_sum += i + ret += (j,) + return index_sum, ret + + x = Tensor(np.arange(3 * 4 * 5).reshape((3, 4, 5))) + net = Net() + net(x, x, x) + + +def test_enumerate_tuple_parameter(): + class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + + def construct(self, x, y, z): + index_sum = 0 + value = (x, y, z) + ret = () + for i, j in enumerate(value): + index_sum += i + ret += (j,) + return index_sum, ret + x = Tensor(np.arange(3 * 4 * 5).reshape((3, 4, 5))) + net = Net() + net(x, x, x) + + +def test_enumerate_tuple_const_1(): + class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.value = (11, 22, 33, 44) + + def construct(self): + index_sum = 0 + value_sum = 0 + for i in enumerate(self.value): + index_sum += i[0] + value_sum += i[1] + return index_sum, value_sum + + net = Net() + assert net() == (6, 110) + + +def test_enumerate_tuple_parameter_1(): + class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + + def construct(self, x, y, z): + index_sum = 0 + value = (x, y, z) + ret = () + for i in enumerate(value): + index_sum += i[0] + ret += (i[1],) + return index_sum, ret + x = Tensor(np.arange(3 * 4 * 5).reshape((3, 4, 5))) + net = Net() + net(x, x, x) + +def test_enumerate_tuple_const_2(): + class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.value = (11, 22, 33, 44) + + def construct(self): + index_sum = 0 + value_sum = 0 + for i in enumerate(self.value, 1): + index_sum += i[0] + value_sum += i[1] + return index_sum, value_sum + + net = Net() + assert net() == (10, 110) + + +def test_enumerate_tuple_parameter_2(): + class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + + def construct(self, x, y, z): + index_sum = 0 + value = (x, y, z) + ret = () + for i in enumerate(value, 2): + index_sum += i[0] + ret += (i[1],) + return index_sum, ret + x = Tensor(np.arange(3 * 4 * 5).reshape((3, 4, 5))) + net = Net() + net(x, x, x) + + +def test_enumerate_parameter_type_error(): + class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + + def construct(self, x): + return enumerate(x) + x = Tensor(np.arange(3 * 4 * 5).reshape((3, 4, 5))) + net = Net() + with pytest.raises(TypeError) as ex: + net(x) + assert "For 'enumerate', the input parameter should be tuple or list" in str(ex.value) From e53b41bad8e1a037e843936655bd207726f8d6bc Mon Sep 17 00:00:00 2001 From: kswang Date: Fri, 3 Jul 2020 15:25:23 +0800 Subject: [PATCH 237/254] optimize cpu input mem --- mindspore/ccsrc/device/cpu/cpu_simple_mem_plan.cc | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/mindspore/ccsrc/device/cpu/cpu_simple_mem_plan.cc b/mindspore/ccsrc/device/cpu/cpu_simple_mem_plan.cc index 95f0d25f5b..e6cb6ee53a 100644 --- a/mindspore/ccsrc/device/cpu/cpu_simple_mem_plan.cc +++ b/mindspore/ccsrc/device/cpu/cpu_simple_mem_plan.cc @@ -27,7 +27,12 @@ void CPUSimpleMemPlan::MemPlan(const session::KernelGraph *graph) { MS_EXCEPTION_IF_NULL(kernel); size_t input_num = AnfAlgo::GetInputTensorNum(kernel); for (size_t i = 0; i < input_num; ++i) { - auto address = AnfAlgo::GetPrevNodeOutputAddr(kernel, i); + auto kernel_with_index = AnfAlgo::GetPrevNodeOutput(kernel, i); + MS_EXCEPTION_IF_NULL(kernel_with_index.first); + if (kernel_with_index.first->isa()) { + continue; + } + auto address = AnfAlgo::GetOutputAddr(kernel_with_index.first, kernel_with_index.second, true); MS_EXCEPTION_IF_NULL(address); if (address->ptr_ == nullptr) { total_mem_size += address->size_; @@ -73,7 +78,12 @@ void CPUSimpleMemPlan::MemAssign(const session::KernelGraph *graph, uint8_t *bas MS_EXCEPTION_IF_NULL(kernel); size_t input_num = AnfAlgo::GetInputTensorNum(kernel); for (size_t i = 0; i < input_num; ++i) { - auto address = AnfAlgo::GetPrevNodeMutableOutputAddr(kernel, i); + auto kernel_with_index = AnfAlgo::GetPrevNodeOutput(kernel, i); + MS_EXCEPTION_IF_NULL(kernel_with_index.first); + if (kernel_with_index.first->isa()) { + continue; + } + auto address = AnfAlgo::GetMutableOutputAddr(kernel_with_index.first, kernel_with_index.second, true); MS_EXCEPTION_IF_NULL(address); if (address->ptr_ == nullptr) { address->ptr_ = mem_ptr; From 9b7a426c6be0d361f0b4cd8197fa89e31230dca7 Mon Sep 17 00:00:00 2001 From: chenzomi Date: Fri, 3 Jul 2020 10:42:34 +0800 Subject: [PATCH 238/254] bug fix in auto create quant graph in master --- mindspore/nn/layer/quant.py | 4 +- mindspore/train/callback/_loss_monitor.py | 2 +- mindspore/train/quant/quant.py | 43 +++++++++-------- mindspore/train/serialization.py | 5 +- model_zoo/lenet_quant/eval.py | 2 +- model_zoo/lenet_quant/eval_quant.py | 2 +- model_zoo/lenet_quant/export.py | 56 +++++++++++++++++++++++ model_zoo/lenet_quant/train.py | 5 +- model_zoo/lenet_quant/train_quant.py | 13 +++--- 9 files changed, 98 insertions(+), 34 deletions(-) create mode 100644 model_zoo/lenet_quant/export.py diff --git a/mindspore/nn/layer/quant.py b/mindspore/nn/layer/quant.py index 225d37bf84..f0c82937c5 100644 --- a/mindspore/nn/layer/quant.py +++ b/mindspore/nn/layer/quant.py @@ -1193,9 +1193,9 @@ class QuantBlock(Cell): self.dequant = dequant_op self.dequant_scale = dequant_scale self.bias = bias - self.has_bias = bias is None + self.has_bias = bias is not None self.activation = activation - self.has_act = activation is None + self.has_act = activation is not None self.bias_add = P.BiasAdd() def construct(self, x): diff --git a/mindspore/train/callback/_loss_monitor.py b/mindspore/train/callback/_loss_monitor.py index bdd2220441..766777e878 100644 --- a/mindspore/train/callback/_loss_monitor.py +++ b/mindspore/train/callback/_loss_monitor.py @@ -86,7 +86,7 @@ class LossMonitor(Callback): if self._per_print_times != 0 and cb_params.cur_step_num % self._per_print_times == 0: print("Epoch: [{:3d}/{:3d}], step: [{:5d}/{:5d}], " - "loss: [{:5.4f}], avg los: [{:5.4f}], time: [{:5.4f}]".format( + "loss: [{:5.4f}], avg los: [{:5.4f}], time: [{:5.4f}ms]".format( cb_params.cur_epoch_num, cb_params.epoch_num, cur_step_in_epoch, int(cb_params.batch_num), step_loss, np.mean(self.losses), diff --git a/mindspore/train/quant/quant.py b/mindspore/train/quant/quant.py index 3709c171e5..bc44ba22c2 100644 --- a/mindspore/train/quant/quant.py +++ b/mindspore/train/quant/quant.py @@ -33,7 +33,6 @@ from ...ops.operations import _inner_ops as inner from ...train import serialization from . import quant_utils - _ACTIVATION_MAP = {nn.ReLU: quant.ReLUQuant, nn.ReLU6: quant.ReLU6Quant, nn.HSigmoid: quant.HSigmoidQuant, @@ -178,7 +177,6 @@ class ConvertToQuantNetwork: dilation=conv_inner.dilation, group=conv_inner.group, eps=bn_inner.eps, - momentum=1 - bn_inner.momentum, quant_delay=self.weight_qdelay, freeze_bn=self.freeze_bn, per_channel=self.weight_channel, @@ -268,16 +266,16 @@ class ConvertToQuantNetwork: narrow_range=self.act_range) -class ExportQuantNetworkDeploy: +class ExportToQuantInferNetwork: """ - Convert quantization aware network to deploy network. + Convert quantization aware network to infer network. Args: - network (Cell): MindSpore network produced by `convert_quant_network`. - inputs (Tensor): Inputs of the `network`. + network (Cell): MindSpore network API `convert_quant_network`. + inputs (Tensor): Input tensors of the `quantization aware training network`. Returns: - Cell, converted network. + Cell, GEIR backend Infer network. """ __quant_op_name__ = ["TensorAdd", "Sub", "Mul", "RealDiv"] @@ -287,7 +285,7 @@ class ExportQuantNetworkDeploy: network = validator.check_isinstance('network', network, (nn.Cell,)) self.data_type = mstype.int8 self.network = copy.deepcopy(network) - self.all_paramters = {p.name: p for p in self.network.get_parameters()} + self.all_parameters = {p.name: p for p in self.network.get_parameters()} self.get_inputs_table(inputs) def get_inputs_table(self, inputs): @@ -315,8 +313,8 @@ class ExportQuantNetworkDeploy: info = self.quant_info_table.get(w_minq_name, None) if info: fack_quant_a_in_op, minq_name = info - maxq = self.all_paramters[minq_name[:-4] + "maxq"] - minq = self.all_paramters[minq_name] + maxq = self.all_parameters[minq_name[:-4] + "maxq"] + minq = self.all_parameters[minq_name] scale_a_in, zp_a_in = quant_utils.scale_zp_from_data(fack_quant_a_in_op, maxq, minq, np_type) else: logger.warning(f"Do not find `fake_quant` from input with `fack_quant.minq` {w_minq_name}") @@ -357,7 +355,7 @@ class ExportQuantNetworkDeploy: return block def _convert_quant2deploy(self, network): - """Convet network's all quant subcell to deploy subcell.""" + """Convert network's all quant subcell to deploy subcell.""" cells = network.name_cells() change = False for name in cells: @@ -395,18 +393,26 @@ class ExportQuantNetworkDeploy: return network -def export_geir(network, *inputs, file_name): +def export(network, *inputs, file_name, file_format='GEIR'): """ - Exports MindSpore quant predict model to deploy with GEIR. + Exports MindSpore quantization predict model to deploy with GEIR. Args: network (Cell): MindSpore network produced by `convert_quant_network`. - inputs (Tensor): Inputs of the `network`. + inputs (Tensor): Inputs of the `quantization aware training network`. file_name (str): File name of model to export. + file_format (str): MindSpore currently supports 'GEIR' format for exported quantization aware model. + - GEIR: Graph Engine Intermediate Representation. An Intermediate representation format of Ascend model. """ - exporter = ExportQuantNetworkDeploy(network, *inputs) - deploy_net = exporter.run() - serialization.export(deploy_net, *inputs, file_name=file_name, file_format="GEIR") + supported_formats = ['GEIR'] + + if file_format not in supported_formats: + raise ValueError('Illegal file format {}.'.format(file_format)) + + if file_format == 'GEIR': + exporter = ExportToQuantInferNetwork(network, *inputs) + deploy_net = exporter.run() + serialization.export(deploy_net, *inputs, file_name=file_name, file_format=file_format) def convert_quant_network(network, @@ -443,6 +449,7 @@ def convert_quant_network(network, Cell, Network which has change to quantization aware training network cell. """ support_device = ["Ascend", "GPU"] + def convert2list(name, value): if not isinstance(value, list) and not isinstance(value, tuple): value = [value] @@ -457,7 +464,7 @@ def convert_quant_network(network, narrow_range = convert2list("narrow range", narrow_range) if context.get_context('device_target') not in support_device: - raise KeyError("Not support {} backend.".format(context.get_context('device_target'))) + raise KeyError("Unsupported {} device target.".format(context.get_context('device_target'))) net = ConvertToQuantNetwork(network=network, quant_delay=quant_delay, diff --git a/mindspore/train/serialization.py b/mindspore/train/serialization.py index fc135b18a9..d74bee2706 100644 --- a/mindspore/train/serialization.py +++ b/mindspore/train/serialization.py @@ -160,7 +160,10 @@ def load_checkpoint(ckpt_file_name, net=None): if not isinstance(ckpt_file_name, str): raise ValueError("The ckpt_file_name must be string.") - if not os.path.exists(ckpt_file_name) or ckpt_file_name[-5:] != ".ckpt": + if not os.path.exists(ckpt_file_name): + raise ValueError("The checkpoint file is not exist.") + + if ckpt_file_name[-5:] != ".ckpt": raise ValueError("Please input the correct checkpoint file name.") if os.path.getsize(ckpt_file_name) == 0: diff --git a/model_zoo/lenet_quant/eval.py b/model_zoo/lenet_quant/eval.py index c0293ae1f7..df9a5b123b 100644 --- a/model_zoo/lenet_quant/eval.py +++ b/model_zoo/lenet_quant/eval.py @@ -57,7 +57,7 @@ if __name__ == "__main__": model = Model(network, net_loss, net_opt, metrics={"Accuracy": Accuracy()}) # load check point into network - param_dict = load_checkpoint(args.ckpt_path, network.type) + param_dict = load_checkpoint(args.ckpt_path) load_param_into_net(network, param_dict) print("============== Starting Testing ==============") diff --git a/model_zoo/lenet_quant/eval_quant.py b/model_zoo/lenet_quant/eval_quant.py index bc9b62121d..96740dc51f 100644 --- a/model_zoo/lenet_quant/eval_quant.py +++ b/model_zoo/lenet_quant/eval_quant.py @@ -49,7 +49,7 @@ if __name__ == "__main__": # define fusion network network = LeNet5Fusion(cfg.num_classes) - # convert fusion netwrok to quantization aware network + # convert fusion network to quantization aware network network = quant.convert_quant_network(network, quant_delay=0, bn_fold=False, freeze_bn=10000) # define loss diff --git a/model_zoo/lenet_quant/export.py b/model_zoo/lenet_quant/export.py new file mode 100644 index 0000000000..78c94f9286 --- /dev/null +++ b/model_zoo/lenet_quant/export.py @@ -0,0 +1,56 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +export quantization aware training network to infer `GEIR` backend. +""" + +import argparse +import numpy as np + +import mindspore +from mindspore import Tensor +from mindspore import context +from mindspore.train.quant import quant +from mindspore.train.serialization import load_checkpoint, load_param_into_net + +from src.config import mnist_cfg as cfg +from src.lenet_fusion import LeNet5 as LeNet5Fusion + +parser = argparse.ArgumentParser(description='MindSpore MNIST Example') +parser.add_argument('--device_target', type=str, default="Ascend", + choices=['Ascend', 'GPU'], + help='device where the code will be implemented (default: Ascend)') +parser.add_argument('--data_path', type=str, default="./MNIST_Data", + help='path where the dataset is saved') +parser.add_argument('--ckpt_path', type=str, default="", + help='if mode is test, must provide path where the trained ckpt file') +parser.add_argument('--dataset_sink_mode', type=bool, default=True, + help='dataset_sink_mode is False or True') +args = parser.parse_args() + +if __name__ == "__main__": + context.set_context(mode=context.GRAPH_MODE, device_target=args.device_target) + + # define fusion network + network = LeNet5Fusion(cfg.num_classes) + # convert fusion network to quantization aware network + network = quant.convert_quant_network(network, quant_delay=0, bn_fold=False, freeze_bn=10000) + # load quantization aware network checkpoint + param_dict = load_checkpoint(args.ckpt_path) + load_param_into_net(network, param_dict) + + # export network + inputs = Tensor(np.ones([1, 1, cfg.image_height, cfg.image_width]), mindspore.float32) + quant.export(network, inputs, file_name="lenet_quant", file_format='GEIR') diff --git a/model_zoo/lenet_quant/train.py b/model_zoo/lenet_quant/train.py index a34b6d5ed6..2cff465832 100644 --- a/model_zoo/lenet_quant/train.py +++ b/model_zoo/lenet_quant/train.py @@ -22,7 +22,7 @@ import os import argparse import mindspore.nn as nn from mindspore import context -from mindspore.train.callback import ModelCheckpoint, CheckpointConfig, LossMonitor, TimeMonitor +from mindspore.train.callback import ModelCheckpoint, CheckpointConfig, LossMonitor from mindspore.train import Model from mindspore.nn.metrics import Accuracy from src.dataset import create_dataset @@ -54,7 +54,6 @@ if __name__ == "__main__": net_opt = nn.Momentum(network.trainable_params(), cfg.lr, cfg.momentum) # call back and monitor - time_cb = TimeMonitor(data_size=ds_train.get_dataset_size()) config_ckpt = CheckpointConfig(save_checkpoint_steps=cfg.epoch_size * step_size, keep_checkpoint_max=cfg.keep_checkpoint_max) ckpt_callback = ModelCheckpoint(prefix="checkpoint_lenet", config=config_ckpt) @@ -63,6 +62,6 @@ if __name__ == "__main__": model = Model(network, net_loss, net_opt, metrics={"Accuracy": Accuracy()}) print("============== Starting Training ==============") - model.train(cfg['epoch_size'], ds_train, callbacks=[time_cb, ckpt_callback, LossMonitor()], + model.train(cfg['epoch_size'], ds_train, callbacks=[ckpt_callback, LossMonitor()], dataset_sink_mode=args.dataset_sink_mode) print("============== End Training ==============") diff --git a/model_zoo/lenet_quant/train_quant.py b/model_zoo/lenet_quant/train_quant.py index ba54e63d80..6f27cec1e3 100644 --- a/model_zoo/lenet_quant/train_quant.py +++ b/model_zoo/lenet_quant/train_quant.py @@ -23,7 +23,7 @@ import argparse import mindspore.nn as nn from mindspore import context from mindspore.train.serialization import load_checkpoint, load_param_into_net -from mindspore.train.callback import ModelCheckpoint, CheckpointConfig, LossMonitor, TimeMonitor +from mindspore.train.callback import ModelCheckpoint, CheckpointConfig, LossMonitor from mindspore.train import Model from mindspore.nn.metrics import Accuracy from mindspore.train.quant import quant @@ -51,20 +51,19 @@ if __name__ == "__main__": # define fusion network network = LeNet5Fusion(cfg.num_classes) - # convert fusion network to quantization aware network - network = quant.convert_quant_network(network, quant_delay=0, bn_fold=False, freeze_bn=10000) - # load quantization aware network checkpoint - param_dict = load_checkpoint(args.ckpt_path, network.type) + param_dict = load_checkpoint(args.ckpt_path) load_param_into_net(network, param_dict) + # convert fusion network to quantization aware network + network = quant.convert_quant_network(network, quant_delay=0, bn_fold=False, freeze_bn=10000) + # define network loss net_loss = nn.SoftmaxCrossEntropyWithLogits(is_grad=False, sparse=True, reduction="mean") # define network optimization net_opt = nn.Momentum(network.trainable_params(), cfg.lr, cfg.momentum) # call back and monitor - time_cb = TimeMonitor(data_size=ds_train.get_dataset_size()) config_ckpt = CheckpointConfig(save_checkpoint_steps=cfg.epoch_size * step_size, keep_checkpoint_max=cfg.keep_checkpoint_max) ckpt_callback = ModelCheckpoint(prefix="checkpoint_lenet", config=config_ckpt) @@ -73,6 +72,6 @@ if __name__ == "__main__": model = Model(network, net_loss, net_opt, metrics={"Accuracy": Accuracy()}) print("============== Starting Training ==============") - model.train(cfg['epoch_size'], ds_train, callbacks=[time_cb, ckpt_callback, LossMonitor()], + model.train(cfg['epoch_size'], ds_train, callbacks=[ckpt_callback, LossMonitor()], dataset_sink_mode=args.dataset_sink_mode) print("============== End Training ==============") From 67a2c5b7d190972c529592a68131cc1d98e2aac0 Mon Sep 17 00:00:00 2001 From: jiangjinsheng Date: Fri, 3 Jul 2020 16:52:54 +0800 Subject: [PATCH 239/254] fix InvertPermutation --- mindspore/ops/operations/array_ops.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/mindspore/ops/operations/array_ops.py b/mindspore/ops/operations/array_ops.py index 0e1456c250..8e9ecfea95 100644 --- a/mindspore/ops/operations/array_ops.py +++ b/mindspore/ops/operations/array_ops.py @@ -986,11 +986,10 @@ class InvertPermutation(PrimitiveWithInfer): values can not be negative. Inputs: - - **input_x** (Union(tuple[int], Tensor[int])) - The input tuple is constructed by multiple + - **input_x** (Union(tuple[int]) - The input tuple is constructed by multiple integers, i.e., :math:`(y_1, y_2, ..., y_S)` representing the indices. The values must include 0. There can be no duplicate values or negative values. - If the input is Tensor, it must be 1-d and the dtype is int. Only constant value is allowed. - + Only constant value is allowed. Outputs: tuple[int]. the lenth is same as input. @@ -1014,9 +1013,7 @@ class InvertPermutation(PrimitiveWithInfer): raise ValueError(f'For \'{self.name}\' the input value must be const.') validator.check_value_type("shape", x_shp, [tuple, list], self.name) if mstype.issubclass_(x['dtype'], mstype.tensor): - validator.check('x dimension', len(x_shp), '', 1, Rel.EQ, self.name) - validator.check_tensor_type_same({'x dtype': x['dtype']}, mstype.int_type, self.name) - x_value = [int(i) for i in x_value.asnumpy()] + raise ValueError(f'For \'{self.name}\' the input value must be non-Tensor.') z = [x_value[i] for i in range(len(x_value))] z.sort() From edc210dfdc5e4173e05594f806b774ccfa06440e Mon Sep 17 00:00:00 2001 From: dinghao Date: Fri, 3 Jul 2020 17:01:24 +0800 Subject: [PATCH 240/254] fix serving peformance --- .../ccsrc/session/ascend_inference_session.cc | 104 ++++-- serving/core/ms_service_pb2.py | 318 ++++++++++++++++++ .../ms_service_pb2_grpc.py | 0 serving/core/server.cc | 4 +- 4 files changed, 393 insertions(+), 33 deletions(-) create mode 100644 serving/core/ms_service_pb2.py rename serving/{python_example => core}/ms_service_pb2_grpc.py (100%) diff --git a/mindspore/ccsrc/session/ascend_inference_session.cc b/mindspore/ccsrc/session/ascend_inference_session.cc index 6295fca1c3..aef7738d0b 100644 --- a/mindspore/ccsrc/session/ascend_inference_session.cc +++ b/mindspore/ccsrc/session/ascend_inference_session.cc @@ -31,14 +31,69 @@ using mindspore::tensor::TensorPy; namespace mindspore { namespace session { +namespace { +std::set weight_infos; +static TypeId GetDataType(const py::buffer_info &buf) { + if (buf.format.size() == 1) { + switch (buf.format.front()) { + case 'e': + case 'f': + case 'd': + switch (buf.itemsize) { + case 2: + return TypeId::kNumberTypeFloat16; + case 4: + return TypeId::kNumberTypeFloat32; + case 8: + return TypeId::kNumberTypeFloat64; + } + break; + case 'b': + case 'h': + case 'i': + case 'l': + case 'q': + switch (buf.itemsize) { + case 1: + return TypeId::kNumberTypeInt8; + case 2: + return TypeId::kNumberTypeInt16; + case 4: + return TypeId::kNumberTypeInt32; + case 8: + return TypeId::kNumberTypeInt64; + } + break; + case 'B': + case 'H': + case 'I': + case 'L': + case 'Q': + switch (buf.itemsize) { + case 1: + return TypeId::kNumberTypeUInt8; + case 2: + return TypeId::kNumberTypeUInt16; + case 4: + return TypeId::kNumberTypeUInt32; + case 8: + return TypeId::kNumberTypeUInt64; + } + break; + case '?': + return TypeId::kNumberTypeBool; + } + } + MS_LOG(WARNING) << "Unsupported DataType format " << buf.format << " item size " << buf.itemsize; + return TypeId::kTypeUnknown; +} +} // namespace void AscendInferenceSession::LoadInputData(const std::shared_ptr &kernel_graph, const std::vector &inputs_const) const { MS_EXCEPTION_IF_NULL(kernel_graph); std::vector inputs(inputs_const); auto input_nodes = kernel_graph->inputs(); - auto ms_context = MsContext::GetInstance(); - MS_EXCEPTION_IF_NULL(ms_context); size_t no_weight_input = 0; for (size_t i = 0; i < input_nodes.size(); ++i) { tensor::TensorPtr tensor = nullptr; @@ -48,45 +103,32 @@ void AscendInferenceSession::LoadInputData(const std::shared_ptr &k } auto pk_node = input_nodes[i]->cast(); MS_EXCEPTION_IF_NULL(pk_node); + auto device_address = AnfAlgo::GetMutableOutputAddr(pk_node, 0); + MS_EXCEPTION_IF_NULL(device_address); if (AnfAlgo::IsParameterWeight(pk_node)) { + if (weight_infos.count(pk_node) != 0) { + continue; + } auto param_value = std::dynamic_pointer_cast(pk_node->default_param()); MS_EXCEPTION_IF_NULL(param_value); auto py_param = param_value->value(); MS_EXCEPTION_IF_NULL(py_param); py::array py_array = py_param.cast(); - tensor = TensorPy::MakeTensor(py_array); + py::buffer_info buf = py_array.request(); + auto buf_type = GetDataType(buf); + if (!device_address->SyncHostToDevice(trans::GetRuntimePaddingShape(pk_node, 0), + LongToSize(buf.size * buf.itemsize), buf_type, buf.ptr)) { + MS_LOG(EXCEPTION) << "SyncHostToDevice failed."; + } + weight_infos.insert(pk_node); } else { tensor = inputs[no_weight_input++]; - } - MS_EXCEPTION_IF_NULL(tensor); - if (AnfAlgo::OutputAddrExist(pk_node, 0)) { - auto device_address = AnfAlgo::GetMutableOutputAddr(pk_node, 0); - bool need_sync = false; - if (ms_context->enable_pynative_infer()) { - if (tensor->device_address().get() == nullptr || tensor->device_address() != device_address) { - need_sync = true; - } - } else { - if (tensor->is_dirty()) { - need_sync = true; - } else if (tensor->device_address() != device_address) { - (void)tensor->data_sync(); - need_sync = true; - } - } - if (need_sync) { - if (ms_context->execution_mode() == kPynativeMode || AnfAlgo::IsParameterWeight(pk_node)) { - tensor->set_device_address(device_address); - } - MS_EXCEPTION_IF_NULL(device_address); - if (!device_address->SyncHostToDevice(trans::GetRuntimePaddingShape(pk_node, 0), - LongToSize(tensor->data().nbytes()), tensor->data_type(), - tensor->data_c())) { - MS_LOG(EXCEPTION) << "SyncHostToDevice failed."; - } + if (!device_address->SyncHostToDevice(trans::GetRuntimePaddingShape(pk_node, 0), + LongToSize(tensor->data().nbytes()), tensor->data_type(), + tensor->data_c())) { + MS_LOG(EXCEPTION) << "SyncHostToDevice failed."; } } - tensor->set_dirty(false); } } } // namespace session diff --git a/serving/core/ms_service_pb2.py b/serving/core/ms_service_pb2.py new file mode 100644 index 0000000000..9feec026f9 --- /dev/null +++ b/serving/core/ms_service_pb2.py @@ -0,0 +1,318 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: ms_service.proto + +from google.protobuf.internal import enum_type_wrapper +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor.FileDescriptor( + name='ms_service.proto', + package='ms_serving', + syntax='proto3', + serialized_options=None, + serialized_pb=b'\n\x10ms_service.proto\x12\nms_serving\"2\n\x0ePredictRequest\x12 \n\x04\x64\x61ta\x18\x01 \x03(\x0b\x32\x12.ms_serving.Tensor\"2\n\x0cPredictReply\x12\"\n\x06result\x18\x01 \x03(\x0b\x32\x12.ms_serving.Tensor\"\x1b\n\x0bTensorShape\x12\x0c\n\x04\x64ims\x18\x01 \x03(\x03\"p\n\x06Tensor\x12-\n\x0ctensor_shape\x18\x01 \x01(\x0b\x32\x17.ms_serving.TensorShape\x12)\n\x0btensor_type\x18\x02 \x01(\x0e\x32\x14.ms_serving.DataType\x12\x0c\n\x04\x64\x61ta\x18\x03 \x01(\x0c*\xc9\x01\n\x08\x44\x61taType\x12\x0e\n\nMS_UNKNOWN\x10\x00\x12\x0b\n\x07MS_BOOL\x10\x01\x12\x0b\n\x07MS_INT8\x10\x02\x12\x0c\n\x08MS_UINT8\x10\x03\x12\x0c\n\x08MS_INT16\x10\x04\x12\r\n\tMS_UINT16\x10\x05\x12\x0c\n\x08MS_INT32\x10\x06\x12\r\n\tMS_UINT32\x10\x07\x12\x0c\n\x08MS_INT64\x10\x08\x12\r\n\tMS_UINT64\x10\t\x12\x0e\n\nMS_FLOAT16\x10\n\x12\x0e\n\nMS_FLOAT32\x10\x0b\x12\x0e\n\nMS_FLOAT64\x10\x0c\x32\x8e\x01\n\tMSService\x12\x41\n\x07Predict\x12\x1a.ms_serving.PredictRequest\x1a\x18.ms_serving.PredictReply\"\x00\x12>\n\x04Test\x12\x1a.ms_serving.PredictRequest\x1a\x18.ms_serving.PredictReply\"\x00\x62\x06proto3' +) + +_DATATYPE = _descriptor.EnumDescriptor( + name='DataType', + full_name='ms_serving.DataType', + filename=None, + file=DESCRIPTOR, + values=[ + _descriptor.EnumValueDescriptor( + name='MS_UNKNOWN', index=0, number=0, + serialized_options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='MS_BOOL', index=1, number=1, + serialized_options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='MS_INT8', index=2, number=2, + serialized_options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='MS_UINT8', index=3, number=3, + serialized_options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='MS_INT16', index=4, number=4, + serialized_options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='MS_UINT16', index=5, number=5, + serialized_options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='MS_INT32', index=6, number=6, + serialized_options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='MS_UINT32', index=7, number=7, + serialized_options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='MS_INT64', index=8, number=8, + serialized_options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='MS_UINT64', index=9, number=9, + serialized_options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='MS_FLOAT16', index=10, number=10, + serialized_options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='MS_FLOAT32', index=11, number=11, + serialized_options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='MS_FLOAT64', index=12, number=12, + serialized_options=None, + type=None), + ], + containing_type=None, + serialized_options=None, + serialized_start=280, + serialized_end=481, +) +_sym_db.RegisterEnumDescriptor(_DATATYPE) + +DataType = enum_type_wrapper.EnumTypeWrapper(_DATATYPE) +MS_UNKNOWN = 0 +MS_BOOL = 1 +MS_INT8 = 2 +MS_UINT8 = 3 +MS_INT16 = 4 +MS_UINT16 = 5 +MS_INT32 = 6 +MS_UINT32 = 7 +MS_INT64 = 8 +MS_UINT64 = 9 +MS_FLOAT16 = 10 +MS_FLOAT32 = 11 +MS_FLOAT64 = 12 + + + +_PREDICTREQUEST = _descriptor.Descriptor( + name='PredictRequest', + full_name='ms_serving.PredictRequest', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='data', full_name='ms_serving.PredictRequest.data', index=0, + number=1, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=32, + serialized_end=82, +) + + +_PREDICTREPLY = _descriptor.Descriptor( + name='PredictReply', + full_name='ms_serving.PredictReply', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='result', full_name='ms_serving.PredictReply.result', index=0, + number=1, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=84, + serialized_end=134, +) + + +_TENSORSHAPE = _descriptor.Descriptor( + name='TensorShape', + full_name='ms_serving.TensorShape', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='dims', full_name='ms_serving.TensorShape.dims', index=0, + number=1, type=3, cpp_type=2, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=136, + serialized_end=163, +) + + +_TENSOR = _descriptor.Descriptor( + name='Tensor', + full_name='ms_serving.Tensor', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='tensor_shape', full_name='ms_serving.Tensor.tensor_shape', index=0, + number=1, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='tensor_type', full_name='ms_serving.Tensor.tensor_type', index=1, + number=2, type=14, cpp_type=8, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='data', full_name='ms_serving.Tensor.data', index=2, + number=3, type=12, cpp_type=9, label=1, + has_default_value=False, default_value=b"", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=165, + serialized_end=277, +) + +_PREDICTREQUEST.fields_by_name['data'].message_type = _TENSOR +_PREDICTREPLY.fields_by_name['result'].message_type = _TENSOR +_TENSOR.fields_by_name['tensor_shape'].message_type = _TENSORSHAPE +_TENSOR.fields_by_name['tensor_type'].enum_type = _DATATYPE +DESCRIPTOR.message_types_by_name['PredictRequest'] = _PREDICTREQUEST +DESCRIPTOR.message_types_by_name['PredictReply'] = _PREDICTREPLY +DESCRIPTOR.message_types_by_name['TensorShape'] = _TENSORSHAPE +DESCRIPTOR.message_types_by_name['Tensor'] = _TENSOR +DESCRIPTOR.enum_types_by_name['DataType'] = _DATATYPE +_sym_db.RegisterFileDescriptor(DESCRIPTOR) + +PredictRequest = _reflection.GeneratedProtocolMessageType('PredictRequest', (_message.Message,), { + 'DESCRIPTOR' : _PREDICTREQUEST, + '__module__' : 'ms_service_pb2' + # @@protoc_insertion_point(class_scope:ms_serving.PredictRequest) + }) +_sym_db.RegisterMessage(PredictRequest) + +PredictReply = _reflection.GeneratedProtocolMessageType('PredictReply', (_message.Message,), { + 'DESCRIPTOR' : _PREDICTREPLY, + '__module__' : 'ms_service_pb2' + # @@protoc_insertion_point(class_scope:ms_serving.PredictReply) + }) +_sym_db.RegisterMessage(PredictReply) + +TensorShape = _reflection.GeneratedProtocolMessageType('TensorShape', (_message.Message,), { + 'DESCRIPTOR' : _TENSORSHAPE, + '__module__' : 'ms_service_pb2' + # @@protoc_insertion_point(class_scope:ms_serving.TensorShape) + }) +_sym_db.RegisterMessage(TensorShape) + +Tensor = _reflection.GeneratedProtocolMessageType('Tensor', (_message.Message,), { + 'DESCRIPTOR' : _TENSOR, + '__module__' : 'ms_service_pb2' + # @@protoc_insertion_point(class_scope:ms_serving.Tensor) + }) +_sym_db.RegisterMessage(Tensor) + + + +_MSSERVICE = _descriptor.ServiceDescriptor( + name='MSService', + full_name='ms_serving.MSService', + file=DESCRIPTOR, + index=0, + serialized_options=None, + serialized_start=484, + serialized_end=626, + methods=[ + _descriptor.MethodDescriptor( + name='Predict', + full_name='ms_serving.MSService.Predict', + index=0, + containing_service=None, + input_type=_PREDICTREQUEST, + output_type=_PREDICTREPLY, + serialized_options=None, + ), + _descriptor.MethodDescriptor( + name='Test', + full_name='ms_serving.MSService.Test', + index=1, + containing_service=None, + input_type=_PREDICTREQUEST, + output_type=_PREDICTREPLY, + serialized_options=None, + ), +]) +_sym_db.RegisterServiceDescriptor(_MSSERVICE) + +DESCRIPTOR.services_by_name['MSService'] = _MSSERVICE + +# @@protoc_insertion_point(module_scope) diff --git a/serving/python_example/ms_service_pb2_grpc.py b/serving/core/ms_service_pb2_grpc.py similarity index 100% rename from serving/python_example/ms_service_pb2_grpc.py rename to serving/core/ms_service_pb2_grpc.py diff --git a/serving/core/server.cc b/serving/core/server.cc index 273f0920c3..5ba7ad36a7 100644 --- a/serving/core/server.cc +++ b/serving/core/server.cc @@ -259,7 +259,7 @@ Status Server::BuildAndStart() { } g_ctx = ctx; #endif - MSServiceImpl msService; + MSServiceImpl ms_service; grpc::EnableDefaultHealthCheckService(true); grpc::reflection::InitProtoReflectionServerBuilderPlugin(); // Set the port is not reuseable @@ -268,7 +268,7 @@ Status Server::BuildAndStart() { serverBuilder.SetOption(std::move(option)); serverBuilder.SetMaxMessageSize(uint32max); serverBuilder.AddListeningPort(server_address, grpc::InsecureServerCredentials()); - serverBuilder.RegisterService(&msService); + serverBuilder.RegisterService(&ms_service); std::unique_ptr server(serverBuilder.BuildAndStart()); if (server == nullptr) { MS_LOG(ERROR) << "The serving server create failed"; From 193c583cf628ff50271e6477fb651dd5df807bc6 Mon Sep 17 00:00:00 2001 From: chenfei Date: Tue, 30 Jun 2020 15:59:31 +0800 Subject: [PATCH 241/254] add unreuse args of kernel graph insert assign at the end of graph clear log clear log 2 handle replace call of unreuse args handle bug of replace node --- .../device/ascend/ascend_label_assign.cc | 3 +- .../ccsrc/session/anf_runtime_algorithm.h | 1 - .../ccsrc/session/ascend_control_parser.cc | 87 +++++++++++++------ .../ccsrc/session/ascend_control_parser.h | 1 + mindspore/ccsrc/session/ascend_session.cc | 57 ++++++------ mindspore/ccsrc/session/ascend_session.h | 4 +- mindspore/ccsrc/session/kernel_graph.cc | 52 ++++++++++- mindspore/ccsrc/session/kernel_graph.h | 4 + tests/st/control/test_ascend_control_sink.py | 29 +++++++ 9 files changed, 173 insertions(+), 65 deletions(-) diff --git a/mindspore/ccsrc/device/ascend/ascend_label_assign.cc b/mindspore/ccsrc/device/ascend/ascend_label_assign.cc index 7af615f448..2db81a1725 100644 --- a/mindspore/ccsrc/device/ascend/ascend_label_assign.cc +++ b/mindspore/ccsrc/device/ascend/ascend_label_assign.cc @@ -102,7 +102,7 @@ static void AssignLabelForGotoSwitch(NotNullinsert(graph.get()); MS_LOG(INFO) << "Process label goto/switch for " << graph->ToString(); - graph->SetExecOrderByDefault(); + auto nodes = graph->execution_order(); auto end_goto = graph->get_end_goto(); if (end_goto != nullptr) { @@ -128,6 +128,7 @@ static void AssignLabelForGotoSwitch(NotNullchild_graph_order()) { AssignLabelForGotoSwitch(NOT_NULL(cg), memo); } + graph->SetExecOrderByDefault(); } void AscendLabelAssign::AssignLabel(NotNull> graph) { diff --git a/mindspore/ccsrc/session/anf_runtime_algorithm.h b/mindspore/ccsrc/session/anf_runtime_algorithm.h index ae8b450e2f..8205619793 100644 --- a/mindspore/ccsrc/session/anf_runtime_algorithm.h +++ b/mindspore/ccsrc/session/anf_runtime_algorithm.h @@ -199,7 +199,6 @@ class AnfRuntimeAlgorithm { static bool IsScalarInput(const CNodePtr &cnode, size_t index); static bool IsScalarOutput(const CNodePtr &cnode, size_t index); static void ReorderExecList(NotNull *> node_list); - static bool IsWhileTrueGraph(const KernelGraphPtr &child_graph); // get fix output precision of cnode. static TypeId GetCNodeOutputPrecision(const AnfNodePtr &node); // get fix output precision from prev node, input_idx is the input index of current node related to prev node. diff --git a/mindspore/ccsrc/session/ascend_control_parser.cc b/mindspore/ccsrc/session/ascend_control_parser.cc index 166d4cc97a..18e71d74e3 100644 --- a/mindspore/ccsrc/session/ascend_control_parser.cc +++ b/mindspore/ccsrc/session/ascend_control_parser.cc @@ -19,6 +19,7 @@ #include #include "session/anf_runtime_algorithm.h" #include "utils/union_find_set.h" +#include "device/ascend/ascend_label_assign.h" static constexpr size_t kCNodePrim = 0; static constexpr size_t kCNodeCallArg = 1; @@ -35,17 +36,25 @@ namespace mindspore { namespace session { static CNodePtr GetJumpNode(NotNull parent_graph, NotNull child_graph) { auto &nodes = parent_graph->execution_order(); + CNodePtr last_jump_node = nullptr; for (auto &node : nodes) { - if (IsPrimitiveCNode(node, prim::kPrimLabelGoto) && child_graph->get_start_label() == node->input(kCNodeCallArg)) { - return node; - } else if (IsPrimitiveCNode(node, prim::kPrimLabelSwitch) && - (child_graph->get_start_label() == node->input(kCNodeSwitchFalse) || - child_graph->get_start_label() == node->input(kCNodeSwitchTrue))) { - return node; + if (IsPrimitiveCNode(node, prim::kPrimLabelGoto)) { + if (child_graph->get_start_label() == node->input(kCNodeCallArg)) { + return node; + } + last_jump_node = node; + } else if (IsPrimitiveCNode(node, prim::kPrimLabelSwitch)) { + if (child_graph->get_start_label() == node->input(kCNodeSwitchFalse) || + child_graph->get_start_label() == node->input(kCNodeSwitchTrue)) { + return node; + } + last_jump_node = node; } } - MS_LOG(INFO) << "Cannot find jump node from " << parent_graph->ToString() << " to " << child_graph->ToString(); - return nullptr; + if (last_jump_node == nullptr) { + MS_LOG(EXCEPTION) << "Cannot find jump node from " << parent_graph->ToString() << " to " << child_graph->ToString(); + } + return last_jump_node; } static void InitUnionFindSet(NotNull kg, const NotNull *> union_find_set, @@ -90,6 +99,9 @@ static void UnionParentParameter(NotNull kg, const NotNullisa()) { continue; } + if (kg->unreuse_args().find(arg) != kg->unreuse_args().end()) { + continue; + } union_find_set->Union(arg, para); } } @@ -133,24 +145,28 @@ static void RecursiveReplaceNode(NotNull kg, NotNull } } +static AnfNodePtr GetMainParameter(NotNull root_kg, const AnfNodePtr key, + const std::set ¶meter_reuse_set) { + AnfNodePtr main_parameter = key; + std::set root_inputs_set; + const auto &root_inputs_vector = root_kg->inputs(); + root_inputs_set.insert(root_inputs_vector.begin(), root_inputs_vector.end()); + for (auto &node : parameter_reuse_set) { + if (root_inputs_set.find(node) != root_inputs_set.end()) { + main_parameter = node; + break; + } + } + return main_parameter; +} + static void ReuseParameter(NotNull root_kg, NotNull *> parameter_set) { auto parameter_reuse_sets = parameter_set->GetSets(); for (auto &[key, parameter_reuse_set] : parameter_reuse_sets) { if (parameter_reuse_set.size() <= 1) { continue; } - - AnfNodePtr main_parameter = key; - std::set root_inputs_set; - const auto &root_inputs_vector = root_kg->inputs(); - root_inputs_set.insert(root_inputs_vector.begin(), root_inputs_vector.end()); - for (auto &node : parameter_reuse_set) { - if (root_inputs_set.find(node) != root_inputs_set.end()) { - main_parameter = node; - break; - } - } - + auto main_parameter = GetMainParameter(root_kg, key, parameter_reuse_set); std::set memo; RecursiveReplaceNode(root_kg, NOT_NULL(main_parameter), parameter_reuse_set, NOT_NULL(&memo)); } @@ -168,6 +184,7 @@ CNodePtr GetNextRealKernel(const std::vector &list, size_t start) { void AscendControlParser::LinkGraph(NotNull kg) { std::set memo; (void)ProcessKernelGraph(kg, nullptr, nullptr, NOT_NULL(&memo)); + device::ascend::AscendLabelAssign::GetInstance().AssignLabel(kg); std::map graph_id_map; for (auto &g : memo) { MS_EXCEPTION_IF_NULL(g); @@ -177,12 +194,13 @@ void AscendControlParser::LinkGraph(NotNull kg) { } graph_id_map[g->graph_id()] = g; } + + // Insert Assign + ChildGraphDataAssign(graph_id_map); // Make UnionFindSet UnionFindSet parameter_set = MakeUnionFindSet(kg); // Reuse Parameter ReuseParameter(kg, NOT_NULL(¶meter_set)); - // Insert Assign - ChildGraphDataAssign(graph_id_map); } void AscendControlParser::ExecutorValidate(NotNull root_graph) { @@ -193,6 +211,7 @@ void AscendControlParser::ExecutorValidate(NotNull root_graph) { void AscendControlParser::ChildGraphDataAssign(const std::map &graph_id_map) { for (auto &iter : graph_id_map) { auto &kg = iter.second; + MS_LOG(INFO) << "Data assign graph:" << kg->graph_id(); MS_EXCEPTION_IF_NULL(kg); std::set> memo; const std::vector>> &real_inputs = kg->real_inputs(); @@ -206,8 +225,14 @@ void AscendControlParser::ChildGraphDataAssign(const std::mapisa()) { + auto unreuse_args_map = kg->unreuse_args(); + auto unreuse_arg_iter = unreuse_args_map.find(arg); + if (unreuse_arg_iter == unreuse_args_map.end()) { + MS_EXCEPTION_IF_NULL(arg); MS_EXCEPTION_IF_NULL(parameter); + if (!arg->isa()) { + MS_LOG(EXCEPTION) << "Reused arg must be parameter, arg:" << arg->DebugString() << "."; + } MS_LOG(DEBUG) << "Parameter should be reused, no need insert assign, parameter: " << parameter->DebugString() << ", arg:" << arg->DebugString(); continue; @@ -220,6 +245,7 @@ void AscendControlParser::ChildGraphDataAssign(const std::mapSetExecOrderByDefault(); } } @@ -353,7 +379,6 @@ void AscendControlParser::RecurseCall(NotNull kg, NotNullset_inputs(new_inputs); cur_node->set_abstract(nullptr); MS_LOG(INFO) << "Succeed processing call func " << cur_node->DebugString(); @@ -394,7 +419,6 @@ void AscendControlParser::RecurseSwitch(NotNull kg, NotNullset_inputs(new_switch_inputs); cur_node->set_abstract(nullptr); MS_LOG(INFO) << "Succeed processing switch func " << cur_node->DebugString(); @@ -477,6 +501,16 @@ void AscendControlParser::InsertMultipleAssignToGraph(NotNull fr auto assign_node = InsertAssignToGraph(from_graph, NOT_NULL(from_outputs[i]), NOT_NULL(to_outputs[i])); if (assign_node != nullptr) { auto jump_node = GetJumpNode(from_graph, to_graph); + const auto &from_graph_exe_order = from_graph->execution_order(); + auto jump_node_iter = std::find(from_graph_exe_order.begin(), from_graph_exe_order.end(), jump_node); + if (jump_node_iter == from_graph_exe_order.end()) { + MS_EXCEPTION_IF_NULL(jump_node); + MS_LOG(EXCEPTION) << "Can't find node:" << jump_node->DebugString() << " in graph:" << from_graph->graph_id(); + } + // insert assign between jump_node -1 and jump_node + if (jump_node_iter != from_graph_exe_order.begin()) { + InsertControlDependToGraph(from_graph, NOT_NULL(*(jump_node_iter - 1)), NOT_NULL(assign_node)); + } if (jump_node != nullptr) { InsertControlDependToGraph(from_graph, NOT_NULL(assign_node), NOT_NULL(jump_node)); } @@ -501,8 +535,6 @@ AnfNodePtr AscendControlParser::InsertAssignToGraph(NotNull kg, auto assign_node = kg->NewCNode(inputs); MS_EXCEPTION_IF_NULL(assign_node); assign_node->set_abstract(to->abstract()); - // append the assign at the end of from graph - InsertDependToGraph(kg, NOT_NULL(assign_node)); return assign_node; } @@ -527,7 +559,6 @@ std::vector AscendControlParser::RecurseGraph(NotNull std::vector execution_order; uint32_t child_order_index = 0; - for (auto &node : cnodes) { execution_order.push_back(node); if (node == graph->get_end_goto()) { diff --git a/mindspore/ccsrc/session/ascend_control_parser.h b/mindspore/ccsrc/session/ascend_control_parser.h index 2b383d7b14..82479fa527 100644 --- a/mindspore/ccsrc/session/ascend_control_parser.h +++ b/mindspore/ccsrc/session/ascend_control_parser.h @@ -23,6 +23,7 @@ #include "session/kernel_graph.h" #include "utils/base_ref.h" #include "utils/contract.h" +#include "utils/union_find_set.h" namespace mindspore { namespace session { diff --git a/mindspore/ccsrc/session/ascend_session.cc b/mindspore/ccsrc/session/ascend_session.cc index f669f89b66..0f6b34c4d9 100644 --- a/mindspore/ccsrc/session/ascend_session.cc +++ b/mindspore/ccsrc/session/ascend_session.cc @@ -202,7 +202,8 @@ static std::vector> GetChildList(const std::vector ¶meters, const std::vector &args, - KernelGraph *child_graph) { + const KernelGraphPtr &graph, KernelGraphPtr child_graph, + const NotNull *> memo) { MS_EXCEPTION_IF_NULL(child_graph); MS_LOG(INFO) << "Start bind parameter of child graph:" << child_graph->graph_id(); if (args.empty()) { @@ -214,18 +215,25 @@ static void BindCallArgsWithParameter(const std::vector ¶meters, } child_graph->SetExecOrderByDefault(); for (size_t i = 0; i < parameters.size(); i++) { + MS_LOG(INFO) << "parameters[" << i << "]" << parameters[i]->DebugString() << ",args[" << i << "]" + << args[i]->DebugString(); if (args[i] == parameters[i]) { - child_graph->SetRealInput(parameters[i], args[i]); MS_LOG(INFO) << "Parameter and arg are same."; continue; } child_graph->SetRealInput(parameters[i], args[i]); + if (memo->find(child_graph) != memo->end() || !args[i]->isa()) { + MS_LOG(INFO) << "Add unreused arg,graph:" << graph->graph_id(); + child_graph->AddUnreuseArgs(args[i], graph); + } } } // if a call has kernel input, it's a child graph split from ME, so these kernel input should be set into real input of // graph.For example, call input = (prim,graph,kernel1,kernel2),then real_input = [kernel1,kernel2] -static void UpdateRealInput(NotNull graph, bool split_flag) { +static void UpdateRealInput(NotNull graph, bool split_flag, + const NotNull *> memo) { + MS_EXCEPTION_IF_NULL(memo.get()); auto call_nodes = graph->FindNodeByPrimitive(prim::kPrimCall); for (auto &call_node : call_nodes) { MS_EXCEPTION_IF_NULL(call_node); @@ -235,7 +243,7 @@ static void UpdateRealInput(NotNull graph, bool split_flag) { std::vector real_args = std::vector(call_node->inputs().begin() + 2, call_node->inputs().end()); std::vector child_inputs = child_graphs[0]->inputs(); - BindCallArgsWithParameter(child_inputs, real_args, child_graphs[0].get()); + BindCallArgsWithParameter(child_inputs, real_args, graph, child_graphs[0], memo); if (split_flag) { call_node->set_inputs(std::vector(call_node->inputs().begin(), call_node->inputs().begin() + 2)); } @@ -256,8 +264,8 @@ static void UpdateRealInput(NotNull graph, bool split_flag) { } return ret; }; - BindCallArgsWithParameter(child_graphs[0]->inputs(), get_partial_args(2), child_graphs[0].get()); - BindCallArgsWithParameter(child_graphs[1]->inputs(), get_partial_args(3), child_graphs[1].get()); + BindCallArgsWithParameter(child_graphs[0]->inputs(), get_partial_args(2), graph, child_graphs[0], memo); + BindCallArgsWithParameter(child_graphs[1]->inputs(), get_partial_args(3), graph, child_graphs[1], memo); } } } @@ -306,8 +314,6 @@ GraphId AscendSession::CompileGraph(NotNull func_graph) { LinkChildGraphs(NOT_NULL(root_graph)); // resource initialize InitRuntimeResource(); - // assign label - AssignLabel(NOT_NULL(root_graph)); // recurse compile child root_graph std::set memo; RecurseCompileGraph(NOT_NULL(root_graph), NOT_NULL(&memo)); @@ -665,12 +671,6 @@ void AscendSession::AssignStream(NotNull kernel_graph) const { MS_LOG(INFO) << "Finish!"; } -void AscendSession::AssignLabel(NotNull kernel_graph) const { - MS_LOG(INFO) << "Start!"; - device::ascend::AscendLabelAssign::GetInstance().AssignLabel(kernel_graph); - MS_LOG(INFO) << "Finish!"; -} - void AscendSession::BuildKernel(const std::shared_ptr &kernel_graph) const { MS_LOG(INFO) << "Start!"; struct timeval start_time, end_time; @@ -1591,14 +1591,17 @@ std::vector AscendSession::ConstructSplitedGraph(const KernelGraphPt auto input = cnode->inputs()[input_idx]; MS_EXCEPTION_IF_NULL(input); AnfNodePtr new_parameter = nullptr; + // check whether input has been put into args of call, if mulptiple use of one parameter or cnode, only set one + // parameter in graph inputs and one arg in call node + auto call_input_it = std::find(call_node_inputs.begin(), call_node_inputs.end(), input); + if (call_input_it != call_node_inputs.end()) { + cnode->set_input(input_idx, new_graph_inputs[std::distance(call_node_inputs.begin(), call_input_it)]); + continue; + } // value node consider move to new graph if (input->isa()) { cnode->set_input(input_idx, input); continue; - } else if (input->isa()) { - // parameter reuse and should attention mulptiple use of one parameter - cnode->set_input(input_idx, input); - new_parameter = input; } else if (AnfAlgo::GetGraphId(input.get()) != new_kernel_graph->graph_id()) { // if is cnode and not in current child graph new_parameter = CreateNewParameterFromCNode(input, true, new_kernel_graph.get()); @@ -1607,12 +1610,8 @@ std::vector AscendSession::ConstructSplitedGraph(const KernelGraphPt // if is a cnode and in current graph continue; } - // if mulptiple use of one parameter or cnode, only set one parameter in graph inputs and one arg in call node - // args - if (std::find(call_node_inputs.begin(), call_node_inputs.end(), new_parameter) == call_node_inputs.end()) { - new_graph_inputs.push_back(new_parameter); - call_node_inputs.push_back(input); - } + new_graph_inputs.push_back(new_parameter); + call_node_inputs.push_back(input); } } // set graph inputs of new graph @@ -1640,7 +1639,7 @@ void AscendSession::SplitGraphs(NotNull root_graph) { // if root graph output is a call node ,the root graph is condition graph of 'if' sentence auto root_graph_output = AnfAlgo::VisitKernelWithReturnType(root_graph->output(), 0).first; if (AnfAlgo::CheckPrimitiveType(root_graph_output, prim::kPrimCall)) { - SplitGraph(root_graph, {prim::kPrimReturn}); + SplitGraph(root_graph, {prim::kPrimReturn}, NOT_NULL(&memo)); for (auto &child_graph : root_graph->child_graph_order()) { RecurseSplitGraph(NOT_NULL(child_graph), NOT_NULL(&memo)); } @@ -1681,7 +1680,8 @@ AnfNodePtr AscendSession::BindNewCallToNewGraph(NotNull graph, return new_call; } -void AscendSession::SplitGraph(NotNull graph, const std::set &cut_prims) { +void AscendSession::SplitGraph(NotNull graph, const std::set &cut_prims, + const NotNull *> memo) { MS_LOG(INFO) << "Start,graph_id:" << graph->graph_id(); bool split_flag = false; auto apply_list = GetCNodes(TopoSort(graph->get_return())); @@ -1719,14 +1719,13 @@ void AscendSession::SplitGraph(NotNull graph, const std::setgraph_id() << "] end"; - // recurse to split child graph } void AscendSession::RecurseSplitGraph(NotNull graph, const NotNull *> memo) { memo->insert(graph.get()); - SplitGraph(graph, {prim::kPrimCall}); + SplitGraph(graph, {prim::kPrimCall}, memo); for (auto &child_graph : graph->child_graph_order()) { if (memo->find(child_graph) == memo->end()) { RecurseSplitGraph(NOT_NULL(child_graph), memo); diff --git a/mindspore/ccsrc/session/ascend_session.h b/mindspore/ccsrc/session/ascend_session.h index 904b011077..0d1ab75ee1 100755 --- a/mindspore/ccsrc/session/ascend_session.h +++ b/mindspore/ccsrc/session/ascend_session.h @@ -77,7 +77,6 @@ class AscendSession : public SessionBasic { void AdjustKernel(const std::shared_ptr &kernel_graph) const; void RunOpAdjustKernel(const std::shared_ptr &kernel_graph) const; void AssignStream(NotNull kernel_graph) const; - void AssignLabel(NotNull kernel_graph) const; void BuildKernel(const std::shared_ptr &kernel_graph) const; void MemoryAlloc(KernelGraph *kernel_graph) const; void RunOpMemoryAlloc(const std::vector &input_tensors, KernelGraph *kernel_graph) const; @@ -100,7 +99,8 @@ class AscendSession : public SessionBasic { void SetFinalGraphOutput(const ValuePtr &value); void SetFinalGraphOutput(const VectorRef &vec_output); - void SplitGraph(NotNull graph, const std::set &cut_prims); + void SplitGraph(NotNull graph, const std::set &cut_prims, + const NotNull *> memo); // split graphs with recurse from root graph void SplitGraphs(NotNull root_graph); void BackendOptimization(const std::vector &all_graphs); diff --git a/mindspore/ccsrc/session/kernel_graph.cc b/mindspore/ccsrc/session/kernel_graph.cc index 306e3351e3..b1f69cfc52 100644 --- a/mindspore/ccsrc/session/kernel_graph.cc +++ b/mindspore/ccsrc/session/kernel_graph.cc @@ -103,6 +103,23 @@ AnfNodePtr MakeValueNode(const AnfNodePtr &node) { AnfAlgo::SetSelectKernelBuildInfo(kernel_build_info_builder->Build(), new_value_node.get()); return new_value_node; } + +bool IsSameLabel(const CNodePtr &left, const CNodePtr &right) { + if (left == right) { + return true; + } + if (left == nullptr || right == nullptr) { + return false; + } + if (!IsPrimitiveCNode(left, GetCNodePrimitive(right))) { + return false; + } + if (AnfAlgo::HasNodeAttr(kAttrLabelIndex, left) && AnfAlgo::HasNodeAttr(kAttrLabelIndex, right)) { + return AnfAlgo::GetNodeAttr(left, kAttrLabelIndex) == + AnfAlgo::GetNodeAttr(right, kAttrLabelIndex); + } + return false; +} } // namespace std::vector KernelGraph::outputs() const { auto graph_output = output(); @@ -219,6 +236,19 @@ void KernelGraph::SetExecOrderByDefault() { if (node == start_label_ || node == end_goto_) { continue; } + + if (IsSameLabel(node, end_goto_)) { + end_goto_ = node; + MS_LOG(INFO) << "Replace end_goto_ in kernel graph:" << graph_id(); + continue; + } + + if (IsSameLabel(node, start_label_)) { + start_label_ = node; + MS_LOG(INFO) << "Replace start_label_ in kernel graph:" << graph_id(); + continue; + } + re_order.push_back(node); } if (end_goto_ != nullptr) { @@ -748,10 +778,9 @@ void KernelGraph::ReplaceNode(NotNull old_anf_node, NotNullsecond; - (void)node_output_edges_.erase(old_anf_node); } + // if change the ir of graph, regenerate execution order of graph + SetExecOrderByDefault(); // update graph inputs in child graph auto it_real_inputs = std::find_if(real_inputs_.begin(), real_inputs_.end(), [&old_anf_node](const std::pair> &n) -> bool { @@ -767,7 +796,7 @@ void KernelGraph::ReplaceNode(NotNull old_anf_node, NotNullDebugString() << " already exist in real inputs, will be rewrited."; + MS_LOG(WARNING) << new_anf_node->DebugString() << " Already exist in real inputs, will be rewrited."; iter->second = old_args; } else { real_inputs_.emplace_back(new_anf_node, old_args); @@ -824,6 +853,10 @@ void KernelGraph::SetRealInput(const AnfNodePtr ¶meter, const AnfNodePtr &ar } } +void KernelGraph::AddUnreuseArgs(const AnfNodePtr &arg, const std::shared_ptr &from_graph) { + unreuse_args_[arg] = from_graph; +} + void KernelGraph::UpdateCallRealInput() { MS_LOG(INFO) << "Update graph id: " << graph_id_; std::vector>> real_inputs_map; @@ -836,6 +869,17 @@ void KernelGraph::UpdateCallRealInput() { // if real input is a call node ,find the child graph output act as the new real input auto tmp_real_input = GetCallRealOutputs(real_input); std::copy(tmp_real_input.begin(), tmp_real_input.end(), std::back_inserter(new_real_inputs)); + // replace the call in unreuse_args_ + auto unreuse_arg_it = unreuse_args_.find(real_input); + if (unreuse_arg_it != unreuse_args_.end()) { + auto old_graph = unreuse_arg_it->second; + for (auto new_real_input : new_real_inputs) { + // if call reference graph output is parameter, it will be allowed to reuse + if (!new_real_input->isa()) { + unreuse_args_[new_real_input] = old_graph; + } + } + } } real_inputs_map.emplace_back(parameter, new_real_inputs); } diff --git a/mindspore/ccsrc/session/kernel_graph.h b/mindspore/ccsrc/session/kernel_graph.h index c7a826e5fe..890fc67dec 100644 --- a/mindspore/ccsrc/session/kernel_graph.h +++ b/mindspore/ccsrc/session/kernel_graph.h @@ -130,6 +130,9 @@ class KernelGraph : public FuncGraph { // get real inputs const std::vector>> &real_inputs() const { return real_inputs_; } void SetRealInput(const AnfNodePtr ¶meter, const AnfNodePtr &arg); + // mark unreused args + void AddUnreuseArgs(const AnfNodePtr &arg, const std::shared_ptr &from_graph); + const std::map> &unreuse_args() const { return unreuse_args_; } // used to dump ir std::string ToString() const override; // update the real input if the node is a call @@ -198,6 +201,7 @@ class KernelGraph : public FuncGraph { std::shared_ptr parent_graph_; // record real parameters,inputs_ is the formal parameters std::vector>> real_inputs_; + std::map> unreuse_args_; CNodePtr start_label_; CNodePtr end_goto_; diff --git a/tests/st/control/test_ascend_control_sink.py b/tests/st/control/test_ascend_control_sink.py index b38668cd25..39af571c14 100644 --- a/tests/st/control/test_ascend_control_sink.py +++ b/tests/st/control/test_ascend_control_sink.py @@ -99,6 +99,19 @@ class ControlIfbyIfbyIf(nn.Cell): return out +class ControlSimpleWhile(nn.Cell): + def __init__(self): + super().__init__() + self.addn = op.AddN() + + def construct(self, x, y, input_data): + out = input_data + while x: + out = self.addn([input_data, input_data, input_data]) + x = y + return out + + class ControlMixedWhileIf(nn.Cell): def __init__(self): super().__init__() @@ -204,6 +217,22 @@ def test_if_by_if_by_if(): assert np.allclose(expect, output.asnumpy(), 0.0001, 0.0001) +@pytest.mark.level0 +@pytest.mark.platform_arm_ascend_training +@pytest.mark.platform_x86_ascend_training +@pytest.mark.env_onecard +def test_simple_while(): + context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") + x = np.array(True).astype(np.bool) + y = np.array(False).astype(np.bool) + input_shape = (127, 7, 53, 31) + input_data = np.random.randn(*input_shape).astype(np.float32) + net = ControlSimpleWhile() + output = net(Tensor(x), Tensor(y), Tensor(input_data)) + expect = input_data * 3 + assert np.allclose(expect, output.asnumpy(), 0.0001, 0.0001) + + @pytest.mark.level0 @pytest.mark.platform_arm_ascend_training @pytest.mark.platform_x86_ascend_training From af1bee3cbb22a3e189f5b3d13d41af4915eb3859 Mon Sep 17 00:00:00 2001 From: wuyongkang Date: Fri, 3 Jul 2020 17:20:44 +0800 Subject: [PATCH 242/254] Add record for transform status --- mindspore/ccsrc/optimizer/opt.cc | 37 +++++++++++++++++++++++++-- mindspore/ccsrc/optimizer/optimizer.h | 11 +++++++- 2 files changed, 45 insertions(+), 3 deletions(-) diff --git a/mindspore/ccsrc/optimizer/opt.cc b/mindspore/ccsrc/optimizer/opt.cc index 4c2e85157f..462d08ad3c 100644 --- a/mindspore/ccsrc/optimizer/opt.cc +++ b/mindspore/ccsrc/optimizer/opt.cc @@ -20,6 +20,7 @@ #include #include #include +#include #include "ir/anf.h" #include "ir/manager.h" @@ -191,15 +192,30 @@ bool SubstitutionList::operator()(const FuncGraphPtr &func_graph, const Optimize FuncGraphManagerPtr manager = optimizer->manager(); manager->AddFuncGraph(func_graph); + // for transform status counting + size_t space = 0; + std::unordered_map> status; + if (optimizer->is_on_debug_) { + for (size_t i = 0; i < list_.size(); i++) { + status[list_[i]->name_ + std::to_string(i)] = {}; + } + } + bool loop = false; bool changes = false; do { loop = false; - for (auto const &transform : list_) { - auto change = ApplyTransform(optimizer, func_graph->output(), transform); + for (size_t i = 0; i < list_.size(); i++) { + auto change = ApplyTransform(optimizer, func_graph->output(), list_[i]); changes = changes || change; loop = loop || change; + + // record the status of each transform + if (optimizer->is_on_debug_) { + status[list_[i]->name_ + std::to_string(i)].push_back(change); + space = std::max(list_[i]->name_.size(), space); + } } if (is_once_) { @@ -207,6 +223,23 @@ bool SubstitutionList::operator()(const FuncGraphPtr &func_graph, const Optimize } } while (loop); + // display the status of each transform + if (optimizer->is_on_debug_) { + std::stringstream ss; + ss << std::endl + << "Pass: " << optimizer->name() << "(" << optimizer->CurPass_.counter << ")_" << optimizer->CurPass_.name + << std::endl; + for (size_t i = 0; i < list_.size(); i++) { + auto name = list_[i]->name_; + ss << std::left << std::setw(space + 4) << name << "\t"; + for (auto change : status[name + std::to_string(i)]) { + ss << change << " "; + } + ss << std::endl; + } + MS_LOG(DEBUG) << ss.str(); + } + return changes; } } // namespace opt diff --git a/mindspore/ccsrc/optimizer/optimizer.h b/mindspore/ccsrc/optimizer/optimizer.h index 77d4589c5e..dc423ed314 100644 --- a/mindspore/ccsrc/optimizer/optimizer.h +++ b/mindspore/ccsrc/optimizer/optimizer.h @@ -95,6 +95,7 @@ class Optimizer : public std::enable_shared_from_this { void Init(const OptPassGroupMap &passes, bool run_only_once) { run_only_once_ = run_only_once; is_watch_renormalize_ = false; + is_on_debug_ = IS_OUTPUT_ON(mindspore::DEBUG); for (auto &iter : passes) { const std::string &name = iter.first; @@ -144,6 +145,7 @@ class Optimizer : public std::enable_shared_from_this { auto run_runc = [&counter, &func_graph, &changes, use_profile, this]() { for (size_t i = 0; i < passes_.size(); ++i) { const OptPass &opt = passes_[i]; + CurPass_ = {counter, pass_names_[i]}; auto opt_func = [&func_graph, &changes, &opt, this]() { if (opt.is_renormalize()) { auto resource_ptr = std::dynamic_pointer_cast(resource_); @@ -173,7 +175,7 @@ class Optimizer : public std::enable_shared_from_this { } }; use_profile ? (WITH(MsProfile::GetProfile()->Step(pass_names_[i])) opt_func) : opt_func(); - if (IS_OUTPUT_ON(mindspore::DEBUG) && MsContext::GetInstance()->save_graphs_flag()) { + if (is_on_debug_ && MsContext::GetInstance()->save_graphs_flag()) { MS_LOG(DEBUG) << "The opt " << name_ << " round " << counter << " OptPass " << pass_names_[i] << " end."; auto fg_name = "opt_substep_" + name_ + "_r" + std::to_string(counter) + "_" + std::to_string(i) + "_" + pass_names_[i]; @@ -217,6 +219,13 @@ class Optimizer : public std::enable_shared_from_this { bool is_watch_renormalize() { return is_watch_renormalize_; } void set_enable(bool enable) { is_enable_ = enable; } + struct { + int counter; + std::string name; + } CurPass_; + + bool is_on_debug_{false}; + private: const std::string name_; pipeline::ResourceBasePtr resource_; From 655d5174ea59abef40b75104d1232619ab0434ef Mon Sep 17 00:00:00 2001 From: yangyongjie Date: Fri, 3 Jul 2020 18:48:07 +0800 Subject: [PATCH 243/254] fix data preprocess bug --- model_zoo/yolov3_darknet53/src/transforms.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/model_zoo/yolov3_darknet53/src/transforms.py b/model_zoo/yolov3_darknet53/src/transforms.py index c654a94c0d..837d1a25ea 100644 --- a/model_zoo/yolov3_darknet53/src/transforms.py +++ b/model_zoo/yolov3_darknet53/src/transforms.py @@ -166,7 +166,7 @@ def _preprocess_true_boxes(true_boxes, anchors, in_shape, num_classes, valid_mask = boxes_wh[..., 0] > 0 wh = boxes_wh[valid_mask] - if wh: + if wh.size > 0: wh = np.expand_dims(wh, -2) boxes_max = wh / 2. boxes_min = -boxes_max @@ -319,7 +319,7 @@ def _choose_candidate_by_constraints(max_trial, input_w, input_h, image_w, image dx = int(_rand(0, input_w - nw)) dy = int(_rand(0, input_h - nh)) - if box: + if box.size > 0: t_box = copy.deepcopy(box) t_box[:, [0, 2]] = t_box[:, [0, 2]] * float(nw) / float(image_w) + dx t_box[:, [1, 3]] = t_box[:, [1, 3]] * float(nh) / float(image_h) + dy From 5a9673ee6f68fb5af268f17be8e2dca7d84b189b Mon Sep 17 00:00:00 2001 From: Danish Farid Date: Fri, 3 Jul 2020 11:17:13 -0400 Subject: [PATCH 244/254] fix_annotate bug fix --- .../python/dataset/test_bounding_box_augment.py | 16 ++++++++++------ .../test_random_crop_and_resize_with_bbox.py | 16 ++++++++++------ .../python/dataset/test_random_crop_with_bbox.py | 16 ++++++++++------ .../test_random_horizontal_flip_with_bbox.py | 16 ++++++++++------ .../dataset/test_random_resize_with_bbox.py | 6 +++++- .../test_random_vertical_flip_with_bbox.py | 16 ++++++++++------ tests/ut/python/dataset/test_resize_with_bbox.py | 6 +++++- 7 files changed, 60 insertions(+), 32 deletions(-) diff --git a/tests/ut/python/dataset/test_bounding_box_augment.py b/tests/ut/python/dataset/test_bounding_box_augment.py index fcdfbe7928..fbcb56514f 100644 --- a/tests/ut/python/dataset/test_bounding_box_augment.py +++ b/tests/ut/python/dataset/test_bounding_box_augment.py @@ -34,12 +34,16 @@ def fix_annotate(bboxes): :return: annotation in [x_min, y_min, w, h, label, truncate, difficult] format """ for bbox in bboxes: - tmp = bbox[0] - bbox[0] = bbox[1] - bbox[1] = bbox[2] - bbox[2] = bbox[3] - bbox[3] = bbox[4] - bbox[4] = tmp + if bbox.size == 7: + tmp = bbox[0] + bbox[0] = bbox[1] + bbox[1] = bbox[2] + bbox[2] = bbox[3] + bbox[3] = bbox[4] + bbox[4] = tmp + else: + print("ERROR: Invalid Bounding Box size provided") + break return bboxes diff --git a/tests/ut/python/dataset/test_random_crop_and_resize_with_bbox.py b/tests/ut/python/dataset/test_random_crop_and_resize_with_bbox.py index 359b527dd1..b13dc466f7 100644 --- a/tests/ut/python/dataset/test_random_crop_and_resize_with_bbox.py +++ b/tests/ut/python/dataset/test_random_crop_and_resize_with_bbox.py @@ -36,12 +36,16 @@ def fix_annotate(bboxes): :return: annotation in [x_min, y_min, w, h, label, truncate, difficult] format """ for bbox in bboxes: - tmp = bbox[0] - bbox[0] = bbox[1] - bbox[1] = bbox[2] - bbox[2] = bbox[3] - bbox[3] = bbox[4] - bbox[4] = tmp + if bbox.size == 7: + tmp = bbox[0] + bbox[0] = bbox[1] + bbox[1] = bbox[2] + bbox[2] = bbox[3] + bbox[3] = bbox[4] + bbox[4] = tmp + else: + print("ERROR: Invalid Bounding Box size provided") + break return bboxes diff --git a/tests/ut/python/dataset/test_random_crop_with_bbox.py b/tests/ut/python/dataset/test_random_crop_with_bbox.py index 08233cb3bc..9262dfd65d 100644 --- a/tests/ut/python/dataset/test_random_crop_with_bbox.py +++ b/tests/ut/python/dataset/test_random_crop_with_bbox.py @@ -37,12 +37,16 @@ def fix_annotate(bboxes): :return: annotation in [x_min, y_min, w, h, label, truncate, difficult] format """ for bbox in bboxes: - tmp = bbox[0] - bbox[0] = bbox[1] - bbox[1] = bbox[2] - bbox[2] = bbox[3] - bbox[3] = bbox[4] - bbox[4] = tmp + if bbox.size == 7: + tmp = bbox[0] + bbox[0] = bbox[1] + bbox[1] = bbox[2] + bbox[2] = bbox[3] + bbox[3] = bbox[4] + bbox[4] = tmp + else: + print("ERROR: Invalid Bounding Box size provided") + break return bboxes diff --git a/tests/ut/python/dataset/test_random_horizontal_flip_with_bbox.py b/tests/ut/python/dataset/test_random_horizontal_flip_with_bbox.py index 37e9fa710f..94ab843ce1 100644 --- a/tests/ut/python/dataset/test_random_horizontal_flip_with_bbox.py +++ b/tests/ut/python/dataset/test_random_horizontal_flip_with_bbox.py @@ -34,12 +34,16 @@ def fix_annotate(bboxes): :return: annotation in [x_min, y_min, w, h, label, truncate, difficult] format """ for bbox in bboxes: - tmp = bbox[0] - bbox[0] = bbox[1] - bbox[1] = bbox[2] - bbox[2] = bbox[3] - bbox[3] = bbox[4] - bbox[4] = tmp + if bbox.size == 7: + tmp = bbox[0] + bbox[0] = bbox[1] + bbox[1] = bbox[2] + bbox[2] = bbox[3] + bbox[3] = bbox[4] + bbox[4] = tmp + else: + print("ERROR: Invalid Bounding Box size provided") + break return bboxes diff --git a/tests/ut/python/dataset/test_random_resize_with_bbox.py b/tests/ut/python/dataset/test_random_resize_with_bbox.py index b0c1c40a80..4aadf9ef01 100644 --- a/tests/ut/python/dataset/test_random_resize_with_bbox.py +++ b/tests/ut/python/dataset/test_random_resize_with_bbox.py @@ -35,7 +35,11 @@ def fix_annotate(bboxes): :return: annotation in [x_min, y_min, w, h, label, truncate, difficult] format """ for (i, box) in enumerate(bboxes): - bboxes[i] = np.roll(box, -1) + if box.size == 7: + bboxes[i] = np.roll(box, -1) + else: + print("ERROR: Invalid Bounding Box size provided") + break return bboxes diff --git a/tests/ut/python/dataset/test_random_vertical_flip_with_bbox.py b/tests/ut/python/dataset/test_random_vertical_flip_with_bbox.py index 72c40c0cad..f746bd50b0 100644 --- a/tests/ut/python/dataset/test_random_vertical_flip_with_bbox.py +++ b/tests/ut/python/dataset/test_random_vertical_flip_with_bbox.py @@ -36,12 +36,16 @@ def fix_annotate(bboxes): :return: annotation in [x_min, y_min, w, h, label, truncate, difficult] format """ for bbox in bboxes: - tmp = bbox[0] - bbox[0] = bbox[1] - bbox[1] = bbox[2] - bbox[2] = bbox[3] - bbox[3] = bbox[4] - bbox[4] = tmp + if bbox.size == 7: + tmp = bbox[0] + bbox[0] = bbox[1] + bbox[1] = bbox[2] + bbox[2] = bbox[3] + bbox[3] = bbox[4] + bbox[4] = tmp + else: + print("ERROR: Invalid Bounding Box size provided") + break return bboxes diff --git a/tests/ut/python/dataset/test_resize_with_bbox.py b/tests/ut/python/dataset/test_resize_with_bbox.py index 75500de653..06f3937958 100644 --- a/tests/ut/python/dataset/test_resize_with_bbox.py +++ b/tests/ut/python/dataset/test_resize_with_bbox.py @@ -35,7 +35,11 @@ def fix_annotate(bboxes): :return: annotation in [x_min, y_min, w, h, label, truncate, difficult] format """ for (i, box) in enumerate(bboxes): - bboxes[i] = np.roll(box, -1) + if box.size == 7: + bboxes[i] = np.roll(box, -1) + else: + print("ERROR: Invalid Bounding Box size provided") + break return bboxes From dc030192009c65d146cd9928946aa8451e0f3f7f Mon Sep 17 00:00:00 2001 From: tinazhang Date: Wed, 17 Jun 2020 16:31:42 -0400 Subject: [PATCH 245/254] unskip md5 testcase for RandomPerspective --- .../golden/random_perspective_01_result.npz | Bin 644 -> 644 bytes .../python/dataset/test_random_perspective.py | 5 +++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/ut/data/dataset/golden/random_perspective_01_result.npz b/tests/ut/data/dataset/golden/random_perspective_01_result.npz index 2f3abd2230d450a24ee575c91fe689373e612b83..7a1d3f5562886569d729934ffb0a60136eeea51a 100644 GIT binary patch delta 99 zcmZo+ZDAD(@MdNaVSoTdh8rET=WY~Qz$n1A>gan1hNnM%``N_SZ|pxa`4FRuKxW;J y1CQ_Yrn#=#`u&v5gYL=SfP&tevh~g-)wM1CRO47Hs~a|1i%FIRWX@!7CQASxK`09V delta 99 zcmZo+ZDAD(@MdNaVSoTd2HOor*ER|*U=(1CI=Ud}i@m0EZM@f;_SH#~4>76;EMn$9 yQ{3@)f8@1iQa|EucuoEW6ihGK?H$+tLEXwT`)Wb0&K;Spoo4Hzc Date: Fri, 3 Jul 2020 13:42:11 -0400 Subject: [PATCH 246/254] Debt Task --- .../dataset/kernels/image/image_utils.cc | 34 ++++++++++++------- 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/mindspore/ccsrc/dataset/kernels/image/image_utils.cc b/mindspore/ccsrc/dataset/kernels/image/image_utils.cc index 9a20743e6c..ded9a8db11 100644 --- a/mindspore/ccsrc/dataset/kernels/image/image_utils.cc +++ b/mindspore/ccsrc/dataset/kernels/image/image_utils.cc @@ -108,7 +108,8 @@ Status Resize(const std::shared_ptr &input, std::shared_ptr *out } try { TensorShape shape{output_height, output_width}; - if (input_cv->Rank() == 3) shape = shape.AppendDim(input_cv->shape()[2]); + int num_channels = input_cv->shape()[2]; + if (input_cv->Rank() == 3) shape = shape.AppendDim(num_channels); std::shared_ptr output_cv = std::make_shared(shape, input_cv->type()); RETURN_UNEXPECTED_IF_NULL(output_cv); auto cv_mode = GetCVInterpolationMode(mode); @@ -351,7 +352,8 @@ Status Crop(const std::shared_ptr &input, std::shared_ptr *outpu } try { TensorShape shape{h, w}; - if (input_cv->Rank() == 3) shape = shape.AppendDim(input_cv->shape()[2]); + int num_channels = input_cv->shape()[2]; + if (input_cv->Rank() == 3) shape = shape.AppendDim(num_channels); std::shared_ptr output_cv = std::make_shared(shape, input_cv->type()); RETURN_UNEXPECTED_IF_NULL(output_cv); cv::Rect roi(x, y, w, h); @@ -374,15 +376,15 @@ Status HwcToChw(std::shared_ptr input, std::shared_ptr *output) *output = input; return Status::OK(); } + int num_channels = input_cv->shape()[2]; if (input_cv->shape().Size() < 2 || input_cv->shape().Size() > 3 || - (input_cv->shape().Size() == 3 && input_cv->shape()[2] != 3 && input_cv->shape()[2] != 1)) { + (input_cv->shape().Size() == 3 && num_channels != 3 && num_channels != 1)) { RETURN_STATUS_UNEXPECTED("The shape is incorrect: number of channels does not equal 3 nor 1"); } cv::Mat output_img; int height = input_cv->shape()[0]; int width = input_cv->shape()[1]; - int num_channels = input_cv->shape()[2]; auto output_cv = std::make_unique(TensorShape{num_channels, height, width}, input_cv->type()); for (int i = 0; i < num_channels; ++i) { @@ -400,7 +402,8 @@ Status HwcToChw(std::shared_ptr input, std::shared_ptr *output) Status SwapRedAndBlue(std::shared_ptr input, std::shared_ptr *output) { try { std::shared_ptr input_cv = CVTensor::AsCVTensor(std::move(input)); - if (input_cv->shape().Size() != 3 || input_cv->shape()[2] != 3) { + int num_channels = input_cv->shape()[2]; + if (input_cv->shape().Size() != 3 || num_channels != 3) { RETURN_STATUS_UNEXPECTED("The shape is incorrect: number of channels does not equal 3"); } auto output_cv = std::make_shared(input_cv->shape(), input_cv->type()); @@ -435,7 +438,8 @@ Status CropAndResize(const std::shared_ptr &input, std::shared_ptrmat(); TensorShape shape{target_height, target_width}; - if (input_cv->Rank() == 3) shape = shape.AppendDim(input_cv->shape()[2]); + int num_channels = input_cv->shape()[2]; + if (input_cv->Rank() == 3) shape = shape.AppendDim(num_channels); std::shared_ptr cvt_out = std::make_shared(shape, input_cv->type()); RETURN_UNEXPECTED_IF_NULL(cvt_out); cv::resize(cv_in(roi), cvt_out->mat(), cv::Size(target_width, target_height), 0, 0, cv_mode); @@ -540,7 +544,8 @@ Status AdjustBrightness(const std::shared_ptr &input, std::shared_ptrmat().data) { RETURN_STATUS_UNEXPECTED("Could not convert to CV Tensor"); } - if (input_cv->Rank() != 3 || input_cv->shape()[2] != 3) { + int num_channels = input_cv->shape()[2]; + if (input_cv->Rank() != 3 || num_channels != 3) { RETURN_STATUS_UNEXPECTED("The shape is incorrect: number of channels does not equal 3"); } auto output_cv = std::make_shared(input_cv->shape(), input_cv->type()); @@ -560,7 +565,8 @@ Status AdjustContrast(const std::shared_ptr &input, std::shared_ptrmat().data) { RETURN_STATUS_UNEXPECTED("Could not convert to CV Tensor"); } - if (input_cv->Rank() != 3 || input_cv->shape()[2] != 3) { + int num_channels = input_cv->shape()[2]; + if (input_cv->Rank() != 3 || num_channels != 3) { RETURN_STATUS_UNEXPECTED("The shape is incorrect: number of channels does not equal 3"); } cv::Mat gray, output_img; @@ -586,7 +592,8 @@ Status AdjustSaturation(const std::shared_ptr &input, std::shared_ptrmat().data) { RETURN_STATUS_UNEXPECTED("Could not convert to CV Tensor"); } - if (input_cv->Rank() != 3 || input_cv->shape()[2] != 3) { + int num_channels = input_cv->shape()[2]; + if (input_cv->Rank() != 3 || num_channels != 3) { RETURN_STATUS_UNEXPECTED("The shape is incorrect: number of channels does not equal 3"); } auto output_cv = std::make_shared(input_cv->shape(), input_cv->type()); @@ -614,7 +621,8 @@ Status AdjustHue(const std::shared_ptr &input, std::shared_ptr * if (!input_cv->mat().data) { RETURN_STATUS_UNEXPECTED("Could not convert to CV Tensor"); } - if (input_cv->Rank() != 3 || input_cv->shape()[2] != 3) { + int num_channels = input_cv->shape()[2]; + if (input_cv->Rank() != 3 || num_channels != 3) { RETURN_STATUS_UNEXPECTED("The shape is incorrect: number of channels does not equal 3"); } auto output_cv = std::make_shared(input_cv->shape(), input_cv->type()); @@ -643,7 +651,8 @@ Status Erase(const std::shared_ptr &input, std::shared_ptr *outp uint8_t fill_g, uint8_t fill_b) { try { std::shared_ptr input_cv = CVTensor::AsCVTensor(input); - if (input_cv->mat().data == nullptr || input_cv->Rank() != 3 || input_cv->shape()[2] != 3) { + int num_channels = input_cv->shape()[2]; + if (input_cv->mat().data == nullptr || input_cv->Rank() != 3 || num_channels != 3) { RETURN_STATUS_UNEXPECTED("bad CV Tensor input for erase"); } cv::Mat input_img = input_cv->mat(); @@ -717,7 +726,8 @@ Status Pad(const std::shared_ptr &input, std::shared_ptr *output std::shared_ptr output_cv = std::make_shared(out_image); RETURN_UNEXPECTED_IF_NULL(output_cv); // pad the dimension if shape information is only 2 dimensional, this is grayscale - if (input_cv->Rank() == 3 && input_cv->shape()[2] == 1 && output_cv->Rank() == 2) output_cv->ExpandDim(2); + int num_channels = input_cv->shape()[2]; + if (input_cv->Rank() == 3 && num_channels == 1 && output_cv->Rank() == 2) output_cv->ExpandDim(2); *output = std::static_pointer_cast(output_cv); return Status::OK(); From 0ee568b733b030057e0314c60a34248199275cb2 Mon Sep 17 00:00:00 2001 From: ougongchang Date: Sat, 4 Jul 2020 09:43:26 +0800 Subject: [PATCH 247/254] Update the Api document of SummaryCollector and SummaryRecord. Add more detail note for SummaryCollector and SummaryRecord, else if it is used not right, some proplem will be caused. --- .../train/callback/_summary_collector.py | 1 + mindspore/train/summary/summary_record.py | 28 ++++++++++--------- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/mindspore/train/callback/_summary_collector.py b/mindspore/train/callback/_summary_collector.py index c77a41fedc..1550c3c55c 100644 --- a/mindspore/train/callback/_summary_collector.py +++ b/mindspore/train/callback/_summary_collector.py @@ -83,6 +83,7 @@ class SummaryCollector(Callback): The data that supports control is shown below. - collect_metric: Whether to collect training metrics, currently only loss is collected. + The first output will be treated as loss, and it will be averaged. Optional: True/False. Default: True. - collect_graph: Whether to collect computational graph, currently only training computational graph is collected. Optional: True/False. Default: True. diff --git a/mindspore/train/summary/summary_record.py b/mindspore/train/summary/summary_record.py index 065d281355..21c8c58d3b 100644 --- a/mindspore/train/summary/summary_record.py +++ b/mindspore/train/summary/summary_record.py @@ -62,14 +62,16 @@ def _dictlist(): class SummaryRecord: """ SummaryRecord is used to record the summary data and lineage data. - The API will create a summary file and a lineage file lazily in a given directory and writes data to them. - It writes the data to files by executing the record method. In addition to record the data bubbled up from + + The API will create a summary file and lineage files lazily in a given directory and writes data to them. + It writes the data to files by executing the 'record' method. In addition to record the data bubbled up from the network by defining the summary operators, SummaryRecord also supports to record extra data which can be added by calling add_value. Note: - Make sure to close the SummaryRecord at the end, or the process will NOT exit. - Please see the Example section below on how to properly close with two ways. + 1. Make sure to close the SummaryRecord at the end, or the process will not exit. + Please see the Example section below on how to properly close with two ways. + 2. The SummaryRecord instance can only allow one at a time, otherwise it will cause problems with data writes. Args: log_dir (str): The log_dir is a directory location to save the summary. @@ -85,12 +87,12 @@ class SummaryRecord: Examples: >>> # use in with statement to auto close - >>> with SummaryRecord(log_dir="/opt/log", file_prefix="xxx_", file_suffix="_yyy") as summary_record: + >>> with SummaryRecord(log_dir="./summary_dir") as summary_record: >>> pass >>> >>> # use in try .. finally .. to ensure closing >>> try: - >>> summary_record = SummaryRecord(log_dir="/opt/log") + >>> summary_record = SummaryRecord(log_dir="./summary_dir") >>> finally: >>> summary_record.close() """ @@ -165,7 +167,7 @@ class SummaryRecord: ValueError: When the mode is not recognized. Examples: - >>> with SummaryRecord(log_dir="/opt/log", file_prefix="xxx_", file_suffix="_yyy") as summary_record: + >>> with SummaryRecord(log_dir="./summary_dir", file_prefix="xxx_", file_suffix="_yyy") as summary_record: >>> summary_record.set_mode('eval') """ mode_spec = 'train', 'eval' @@ -204,7 +206,7 @@ class SummaryRecord: TypeError: When the value is not a Tensor. Examples: - >>> with SummaryRecord(log_dir="/opt/log", file_prefix="xxx_", file_suffix="_yyy") as summary_record: + >>> with SummaryRecord(log_dir="./summary_dir", file_prefix="xxx_", file_suffix="_yyy") as summary_record: >>> summary_record.add_value('scalar', 'loss', Tensor(0.1)) """ if plugin in ('tensor', 'scalar', 'image', 'histogram'): @@ -239,10 +241,10 @@ class SummaryRecord: bool, whether the record process is successful or not. Examples: - >>> with SummaryRecord(log_dir="/opt/log", file_prefix="xxx_", file_suffix="_yyy") as summary_record: + >>> with SummaryRecord(log_dir="./summary_dir", file_prefix="xxx_", file_suffix="_yyy") as summary_record: >>> summary_record.record(step=2) """ - logger.info("SummaryRecord step is %r.", step) + logger.debug("SummaryRecord step is %r.", step) if self._closed: logger.error("The record writer is closed.") return False @@ -296,7 +298,7 @@ class SummaryRecord: str, the full path of log file. Examples: - >>> with SummaryRecord(log_dir="/opt/log", file_prefix="xxx_", file_suffix="_yyy") as summary_record: + >>> with SummaryRecord(log_dir="./summary_dir", file_prefix="xxx_", file_suffix="_yyy") as summary_record: >>> print(summary_record.log_dir) """ return self.full_file_name @@ -308,7 +310,7 @@ class SummaryRecord: Call it to make sure that all pending events have been written to disk. Examples: - >>> with SummaryRecord(log_dir="/opt/log", file_prefix="xxx_", file_suffix="_yyy") as summary_record: + >>> with SummaryRecord(log_dir="./summary_dir", file_prefix="xxx_", file_suffix="_yyy") as summary_record: >>> summary_record.flush() """ if self._closed: @@ -322,7 +324,7 @@ class SummaryRecord: Examples: >>> try: - >>> summary_record = SummaryRecord(log_dir="/opt/log") + >>> summary_record = SummaryRecord(log_dir="./summary_dir") >>> finally: >>> summary_record.close() """ From 1f8b00dff2136dc119694fe2c42680403fe6641f Mon Sep 17 00:00:00 2001 From: zhoufeng Date: Fri, 3 Jul 2020 16:11:44 +0800 Subject: [PATCH 248/254] Fix empty graph dump ir Signed-off-by: zhoufeng --- mindspore/ccsrc/session/anf_runtime_algorithm.cc | 5 ++++- mindspore/ccsrc/session/ascend_control_parser.cc | 13 +++++++------ mindspore/ccsrc/session/ascend_control_parser.h | 2 +- mindspore/ccsrc/session/ascend_session.cc | 3 +++ mindspore/ccsrc/session/kernel_graph.cc | 10 ++++------ tests/ut/cpp/session/anf_runtime_algorithm_test.cc | 3 ++- 6 files changed, 21 insertions(+), 15 deletions(-) diff --git a/mindspore/ccsrc/session/anf_runtime_algorithm.cc b/mindspore/ccsrc/session/anf_runtime_algorithm.cc index 5f896282dc..81ad02e787 100644 --- a/mindspore/ccsrc/session/anf_runtime_algorithm.cc +++ b/mindspore/ccsrc/session/anf_runtime_algorithm.cc @@ -694,7 +694,7 @@ void AnfRuntimeAlgorithm::SetOutputInferTypeAndShape(const std::vector & MS_LOG(EXCEPTION) << "Types size " << types.size() << "should be same with shapes size " << shapes.size(); } if (shapes.empty()) { - MS_LOG(EXCEPTION) << "Illegal empty output_types_shapes"; + node->set_abstract(std::make_shared()); } else if (shapes.size() == 1) { // single output handle std::vector shape_int; @@ -1012,6 +1012,9 @@ std::vector AnfRuntimeAlgorithm::GetCallNodeKernelGraph(const CN auto get_switch_kernel_graph = [switch_node](size_t input_index) -> KernelGraphPtr { auto partial = switch_node->input(input_index); MS_EXCEPTION_IF_NULL(partial); + if (IsValueNode(partial)) { + return GetValueNode(partial); + } auto partial_cnode = partial->cast(); MS_EXCEPTION_IF_NULL(partial_cnode); auto graph_node = partial_cnode->input(1); diff --git a/mindspore/ccsrc/session/ascend_control_parser.cc b/mindspore/ccsrc/session/ascend_control_parser.cc index 166d4cc97a..56f9550b91 100644 --- a/mindspore/ccsrc/session/ascend_control_parser.cc +++ b/mindspore/ccsrc/session/ascend_control_parser.cc @@ -386,8 +386,7 @@ void AscendControlParser::RecurseSwitch(NotNull kg, NotNull kg, NotNull origin_switch_inputs[kCNodeSwitchCond]}; for (size_t i = 0; i < branch_partial.size(); ++i) { // 3.1 branch kernel graph and args - KernelGraphPtr branch_fg; - std::tie(std::ignore, branch_fg) = ParsePartial(NOT_NULL(origin_switch_inputs[i])); + KernelGraphPtr branch_fg = ParsePartial(NOT_NULL(origin_switch_inputs[i])); // 3.2 recurse sub graph CNodePtr branch_label = ProcessKernelGraph(NOT_NULL(branch_fg), cur_node, back_label, memo); new_switch_inputs.push_back(branch_label); @@ -444,8 +442,11 @@ void AscendControlParser::RecurseSwitchLayer(NotNull kg, NotNull MS_LOG(INFO) << "Succeed processing switch layer " << cur_node->DebugString(); } -std::tuple AscendControlParser::ParsePartial(NotNull node) { +KernelGraphPtr AscendControlParser::ParsePartial(NotNull node) { if (!node.get()->isa()) { + if (IsValueNode(node)) { + return GetValueNode(node); + } MS_LOG(EXCEPTION) << "Switch branches must be partial, node: " << node->DebugString(); } // 2.1 branch kernel graph and args @@ -460,7 +461,7 @@ std::tuple AscendControlParser::ParsePartial(NotNull(partial_inputs[kCNodePartialFunc]); - return {partial_cnode, branch_kg}; + return branch_kg; } void AscendControlParser::InsertMultipleAssignToGraph(NotNull from_graph, diff --git a/mindspore/ccsrc/session/ascend_control_parser.h b/mindspore/ccsrc/session/ascend_control_parser.h index 2b383d7b14..ba1217c38d 100644 --- a/mindspore/ccsrc/session/ascend_control_parser.h +++ b/mindspore/ccsrc/session/ascend_control_parser.h @@ -52,7 +52,7 @@ class AscendControlParser { static void LinkParentGraph(NotNull kg, const CNodePtr &from_graph_call_node, const CNodePtr &last_label); - static std::tuple ParsePartial(NotNull node); + static KernelGraphPtr ParsePartial(NotNull node); static void InsertMultipleAssignToGraph(NotNull from_graph, NotNull to_graph, NotNull from, NotNull to); diff --git a/mindspore/ccsrc/session/ascend_session.cc b/mindspore/ccsrc/session/ascend_session.cc index 01b5691029..c0bb5d4b12 100644 --- a/mindspore/ccsrc/session/ascend_session.cc +++ b/mindspore/ccsrc/session/ascend_session.cc @@ -247,6 +247,9 @@ static void UpdateRealInput(NotNull graph, bool split_flag) { MS_EXCEPTION_IF_NULL(switch_cnode); auto partial = switch_cnode->input(input_index); MS_EXCEPTION_IF_NULL(partial); + if (IsValueNode(partial)) { + return {}; + } auto partial_cnode = partial->cast(); MS_EXCEPTION_IF_NULL(partial_cnode); auto ret = std::vector(partial_cnode->inputs().begin() + 2, partial_cnode->inputs().end()); diff --git a/mindspore/ccsrc/session/kernel_graph.cc b/mindspore/ccsrc/session/kernel_graph.cc index 7b53afac2a..13c5a64ae2 100644 --- a/mindspore/ccsrc/session/kernel_graph.cc +++ b/mindspore/ccsrc/session/kernel_graph.cc @@ -357,18 +357,16 @@ ParameterPtr KernelGraph::NewParameter(const ParameterPtr ¶meter) { } else { kernel_info->SetFeatureMapFlag(true); } - // if output is a tuple tensor,now can use for loop to handle tuple tensor - output_tensor_num = AnfAlgo::GetOutputTensorNum(parameter); } new_parameter->set_kernel_info(kernel_info); // create kernel_build_info for new parameter auto kernel_build_info_builder = std::make_shared(); // create init data type, std::vector init_data_type = {}; - for (size_t i = 0; i < output_tensor_num; i++) { - TypeId infer_data_type = AnfAlgo::GetOutputInferDataType(new_parameter, i); - init_data_type.push_back(AnfAlgo::IsParameterWeight(new_parameter) ? kTypeUnknown : infer_data_type); - } + + TypeId infer_data_type = AnfAlgo::GetOutputInferDataType(new_parameter, 0); + init_data_type.push_back(AnfAlgo::IsParameterWeight(new_parameter) ? kTypeUnknown : infer_data_type); + // set the format of parameter to DEFAULT_FORMAT kernel_build_info_builder->SetOutputsFormat(std::vector(output_tensor_num, kOpFormat_DEFAULT)); // set parameter initaial device data type diff --git a/tests/ut/cpp/session/anf_runtime_algorithm_test.cc b/tests/ut/cpp/session/anf_runtime_algorithm_test.cc index 2ea2453381..4c94cdde57 100644 --- a/tests/ut/cpp/session/anf_runtime_algorithm_test.cc +++ b/tests/ut/cpp/session/anf_runtime_algorithm_test.cc @@ -590,7 +590,8 @@ TEST_F(AnfRuntimeAlgorithmTest, SetOutputInferTypeAndShape) { std::vector none_types = {}; std::vector> none_shapes = {}; EXPECT_THROW(AnfAlgo::SetOutputInferTypeAndShape(none_types, none_shapes, nullptr), std::runtime_error); - EXPECT_THROW(AnfAlgo::SetOutputInferTypeAndShape(none_types, none_shapes, add.get()), std::runtime_error); + AnfAlgo::SetOutputInferTypeAndShape(none_types, none_shapes, add.get()); + EXPECT_EQ((*add->abstract()), abstract::AbstractNone()); // set single input std::vector single_types = {kFloat32->type_id()}; std::vector> single_shapes = {{2, 32, 224, 224}}; From 5cccfbc61ba4e67de63eecacd564373b7ddb0e3a Mon Sep 17 00:00:00 2001 From: leilei_snow Date: Fri, 8 May 2020 15:43:17 +0800 Subject: [PATCH 249/254] Add TransShape Operator. --- mindspore/ccsrc/transform/convert.cc | 2 ++ mindspore/ccsrc/transform/op_declare.cc | 6 ++++++ mindspore/ccsrc/transform/op_declare.h | 3 +++ mindspore/ops/_grad/grad_array_ops.py | 10 ++++++++++ mindspore/ops/operations/__init__.py | 2 +- mindspore/ops/operations/array_ops.py | 25 +++++++++++++++++++++++++ tests/ut/python/ops/test_ops.py | 6 ++++++ 7 files changed, 53 insertions(+), 1 deletion(-) diff --git a/mindspore/ccsrc/transform/convert.cc b/mindspore/ccsrc/transform/convert.cc index 3f6b31303c..f88e31fcd2 100644 --- a/mindspore/ccsrc/transform/convert.cc +++ b/mindspore/ccsrc/transform/convert.cc @@ -134,6 +134,7 @@ const char kNameAssignSub[] = "AssignSub"; const char kNameNPUAllocFloatStatus[] = "NPUAllocFloatStatus"; const char kNameNPUClearFloatStatus[] = "NPUClearFloatStatus"; const char kNameReshape[] = "Reshape"; +const char kNameTransShape[] = "TransShape"; const char kNameRealDiv[] = "RealDiv"; const char kNameTile[] = "Tile"; const char kNameCos[] = "Cos"; @@ -242,6 +243,7 @@ std::unordered_map &DfGraphConvertor::get_adpt_ma {string(kNameBatchNorm), ADPT_DESC(BatchNorm)}, {string(kNameBatchNormGrad), ADPT_DESC(BatchNormGrad)}, {string(kNameReshape), ADPT_DESC(Reshape)}, + {string(kNameTransShape), ADPT_DESC(TransShape)}, {string(kNameFlattenGrad), ADPT_DESC(Reshape)}, {prim::kPrimFlatten->name(), ADPT_DESC(Flatten)}, {string(kNameAddN), ADPT_DESC(AddN)}, diff --git a/mindspore/ccsrc/transform/op_declare.cc b/mindspore/ccsrc/transform/op_declare.cc index cac526f1fb..fd8ce624a9 100644 --- a/mindspore/ccsrc/transform/op_declare.cc +++ b/mindspore/ccsrc/transform/op_declare.cc @@ -442,6 +442,12 @@ INPUT_MAP(Reshape) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(shape)}}; ATTR_MAP(Reshape) = EMPTY_ATTR_MAP; OUTPUT_MAP(Reshape) = {{0, OUTPUT_DESC(y)}}; +// TransShape +INPUT_MAP(TransShape) = {{1, INPUT_DESC(x)}}; +INPUT_ATTR_MAP(TransShape) = {{2, ATTR_DESC(outShape, AnyTraits(), AnyTraits>())}}; +ATTR_MAP(TransShape) = EMPTY_ATTR_MAP; +OUTPUT_MAP(TransShape) = {{0, OUTPUT_DESC(y)}}; + // BiasAdd INPUT_MAP(BiasAdd) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(bias)}}; ATTR_MAP(BiasAdd) = {{"data_format", ATTR_DESC(data_format, AnyTraits())}}; diff --git a/mindspore/ccsrc/transform/op_declare.h b/mindspore/ccsrc/transform/op_declare.h index f64dc7b671..baa819f71f 100755 --- a/mindspore/ccsrc/transform/op_declare.h +++ b/mindspore/ccsrc/transform/op_declare.h @@ -112,6 +112,9 @@ DECLARE_OP_USE_INPUT_ATTR(DepthwiseConv2DBackpropInputD) DECLARE_OP_USE_OUTPUT(DepthwiseConv2DBackpropInputD) DECLARE_OP_ADAPTER(Reshape) DECLARE_OP_USE_OUTPUT(Reshape) +DECLARE_OP_ADAPTER(TransShape) +DECLARE_OP_USE_INPUT_ATTR(TransShape) +DECLARE_OP_USE_OUTPUT(TransShape) DECLARE_OP_ADAPTER(Iou) DECLARE_OP_USE_OUTPUT(Iou) DECLARE_OP_ADAPTER(ResizeNearestNeighborV2D) diff --git a/mindspore/ops/_grad/grad_array_ops.py b/mindspore/ops/_grad/grad_array_ops.py index a2a808781e..e216a4f0d0 100644 --- a/mindspore/ops/_grad/grad_array_ops.py +++ b/mindspore/ops/_grad/grad_array_ops.py @@ -696,3 +696,13 @@ def get_bprop_reverse_sequence(self): dx = reverse_sequence_grad(dout, seq_lengths) return dx, zeros_like(seq_lengths) return bprop + + +@bprop_getters.register(P.TransShape) +def get_bprop_trans_shape(self): + """Generate bprop for TransShape""" + op = P.TransShape() + def bprop(x, shape, out, dout): + dx = op(dout, shape_op(x)) + return (dx, zeros_like(shape)) + return bprop diff --git a/mindspore/ops/operations/__init__.py b/mindspore/ops/operations/__init__.py index b2d0fc7382..9053abab06 100644 --- a/mindspore/ops/operations/__init__.py +++ b/mindspore/ops/operations/__init__.py @@ -27,7 +27,7 @@ from .array_ops import (Argmax, Argmin, Cast, Concat, Pack, Unpack, Rank, Reshape, ResizeNearestNeighbor, ArgMinWithValue, SameTypeShape, ScatterAdd, ScatterSub, ScatterMul, ScatterDiv, ScatterMax, ScatterMin, ScatterUpdate, ScalarToArray, ScalarToTensor, ScatterNd, ScatterNdUpdate, Select, - Shape, Size, Slice, Split, + Shape, Size, Slice, Split, TransShape, Squeeze, StridedSlice, Tile, TensorScatterUpdate, Transpose, TruncatedNormal, TupleToArray, UnsortedSegmentMin, UnsortedSegmentSum, SpaceToDepth, DepthToSpace, SpaceToBatch, BatchToSpace, diff --git a/mindspore/ops/operations/array_ops.py b/mindspore/ops/operations/array_ops.py index 8e9ecfea95..70665d3367 100644 --- a/mindspore/ops/operations/array_ops.py +++ b/mindspore/ops/operations/array_ops.py @@ -3106,3 +3106,28 @@ class ReverseSequence(PrimitiveWithInfer): validator.check_tensor_type_same({"x_dtype": x}, mstype.number_type + (mstype.bool_,), self.name) validator.check_tensor_type_same({"seq_lengths_dtype": seq_lengths}, [mstype.int32, mstype.int64], self.name) return x + + +class TransShape(PrimitiveWithInfer): + """ + Transform the shape of input tensor to target shape. + + Inputs: + - **input_x** (Tensor) - A input tensor. + - **out_shape** (tuple[int]) - The shape of output data. + + Outputs: + Tensor, a tensor whose data type is same as 'input_x', and the shape is same as the `out_shape`. + """ + @prim_attr_register + def __init__(self): + self.__setattr_flag__ = True + + def __infer__(self, x, shape): + shp = shape['value'] + dtype = x['dtype'] + validator.check_tensor_type_same({'x': dtype}, mstype.number_type + (mstype.bool_,), self.name) + self.add_prim_attr('out_shape', tuple(shp)) + return {'shape': shp, + 'dtype': dtype, + 'value': None} diff --git a/tests/ut/python/ops/test_ops.py b/tests/ut/python/ops/test_ops.py index 5262145c80..c4330e3f65 100755 --- a/tests/ut/python/ops/test_ops.py +++ b/tests/ut/python/ops/test_ops.py @@ -1865,6 +1865,12 @@ test_case_array_ops = [ Tensor(np.arange(-12, 0).reshape(3, 2, 2), mstype.float32)], 'skip': ['backward'], }), + ('TransShape', { + 'block': P.TransShape(), + 'desc_const': [(1, 12, 24, 24)], + 'desc_inputs': [[1, 3, 24, 24]], + 'desc_bprop': [[1, 12, 24, 24]], + }), ] test_case_other_ops = [ From 5897793b01697bf1ca23fae0b760efcdd4e60301 Mon Sep 17 00:00:00 2001 From: liuwenhao4 Date: Fri, 3 Jul 2020 21:37:04 +0800 Subject: [PATCH 250/254] Fixing some tiny mistake of InplaceAdd, InplaceSub and InplaceUpdate vm ops and apply new character of dynamic format --- mindspore/ops/_op_impl/tbe/accumulate_n_v2.py | 10 ++++---- .../ops/_op_impl/tbe/approximate_equal.py | 6 ++--- .../ops/_op_impl/tbe/binary_cross_entropy.py | 6 ++--- mindspore/ops/_op_impl/tbe/lin_space.py | 4 ++-- mindspore/ops/_op_impl/tbe/mod.py | 16 +++++-------- mindspore/ops/_op_impl/tbe/reduce_mean_d.py | 9 ++++---- mindspore/ops/_op_impl/tbe/softsign.py | 4 ++-- mindspore/ops/_op_impl/tbe/splitv.py | 23 +------------------ mindspore/ops/operations/array_ops.py | 2 +- mindspore/ops/operations/math_ops.py | 4 ++-- 10 files changed, 28 insertions(+), 56 deletions(-) diff --git a/mindspore/ops/_op_impl/tbe/accumulate_n_v2.py b/mindspore/ops/_op_impl/tbe/accumulate_n_v2.py index fdd72a9494..b16233c37e 100644 --- a/mindspore/ops/_op_impl/tbe/accumulate_n_v2.py +++ b/mindspore/ops/_op_impl/tbe/accumulate_n_v2.py @@ -27,11 +27,11 @@ accumulate_n_v2_op_info = TBERegOp("AccumulateNV2") \ .input(0, "x", False, "dynamic", "all") \ .output(0, "y", False, "required", "all") \ .op_pattern("broadcast") \ - .dtype_format(DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default) \ - .dtype_format(DataType.I32_Default, DataType.I32_Default) \ - .dtype_format(DataType.I8_Default, DataType.I8_Default) \ - .dtype_format(DataType.U8_Default, DataType.U8_Default) \ + .dtype_format(DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None) \ + .dtype_format(DataType.I32_None, DataType.I32_None) \ + .dtype_format(DataType.I8_None, DataType.I8_None) \ + .dtype_format(DataType.U8_None, DataType.U8_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/approximate_equal.py b/mindspore/ops/_op_impl/tbe/approximate_equal.py index 62b8a0c16d..195918a19d 100644 --- a/mindspore/ops/_op_impl/tbe/approximate_equal.py +++ b/mindspore/ops/_op_impl/tbe/approximate_equal.py @@ -28,10 +28,8 @@ approximate_equal_op_info = TBERegOp("ApproximateEqual") \ .input(0, "x1", False, "required", "all") \ .input(1, "x2", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.F16_Default, DataType.F16_Default, DataType.BOOL_Default) \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD, DataType.BOOL_5HD) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default, DataType.BOOL_Default) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD, DataType.BOOL_5HD) \ + .dtype_format(DataType.F16_None, DataType.F16_None, DataType.BOOL_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None, DataType.BOOL_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/binary_cross_entropy.py b/mindspore/ops/_op_impl/tbe/binary_cross_entropy.py index bbb4dcab0b..1f73d3bfe7 100644 --- a/mindspore/ops/_op_impl/tbe/binary_cross_entropy.py +++ b/mindspore/ops/_op_impl/tbe/binary_cross_entropy.py @@ -28,10 +28,8 @@ binary_cross_entropy_op_info = TBERegOp("BinaryCrossEntropy") \ .input(1, "y", False, "required", "all") \ .input(2, "weight", False, "optional", "all") \ .output(0, "output", False, "required", "all") \ - .dtype_format(DataType.F16_Default, DataType.F16_Default, DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD, DataType.F16_5HD, DataType.F16_5HD) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default, DataType.F32_Default, DataType.F32_Default) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD, DataType.F32_5HD, DataType.F32_5HD) \ + .op_pattern("dynamicFormat") \ + .dtype_format(DataType.None_None, DataType.None_None, DataType.None_None, DataType.None_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/lin_space.py b/mindspore/ops/_op_impl/tbe/lin_space.py index aed41e80d4..6e474c50ea 100644 --- a/mindspore/ops/_op_impl/tbe/lin_space.py +++ b/mindspore/ops/_op_impl/tbe/lin_space.py @@ -29,8 +29,8 @@ lin_space_op_info = TBERegOp("LinSpace") \ .input(2, "stop", False, "required", "all") \ .input(3, "num", False, "required", "all") \ .output(0, "output", False, "required", "all") \ - .dtype_format(DataType.F32_Default, DataType.F32_Default, DataType.F32_Default, DataType.I32_Default, - DataType.F32_Default,) \ + .dtype_format(DataType.F32_None, DataType.F32_None, DataType.F32_None, DataType.I32_None, + DataType.F32_None,) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/mod.py b/mindspore/ops/_op_impl/tbe/mod.py index c8fecd697a..334a3e3820 100644 --- a/mindspore/ops/_op_impl/tbe/mod.py +++ b/mindspore/ops/_op_impl/tbe/mod.py @@ -26,16 +26,12 @@ mod_op_info = TBERegOp("Mod") \ .input(0, "x1", False, "required", "all") \ .input(1, "x2", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.I8_Default, DataType.I8_Default, DataType.I8_Default) \ - .dtype_format(DataType.I8_5HD, DataType.I8_5HD, DataType.I8_5HD) \ - .dtype_format(DataType.U8_Default, DataType.U8_Default, DataType.U8_Default) \ - .dtype_format(DataType.U8_5HD, DataType.U8_5HD, DataType.U8_5HD) \ - .dtype_format(DataType.I32_Default, DataType.I32_Default, DataType.I32_Default) \ - .dtype_format(DataType.I32_5HD, DataType.I32_5HD, DataType.I32_5HD) \ - .dtype_format(DataType.F16_Default, DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F16_5HD, DataType.F16_5HD, DataType.F16_5HD) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default, DataType.F32_Default) \ - .dtype_format(DataType.F32_5HD, DataType.F32_5HD, DataType.F32_5HD) \ + .op_pattern("broadcast") \ + .dtype_format(DataType.I8_None, DataType.I8_None, DataType.I8_None) \ + .dtype_format(DataType.U8_None, DataType.U8_None, DataType.U8_None) \ + .dtype_format(DataType.I32_None, DataType.I32_None, DataType.I32_None) \ + .dtype_format(DataType.F16_None, DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/reduce_mean_d.py b/mindspore/ops/_op_impl/tbe/reduce_mean_d.py index e427b34869..a0890816d2 100644 --- a/mindspore/ops/_op_impl/tbe/reduce_mean_d.py +++ b/mindspore/ops/_op_impl/tbe/reduce_mean_d.py @@ -27,10 +27,11 @@ reduce_mean_d_op_info = TBERegOp("ReduceMeanD") \ .attr("keep_dims", "optional", "bool", "all") \ .input(0, "x", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.I8_Default, DataType.I8_Default) \ - .dtype_format(DataType.U8_Default, DataType.U8_Default) \ - .dtype_format(DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default) \ + .op_pattern("reduce") \ + .dtype_format(DataType.I8_None, DataType.I8_None) \ + .dtype_format(DataType.U8_None, DataType.U8_None) \ + .dtype_format(DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/softsign.py b/mindspore/ops/_op_impl/tbe/softsign.py index 9f1609bf0a..97f560c939 100644 --- a/mindspore/ops/_op_impl/tbe/softsign.py +++ b/mindspore/ops/_op_impl/tbe/softsign.py @@ -26,8 +26,8 @@ softsign_op_info = TBERegOp("Softsign") \ .op_pattern("formatAgnostic") \ .input(0, "x", False, "required", "all") \ .output(0, "y", False, "required", "all") \ - .dtype_format(DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default) \ + .dtype_format(DataType.F16_None, DataType.F16_None) \ + .dtype_format(DataType.F32_None, DataType.F32_None) \ .get_op_info() diff --git a/mindspore/ops/_op_impl/tbe/splitv.py b/mindspore/ops/_op_impl/tbe/splitv.py index 29f65c7e87..6948524bd7 100644 --- a/mindspore/ops/_op_impl/tbe/splitv.py +++ b/mindspore/ops/_op_impl/tbe/splitv.py @@ -29,28 +29,7 @@ split_v_op_info = TBERegOp("SplitV") \ .input(0, "x", False, "required", "all") \ .output(0, "y", False, "dynamic", "all") \ .op_pattern("dynamicFormat") \ - .dtype_format(DataType.BOOL_Default, DataType.BOOL_Default) \ - .dtype_format(DataType.BOOL_NHWC, DataType.BOOL_NHWC) \ - .dtype_format(DataType.I8_Default, DataType.I8_Default) \ - .dtype_format(DataType.I8_NHWC, DataType.I8_NHWC) \ - .dtype_format(DataType.U8_Default, DataType.U8_Default) \ - .dtype_format(DataType.U8_NHWC, DataType.U8_NHWC) \ - .dtype_format(DataType.I16_Default, DataType.I16_Default) \ - .dtype_format(DataType.I16_NHWC, DataType.I16_NHWC) \ - .dtype_format(DataType.U16_Default, DataType.U16_Default) \ - .dtype_format(DataType.U16_NHWC, DataType.U16_NHWC) \ - .dtype_format(DataType.I32_Default, DataType.I32_Default) \ - .dtype_format(DataType.I32_NHWC, DataType.I32_NHWC) \ - .dtype_format(DataType.U32_Default, DataType.U32_Default) \ - .dtype_format(DataType.U32_NHWC, DataType.U32_NHWC) \ - .dtype_format(DataType.I64_Default, DataType.I64_Default) \ - .dtype_format(DataType.I64_NHWC, DataType.I64_NHWC) \ - .dtype_format(DataType.U64_Default, DataType.U64_Default) \ - .dtype_format(DataType.U64_NHWC, DataType.U64_NHWC) \ - .dtype_format(DataType.F16_Default, DataType.F16_Default) \ - .dtype_format(DataType.F16_NHWC, DataType.F16_NHWC) \ - .dtype_format(DataType.F32_Default, DataType.F32_Default) \ - .dtype_format(DataType.F32_NHWC, DataType.F32_NHWC) \ + .dtype_format(DataType.None_None, DataType.None_None) \ .get_op_info() diff --git a/mindspore/ops/operations/array_ops.py b/mindspore/ops/operations/array_ops.py index 8e9ecfea95..125a2a0fe0 100644 --- a/mindspore/ops/operations/array_ops.py +++ b/mindspore/ops/operations/array_ops.py @@ -3055,7 +3055,7 @@ class InplaceUpdate(PrimitiveWithInfer): raise ValueError(f'The value of indices must be in [0, {x_shape[0]}), but got {i}.') x_rank = len(x_shape) for idx in range(x_rank)[1:]: - validator.check("x dim %d" % idx, x_shape[idx], 'v dim %d' % idx, v_shape[idx], Rel.EQ, self.name) + validator.check('v dim %d' % idx, v_shape[idx], "x dim %d" % idx, x_shape[idx], Rel.EQ, self.name) return x_shape diff --git a/mindspore/ops/operations/math_ops.py b/mindspore/ops/operations/math_ops.py index d7fa77787e..b4a684d2f7 100644 --- a/mindspore/ops/operations/math_ops.py +++ b/mindspore/ops/operations/math_ops.py @@ -947,7 +947,7 @@ class InplaceAdd(PrimitiveWithInfer): raise ValueError(f'The value of indices must be in [0, {x_shape[0]}), but got {i}.') x_rank = len(x_shape) for idx in range(x_rank)[1:]: - validator.check("x dim %d" % idx, x_shape[idx], 'v dim %d' % idx, v_shape[idx], Rel.EQ, self.name) + validator.check('v dim %d' % idx, v_shape[idx], "x dim %d" % idx, x_shape[idx], Rel.EQ, self.name) return x_shape @@ -1005,7 +1005,7 @@ class InplaceSub(PrimitiveWithInfer): raise ValueError(f'The value of indices must be in [0, {x_shape[0]}), but got {i}.') x_rank = len(x_shape) for idx in range(x_rank)[1:]: - validator.check("x dim %d" % idx, x_shape[idx], 'v dim %d' % idx, v_shape[idx], Rel.EQ, self.name) + validator.check('v dim %d' % idx, v_shape[idx], "x dim %d" % idx, x_shape[idx], Rel.EQ, self.name) return x_shape From df0d89ed8bd7bbb87c2cbc2f35a50e3733717399 Mon Sep 17 00:00:00 2001 From: zhousiyi Date: Mon, 6 Jul 2020 01:11:17 +0000 Subject: [PATCH 251/254] use addparam to optimize setparam to reduce manager overhead --- mindspore/ccsrc/ir/func_graph.cc | 10 ++---- mindspore/ccsrc/ir/func_graph.h | 1 + mindspore/ccsrc/ir/manager.cc | 60 ++++++++++++++++++++++---------- mindspore/ccsrc/ir/manager.h | 16 ++++++++- 4 files changed, 59 insertions(+), 28 deletions(-) diff --git a/mindspore/ccsrc/ir/func_graph.cc b/mindspore/ccsrc/ir/func_graph.cc index 803c910d1d..4e01e9003f 100644 --- a/mindspore/ccsrc/ir/func_graph.cc +++ b/mindspore/ccsrc/ir/func_graph.cc @@ -68,9 +68,7 @@ ParameterPtr FuncGraph::add_parameter() { void FuncGraph::add_parameter(const ParameterPtr &p) { if (manager_.lock()) { - std::vector new_params = parameters_; - new_params.push_back(p); - manager_.lock()->SetParameters(shared_from_base(), new_params); + manager_.lock()->AddParameter(shared_from_base(), p); } else { parameters_.push_back(p); } @@ -82,12 +80,8 @@ ParameterPtr FuncGraph::AddWeightParameter(const std::string &name) { p->set_name(name); p->debug_info()->set_name(name); - std::vector new_params = parameters_; - // append parameter - new_params.push_back(p); - if (manager_.lock()) { - manager_.lock()->SetParameters(shared_from_base(), new_params); + manager_.lock()->AddParameter(shared_from_base(), p); } else { parameters_.push_back(p); } diff --git a/mindspore/ccsrc/ir/func_graph.h b/mindspore/ccsrc/ir/func_graph.h index 5f09dfe6b5..b1be892a53 100644 --- a/mindspore/ccsrc/ir/func_graph.h +++ b/mindspore/ccsrc/ir/func_graph.h @@ -158,6 +158,7 @@ class FuncGraph : public FuncGraphBase { const std::vector ¶meters() const { return parameters_; } virtual ParameterPtr add_parameter(); void add_parameter(const ParameterPtr &p); + void append_parameter(const ParameterPtr &p) { parameters_.push_back(p); } void set_parameters(const std::vector ¶ms) { parameters_ = params; } // add a weight parameter with specific name ParameterPtr AddWeightParameter(const std::string &name); diff --git a/mindspore/ccsrc/ir/manager.cc b/mindspore/ccsrc/ir/manager.cc index 291a752405..cf56500aea 100644 --- a/mindspore/ccsrc/ir/manager.cc +++ b/mindspore/ccsrc/ir/manager.cc @@ -420,6 +420,12 @@ void FuncGraphManager::SetParameters(const FuncGraphPtr &fg, const std::vector &changes, EdgeTupl for (auto &iter : changes) { auto operation = iter.op; auto args = iter.args; - if (operation == Change::kTxSetEdge) { - auto edge = args.cast(); - auto old_node = edge.root_node->input(edge.index); - (*rm_edges)[std::make_pair(edge.root_node, std::make_pair(edge.index, old_node))] += 1; - (*add_edges)[std::make_pair(edge.root_node, std::make_pair(edge.index, edge.new_node))] += 1; - (*rms)[old_node] += 1; - (*adds)[edge.new_node] += 1; - edge.root_node->set_input(edge.index, edge.new_node); - } else if (operation == Change::kTxSetParams) { - auto param = args.cast(); - MS_EXCEPTION_IF_NULL(param.func_graph); - auto old_parameters = param.func_graph->parameters(); - for (auto &p : param.params) { - (*adds)[p] += 1; - } - for (auto &p : old_parameters) { - (*rms)[p] += 1; - } - param.func_graph->set_parameters(param.params); + switch (operation) { + case Change::kTxSetEdge: { + auto edge = args.cast(); + auto old_node = edge.root_node->input(edge.index); + (*rm_edges)[std::make_pair(edge.root_node, std::make_pair(edge.index, old_node))] += 1; + (*add_edges)[std::make_pair(edge.root_node, std::make_pair(edge.index, edge.new_node))] += 1; + (*rms)[old_node] += 1; + (*adds)[edge.new_node] += 1; + edge.root_node->set_input(edge.index, edge.new_node); + } break; + case Change::kTxSetParams: { + auto param = args.cast(); + MS_EXCEPTION_IF_NULL(param.func_graph); + auto old_parameters = param.func_graph->parameters(); + for (auto &p : param.params) { + (*adds)[p] += 1; + } + for (auto &p : old_parameters) { + (*rms)[p] += 1; + } + param.func_graph->set_parameters(param.params); + } break; + case Change::kTxAddParam: { + auto param = args.cast(); + MS_EXCEPTION_IF_NULL(param.func_graph); + (*adds)[param.param] += 1; + auto param_node = param.param->cast(); + param.func_graph->append_parameter(param_node); + } break; + default: + break; } } } @@ -599,6 +617,10 @@ void FuncGraphTransaction::SetParameters(FuncGraphPtr fg, const std::vector { void KeepRoots(const std::vector &roots = {}); void RemoveRoots(); void SetParameters(const FuncGraphPtr &fg, const std::vector ¶meters); + void AddParameter(const FuncGraphPtr &fg, const AnfNodePtr ¶meter); void MaybeDropFuncGraphs(const FuncGraphSet &func_graphs, bool ignore_users = false); bool Replace(const AnfNodePtr &old_node, const AnfNodePtr &new_node); void SetEdge(const AnfNodePtr &node, int index, const AnfNodePtr &value); @@ -400,6 +401,7 @@ class FuncGraphTransaction { // set parameters of a func graph void SetParameters(FuncGraphPtr fg, const std::vector ¶ms); + void AddParameter(FuncGraphPtr fg, const AnfNodePtr ¶m); // replace old_node with new_node bool Replace(const AnfNodePtr &old_node, const AnfNodePtr &new_node); @@ -427,6 +429,18 @@ struct ArgsOfSetParams { } }; +// args for add param +struct ArgsOfAddParam { + FuncGraphPtr func_graph; + AnfNodePtr param; + bool operator==(const ArgsOfAddParam &other) const { return &other == this; } + + friend std::ostream &operator<<(std::ostream &os, const ArgsOfAddParam &) { + os << "[ArgsOfAddParam]"; + return os; + } +}; + // args for set edge struct ArgsOfSetEdge { CNodePtr root_node; @@ -441,7 +455,7 @@ struct ArgsOfSetEdge { }; struct Change { - enum OpName { kTxSetParams, kTxSetEdge }; + enum OpName { kTxSetParams, kTxSetEdge, kTxAddParam }; OpName op; Any args; Change(OpName name, const Any ¶) : op(name), args(para) {} From 9523ab19ef1c0598d696d8439b142d06dc25b62b Mon Sep 17 00:00:00 2001 From: limingqi107 Date: Sat, 4 Jul 2020 10:24:26 +0800 Subject: [PATCH 252/254] Revert "optimize the graph output of all nop node" This reverts commit 4949f6ca3989d4cd2abecc2264e643d037d2b9ca. --- mindspore/ccsrc/device/kernel_runtime.cc | 27 ++++++------------------ mindspore/ccsrc/device/kernel_runtime.h | 4 ++-- mindspore/ccsrc/session/session_basic.cc | 10 +-------- 3 files changed, 9 insertions(+), 32 deletions(-) diff --git a/mindspore/ccsrc/device/kernel_runtime.cc b/mindspore/ccsrc/device/kernel_runtime.cc index 9672384e70..27cf1dfc92 100644 --- a/mindspore/ccsrc/device/kernel_runtime.cc +++ b/mindspore/ccsrc/device/kernel_runtime.cc @@ -30,7 +30,6 @@ #include "kernel/common_utils.h" #include "kernel/oplib/oplib.h" #include "ir/value.h" -#include "pre_activate/common/helper.h" using mindspore::kernel::Address; using mindspore::kernel::AddressPtr; @@ -637,7 +636,7 @@ void KernelRuntime::AssignWorkSpaceMem(int flag, const AnfNodePtr &node) { } } -void KernelRuntime::GenLaunchArgs(const session::KernelGraph &graph, const mindspore::AnfNodePtr &kernel, +void KernelRuntime::GenLaunchArgs(const mindspore::kernel::KernelMod &kernel_mod, const mindspore::AnfNodePtr &kernel, AddressPtrList *kernel_inputs, AddressPtrList *const kernel_workspaces, AddressPtrList *kernel_outputs) { MS_EXCEPTION_IF_NULL(kernel); @@ -649,15 +648,9 @@ void KernelRuntime::GenLaunchArgs(const session::KernelGraph &graph, const minds if (AnfAlgo::GetCNodeName(cnode) == kAtomicAddrCleanOpName) { return GenAddrCleanLaunchArgs(cnode, kernel_inputs); } - auto is_all_nop_node = opt::IsAllNopNode(&graph); for (size_t i = 0; i < AnfAlgo::GetInputTensorNum(kernel); ++i) { auto real_input = AnfAlgo::GetRealInputIndex(kernel, i); - DeviceAddressPtr device_address; - if (is_all_nop_node) { - device_address = AnfAlgo::GetPrevNodeMutableOutputAddr(kernel, real_input, false); - } else { - device_address = AnfAlgo::GetPrevNodeMutableOutputAddr(kernel, real_input, true); - } + auto device_address = AnfAlgo::GetPrevNodeOutputAddr(kernel, real_input); MS_EXCEPTION_IF_NULL(device_address); kernel::AddressPtr input = std::make_shared(); MS_EXCEPTION_IF_NULL(input); @@ -667,16 +660,8 @@ void KernelRuntime::GenLaunchArgs(const session::KernelGraph &graph, const minds kernel_inputs->emplace_back(input); } - auto kernel_mod = AnfAlgo::GetKernelMod(kernel); - MS_EXCEPTION_IF_NULL(kernel_mod); - for (size_t i = 0; i < kernel_mod->GetOutputSizeList().size(); ++i) { - DeviceAddressPtr device_address; - if (is_all_nop_node) { - device_address = AnfAlgo::GetMutableOutputAddr(kernel, i, false); - } else { - device_address = AnfAlgo::GetMutableOutputAddr(kernel, i, true); - } - MS_EXCEPTION_IF_NULL(device_address); + for (size_t i = 0; i < kernel_mod.GetOutputSizeList().size(); ++i) { + auto device_address = AnfAlgo::GetOutputAddr(kernel, i); kernel::AddressPtr output = std::make_shared(); MS_EXCEPTION_IF_NULL(output); output->addr = device_address->ptr_; @@ -685,7 +670,7 @@ void KernelRuntime::GenLaunchArgs(const session::KernelGraph &graph, const minds kernel_outputs->emplace_back(output); } - for (size_t i = 0; i < kernel_mod->GetWorkspaceSizeList().size(); ++i) { + for (size_t i = 0; i < kernel_mod.GetWorkspaceSizeList().size(); ++i) { auto device_address = AnfAlgo::GetWorkspaceAddr(kernel, i); kernel::AddressPtr workspace = std::make_shared(); MS_EXCEPTION_IF_NULL(workspace); @@ -740,7 +725,7 @@ bool KernelRuntime::LaunchKernelMod(const session::KernelGraph &graph) { AddressPtrList kernel_inputs; AddressPtrList kernel_workspaces; AddressPtrList kernel_outputs; - GenLaunchArgs(graph, kernel, &kernel_inputs, &kernel_workspaces, &kernel_outputs); + GenLaunchArgs(*kernel_mod, kernel, &kernel_inputs, &kernel_workspaces, &kernel_outputs); auto ret = kernel_mod->Launch(kernel_inputs, kernel_workspaces, kernel_outputs, stream_); if (!ret) { MS_LOG(ERROR) << "Launch kernel failed."; diff --git a/mindspore/ccsrc/device/kernel_runtime.h b/mindspore/ccsrc/device/kernel_runtime.h index 656ef8e2e6..8c6a5eb19b 100644 --- a/mindspore/ccsrc/device/kernel_runtime.h +++ b/mindspore/ccsrc/device/kernel_runtime.h @@ -96,8 +96,8 @@ class KernelRuntime { private: void AssignStaticMemoryOutput(session::KernelGraph *graph); - void GenLaunchArgs(const session::KernelGraph &graph, const AnfNodePtr &kernel, AddressPtrList *kernel_inputs, - AddressPtrList *kernel_workspaces, AddressPtrList *kernel_outputs); + void GenLaunchArgs(const mindspore::kernel::KernelMod &kernel_mod, const AnfNodePtr &kernel, + AddressPtrList *kernel_inputs, AddressPtrList *kernel_workspaces, AddressPtrList *kernel_outputs); bool LaunchKernelMod(const session::KernelGraph &graph); void GenAddrCleanLaunchArgs(const CNodePtr &cnode, AddressPtrList *kernel_inputs); size_t CountNodeDeviceMemorySize(const AnfNodePtr &node, size_t output_index); diff --git a/mindspore/ccsrc/session/session_basic.cc b/mindspore/ccsrc/session/session_basic.cc index 3f893b6090..91e430182c 100644 --- a/mindspore/ccsrc/session/session_basic.cc +++ b/mindspore/ccsrc/session/session_basic.cc @@ -81,15 +81,7 @@ BaseRef CreateOneTensor(const AnfNodePtr &node, size_t output_index, const Kerne } } // if proccess reach here,it remarks item_with_index is a real node(Parameter,or executable CNode) - DeviceAddressPtr address; - auto is_all_nop_node = opt::IsAllNopNode(&graph); - if (is_all_nop_node) { - // The graph does not remove the nop node. - address = AnfAlgo::GetMutableOutputAddr(node, output_index, false); - } else { - // The graph removes the nop node. - address = AnfAlgo::GetMutableOutputAddr(node, output_index, true); - } + auto address = AnfAlgo::GetMutableOutputAddr(node, output_index); MS_EXCEPTION_IF_NULL(address); auto shape = AnfAlgo::GetOutputInferShape(node, output_index); TypeId type_id = kNumberTypeFloat32; From f9088f46bd4a442137ec80143747d37e092616ad Mon Sep 17 00:00:00 2001 From: xutianchun Date: Fri, 3 Jul 2020 16:56:06 +0800 Subject: [PATCH 253/254] change crop_size from tensor to tuple --- mindspore/ops/operations/image_ops.py | 27 ++++++++++--------- .../test_aicpu_ops/test_crop_and_reszie.py | 4 +-- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/mindspore/ops/operations/image_ops.py b/mindspore/ops/operations/image_ops.py index bd26eba906..1e366b5ea6 100644 --- a/mindspore/ops/operations/image_ops.py +++ b/mindspore/ops/operations/image_ops.py @@ -45,10 +45,9 @@ class CropAndResize(PrimitiveWithInfer): extrapolate the input image values. Types allowd: float32. - **box_index** (Tensor) - A 1-D tensor of shape [num_boxes] with int32 values in [0, batch). The value of box_ind[i] specifies the image that the i-th box refers to. Types allowd: int32. - - **crop_size** (Tensor) - Only constant value is allowd. Types allowed: int32. - A 1-D tensor of 2 elements, size = [crop_height, crop_width]. - All cropped image patches are resized to this size. The aspect ratio of the image content is not preserved. - Both crop_height and crop_width need to be positive. + - **crop_size** (Tuple[int]) - A tuple of two int32 elements: (crop_height, crop_width). + Only constant value is allowed. All cropped image patches are resized to this size. + The aspect ratio of the image content is not preserved. Both crop_height and crop_width need to be positive. Outputs: A 4-D tensor of shape [num_boxes, crop_height, crop_width, depth] with type: float32. @@ -70,8 +69,8 @@ class CropAndResize(PrimitiveWithInfer): >>> image = np.random.normal(size=[BATCH_SIZE, IMAGE_HEIGHT, IMAGE_WIDTH, CHANNELS]).astype(np.float32) >>> boxes = np.random.uniform(size=[NUM_BOXES, 4]).astype(np.float32) >>> box_index = np.random.uniform(size=[NUM_BOXES], low=0, high=BATCH_SIZE).astype(np.int32) - >>> crop_size = np.array([24, 24]).astype(np.int32) - >>> crop_and_resize = CropAndResizeNet(crop_size=Tensor(crop_size)) + >>> crop_size = (24, 24) + >>> crop_and_resize = CropAndResizeNet(crop_size=crop_size) >>> output = crop_and_resize(Tensor(image), Tensor(boxes), Tensor(box_index)) >>> print(output.asnumpy()) """ @@ -91,11 +90,10 @@ class CropAndResize(PrimitiveWithInfer): x_shape = list(x['shape']) boxes_shape = list(boxes['shape']) box_index_shape = list(box_index['shape']) - crop_size_shape = list(crop_size['shape']) # get value if crop_size['value'] is None: - raise ValueError(f"For {self.name}, crop_size must be const.") - crop_size_value = crop_size['value'].asnumpy() + raise ValueError(f"For {self.name}, crop_size must be constant.") + crop_size_value = crop_size['value'] # get dtype x_dtype = x['dtype'] boxes_dtype = boxes['dtype'] @@ -107,15 +105,20 @@ class CropAndResize(PrimitiveWithInfer): mstype.float32, mstype.float64, mstype.uint8, mstype.uint16], self.name) validator.check_tensor_type_same({"boxes": boxes_dtype}, [mstype.float32], self.name) validator.check_tensor_type_same({"box_index": box_index_dtype}, [mstype.int32], self.name) - validator.check_tensor_type_same({"crop_size": crop_size_dtype}, [mstype.int32], self.name) + validator.check_value_type("crop_size", crop_size_value, [tuple], self.name) # check input shape rank validator.check("x rank", len(x_shape), "expected", 4, Rel.EQ, self.name) validator.check("boxes rank", len(boxes_shape), "expected", 2, Rel.EQ, self.name) validator.check("box_index rank", len(box_index_shape), "expected", 1, Rel.EQ, self.name) - validator.check("crop_size rank", len(crop_size_shape), "expected", 1, Rel.EQ, self.name) - + validator.check("crop_size size", len(crop_size_value), "expected", 2, Rel.EQ, self.name) validator.check("boxes dim_0", boxes_shape[0], "box_index dim_0", box_index_shape[0], Rel.EQ, self.name) validator.check("boxes dim_1", boxes_shape[1], "expected", 4, Rel.EQ, self.name) + # check crop_size_value + validator.check("crop_height", crop_size_value[0], "minimum", 0, Rel.GT, self.name) + validator.check("crop_width", crop_size_value[1], "minimum", 0, Rel.GT, self.name) + # check crop_size element type + validator.check("crop_height dtype", crop_size_dtype[0], mstype.int32, self.name) + validator.check("crop_width dtype", crop_size_dtype[1], mstype.int32, self.name) num_boxes = boxes_shape[0] crop_height = crop_size_value[0] diff --git a/tests/st/ops/ascend/test_aicpu_ops/test_crop_and_reszie.py b/tests/st/ops/ascend/test_aicpu_ops/test_crop_and_reszie.py index 92f7fb6b42..b1082f7226 100644 --- a/tests/st/ops/ascend/test_aicpu_ops/test_crop_and_reszie.py +++ b/tests/st/ops/ascend/test_aicpu_ops/test_crop_and_reszie.py @@ -43,7 +43,7 @@ def test_net_float32(): image = np.random.normal(size=[batch_size, image_height, image_width, channels]).astype(np.float32) boxes = np.random.uniform(size=[num_boxes, 4]).astype(np.float32) box_index = np.random.uniform(size=[num_boxes], low=0, high=batch_size).astype(np.int32) - crop_size = np.array([24, 24]).astype(np.int32) - net = Net(crop_size=Tensor(crop_size)) + crop_size = (24, 24) + net = Net(crop_size=crop_size) output = net(Tensor(image), Tensor(boxes), Tensor(box_index)) print(output.asnumpy()) From b1d6ef0e888e5481071e012958ba9a30839fb3db Mon Sep 17 00:00:00 2001 From: baihuawei Date: Mon, 6 Jul 2020 09:39:30 +0800 Subject: [PATCH 254/254] fix GatherV2 index out of bounds --- .../ccsrc/kernel/cpu/gather_cpu_kernel.cc | 46 ++++++++++--------- 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/mindspore/ccsrc/kernel/cpu/gather_cpu_kernel.cc b/mindspore/ccsrc/kernel/cpu/gather_cpu_kernel.cc index 9117a533c8..28090817cb 100644 --- a/mindspore/ccsrc/kernel/cpu/gather_cpu_kernel.cc +++ b/mindspore/ccsrc/kernel/cpu/gather_cpu_kernel.cc @@ -21,17 +21,14 @@ namespace mindspore { namespace kernel { void GatherV2CPUKernel::InitKernel(const CNodePtr &kernel_node) { CheckParam(kernel_node); - input_shape_ = AnfAlgo::GetPrevNodeOutputInferShape(kernel_node, 0); indices_shape_ = AnfAlgo::GetPrevNodeOutputInferShape(kernel_node, 1); output_shape_ = AnfAlgo::GetOutputInferShape(kernel_node, 0); - axis_ = AnfAlgo::GetNodeAttr(kernel_node, AXIS); if (axis_ < 0) { axis_ = axis_ + SizeToInt(input_shape_.size()); } axis_ += 4 - input_shape_.size(); - CPUKernelUtils::ExpandDimsTo4(&input_shape_); CPUKernelUtils::ExpandDimsTo4(&output_shape_); } @@ -44,7 +41,6 @@ bool GatherV2CPUKernel::Launch(const std::vector &inputs, size_t dim0 = input_shape_[0]; size_t dim1 = input_shape_[1]; size_t dim2 = input_shape_[2]; - if (axis_ == 3) { for (size_t i = 0; i < dim0; ++i) { for (size_t j = 0; j < dim1; ++j) { @@ -66,7 +62,6 @@ bool GatherV2CPUKernel::Launch(const std::vector &inputs, } else if (axis_ == 0) { CopyDataToOutput(inputs, 0, 0, 0, &output_addr, &buff_size); } - return true; } @@ -75,34 +70,43 @@ void GatherV2CPUKernel::CopyDataToOutput(const std::vector & auto input_addr = reinterpret_cast(inputs[0]->addr); auto indices_addr = reinterpret_cast(inputs[1]->addr); size_t elem_num = inputs[1]->size / 4; + size_t num = CPUKernelUtils::GetElementNumOnAxis(input_shape_, axis_); for (size_t i = 0; i < elem_num; ++i) { - size_t index = IntToSize(indices_addr[i]); - size_t pos = 0; - if (axis_ == 3) { - pos = CPUKernelUtils::CalcOffset(input_shape_, dim0, dim1, dim2, index); - } else if (axis_ == 2) { - pos = CPUKernelUtils::CalcOffset(input_shape_, dim0, dim1, index, 0); - } else if (axis_ == 1) { - pos = CPUKernelUtils::CalcOffset(input_shape_, dim0, index, 0, 0); - } else if (axis_ == 0) { - pos = CPUKernelUtils::CalcOffset(input_shape_, index, 0, 0, 0); + if (indices_addr[i] < 0) { + MS_LOG(EXCEPTION) << "The indices value is less than 0."; } - size_t num = CPUKernelUtils::GetElementNumOnAxis(input_shape_, axis_); - auto ret = memcpy_s(*output_addr, *buff_size, input_addr + pos, num * sizeof(float)); - if (ret != EOK) { - MS_LOG(EXCEPTION) << "memcpy failed."; + size_t index = IntToSize(indices_addr[i]); + if (index >= input_shape_[IntToSize(axis_)]) { + auto ret = memset_s(*output_addr, *buff_size, 0., num * sizeof(float)); + if (ret != EOK) { + MS_LOG(EXCEPTION) << "memset failed."; + } + } else { + size_t pos = 0; + if (axis_ == 3) { + pos = CPUKernelUtils::CalcOffset(input_shape_, dim0, dim1, dim2, index); + } else if (axis_ == 2) { + pos = CPUKernelUtils::CalcOffset(input_shape_, dim0, dim1, index, 0); + } else if (axis_ == 1) { + pos = CPUKernelUtils::CalcOffset(input_shape_, dim0, index, 0, 0); + } else if (axis_ == 0) { + pos = CPUKernelUtils::CalcOffset(input_shape_, index, 0, 0, 0); + } + auto ret = memcpy_s(*output_addr, *buff_size, input_addr + pos, num * sizeof(float)); + if (ret != EOK) { + MS_LOG(EXCEPTION) << "memcpy failed."; + } } *output_addr += num; *buff_size -= num * sizeof(float); } -} +} // namespace kernel void GatherV2CPUKernel::CheckParam(const CNodePtr &kernel_node) { auto input_shape = AnfAlgo::GetPrevNodeOutputInferShape(kernel_node, 0); if (input_shape.size() > 4) { MS_LOG(EXCEPTION) << "Input dims is " << input_shape.size() << ", but GatherV2CPUKernel olny support 4d or lower."; } - size_t input_num = AnfAlgo::GetInputTensorNum(kernel_node); if (input_num != 2) { MS_LOG(EXCEPTION) << "Argument number is " << input_num << ", but GatherV2CPUKernel needs 2.";