!18220 add code of basic Communication for federated learning

Merge pull request !18220 from zhoushan33/flclient0611

mindspore/lite/java/java/fl_client/build.gradle

@@ -0,0 +1,128 @@
+/*
+ * This file was generated by the Gradle 'init' task.
+ *
+ * This generated file contains a sample Java Library project to get you started.
+ * For more details take a look at the Java Libraries chapter in the Gradle
+ * User Manual available at https://docs.gradle.org/6.3/userguide/java_library_plugin.html
+ */
+
+buildscript {
+    repositories {
+        maven {
+            url "https://plugins.gradle.org/m2/"
+        }
+    }
+    dependencies {
+        classpath "gradle.plugin.io.netifi:gradle-flatbuffers-plugin:1.0.7"
+    }
+}
+
+
+plugins {
+    // Apply the java-library plugin to add support for Java Library
+    id 'java-library'
+}
+
+apply plugin: "io.netifi.flatbuffers"
+
+repositories {
+    // Use jcenter for resolving dependencies.
+    // You can declare any Maven/Ivy/file repository here.
+    jcenter()
+    maven {
+        url "https://maven.springframework.org/release"
+    }
+    maven {
+        url "https://maven.restlet.com"
+    }
+    maven {
+        url "https://plugins.gradle.org/m2/"
+    }
+}
+
+dependencies {
+    // This dependency is exported to consumers, that is to say found on their compile classpath.
+    compile 'org.apache.commons:commons-math3:3.6.1'
+
+    // This dependency is used internally, and not exposed to consumers on their own compile classpath.
+    compile 'com.google.guava:guava:28.2-jre'
+
+    // Use JUnit test framework
+    testImplementation 'junit:junit:4.12'
+
+    // https://mvnrepository.com/artifact/com.squareup.okhttp3/okhttp
+    compile group: 'com.squareup.okhttp3', name: 'okhttp', version: '3.14.4'
+    testCompile group: 'com.squareup.okhttp3', name: 'mockwebserver', version: '3.14.4'
+
+    // https://mvnrepository.com/artifact/com.google.flatbuffers/flatbuffers-java
+    compile group: 'com.google.flatbuffers', name: 'flatbuffers-java', version: '1.11.0'
+
+    compile(group: 'org.bouncycastle',name: 'bcprov-jdk15on', version: '1.63')
+    implementation project(':common')
+    implementation project(':linux_x86')
+
+}
+
+import io.netifi.flatbuffers.plugin.tasks.FlatBuffers
+flatbuffers {
+    flatcPath = '../../../build/_deps/flatbuffers-src/_build/flatc'
+}
+task createFlatBuffers(type: FlatBuffers) {
+    println("-----------------executing task: createFlatBuffers-----------------")
+    inputDir = file("../../../../schema")
+    outputDir = file("src/main/java/")
+    language = 'java'
+    println("-----------------Finish task: createFlatBuffers-----------------")
+}
+
+
+task clearJar(type: Delete) {
+    delete 'build/libs/jarAAR/mindspore-lite-java-flclient.jar'
+    delete 'build/libs/jarX86/mindspore-lite-java-flclient.jar'
+}
+
+archivesBaseName ='mindspore-lite-java-flclient'
+
+task packFLJarAAR(type: Jar){
+    println("-----------------executing task: packFLJarAAR-----------------")
+    manifest {
+        attributes "Main-Class": "com.huawei.flclient.SyncFLJob"
+    }
+    from('build/classes/java/main')
+    from {
+        configurations.compile.collect {
+            it.isDirectory() ? it : zipTree(it)
+        }
+    }
+    destinationDirectory = file('build/libs/jarAAR')
+    println("-----------------Finish task: packFLJarAAR-----------------")
+
+}
+
+task flReleaseJarAAR (type:Exec, dependsOn: ['packFLJarAAR']){
+    println("-----------------executing task: flReleaseJarAAR-----------------")
+    commandLine "zip", "-d", "./build/libs/jarAAR/mindspore-lite-java-flclient.jar",'META-INF/.SF',  'META-INF/.RSA',  'META-INF/*SF'
+    println("-----------------zip -d ./build/libs/jarAAR/mindspore-lite-java-flclient.jar-----------------")
+}
+
+task packFLJarX86(type: Jar){
+    println("-----------------executing task: packFLJarX86-----------------")
+    manifest {
+        attributes "Main-Class": "com.huawei.flclient.SyncFLJob"
+    }
+    from ('build/classes/java/main')
+    from ('../common/build/classes/java/main')
+    from ('../linux_x86/build/classes/java/main')
+    from {
+        configurations.compile.collect {
+            it.isDirectory() ? it : zipTree(it) }
+    }
+    destinationDirectory = file('build/libs/jarX86')
+    println("-----------------Finish task: packFLJarX86-----------------")
+}
+
+task flReleaseJarX86 (type:Exec, dependsOn: ['packFLJarX86']){
+    println("-----------------executing task: flReleaseJarX86-----------------")
+    commandLine "zip", "-d", "./build/libs/jarX86/mindspore-lite-java-flclient.jar",'META-INF/.SF',  'META-INF/.RSA',  'META-INF/*SF'
+    println("-----------------zip -d ./build/libs/jarX86/mindspore-lite-java-flclient.jar-----------------")
+}

mindspore/lite/java/java/fl_client/settings.gradle

@@ -0,0 +1,12 @@
+/*
+ * This file was generated by the Gradle 'init' task.
+ *
+ * The settings file is used to specify which projects to include in your build.
+ *
+ * Detailed information about configuring a multi-project build in Gradle can be found
+ * in the user manual at https://docs.gradle.org/6.6/userguide/multi_project_builds.html
+ */
+
+includeFlat('common')
+includeFlat('linux_x86')
+rootProject.name = 'flclient'

mindspore/lite/java/java/fl_client/src/main/java/com/huawei/flclient/EarlyStopMod.java

@@ -0,0 +1,25 @@
+/**
+ * Copyright 2021 Huawei Technologies Co., Ltd
+ * <p>
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.huawei.flclient;
+
+public enum EarlyStopMod {
+    LOSS_DIFF,
+    LOSS_ABS,
+    WEIGHT_DIFF,
+    NOT_EARLY_STOP
+}
+
+

mindspore/lite/java/java/fl_client/src/main/java/com/huawei/flclient/EncryptLevel.java

@@ -0,0 +1,24 @@
+/**
+ * Copyright 2021 Huawei Technologies Co., Ltd
+ * <p>
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.huawei.flclient;
+
+public enum EncryptLevel {
+    PW_ENCRYPT,
+    DP_ENCRYPT,
+    NOT_ENCRYPT
+}
+
+

mindspore/lite/java/java/fl_client/src/main/java/com/huawei/flclient/FLClientStatus.java

@@ -0,0 +1,24 @@
+/**
+  * Copyright 2021 Huawei Technologies Co., Ltd
+  *
+  * Licensed under the Apache License, Version 2.0 (the "License");
+  * you may not use this file except in compliance with the License.
+  * You may obtain a copy of the License at
+  *
+  * http://www.apache.org/licenses/LICENSE-2.0
+  *
+  * Unless required by applicable law or agreed to in writing, software                                                           
+  * distributed under the License is distributed on an "AS IS" BASIS,                                                             
+  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.                                                      
+  * See the License for the specific language governing permissions and                                                           
+  * limitations under the License.
+  */
+
+package com.huawei.flclient;
+
+public enum FLClientStatus {
+    SUCCESS,
+    FAILED,
+    WAIT,
+    RESTART
+}

mindspore/lite/java/java/fl_client/src/main/java/com/huawei/flclient/FLJobResultCallback.java

@@ -0,0 +1,29 @@
+/**
+ * Copyright 2021 Huawei Technologies Co., Ltd
+ * <p>
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.huawei.flclient;
+
+import java.util.logging.Logger;
+
+public class FLJobResultCallback  implements IFLJobResultCallback{
+    private static final Logger logger = Logger.getLogger(FLJobResultCallback.class.toString());
+    public void onFlJobIterationFinished(String modelName, int iterationSeq, int resultCode) {
+        logger.info(Common.addTag("[onFlJobIterationFinished] modelName: " + modelName + " iterationSeq: " + iterationSeq + " resultCode: " + resultCode));
+    }
+    public void onFlJobFinished(String modelName, int iterationCount, int resultCode) {
+        logger.info(Common.addTag("[onFlJobFinished] modelName: " + modelName + " iterationCount: " + iterationCount + " resultCode: " + resultCode));
+    }
+
+}

mindspore/lite/java/java/fl_client/src/main/java/com/huawei/flclient/IAsyncCallBack.java

@@ -0,0 +1,25 @@
+/**
+  * Copyright 2021 Huawei Technologies Co., Ltd
+  *
+  * Licensed under the Apache License, Version 2.0 (the "License");
+  * you may not use this file except in compliance with the License.
+  * You may obtain a copy of the License at
+  *
+  * http://www.apache.org/licenses/LICENSE-2.0
+  *
+  * Unless required by applicable law or agreed to in writing, software                                                           
+  * distributed under the License is distributed on an "AS IS" BASIS,                                                             
+  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.                                                      
+  * See the License for the specific language governing permissions and                                                           
+  * limitations under the License.
+  */
+
+package com.huawei.flclient;
+
+import java.io.IOException;
+
+public interface IAsyncCallBack {
+    public FLClientStatus onFailure(IOException exception);
+
+    public FLClientStatus onResponse(byte[] msg);
+}

mindspore/lite/java/java/fl_client/src/main/java/com/huawei/flclient/IFLCommunication.java

@@ -0,0 +1,33 @@
+/**
+  * Copyright 2021 Huawei Technologies Co., Ltd
+  *
+  * Licensed under the Apache License, Version 2.0 (the "License");
+  * you may not use this file except in compliance with the License.
+  * You may obtain a copy of the License at
+  *
+  * http://www.apache.org/licenses/LICENSE-2.0
+  *
+  * Unless required by applicable law or agreed to in writing, software                                                           
+  * distributed under the License is distributed on an "AS IS" BASIS,                                                             
+  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.                                                      
+  * See the License for the specific language governing permissions and                                                           
+  * limitations under the License.
+  */
+
+package com.huawei.flclient;
+
+import java.util.concurrent.TimeoutException;
+
+/**
+ * @author smurf
+ *
+ */
+public interface IFLCommunication {
+
+    public void setTimeOut(int timeout) throws TimeoutException;
+
+    public byte[] syncRequest(String url, byte[] msg) throws Exception;
+
+    public void asyncRequest(String url, byte[] msg, IAsyncCallBack callBack) throws Exception;
+}
+

mindspore/lite/java/java/fl_client/src/main/java/com/huawei/flclient/IFLJobResultCallback.java

@@ -0,0 +1,34 @@
+/**
+ * Copyright 2021 Huawei Technologies Co., Ltd
+ * <p>
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.huawei.flclient;
+
+public interface IFLJobResultCallback {
+    /**
+     *  Called at the end of an iteration for Fl job
+     * @param modelName the name of model
+     * @param iterationSeq Iteration number
+     * @param resultCode Status Code
+     */
+    public void onFlJobIterationFinished(String modelName, int iterationSeq, int resultCode);
+
+    /**
+     * Called on completion for Fl job
+     * @param modelName the name of model
+     * @param iterationCount total Iteration numbers
+     * @param resultCode Status Code
+     */
+    public void onFlJobFinished(String modelName, int iterationCount, int resultCode);
+}

mindspore/lite/java/java/fl_client/src/main/java/com/huawei/flclient/SSLSocketFactoryTools.java

@@ -0,0 +1,151 @@
+/**
+ * Copyright 2021 Huawei Technologies Co., Ltd
+ * <p>
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.huawei.flclient;
+
+import javax.net.ssl.*;
+import java.io.FileInputStream;
+import java.io.InputStream;
+import java.security.*;
+import java.security.cert.CertificateException;
+import java.security.cert.CertificateFactory;
+import java.security.cert.X509Certificate;
+import java.util.logging.Logger;
+
+public class SSLSocketFactoryTools {
+    private static final Logger logger = Logger.getLogger(SSLSocketFactory.class.toString());
+    private FLParameter flParameter = FLParameter.getInstance();
+    private X509Certificate x509Certificate;
+    private SSLSocketFactory sslSocketFactory;
+    private SSLContext sslContext;
+    private MyTrustManager myTrustManager;
+    private static SSLSocketFactoryTools instance;
+    private SSLSocketFactoryTools() {
+        initSslSocketFactory();
+    }
+
+    private  void initSslSocketFactory(){
+        try {
+            sslContext = SSLContext.getInstance("TLS");
+            x509Certificate = readCert(flParameter.getCertPath());
+            myTrustManager = new MyTrustManager(x509Certificate);
+            sslContext.init(null, new TrustManager[]{
+                    myTrustManager
+            }, new java.security.SecureRandom());
+            sslSocketFactory = sslContext.getSocketFactory();
+
+        } catch (Exception e) {
+            logger.severe(Common.addTag("[SSLSocketFactoryTools]catch Exception in initSslSocketFactory: " + e.getMessage()));
+        }
+    }
+
+
+    public static SSLSocketFactoryTools getInstance() {
+        if (instance == null) {
+            instance=new SSLSocketFactoryTools();
+        }
+        return instance;
+    }
+
+
+    public  X509Certificate readCert(String assetName) {
+        InputStream inputStream = null;
+        try {
+            inputStream = new FileInputStream(assetName);
+        } catch (Exception e) {
+            logger.severe(Common.addTag("[SSLSocketFactoryTools] catch Exception of read inputStream in readCert: " + e.getMessage()));
+            return null;
+        }
+        X509Certificate cert = null;
+        try {
+            CertificateFactory cf = CertificateFactory.getInstance("X.509");
+            cert = (X509Certificate) cf.generateCertificate(inputStream);
+        } catch (Exception e) {
+            logger.severe(Common.addTag("[SSLSocketFactoryTools] catch Exception of creating CertificateFactory in readCert: " + e.getMessage()));
+        } finally {
+            try {
+                if (inputStream != null) {
+                    inputStream.close();
+                }
+            } catch (Throwable ex) {
+            }
+        }
+
+        return cert;
+    }
+
+    public HostnameVerifier getHostnameVerifier() {
+        return hostnameVerifier;
+    }
+
+    public SSLSocketFactory getmSslSocketFactory() {
+        return sslSocketFactory;
+    }
+
+    public MyTrustManager getmTrustManager() {
+        return myTrustManager;
+    }
+
+
+    private static final class MyTrustManager implements X509TrustManager {
+        X509Certificate cert;
+
+        MyTrustManager(X509Certificate cert) {
+            this.cert = cert;
+        }
+
+        @Override
+        public void checkClientTrusted(X509Certificate[] chain, String authType) throws CertificateException {
+        }
+
+
+        @Override
+        public void checkServerTrusted(X509Certificate[] chain, String authType) throws CertificateException {
+            for (X509Certificate cert : chain) {
+
+                // Make sure that it hasn't expired.
+                cert.checkValidity();
+
+                // Verify the certificate's public key chain.
+                try {
+                    cert.verify(((X509Certificate) this.cert).getPublicKey());
+                } catch (NoSuchAlgorithmException e) {
+                    logger.severe(Common.addTag("[SSLSocketFactoryTools] catch NoSuchAlgorithmException in checkServerTrusted: " + e.getMessage()));
+                } catch (InvalidKeyException e) {
+                    logger.severe(Common.addTag("[SSLSocketFactoryTools] catch InvalidKeyException in checkServerTrusted: " + e.getMessage()));
+                } catch (NoSuchProviderException e) {
+                    logger.severe(Common.addTag("[SSLSocketFactoryTools] catch NoSuchProviderException in checkServerTrusted: " + e.getMessage()));
+                } catch (SignatureException e) {
+                    logger.severe(Common.addTag("[SSLSocketFactoryTools] catch SignatureException in checkServerTrusted: " + e.getMessage()));
+                }
+            }
+        }
+
+        @Override
+        public X509Certificate[] getAcceptedIssuers() {
+            return new java.security.cert.X509Certificate[0];
+        }
+    }
+
+    private final HostnameVerifier hostnameVerifier = new HostnameVerifier() {
+        @Override
+        public boolean verify(String hostname, SSLSession session) {
+            logger.info(Common.addTag("[SSLSocketFactoryTools] server hostname: " + flParameter.getHostName()));
+            logger.info(Common.addTag("[SSLSocketFactoryTools] client request hostname: " + hostname));
+            return hostname.equals(flParameter.getHostName());
+        }
+    };
+
+}

mindspore/lite/java/java/fl_client/src/main/java/com/huawei/flclient/ServerMod.java

@@ -0,0 +1,21 @@
+/**
+ * Copyright 2021 Huawei Technologies Co., Ltd
+ * <p>
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.huawei.flclient;
+
+public enum ServerMod {
+    FEDERATED_LEARNING,
+    HYBRID_TRAINING
+}